feat: implement metrics tracking system[proof of concept]

package.json

@@ -43,10 +43,12 @@
     "@nestjs/mongoose": "^10.0.0",
     "@nestjs/passport": "^10.0.0",
     "@nestjs/platform-express": "^10.3.8",
+    "@nestjs/schedule": "^4.1.1",
     "@nestjs/swagger": "^7.1.7",
     "@nestjs/terminus": "^10.1.1",
     "@user-office-software/duo-logger": "^2.1.1",
     "@user-office-software/duo-message-broker": "^1.4.0",
+    "@willsoto/nestjs-prometheus": "^6.0.1",
     "bcrypt": "^5.1.0",
     "class-transformer": "^0.5.1",
     "class-validator": "^0.14.0",

src/app.module.ts

@@ -32,6 +32,8 @@ import { EventEmitterModule } from "@nestjs/event-emitter";
 import { AdminModule } from "./admin/admin.module";
 import { HealthModule } from "./health/health.module";
 import { LoggerModule } from "./loggers/logger.module";
+import { MetricsModule } from "./metrics/metrics.module";
+import { ScheduleModule } from "@nestjs/schedule";
 
 @Module({
   imports: [
@@ -94,6 +96,8 @@ import { LoggerModule } from "./loggers/logger.module";
     UsersModule,
     AdminModule,
     HealthModule,
+    MetricsModule,
+    ScheduleModule.forRoot(),
   ],
   controllers: [],
   providers: [

src/datasets/datasets.controller.ts

@@ -104,6 +104,7 @@ import {
   PartialUpdateDatasetDto,
   UpdateDatasetDto,
 } from "./dto/update-dataset.dto";
+import { MetricsTrackInterceptor } from "src/metrics/interceptors/metrics-track.interceptor";
 
 @ApiBearerAuth()
 @ApiExtraModels(
@@ -1144,6 +1145,7 @@ export class DatasetsController {
   // GET /datasets/:id
   //@UseGuards(PoliciesGuard)
   @UseGuards(PoliciesGuard)
+  @UseInterceptors(MetricsTrackInterceptor)
   @CheckPolicies("datasets", (ability: AppAbility) =>
     ability.can(Action.DatasetRead, DatasetClass),
   )

src/datasets/datasets.module.ts

@@ -13,9 +13,11 @@ import { LogbooksModule } from "src/logbooks/logbooks.module";
 import { PoliciesService } from "src/policies/policies.service";
 import { PoliciesModule } from "src/policies/policies.module";
 import { ElasticSearchModule } from "src/elastic-search/elastic-search.module";
+import { MetricsModule } from "src/metrics/metrics.module";
 
 @Module({
   imports: [
+    MetricsModule,
     AttachmentsModule,
     ConfigModule,
     DatablocksModule,

src/metrics/interceptors/metrics-track.interceptor.ts

@@ -0,0 +1,39 @@
+import {
+  Injectable,
+  CallHandler,
+  ExecutionContext,
+  NestInterceptor,
+} from "@nestjs/common";
+import { Observable } from "rxjs";
+import { map } from "rxjs/operators";
+import { MetricsService } from "../metrics.service";
+
+@Injectable()
+export class MetricsTrackInterceptor implements NestInterceptor {
+  constructor(private metricsService: MetricsService) {}
+
+  intercept(context: ExecutionContext, next: CallHandler): Observable<unknown> {
+    const request = context.switchToHttp().getRequest();
+    const response = context.switchToHttp().getResponse();
+
+    const requestPath = request.path.toLowerCase();
+    // Check if the route matches, e.g., "/datasets/"
+
+    // TODO: MUST: It will match any path after datasets/
+    // Implement with cautious
+    if (requestPath.includes("/datasets/")) {
+      return next.handle().pipe(
+        map((data) => {
+          if (response.statusCode === 200) {
+            this.metricsService.incrementViewCount(data.pid, data.isPublished);
+          }
+
+          return data;
+        }),
+      );
+    }
+
+    // Proceed with the request normally if route doesn't match
+    return next.handle();
+  }
+}

src/metrics/interfaces/metrics-type.enum.ts

@@ -0,0 +1,5 @@
+export enum MetricsType {
+  VIEWS = "METRICS_DATASET_VIEWS",
+  DOWNLOADS = "METRICS_DATASET_DOWNLOADS",
+  QUERIES = "METRICS_DATASET_QUERIES",
+}

src/metrics/metrics-daily-sync.service.ts

@@ -0,0 +1,113 @@
+import { Injectable } from "@nestjs/common";
+import { Cron } from "@nestjs/schedule";
+import { InjectModel } from "@nestjs/mongoose";
+import { Model } from "mongoose";
+import { MetricsClass, MetricsDocument } from "./schemas/metrics.schema";
+import axios from "axios";
+import { MetricsType } from "./interfaces/metrics-type.enum";
+
+@Injectable()
+export class MetricsDailySyncService {
+  constructor(
+    @InjectModel(MetricsClass.name)
+    private metricsModel: Model<MetricsDocument>,
+  ) {
+    this.aggregateMetrics();
+  }
+
+  private getEmptyMetricsTemplate(): Record<
+    string,
+    { public: number; private: number }
+  > {
+    return {}; // Empty object to start from scratch
+  }
+
+  @Cron("*/10 * * * * *") // Runs every 5 seconds
+  async aggregateMetrics() {
+    console.log("Aggregating metrics...");
+    try {
+      // Start with a fresh, empty template each time
+      const counts = this.getEmptyMetricsTemplate();
+
+      // TODO: MUST: change the URL!!!
+      const { data } = await axios.get("http://localhost:3000/api/v3/metrics");
+
+      // Process Prometheus metrics for view, download, and query events
+      const viewCounts = this.parseMetric(data, MetricsType.VIEWS);
+      const downloadCounts = this.parseMetric(data, MetricsType.DOWNLOADS);
+      const queryCounts = this.parseMetric(data, MetricsType.QUERIES);
+
+      // Update counts based on parsed data
+      Object.assign(counts, viewCounts, downloadCounts, queryCounts);
+
+      // Aggregate each metric type per dataset and update MongoDB
+      await this.updateMetrics("view", viewCounts);
+      await this.updateMetrics("download", downloadCounts);
+      await this.updateMetrics("query", queryCounts);
+    } catch (error) {
+      console.error("Failed to aggregate metrics:", error);
+    }
+  }
+  private async updateMetrics(
+    eventType: string,
+    counts: Record<string, { public: number; private: number }>,
+  ) {
+    const date = new Date().toISOString().slice(11, 19);
+
+    console.log("===counts", counts);
+
+    for (const [
+      datasetId,
+      { public: publicCount, private: privateCount },
+    ] of Object.entries(counts)) {
+      // Create a document with unique identifier as date
+      this.createDocument(eventType, date);
+    }
+  }
+
+  // Parses Prometheus metrics data and returns counts for public and private datasets
+  private parseMetric(
+    data: string,
+    metricName: string,
+  ): Record<string, { public: number; private: number }> {
+    const lines = data.split("\n");
+    const counts: Record<string, { public: number; private: number }> = {};
+
+    for (const line of lines) {
+      if (line.startsWith(`${metricName}{`)) {
+        const datasetIdMatch = line.match(/datasetPid="([^"]+)"/);
+        const isPublished = /isPublished="true"/.test(line);
+        const value = parseFloat(line.split(" ")[1]);
+
+        if (datasetIdMatch && !isNaN(value)) {
+          const datasetId = datasetIdMatch[1];
+          if (!counts[datasetId]) {
+            counts[datasetId] = { public: 0, private: 0 };
+          }
+          if (isPublished) counts[datasetId].public += value;
+          else counts[datasetId].private += value;
+        }
+      }
+    }
+
+    return counts;
+  }
+
+  private createDocument = async (eventType: string, date: string) => {
+    await this.metricsModel.create(
+      {
+        date,
+      },
+      {
+        $setOnInsert: {
+          eventType,
+          date,
+          publicDataCount: 0, // Start with zero counts initially
+          privateDataCount: 0,
+          datasetCounts: [], // Start with an empty array for datasetCounts
+        },
+      },
+      {},
+    );
+  };
+}

src/metrics/metrics.module.ts

@@ -0,0 +1,43 @@
+import { Module } from "@nestjs/common";
+import { MetricsService } from "./metrics.service";
+import {
+  makeCounterProvider,
+  PrometheusModule,
+} from "@willsoto/nestjs-prometheus";
+import { MongooseModule } from "@nestjs/mongoose";
+import { MetricsClass, MetricsSchema } from "./schemas/metrics.schema";
+import { MetricsDailySyncService } from "./metrics-daily-sync.service";
+import { MetricsType } from "./interfaces/metrics-type.enum";
+
+@Module({
+  imports: [
+    PrometheusModule.register(),
+    MongooseModule.forFeature([
+      {
+        name: MetricsClass.name,
+        schema: MetricsSchema,
+      },
+    ]),
+  ],
+  providers: [
+    MetricsService,
+    MetricsDailySyncService,
+    makeCounterProvider({
+      name: MetricsType.VIEWS,
+      help: "Counter for dataset views",
+      labelNames: ["datasetPid", "isPublished"], // TODO: is it possible to not hardcode this here?
+    }),
+    makeCounterProvider({
+      name: MetricsType.DOWNLOADS,
+      help: "Counter for dataset downloads",
+      labelNames: ["datasetPid", "isPublished"],
+    }),
+    makeCounterProvider({
+      name: MetricsType.QUERIES,
+      help: "Counter for dataset queries",
+      labelNames: ["datasetPid", "isPublished"],
+    }),
+  ],
+  exports: [MetricsService],
+})
+export class MetricsModule {}