[WIP] [POC] feat(metrics): add metrics logging and configuration support

.gitignore

@@ -5,6 +5,7 @@ functionalAccounts.json
 datasetTypes.json
 proposalTypes.json
 loggers.json
+metricsConfig.json
 
 # Configs
 .env

metricsConfig.example.json

@@ -0,0 +1,31 @@
+{
+  "include": [
+    {
+      "path": "*",
+      "method": 0,
+      "version": "3"
+    },
+    {
+      "path": "datasets/fullquery",
+      "method": 0,
+      "version": "3"
+    },
+    {
+      "path": "datasets/other-endpoint",
+      "method": 0,
+      "version": "3"
+    }
+  ],
+  "exclude": [
+    {
+      "path": "datasets/fullfacet",
+      "method": 0,
+      "version": "3"
+    },
+    {
+      "path": "datasets/metadataKeys",
+      "method": 0,
+      "version": "3"
+    }
+  ]
+}

package.json

@@ -43,6 +43,7 @@
     "@nestjs/mongoose": "^10.0.0",
     "@nestjs/passport": "^10.0.0",
     "@nestjs/platform-express": "^10.3.8",
+    "@nestjs/schedule": "^4.1.2",
     "@nestjs/swagger": "^7.1.7",
     "@nestjs/terminus": "^10.1.1",
     "@user-office-software/duo-logger": "^2.1.1",

src/app.module.ts

@@ -3,7 +3,7 @@ import { MongooseModule } from "@nestjs/mongoose";
 import { DatasetsModule } from "./datasets/datasets.module";
 import { AuthModule } from "./auth/auth.module";
 import { UsersModule } from "./users/users.module";
-import { ConfigModule, ConfigService } from "@nestjs/config";
+import { ConditionalModule, ConfigModule, ConfigService } from "@nestjs/config";
 import { CaslModule } from "./casl/casl.module";
 import configuration from "./config/configuration";
 import { APP_GUARD, Reflector } from "@nestjs/core";
@@ -32,6 +32,7 @@ import { EventEmitterModule } from "@nestjs/event-emitter";
 import { AdminModule } from "./admin/admin.module";
 import { HealthModule } from "./health/health.module";
 import { LoggerModule } from "./loggers/logger.module";
+import { MetricsAndLogsModule } from "./metrics-and-logs/metrics-and-logs.module";
 
 @Module({
   imports: [
@@ -42,6 +43,10 @@ import { LoggerModule } from "./loggers/logger.module";
     ConfigModule.forRoot({
       load: [configuration],
     }),
+    ConditionalModule.registerWhen(
+      MetricsAndLogsModule,
+      (env: NodeJS.ProcessEnv) => env.METRICS_ENABLED === "yes",
+    ),
     LoggerModule,
     DatablocksModule,
     DatasetsModule,

src/config/configuration.ts

@@ -48,6 +48,7 @@ const configuration = () => {
     loggers: process.env.LOGGERS_CONFIG_FILE || "loggers.json",
     datasetTypes: process.env.DATASET_TYPES_FILE || "datasetTypes.json",
     proposalTypes: process.env.PROPOSAL_TYPES_FILE || "proposalTypes.json",
+    metricsConfig: process.env.METRICS_CONFIG_FILE || "metricsConfig.json",
   };
   Object.keys(jsonConfigFileList).forEach((key) => {
     const filePath = jsonConfigFileList[key];
@@ -81,6 +82,7 @@ const configuration = () => {
     },
     swaggerPath: process.env.SWAGGER_PATH || "explorer",
     loggerConfigs: jsonConfigMap.loggers || [defaultLogger],
+    metricsConfig: jsonConfigMap.metricsConfig,
     adminGroups: adminGroups.split(",").map((v) => v.trim()) ?? [],
     deleteGroups: deleteGroups.split(",").map((v) => v.trim()) ?? [],
     createDatasetGroups: createDatasetGroups.split(",").map((v) => v.trim()),

src/main.ts

@@ -15,9 +15,7 @@ async function bootstrap() {
   const app = await NestFactory.create<NestExpressApplication>(AppModule, {
     bufferLogs: true,
   });
-  const configService: ConfigService<Record<string, unknown>, false> = app.get(
-    ConfigService,
-  );
+  const configService = app.get(ConfigService);
   const apiVersion = configService.get<string>("versions.api");
   const swaggerPath = `${configService.get<string>("swaggerPath")}`;
 

src/metrics-and-logs/access-logs/access-logs.controller.ts

@@ -0,0 +1,49 @@
+import { Controller, Get, Query } from "@nestjs/common";
+import { ApiOperation, ApiQuery, ApiTags } from "@nestjs/swagger";
+import { AccessLogsService } from "./access-logs.service";
+@ApiTags("access-logs")
+@Controller("access-logs")
+export class AccessLogsController {
+  constructor(private readonly accessLogsService: AccessLogsService) {}
+
+  //TODO: use correct checkpolies
+  // @UseGuards(PoliciesGuard)
+  @Get("/find")
+  @ApiOperation({ summary: "Get access logs" })
+  @ApiQuery({
+    name: "query",
+    description: `{ "endpoint": { "$regex": "20.500.12269", "$options": "i" } } \n
+{ "userId": "example" } \n
+{ "$or": [ { "userId": "example" }, { "endpoint": { "$regex": "20.500.12269", "$options": "i" } } ] }`,
+    required: false,
+    type: String,
+  })
+  @ApiQuery({
+    name: "projection",
+    description: '{"userId": 1, "endpoint": 1, "createdAt": 1}',
+    required: false,
+    type: String,
+  })
+  @ApiQuery({
+    name: "options",
+    description:
+      "{ limit: 10, skip: 5, sort: { createdAt: -1 }, maxTimeMS: 1000, hint: { createdAt: 1 } }",
+    required: false,
+    type: String,
+  })
+  async find(
+    @Query("query") query: string,
+    @Query("projection")
+    projection?: string,
+    @Query("options") options?: string,
+  ) {
+    const parsedQuery = query ? JSON.parse(query) : {};
+    const parsedOption = options ? JSON.parse(options) : {};
+    const parsedProjection = projection ? JSON.parse(projection) : null;
+    return this.accessLogsService.find(
+      parsedQuery,
+      parsedProjection,
+      parsedOption,
+    );
+  }
+}

src/metrics-and-logs/access-logs/access-logs.module.ts

@@ -0,0 +1,22 @@
+import { Module } from "@nestjs/common";
+import { AccessLogsService } from "./access-logs.service";
+import { MongooseModule } from "@nestjs/mongoose";
+import { AccessLog, AccessLogSchema } from "./schemas/access-log.schema";
+import { AccessLogsController } from "./access-logs.controller";
+import { CaslModule } from "src/casl/casl.module";
+
+@Module({
+  imports: [
+    MongooseModule.forFeature([
+      {
+        name: AccessLog.name,
+        schema: AccessLogSchema,
+      },
+    ]),
+    CaslModule,
+  ],
+  providers: [AccessLogsService, AccessLogsController],
+  exports: [AccessLogsService, AccessLogsController],
+  controllers: [AccessLogsController],
+})
+export class AccessLogsModule {}

src/metrics-and-logs/access-logs/access-logs.service.ts

@@ -0,0 +1,24 @@
+import { InjectModel } from "@nestjs/mongoose";
+import { Model, FilterQuery, QueryOptions, ProjectionType } from "mongoose";
+import { AccessLog, AccessLogDocument } from "./schemas/access-log.schema";
+
+export class AccessLogsService {
+  constructor(
+    @InjectModel(AccessLog.name)
+    private accessLogModel: Model<AccessLogDocument>,
+  ) {}
+  // Method to log/store metrics
+
+  async create(accessLogData: AccessLog): Promise<AccessLog> {
+    const accessLog = new this.accessLogModel(accessLogData);
+    return accessLog.save();
+  }
+
+  async find(
+    query: FilterQuery<AccessLogDocument>,
+    projection: ProjectionType<AccessLogDocument> | null | undefined,
+    options: QueryOptions<AccessLogDocument> | null | undefined,
+  ): Promise<AccessLog[]> {
+    return this.accessLogModel.find(query, projection, options).exec();
+  }
+}

src/metrics-and-logs/access-logs/interfaces/accessLogs.interface.ts

@@ -0,0 +1,8 @@
+export interface IAccessLogs {
+  user: string | null;
+  ip: string | undefined;
+  userAgent: string | undefined;
+  endpoint: string;
+  statusCode: number;
+  responseTime: number;
+}

src/metrics-and-logs/access-logs/schemas/access-log.schema.ts

@@ -0,0 +1,65 @@
+import { Prop, Schema, SchemaFactory } from "@nestjs/mongoose";
+import { ApiProperty, ApiPropertyOptional } from "@nestjs/swagger";
+import { Document } from "mongoose";
+
+export type AccessLogDocument = AccessLog & Document;
+
+@Schema({
+  collection: "AccessLog",
+  timestamps: { createdAt: true, updatedAt: false },
+  versionKey: false,
+})
+export class AccessLog {
+  @ApiPropertyOptional({
+    description: "User ID associated with the access log",
+    type: String,
+  })
+  @Prop({ type: String, default: null })
+  userId: string | null;
+
+  @ApiPropertyOptional({
+    description: "The timestamp when the document was created",
+    type: Date,
+  })
+  @Prop({ type: Date, default: Date.now })
+  createdAt?: Date;
+
+  @ApiProperty({
+    description: "HTTP status code for the request",
+    type: Number,
+  })
+  @Prop({ type: Number, required: true })
+  statusCode: number;
+
+  @ApiProperty({
+    description: "Endpoint for the request",
+    type: String,
+  })
+  @Prop({ type: String, required: true })
+  endpoint: string;
+
+  @ApiPropertyOptional({
+    description: "Endpoint for the request",
+    type: Object,
+  })
+  @Prop({ type: Object, default: null })
+  query: object;
+
+  @ApiPropertyOptional({
+    description: "Original IP of the user for the request",
+    type: String,
+  })
+  @Prop({ type: String, default: "Not found" })
+  originIp: string | undefined;
+
+  @ApiProperty({
+    description: "Response time in ms for the request",
+    type: String,
+  })
+  @Prop({ type: Number, required: true })
+  responseTime: number;
+}
+
+export const AccessLogSchema = SchemaFactory.createForClass(AccessLog);
+
+AccessLogSchema.index({ createdAt: 1 });

src/metrics-and-logs/metrics-and-logs.module.ts

@@ -0,0 +1,29 @@
+import { Logger, MiddlewareConsumer, Module, NestModule } from "@nestjs/common";
+import { MetricsModule } from "./metrics/metrics.module";
+import { AccessLogsModule } from "./access-logs/access-logs.module";
+import { ConfigModule, ConfigService } from "@nestjs/config";
+import { MetricsAndLogsMiddleware } from "./middlewares/metrics-and-logs.middleware";
+import { JwtModule } from "@nestjs/jwt";
+
+@Module({
+  imports: [MetricsModule, AccessLogsModule, ConfigModule, JwtModule],
+  exports: [MetricsModule, AccessLogsModule],
+})
+export class MetricsAndLogsModule implements NestModule {
+  constructor(private readonly configService: ConfigService) {}
+
+  configure(consumer: MiddlewareConsumer) {
+    const { include = [], exclude = [] } =
+      this.configService.get("metricsConfig") || {};
+
+    try {
+      consumer
+        .apply(MetricsAndLogsMiddleware)
+        .exclude(...exclude)
+        .forRoutes(...include);
+      Logger.log("Start collecting metrics", "MetricsModule");
+    } catch (error) {
+      Logger.error("Error configuring metrics middleware", error);
+    }
+  }
+}

src/metrics-and-logs/metrics/metrics.controller.ts

@@ -0,0 +1,20 @@
+import { Controller } from "@nestjs/common";
+import { ApiTags } from "@nestjs/swagger";
+
+@ApiTags("metrics")
+@Controller("metrics")
+export class MetricsController {
+  constructor() {}
+  logMetrics(
+    user: string | null,
+    ip: string | undefined,
+    userAgent: string | undefined,
+    endpoint: string,
+    statusCode: number,
+    responseTime: number,
+  ) {
+    console.log(
+      `User ID: ${user}, IP: ${ip}, User-Agent: ${userAgent}, Endpoint: ${endpoint}, Status Code: ${statusCode}, Response Time: ${responseTime}ms`,
+    );
+  }
+}