From 54b6bce293c98e7272339aa8f52212ed415ee104 Mon Sep 17 00:00:00 2001 From: junjiequan Date: Wed, 11 Dec 2024 16:58:47 +0100 Subject: [PATCH 1/5] feat(metrics): add metrics logging and configuration support --- .gitignore | 1 + metricsConfig.example.json | 31 +++++++++++ src/app.module.ts | 15 +++++- src/config/configuration.ts | 2 + src/main.ts | 4 +- src/metrics/interfaces/common.interface.ts | 0 src/metrics/metrics.controller.ts | 18 +++++++ src/metrics/metrics.module.ts | 46 ++++++++++++++++ src/metrics/metrics.service.ts | 17 ++++++ src/metrics/middlewares/metrics.middleware.ts | 52 +++++++++++++++++++ src/metrics/schemas/access-log.schema.ts | 41 +++++++++++++++ src/metrics/schemas/metrics-record.schema.ts | 48 +++++++++++++++++ src/metrics/schemas/metrics.schema.ts | 28 ++++++++++ 13 files changed, 298 insertions(+), 5 deletions(-) create mode 100644 metricsConfig.example.json create mode 100644 src/metrics/interfaces/common.interface.ts create mode 100644 src/metrics/metrics.controller.ts create mode 100644 src/metrics/metrics.module.ts create mode 100644 src/metrics/metrics.service.ts create mode 100644 src/metrics/middlewares/metrics.middleware.ts create mode 100644 src/metrics/schemas/access-log.schema.ts create mode 100644 src/metrics/schemas/metrics-record.schema.ts create mode 100644 src/metrics/schemas/metrics.schema.ts diff --git a/.gitignore b/.gitignore index eaadcb985..7edf11d70 100644 --- a/.gitignore +++ b/.gitignore @@ -5,6 +5,7 @@ functionalAccounts.json datasetTypes.json proposalTypes.json loggers.json +metricsConfig.json # Configs .env diff --git a/metricsConfig.example.json b/metricsConfig.example.json new file mode 100644 index 000000000..9d7fdf13a --- /dev/null +++ b/metricsConfig.example.json @@ -0,0 +1,31 @@ +{ + "include": [ + { + "path": "*", + "method": 0, + "version": "3" + }, + { + "path": "datasets/fullquery", + "method": 0, + "version": "3" + }, + { + "path": "datasets/other-endpoint", + "method": 0, + "version": "3" + } + ], + "exclude": [ + { + "path": "datasets/fullfacet", + "method": 0, + "version": "3" + }, + { + "path": "datasets/metadataKeys", + "method": 0, + "version": "3" + } + ] +} diff --git a/src/app.module.ts b/src/app.module.ts index 04604d824..46da3abfe 100644 --- a/src/app.module.ts +++ b/src/app.module.ts @@ -1,9 +1,14 @@ -import { Logger, Module, OnApplicationBootstrap } from "@nestjs/common"; +import { + Logger, + MiddlewareConsumer, + Module, + OnApplicationBootstrap, +} from "@nestjs/common"; import { MongooseModule } from "@nestjs/mongoose"; import { DatasetsModule } from "./datasets/datasets.module"; import { AuthModule } from "./auth/auth.module"; import { UsersModule } from "./users/users.module"; -import { ConfigModule, ConfigService } from "@nestjs/config"; +import { ConditionalModule, ConfigModule, ConfigService } from "@nestjs/config"; import { CaslModule } from "./casl/casl.module"; import configuration from "./config/configuration"; import { APP_GUARD, Reflector } from "@nestjs/core"; @@ -32,6 +37,8 @@ import { EventEmitterModule } from "@nestjs/event-emitter"; import { AdminModule } from "./admin/admin.module"; import { HealthModule } from "./health/health.module"; import { LoggerModule } from "./loggers/logger.module"; +import { MetricsMiddleware } from "./metrics/middlewares/metrics.middleware"; +import { MetricsModule } from "./metrics/metrics.module"; @Module({ imports: [ @@ -42,6 +49,10 @@ import { LoggerModule } from "./loggers/logger.module"; ConfigModule.forRoot({ load: [configuration], }), + ConditionalModule.registerWhen( + MetricsModule, + (env: NodeJS.ProcessEnv) => env.METRICS_ENABLED === "yes", + ), LoggerModule, DatablocksModule, DatasetsModule, diff --git a/src/config/configuration.ts b/src/config/configuration.ts index f7afe539f..0347fd49e 100644 --- a/src/config/configuration.ts +++ b/src/config/configuration.ts @@ -48,6 +48,7 @@ const configuration = () => { loggers: process.env.LOGGERS_CONFIG_FILE || "loggers.json", datasetTypes: process.env.DATASET_TYPES_FILE || "datasetTypes.json", proposalTypes: process.env.PROPOSAL_TYPES_FILE || "proposalTypes.json", + metricsConfig: process.env.METRICS_CONFIG_FILE || "metricsConfig.json", }; Object.keys(jsonConfigFileList).forEach((key) => { const filePath = jsonConfigFileList[key]; @@ -81,6 +82,7 @@ const configuration = () => { }, swaggerPath: process.env.SWAGGER_PATH || "explorer", loggerConfigs: jsonConfigMap.loggers || [defaultLogger], + metricsConfig: jsonConfigMap.metricsConfig, adminGroups: adminGroups.split(",").map((v) => v.trim()) ?? [], deleteGroups: deleteGroups.split(",").map((v) => v.trim()) ?? [], createDatasetGroups: createDatasetGroups.split(",").map((v) => v.trim()), diff --git a/src/main.ts b/src/main.ts index 327248a93..e7780f415 100644 --- a/src/main.ts +++ b/src/main.ts @@ -15,9 +15,7 @@ async function bootstrap() { const app = await NestFactory.create(AppModule, { bufferLogs: true, }); - const configService: ConfigService, false> = app.get( - ConfigService, - ); + const configService = app.get(ConfigService); const apiVersion = configService.get("versions.api"); const swaggerPath = `${configService.get("swaggerPath")}`; diff --git a/src/metrics/interfaces/common.interface.ts b/src/metrics/interfaces/common.interface.ts new file mode 100644 index 000000000..e69de29bb diff --git a/src/metrics/metrics.controller.ts b/src/metrics/metrics.controller.ts new file mode 100644 index 000000000..1a2ac8e4b --- /dev/null +++ b/src/metrics/metrics.controller.ts @@ -0,0 +1,18 @@ +import { Controller } from "@nestjs/common"; + +@Controller("metrics") +export class MetricsController { + constructor() {} + logMetrics( + user: string | null, + ip: string | undefined, + userAgent: string | undefined, + endpoint: string, + statusCode: number, + responseTime: number, + ) { + console.log( + `User ID: ${user}, IP: ${ip}, User-Agent: ${userAgent}, Endpoint: ${endpoint}, Status Code: ${statusCode}, Response Time: ${responseTime}ms`, + ); + } +} diff --git a/src/metrics/metrics.module.ts b/src/metrics/metrics.module.ts new file mode 100644 index 000000000..d9e5b7800 --- /dev/null +++ b/src/metrics/metrics.module.ts @@ -0,0 +1,46 @@ +import { Logger, MiddlewareConsumer, Module, NestModule } from "@nestjs/common"; +import { ConfigModule, ConfigService } from "@nestjs/config"; +import { MetricsMiddleware } from "./middlewares/metrics.middleware"; +import { JwtModule } from "@nestjs/jwt"; +import { MetricsService } from "./metrics.service"; +import { MetricsController } from "./metrics.controller"; +import { MongooseModule } from "@nestjs/mongoose"; +import { Metrics, MetricsSchema } from "./schemas/metrics.schema"; +import { AccessLogs, AccessLogSchema } from "./schemas/access-log.schema"; + +@Module({ + imports: [ + ConfigModule, + JwtModule, + MongooseModule.forFeature([ + { + name: Metrics.name, + schema: MetricsSchema, + }, + { + name: AccessLogs.name, + schema: AccessLogSchema, + }, + ]), + ], + providers: [MetricsService, MetricsController], + exports: [], +}) +export class MetricsModule implements NestModule { + constructor(private readonly configService: ConfigService) {} + + configure(consumer: MiddlewareConsumer) { + const { include = [], exclude = [] } = + this.configService.get("metricsConfig") || {}; + + try { + consumer + .apply(MetricsMiddleware) + .exclude(...exclude) + .forRoutes(...include); + Logger.log("Start collecting metrics", "MetricsModule"); + } catch (error) { + Logger.error("Error configuring metrics middleware", error); + } + } +} diff --git a/src/metrics/metrics.service.ts b/src/metrics/metrics.service.ts new file mode 100644 index 000000000..711b2decc --- /dev/null +++ b/src/metrics/metrics.service.ts @@ -0,0 +1,17 @@ +import { InjectModel } from "@nestjs/mongoose"; +import { Metrics, MetricsDocument } from "./schemas/metrics.schema"; +import { Model } from "mongoose"; +import { AccessLogs, AccessLogsDocument } from "./schemas/access-log.schema"; + +export class MetricsService { + constructor( + @InjectModel(Metrics.name) + private metricsModel: Model, + @InjectModel(AccessLogs.name) + private accessLogsModel: Model, + ) {} + // Method to log/store metrics + async create() {} + async findById() {} + async findAll() {} +} diff --git a/src/metrics/middlewares/metrics.middleware.ts b/src/metrics/middlewares/metrics.middleware.ts new file mode 100644 index 000000000..def933f65 --- /dev/null +++ b/src/metrics/middlewares/metrics.middleware.ts @@ -0,0 +1,52 @@ +import { Injectable, Logger, NestMiddleware } from "@nestjs/common"; +import { Request, Response, NextFunction } from "express"; +import { JwtService } from "@nestjs/jwt"; +import { MetricsController } from "../metrics.controller"; + +@Injectable() +export class MetricsMiddleware implements NestMiddleware { + constructor( + private readonly jwtService: JwtService, + private readonly metricsController: MetricsController, + ) {} + use(req: Request, res: Response, next: NextFunction) { + const userAgent = req.headers["user-agent"]; + const authHeader = req.headers.authorization; + const ip = req.ip || req.socket.remoteAddress; + const user = this.parseToken(authHeader); + const endpoint = req.originalUrl; + const startTime = Date.now(); + + res.on("finish", () => { + const statusCode = res.statusCode; + const responseTime = Date.now() - startTime; + + this.metricsController.logMetrics( + user, + ip, + userAgent, + endpoint, + statusCode, + responseTime, + ); + }); + + next(); + } + + private parseToken(authHeader?: string) { + try { + if (authHeader && authHeader.startsWith("Bearer ")) { + const token = authHeader.split(" ")[1]; + const { id, username } = this.jwtService.decode(token); + // TODO: changes it to return userid before merge; + return username; + // return id; + } + return null; + } catch (error) { + Logger.error("Error parsing token-> MetricsMiddleware", error); + return null; + } + } +} diff --git a/src/metrics/schemas/access-log.schema.ts b/src/metrics/schemas/access-log.schema.ts new file mode 100644 index 000000000..d959e17c5 --- /dev/null +++ b/src/metrics/schemas/access-log.schema.ts @@ -0,0 +1,41 @@ +import { Prop, Schema, SchemaFactory } from "@nestjs/mongoose"; +import { ApiProperty, ApiPropertyOptional } from "@nestjs/swagger"; +import { Document } from "mongoose"; + +export type AccessLogsDocument = AccessLogs & Document; + +// Define the structure of the AccessLogSchema +@Schema() +export class AccessLogs { + @ApiProperty({ + description: "User ID associated with the access log", + type: String, + }) + @Prop({ type: String, default: null }) + userId: string | null; + + @ApiProperty({ + description: "HTTP status code for the request", + type: Number, + }) + @Prop({ type: Number, required: true }) + statusCode: number; + + @ApiProperty({ + description: "Endpoint for the request", + type: String, + }) + @Prop({ type: String, required: true }) + endpoint: string; + + @ApiPropertyOptional({ + description: "Original IP of the user for the request", + type: String, + }) + @Prop({ type: String, required: true }) + originIp: string; +} + +export const AccessLogSchema = SchemaFactory.createForClass(AccessLogs); + +AccessLogSchema.index({ createdAt: 1 }); diff --git a/src/metrics/schemas/metrics-record.schema.ts b/src/metrics/schemas/metrics-record.schema.ts new file mode 100644 index 000000000..24bc41bea --- /dev/null +++ b/src/metrics/schemas/metrics-record.schema.ts @@ -0,0 +1,48 @@ +import { Prop, Schema, SchemaFactory } from "@nestjs/mongoose"; +import { ApiProperty } from "@nestjs/swagger"; +import { Document } from "mongoose"; + +@Schema() +export class MetricsRecord { + @ApiProperty({ + description: "The endpoint path being tracked", + type: String, + }) + @Prop({ type: String, required: true }) + endpoint: string; + + @ApiProperty({ + description: "The records of individual access logs for this endpoint", + type: [Object], + }) + @Prop({ + type: [ + { + userId: { type: String, required: false }, + statusCode: { type: Number, required: true }, + date: { type: Date, required: true }, + }, + ], + required: true, + }) + records: Array<{ + userId: string | null; + statusCode: number; + date: Date; + }>; + + @ApiProperty({ + description: "Summary statistics of access for this endpoint", + type: Object, + }) + @Prop({ type: Object, required: true }) + summary: { + totalRequests: number; + }; +} + +// Create the schema for MetricsRecord +export const MetricsRecordSchema = SchemaFactory.createForClass(MetricsRecord); + +// If needed, export the type +export type MetricsRecordDocument = MetricsRecord & Document; diff --git a/src/metrics/schemas/metrics.schema.ts b/src/metrics/schemas/metrics.schema.ts new file mode 100644 index 000000000..2a2e0ffe9 --- /dev/null +++ b/src/metrics/schemas/metrics.schema.ts @@ -0,0 +1,28 @@ +import { Prop, Schema, SchemaFactory } from "@nestjs/mongoose"; +import { ApiProperty } from "@nestjs/swagger"; +import { Date, Document } from "mongoose"; +import { MetricsRecord } from "./metrics-record.schema"; + +export type MetricsDocument = Metrics & Document; + +@Schema({ + collection: "metrics", + timestamps: true, +}) +export class Metrics { + @ApiProperty({ + description: "The date the metric was recorded", + type: String, + }) + @Prop({ type: Date, required: false, default: Date.now }) + date: Date; + + @ApiProperty({ + description: "A list of endpoints with their access log compacted details", + type: [Object], + }) + @Prop({ type: [MetricsRecord] }) + endpointMetrics: MetricsRecord[]; +} + +export const MetricsSchema = SchemaFactory.createForClass(Metrics); From f068e2fa3da039addebb0177fcae9785a9f8d64d Mon Sep 17 00:00:00 2001 From: junjiequan Date: Thu, 12 Dec 2024 11:02:54 +0100 Subject: [PATCH 2/5] refactor folder structure --- src/app.module.ts | 12 ++--- .../access-logs/access-logs.controller.ts | 18 ++++++++ .../access-logs/access-logs.module.ts | 19 ++++++++ .../access-logs/access-logs.service.ts | 14 ++++++ .../interfaces/accessLogs.interface.ts | 8 ++++ .../access-logs}/schemas/access-log.schema.ts | 5 +- .../metrics-and-logs.module.ts | 29 ++++++++++++ .../metrics/interfaces/metrics.interface.ts} | 0 .../metrics/metrics.controller.ts | 0 .../metrics/metrics.module.ts | 19 ++++++++ .../metrics/metrics.service.ts | 3 -- .../metrics/schemas/metrics-record.schema.ts | 0 .../metrics/schemas/metrics.schema.ts | 0 .../metrics-and-logs.middleware.ts} | 10 ++-- src/metrics/metrics.module.ts | 46 ------------------- 15 files changed, 119 insertions(+), 64 deletions(-) create mode 100644 src/metrics-and-logs/access-logs/access-logs.controller.ts create mode 100644 src/metrics-and-logs/access-logs/access-logs.module.ts create mode 100644 src/metrics-and-logs/access-logs/access-logs.service.ts create mode 100644 src/metrics-and-logs/access-logs/interfaces/accessLogs.interface.ts rename src/{metrics => metrics-and-logs/access-logs}/schemas/access-log.schema.ts (94%) create mode 100644 src/metrics-and-logs/metrics-and-logs.module.ts rename src/{metrics/interfaces/common.interface.ts => metrics-and-logs/metrics/interfaces/metrics.interface.ts} (100%) rename src/{ => metrics-and-logs}/metrics/metrics.controller.ts (100%) create mode 100644 src/metrics-and-logs/metrics/metrics.module.ts rename src/{ => metrics-and-logs}/metrics/metrics.service.ts (69%) rename src/{ => metrics-and-logs}/metrics/schemas/metrics-record.schema.ts (100%) rename src/{ => metrics-and-logs}/metrics/schemas/metrics.schema.ts (100%) rename src/{metrics/middlewares/metrics.middleware.ts => metrics-and-logs/middlewares/metrics-and-logs.middleware.ts} (79%) delete mode 100644 src/metrics/metrics.module.ts diff --git a/src/app.module.ts b/src/app.module.ts index 46da3abfe..9f70df804 100644 --- a/src/app.module.ts +++ b/src/app.module.ts @@ -1,9 +1,4 @@ -import { - Logger, - MiddlewareConsumer, - Module, - OnApplicationBootstrap, -} from "@nestjs/common"; +import { Logger, Module, OnApplicationBootstrap } from "@nestjs/common"; import { MongooseModule } from "@nestjs/mongoose"; import { DatasetsModule } from "./datasets/datasets.module"; import { AuthModule } from "./auth/auth.module"; @@ -37,8 +32,7 @@ import { EventEmitterModule } from "@nestjs/event-emitter"; import { AdminModule } from "./admin/admin.module"; import { HealthModule } from "./health/health.module"; import { LoggerModule } from "./loggers/logger.module"; -import { MetricsMiddleware } from "./metrics/middlewares/metrics.middleware"; -import { MetricsModule } from "./metrics/metrics.module"; +import { MetricsAndLogsModule } from "./metrics-and-logs/metrics-and-logs.module"; @Module({ imports: [ @@ -50,7 +44,7 @@ import { MetricsModule } from "./metrics/metrics.module"; load: [configuration], }), ConditionalModule.registerWhen( - MetricsModule, + MetricsAndLogsModule, (env: NodeJS.ProcessEnv) => env.METRICS_ENABLED === "yes", ), LoggerModule, diff --git a/src/metrics-and-logs/access-logs/access-logs.controller.ts b/src/metrics-and-logs/access-logs/access-logs.controller.ts new file mode 100644 index 000000000..9c3d5c174 --- /dev/null +++ b/src/metrics-and-logs/access-logs/access-logs.controller.ts @@ -0,0 +1,18 @@ +import { Controller } from "@nestjs/common"; + +@Controller("access-logs") +export class AccessLogsController { + constructor() {} + logMetrics( + user: string | null, + ip: string | undefined, + userAgent: string | undefined, + endpoint: string, + statusCode: number, + responseTime: number, + ) { + console.log( + `User ID: ${user}, IP: ${ip}, User-Agent: ${userAgent}, Endpoint: ${endpoint}, Status Code: ${statusCode}, Response Time: ${responseTime}ms`, + ); + } +} diff --git a/src/metrics-and-logs/access-logs/access-logs.module.ts b/src/metrics-and-logs/access-logs/access-logs.module.ts new file mode 100644 index 000000000..2c4072304 --- /dev/null +++ b/src/metrics-and-logs/access-logs/access-logs.module.ts @@ -0,0 +1,19 @@ +import { Module } from "@nestjs/common"; +import { AccessLogsService } from "./access-logs.service"; +import { MongooseModule } from "@nestjs/mongoose"; +import { AccessLogs, AccessLogSchema } from "./schemas/access-log.schema"; +import { AccessLogsController } from "./access-logs.controller"; + +@Module({ + imports: [ + MongooseModule.forFeature([ + { + name: AccessLogs.name, + schema: AccessLogSchema, + }, + ]), + ], + providers: [AccessLogsService, AccessLogsController], + exports: [AccessLogsController], +}) +export class AccessLogsModule {} diff --git a/src/metrics-and-logs/access-logs/access-logs.service.ts b/src/metrics-and-logs/access-logs/access-logs.service.ts new file mode 100644 index 000000000..c3f6ef4a7 --- /dev/null +++ b/src/metrics-and-logs/access-logs/access-logs.service.ts @@ -0,0 +1,14 @@ +import { InjectModel } from "@nestjs/mongoose"; +import { Model } from "mongoose"; +import { AccessLogs, AccessLogsDocument } from "./schemas/access-log.schema"; + +export class AccessLogsService { + constructor( + @InjectModel(AccessLogs.name) + private accessLogsModel: Model, + ) {} + // Method to log/store metrics + async create() {} + async findById() {} + async findAll() {} +} diff --git a/src/metrics-and-logs/access-logs/interfaces/accessLogs.interface.ts b/src/metrics-and-logs/access-logs/interfaces/accessLogs.interface.ts new file mode 100644 index 000000000..101f928e9 --- /dev/null +++ b/src/metrics-and-logs/access-logs/interfaces/accessLogs.interface.ts @@ -0,0 +1,8 @@ +export interface IAccessLogs { + user: string | null; + ip: string | undefined; + userAgent: string | undefined; + endpoint: string; + statusCode: number; + responseTime: number; +} diff --git a/src/metrics/schemas/access-log.schema.ts b/src/metrics-and-logs/access-logs/schemas/access-log.schema.ts similarity index 94% rename from src/metrics/schemas/access-log.schema.ts rename to src/metrics-and-logs/access-logs/schemas/access-log.schema.ts index d959e17c5..85818af71 100644 --- a/src/metrics/schemas/access-log.schema.ts +++ b/src/metrics-and-logs/access-logs/schemas/access-log.schema.ts @@ -4,7 +4,10 @@ import { Document } from "mongoose"; export type AccessLogsDocument = AccessLogs & Document; -// Define the structure of the AccessLogSchema +@Schema({ + collection: "accessLog", + timestamps: true, +}) @Schema() export class AccessLogs { @ApiProperty({ diff --git a/src/metrics-and-logs/metrics-and-logs.module.ts b/src/metrics-and-logs/metrics-and-logs.module.ts new file mode 100644 index 000000000..1e97d3e16 --- /dev/null +++ b/src/metrics-and-logs/metrics-and-logs.module.ts @@ -0,0 +1,29 @@ +import { Logger, MiddlewareConsumer, Module, NestModule } from "@nestjs/common"; +import { MetricsModule } from "./metrics/metrics.module"; +import { AccessLogsModule } from "./access-logs/access-logs.module"; +import { ConfigModule, ConfigService } from "@nestjs/config"; +import { MetricsAndLogsMiddleware } from "./middlewares/metrics-and-logs.middleware"; +import { JwtModule } from "@nestjs/jwt"; + +@Module({ + imports: [MetricsModule, AccessLogsModule, ConfigModule, JwtModule], + exports: [MetricsModule, AccessLogsModule], +}) +export class MetricsAndLogsModule implements NestModule { + constructor(private readonly configService: ConfigService) {} + + configure(consumer: MiddlewareConsumer) { + const { include = [], exclude = [] } = + this.configService.get("metricsConfig") || {}; + + try { + consumer + .apply(MetricsAndLogsMiddleware) + .exclude(...exclude) + .forRoutes(...include); + Logger.log("Start collecting metrics", "MetricsModule"); + } catch (error) { + Logger.error("Error configuring metrics middleware", error); + } + } +} diff --git a/src/metrics/interfaces/common.interface.ts b/src/metrics-and-logs/metrics/interfaces/metrics.interface.ts similarity index 100% rename from src/metrics/interfaces/common.interface.ts rename to src/metrics-and-logs/metrics/interfaces/metrics.interface.ts diff --git a/src/metrics/metrics.controller.ts b/src/metrics-and-logs/metrics/metrics.controller.ts similarity index 100% rename from src/metrics/metrics.controller.ts rename to src/metrics-and-logs/metrics/metrics.controller.ts diff --git a/src/metrics-and-logs/metrics/metrics.module.ts b/src/metrics-and-logs/metrics/metrics.module.ts new file mode 100644 index 000000000..873a5c306 --- /dev/null +++ b/src/metrics-and-logs/metrics/metrics.module.ts @@ -0,0 +1,19 @@ +import { Module } from "@nestjs/common"; +import { MetricsService } from "./metrics.service"; +import { MetricsController } from "./metrics.controller"; +import { MongooseModule } from "@nestjs/mongoose"; +import { Metrics, MetricsSchema } from "./schemas/metrics.schema"; + +@Module({ + imports: [ + MongooseModule.forFeature([ + { + name: Metrics.name, + schema: MetricsSchema, + }, + ]), + ], + providers: [MetricsService, MetricsController], + exports: [], +}) +export class MetricsModule {} diff --git a/src/metrics/metrics.service.ts b/src/metrics-and-logs/metrics/metrics.service.ts similarity index 69% rename from src/metrics/metrics.service.ts rename to src/metrics-and-logs/metrics/metrics.service.ts index 711b2decc..fa35bc19e 100644 --- a/src/metrics/metrics.service.ts +++ b/src/metrics-and-logs/metrics/metrics.service.ts @@ -1,14 +1,11 @@ import { InjectModel } from "@nestjs/mongoose"; import { Metrics, MetricsDocument } from "./schemas/metrics.schema"; import { Model } from "mongoose"; -import { AccessLogs, AccessLogsDocument } from "./schemas/access-log.schema"; export class MetricsService { constructor( @InjectModel(Metrics.name) private metricsModel: Model, - @InjectModel(AccessLogs.name) - private accessLogsModel: Model, ) {} // Method to log/store metrics async create() {} diff --git a/src/metrics/schemas/metrics-record.schema.ts b/src/metrics-and-logs/metrics/schemas/metrics-record.schema.ts similarity index 100% rename from src/metrics/schemas/metrics-record.schema.ts rename to src/metrics-and-logs/metrics/schemas/metrics-record.schema.ts diff --git a/src/metrics/schemas/metrics.schema.ts b/src/metrics-and-logs/metrics/schemas/metrics.schema.ts similarity index 100% rename from src/metrics/schemas/metrics.schema.ts rename to src/metrics-and-logs/metrics/schemas/metrics.schema.ts diff --git a/src/metrics/middlewares/metrics.middleware.ts b/src/metrics-and-logs/middlewares/metrics-and-logs.middleware.ts similarity index 79% rename from src/metrics/middlewares/metrics.middleware.ts rename to src/metrics-and-logs/middlewares/metrics-and-logs.middleware.ts index def933f65..202c0b3fc 100644 --- a/src/metrics/middlewares/metrics.middleware.ts +++ b/src/metrics-and-logs/middlewares/metrics-and-logs.middleware.ts @@ -1,13 +1,13 @@ import { Injectable, Logger, NestMiddleware } from "@nestjs/common"; import { Request, Response, NextFunction } from "express"; import { JwtService } from "@nestjs/jwt"; -import { MetricsController } from "../metrics.controller"; +import { AccessLogsController } from "../access-logs/access-logs.controller"; @Injectable() -export class MetricsMiddleware implements NestMiddleware { +export class MetricsAndLogsMiddleware implements NestMiddleware { constructor( private readonly jwtService: JwtService, - private readonly metricsController: MetricsController, + private readonly accessLogsController: AccessLogsController, ) {} use(req: Request, res: Response, next: NextFunction) { const userAgent = req.headers["user-agent"]; @@ -21,7 +21,7 @@ export class MetricsMiddleware implements NestMiddleware { const statusCode = res.statusCode; const responseTime = Date.now() - startTime; - this.metricsController.logMetrics( + this.accessLogsController.logMetrics( user, ip, userAgent, @@ -45,7 +45,7 @@ export class MetricsMiddleware implements NestMiddleware { } return null; } catch (error) { - Logger.error("Error parsing token-> MetricsMiddleware", error); + Logger.error("Error parsing token-> MetricsAndLogsMiddleware", error); return null; } } diff --git a/src/metrics/metrics.module.ts b/src/metrics/metrics.module.ts deleted file mode 100644 index d9e5b7800..000000000 --- a/src/metrics/metrics.module.ts +++ /dev/null @@ -1,46 +0,0 @@ -import { Logger, MiddlewareConsumer, Module, NestModule } from "@nestjs/common"; -import { ConfigModule, ConfigService } from "@nestjs/config"; -import { MetricsMiddleware } from "./middlewares/metrics.middleware"; -import { JwtModule } from "@nestjs/jwt"; -import { MetricsService } from "./metrics.service"; -import { MetricsController } from "./metrics.controller"; -import { MongooseModule } from "@nestjs/mongoose"; -import { Metrics, MetricsSchema } from "./schemas/metrics.schema"; -import { AccessLogs, AccessLogSchema } from "./schemas/access-log.schema"; - -@Module({ - imports: [ - ConfigModule, - JwtModule, - MongooseModule.forFeature([ - { - name: Metrics.name, - schema: MetricsSchema, - }, - { - name: AccessLogs.name, - schema: AccessLogSchema, - }, - ]), - ], - providers: [MetricsService, MetricsController], - exports: [], -}) -export class MetricsModule implements NestModule { - constructor(private readonly configService: ConfigService) {} - - configure(consumer: MiddlewareConsumer) { - const { include = [], exclude = [] } = - this.configService.get("metricsConfig") || {}; - - try { - consumer - .apply(MetricsMiddleware) - .exclude(...exclude) - .forRoutes(...include); - Logger.log("Start collecting metrics", "MetricsModule"); - } catch (error) { - Logger.error("Error configuring metrics middleware", error); - } - } -} From 640a41268effc4cfac5c920036cab6c3cf32803c Mon Sep 17 00:00:00 2001 From: junjiequan Date: Thu, 12 Dec 2024 15:49:22 +0100 Subject: [PATCH 3/5] implement accesslogs service methods --- package-lock.json | 26 +++++++- package.json | 1 + .../access-logs/access-logs.controller.ts | 55 +++++++++++++---- .../access-logs/access-logs.module.ts | 9 ++- .../access-logs/access-logs.service.ts | 24 +++++--- .../access-logs/schemas/access-log.schema.ts | 38 +++++++++--- .../metrics/metrics.controller.ts | 2 + .../metrics/metrics.service.ts | 25 ++++++-- .../metrics/schemas/metrics-record.schema.ts | 4 +- .../metrics/schemas/metrics.schema.ts | 5 +- .../metrics/tasks/metrics-cron.task.ts | 13 ++++ .../metrics-and-logs.middleware.ts | 61 ++++++++++++------- 12 files changed, 202 insertions(+), 61 deletions(-) create mode 100644 src/metrics-and-logs/metrics/tasks/metrics-cron.task.ts diff --git a/package-lock.json b/package-lock.json index 9fcc5a8c4..488e110de 100644 --- a/package-lock.json +++ b/package-lock.json @@ -22,6 +22,7 @@ "@nestjs/mongoose": "^10.0.0", "@nestjs/passport": "^10.0.0", "@nestjs/platform-express": "^10.3.8", + "@nestjs/schedule": "^4.1.2", "@nestjs/swagger": "^7.1.7", "@nestjs/terminus": "^10.1.1", "@user-office-software/duo-logger": "^2.1.1", @@ -2557,6 +2558,19 @@ "@nestjs/core": "^10.0.0" } }, + "node_modules/@nestjs/schedule": { + "version": "4.1.2", + "resolved": "https://registry.npmjs.org/@nestjs/schedule/-/schedule-4.1.2.tgz", + "integrity": "sha512-hCTQ1lNjIA5EHxeu8VvQu2Ed2DBLS1GSC6uKPYlBiQe6LL9a7zfE9iVSK+zuK8E2odsApteEBmfAQchc8Hx0Gg==", + "dependencies": { + "cron": "3.2.1", + "uuid": "11.0.3" + }, + "peerDependencies": { + "@nestjs/common": "^8.0.0 || ^9.0.0 || ^10.0.0", + "@nestjs/core": "^8.0.0 || ^9.0.0 || ^10.0.0" + } + }, "node_modules/@nestjs/schematics": { "version": "10.1.4", "resolved": "https://registry.npmjs.org/@nestjs/schematics/-/schematics-10.1.4.tgz", @@ -3247,8 +3261,7 @@ "node_modules/@types/luxon": { "version": "3.4.2", "resolved": "https://registry.npmjs.org/@types/luxon/-/luxon-3.4.2.tgz", - "integrity": "sha512-TifLZlFudklWlMBfhubvgqTXRzLDI5pCbGa4P8a3wPyUQSW+1xQ5eDsreP9DWHX3tjq1ke96uYG/nwundroWcA==", - "dev": true + "integrity": "sha512-TifLZlFudklWlMBfhubvgqTXRzLDI5pCbGa4P8a3wPyUQSW+1xQ5eDsreP9DWHX3tjq1ke96uYG/nwundroWcA==" }, "node_modules/@types/methods": { "version": "1.1.4", @@ -5537,6 +5550,15 @@ "integrity": "sha512-dcKFX3jn0MpIaXjisoRvexIJVEKzaq7z2rZKxf+MSr9TkdmHmsU4m2lcLojrj/FHl8mk5VxMmYA+ftRkP/3oKQ==", "dev": true }, + "node_modules/cron": { + "version": "3.2.1", + "resolved": "https://registry.npmjs.org/cron/-/cron-3.2.1.tgz", + "integrity": "sha512-w2n5l49GMmmkBFEsH9FIDhjZ1n1QgTMOCMGuQtOXs5veNiosZmso6bQGuqOJSYAXXrG84WQFVneNk+Yt0Ua9iw==", + "dependencies": { + "@types/luxon": "~3.4.0", + "luxon": "~3.5.0" + } + }, "node_modules/cross-spawn": { "version": "7.0.6", "resolved": "https://registry.npmjs.org/cross-spawn/-/cross-spawn-7.0.6.tgz", diff --git a/package.json b/package.json index 737cf9ac0..f38220db2 100644 --- a/package.json +++ b/package.json @@ -43,6 +43,7 @@ "@nestjs/mongoose": "^10.0.0", "@nestjs/passport": "^10.0.0", "@nestjs/platform-express": "^10.3.8", + "@nestjs/schedule": "^4.1.2", "@nestjs/swagger": "^7.1.7", "@nestjs/terminus": "^10.1.1", "@user-office-software/duo-logger": "^2.1.1", diff --git a/src/metrics-and-logs/access-logs/access-logs.controller.ts b/src/metrics-and-logs/access-logs/access-logs.controller.ts index 9c3d5c174..0dabb0f5e 100644 --- a/src/metrics-and-logs/access-logs/access-logs.controller.ts +++ b/src/metrics-and-logs/access-logs/access-logs.controller.ts @@ -1,18 +1,49 @@ -import { Controller } from "@nestjs/common"; - +import { Controller, Get, Query } from "@nestjs/common"; +import { ApiOperation, ApiQuery, ApiTags } from "@nestjs/swagger"; +import { AccessLogsService } from "./access-logs.service"; +@ApiTags("access-logs") @Controller("access-logs") export class AccessLogsController { - constructor() {} - logMetrics( - user: string | null, - ip: string | undefined, - userAgent: string | undefined, - endpoint: string, - statusCode: number, - responseTime: number, + constructor(private readonly accessLogsService: AccessLogsService) {} + + //TODO: use correct checkpolies + // @UseGuards(PoliciesGuard) + @Get("/find") + @ApiOperation({ summary: "Get access logs" }) + @ApiQuery({ + name: "query", + description: `{ "endpoint": { "$regex": "20.500.12269", "$options": "i" } } \n +{ "userId": "example" } \n +{ "$or": [ { "userId": "example" }, { "endpoint": { "$regex": "20.500.12269", "$options": "i" } } ] }`, + required: false, + type: String, + }) + @ApiQuery({ + name: "projection", + description: '{"userId": 1, "endpoint": 1, "createdAt": 1}', + required: false, + type: String, + }) + @ApiQuery({ + name: "options", + description: + "{ limit: 10, skip: 5, sort: { createdAt: -1 }, maxTimeMS: 1000, hint: { createdAt: 1 } }", + required: false, + type: String, + }) + async find( + @Query("query") query: string, + @Query("projection") + projection?: string, + @Query("options") options?: string, ) { - console.log( - `User ID: ${user}, IP: ${ip}, User-Agent: ${userAgent}, Endpoint: ${endpoint}, Status Code: ${statusCode}, Response Time: ${responseTime}ms`, + const parsedQuery = query ? JSON.parse(query) : {}; + const parsedOption = options ? JSON.parse(options) : {}; + const parsedProjection = projection ? JSON.parse(projection) : null; + return this.accessLogsService.find( + parsedQuery, + parsedProjection, + parsedOption, ); } } diff --git a/src/metrics-and-logs/access-logs/access-logs.module.ts b/src/metrics-and-logs/access-logs/access-logs.module.ts index 2c4072304..abd383adc 100644 --- a/src/metrics-and-logs/access-logs/access-logs.module.ts +++ b/src/metrics-and-logs/access-logs/access-logs.module.ts @@ -1,19 +1,22 @@ import { Module } from "@nestjs/common"; import { AccessLogsService } from "./access-logs.service"; import { MongooseModule } from "@nestjs/mongoose"; -import { AccessLogs, AccessLogSchema } from "./schemas/access-log.schema"; +import { AccessLog, AccessLogSchema } from "./schemas/access-log.schema"; import { AccessLogsController } from "./access-logs.controller"; +import { CaslModule } from "src/casl/casl.module"; @Module({ imports: [ MongooseModule.forFeature([ { - name: AccessLogs.name, + name: AccessLog.name, schema: AccessLogSchema, }, ]), + CaslModule, ], providers: [AccessLogsService, AccessLogsController], - exports: [AccessLogsController], + exports: [AccessLogsService, AccessLogsController], + controllers: [AccessLogsController], }) export class AccessLogsModule {} diff --git a/src/metrics-and-logs/access-logs/access-logs.service.ts b/src/metrics-and-logs/access-logs/access-logs.service.ts index c3f6ef4a7..974b260b4 100644 --- a/src/metrics-and-logs/access-logs/access-logs.service.ts +++ b/src/metrics-and-logs/access-logs/access-logs.service.ts @@ -1,14 +1,24 @@ import { InjectModel } from "@nestjs/mongoose"; -import { Model } from "mongoose"; -import { AccessLogs, AccessLogsDocument } from "./schemas/access-log.schema"; +import { Model, FilterQuery, QueryOptions, ProjectionType } from "mongoose"; +import { AccessLog, AccessLogDocument } from "./schemas/access-log.schema"; export class AccessLogsService { constructor( - @InjectModel(AccessLogs.name) - private accessLogsModel: Model, + @InjectModel(AccessLog.name) + private accessLogModel: Model, ) {} // Method to log/store metrics - async create() {} - async findById() {} - async findAll() {} + + async create(accessLogData: AccessLog): Promise { + const accessLog = new this.accessLogModel(accessLogData); + return accessLog.save(); + } + + async find( + query: FilterQuery, + projection: ProjectionType | null | undefined, + options: QueryOptions | null | undefined, + ): Promise { + return this.accessLogModel.find(query, projection, options).exec(); + } } diff --git a/src/metrics-and-logs/access-logs/schemas/access-log.schema.ts b/src/metrics-and-logs/access-logs/schemas/access-log.schema.ts index 85818af71..acfe02bb8 100644 --- a/src/metrics-and-logs/access-logs/schemas/access-log.schema.ts +++ b/src/metrics-and-logs/access-logs/schemas/access-log.schema.ts @@ -2,21 +2,29 @@ import { Prop, Schema, SchemaFactory } from "@nestjs/mongoose"; import { ApiProperty, ApiPropertyOptional } from "@nestjs/swagger"; import { Document } from "mongoose"; -export type AccessLogsDocument = AccessLogs & Document; +export type AccessLogDocument = AccessLog & Document; @Schema({ - collection: "accessLog", - timestamps: true, + collection: "AccessLog", + timestamps: { createdAt: true, updatedAt: false }, + versionKey: false, }) @Schema() -export class AccessLogs { - @ApiProperty({ +export class AccessLog { + @ApiPropertyOptional({ description: "User ID associated with the access log", type: String, }) @Prop({ type: String, default: null }) userId: string | null; + @ApiPropertyOptional({ + description: "The timestamp when the document was created", + type: Date, + }) + @Prop({ type: Date, default: Date.now }) + createdAt?: Date; + @ApiProperty({ description: "HTTP status code for the request", type: Number, @@ -31,14 +39,28 @@ export class AccessLogs { @Prop({ type: String, required: true }) endpoint: string; + @ApiPropertyOptional({ + description: "Endpoint for the request", + type: Object, + }) + @Prop({ type: Object, default: null }) + query: object; + @ApiPropertyOptional({ description: "Original IP of the user for the request", type: String, }) - @Prop({ type: String, required: true }) - originIp: string; + @Prop({ type: String, default: "Not found" }) + originIp: string | undefined; + + @ApiProperty({ + description: "Response time in ms for the request", + type: String, + }) + @Prop({ type: Number, required: true }) + responseTime: number; } -export const AccessLogSchema = SchemaFactory.createForClass(AccessLogs); +export const AccessLogSchema = SchemaFactory.createForClass(AccessLog); AccessLogSchema.index({ createdAt: 1 }); diff --git a/src/metrics-and-logs/metrics/metrics.controller.ts b/src/metrics-and-logs/metrics/metrics.controller.ts index 1a2ac8e4b..1ea8794df 100644 --- a/src/metrics-and-logs/metrics/metrics.controller.ts +++ b/src/metrics-and-logs/metrics/metrics.controller.ts @@ -1,5 +1,7 @@ import { Controller } from "@nestjs/common"; +import { ApiTags } from "@nestjs/swagger"; +@ApiTags("metrics") @Controller("metrics") export class MetricsController { constructor() {} diff --git a/src/metrics-and-logs/metrics/metrics.service.ts b/src/metrics-and-logs/metrics/metrics.service.ts index fa35bc19e..86fdc7966 100644 --- a/src/metrics-and-logs/metrics/metrics.service.ts +++ b/src/metrics-and-logs/metrics/metrics.service.ts @@ -1,14 +1,31 @@ import { InjectModel } from "@nestjs/mongoose"; import { Metrics, MetricsDocument } from "./schemas/metrics.schema"; import { Model } from "mongoose"; +import { Logger } from "@nestjs/common"; export class MetricsService { + accessLogsService: any; constructor( @InjectModel(Metrics.name) private metricsModel: Model, ) {} - // Method to log/store metrics - async create() {} - async findById() {} - async findAll() {} + + async createMetric(metricData: Partial): Promise { + const metric = new this.metricsModel(metricData); + return metric.save(); + } + + async generateCompactMetrics() { + Logger.log("Starting metrics compaction..."); + const rawLogs = await this.accessLogsService.findLogs({}); + // Compact logic here + const compactedMetrics = this.compactLogs(rawLogs); + await this.createMetric(compactedMetrics); + Logger.log("Metrics compaction completed."); + } + + private compactLogs(logs: any[]): Partial { + // Implement compaction logic + return {}; + } } diff --git a/src/metrics-and-logs/metrics/schemas/metrics-record.schema.ts b/src/metrics-and-logs/metrics/schemas/metrics-record.schema.ts index 24bc41bea..cea9b46b3 100644 --- a/src/metrics-and-logs/metrics/schemas/metrics-record.schema.ts +++ b/src/metrics-and-logs/metrics/schemas/metrics-record.schema.ts @@ -2,7 +2,9 @@ import { Prop, Schema, SchemaFactory } from "@nestjs/mongoose"; import { ApiProperty } from "@nestjs/swagger"; import { Document } from "mongoose"; -@Schema() +@Schema({ + versionKey: false, +}) export class MetricsRecord { @ApiProperty({ description: "The endpoint path being tracked", diff --git a/src/metrics-and-logs/metrics/schemas/metrics.schema.ts b/src/metrics-and-logs/metrics/schemas/metrics.schema.ts index 2a2e0ffe9..f8029e987 100644 --- a/src/metrics-and-logs/metrics/schemas/metrics.schema.ts +++ b/src/metrics-and-logs/metrics/schemas/metrics.schema.ts @@ -6,8 +6,9 @@ import { MetricsRecord } from "./metrics-record.schema"; export type MetricsDocument = Metrics & Document; @Schema({ - collection: "metrics", - timestamps: true, + collection: "Metrics", + timestamps: { createdAt: true, updatedAt: false }, + versionKey: false, }) export class Metrics { @ApiProperty({ diff --git a/src/metrics-and-logs/metrics/tasks/metrics-cron.task.ts b/src/metrics-and-logs/metrics/tasks/metrics-cron.task.ts new file mode 100644 index 000000000..127ec0607 --- /dev/null +++ b/src/metrics-and-logs/metrics/tasks/metrics-cron.task.ts @@ -0,0 +1,13 @@ +import { Injectable, Logger } from "@nestjs/common"; +import { Cron } from "@nestjs/schedule"; +import { MetricsController } from "../metrics.controller"; + +@Injectable() +export class MetricsCronTask { + constructor(private readonly metricsController: MetricsController) {} + + @Cron("0 0 * * *") // Run daily at midnight + async handleCron() { + Logger.log("Running daily metrics compaction..."); + } +} diff --git a/src/metrics-and-logs/middlewares/metrics-and-logs.middleware.ts b/src/metrics-and-logs/middlewares/metrics-and-logs.middleware.ts index 202c0b3fc..97d8b1c89 100644 --- a/src/metrics-and-logs/middlewares/metrics-and-logs.middleware.ts +++ b/src/metrics-and-logs/middlewares/metrics-and-logs.middleware.ts @@ -1,49 +1,66 @@ import { Injectable, Logger, NestMiddleware } from "@nestjs/common"; import { Request, Response, NextFunction } from "express"; import { JwtService } from "@nestjs/jwt"; -import { AccessLogsController } from "../access-logs/access-logs.controller"; +import { AccessLogsService } from "../access-logs/access-logs.service"; +import { parse } from "url"; @Injectable() export class MetricsAndLogsMiddleware implements NestMiddleware { + private requestCache = new Map(); // Cache to store recent requests + private cacheDuration = 1000; + constructor( private readonly jwtService: JwtService, - private readonly accessLogsController: AccessLogsController, + private readonly accessLogsService: AccessLogsService, ) {} use(req: Request, res: Response, next: NextFunction) { + const { query, pathname } = parse(req.originalUrl, true); const userAgent = req.headers["user-agent"]; - const authHeader = req.headers.authorization; - const ip = req.ip || req.socket.remoteAddress; - const user = this.parseToken(authHeader); - const endpoint = req.originalUrl; + // TODO: Better to use a library for this? + const isBot = userAgent ? /bot|crawl|spider|slurp/i.test(userAgent) : false; + + if (!pathname || isBot) return; + const startTime = Date.now(); + //TODO: Implment the logic for checking if this is human or bot + const authHeader = req.headers.authorization; + const originIp = req.socket.remoteAddress; + const userId = this.parseToken(authHeader); + + const cacheKeyIdentifier = `${userId}-${originIp}-${pathname}`; res.on("finish", () => { const statusCode = res.statusCode; const responseTime = Date.now() - startTime; - this.accessLogsController.logMetrics( - user, - ip, - userAgent, - endpoint, - statusCode, - responseTime, - ); + const lastHitTime = this.requestCache.get(cacheKeyIdentifier); + + // Log only if the request was not recently logged + if (!lastHitTime || Date.now() - lastHitTime > this.cacheDuration) { + this.accessLogsService.create({ + userId, + originIp, + endpoint: pathname, + query: query, + statusCode, + responseTime, + }); + + this.requestCache.set(cacheKeyIdentifier, Date.now()); + } }); next(); } private parseToken(authHeader?: string) { + if (!authHeader) return null; + const token = authHeader.split(" ")[1]; + if (!token) return null; + try { - if (authHeader && authHeader.startsWith("Bearer ")) { - const token = authHeader.split(" ")[1]; - const { id, username } = this.jwtService.decode(token); - // TODO: changes it to return userid before merge; - return username; - // return id; - } - return null; + const { id } = this.jwtService.decode(token); + return id; } catch (error) { Logger.error("Error parsing token-> MetricsAndLogsMiddleware", error); return null; From 6a1cda5da12de08cbf771e921b88245ecaa31255 Mon Sep 17 00:00:00 2001 From: junjiequan Date: Thu, 12 Dec 2024 16:52:46 +0100 Subject: [PATCH 4/5] minor fixes --- .../access-logs/schemas/access-log.schema.ts | 1 - src/metrics-and-logs/metrics/schemas/metrics.schema.ts | 10 +++++----- 2 files changed, 5 insertions(+), 6 deletions(-) diff --git a/src/metrics-and-logs/access-logs/schemas/access-log.schema.ts b/src/metrics-and-logs/access-logs/schemas/access-log.schema.ts index acfe02bb8..523d87e95 100644 --- a/src/metrics-and-logs/access-logs/schemas/access-log.schema.ts +++ b/src/metrics-and-logs/access-logs/schemas/access-log.schema.ts @@ -9,7 +9,6 @@ export type AccessLogDocument = AccessLog & Document; timestamps: { createdAt: true, updatedAt: false }, versionKey: false, }) -@Schema() export class AccessLog { @ApiPropertyOptional({ description: "User ID associated with the access log", diff --git a/src/metrics-and-logs/metrics/schemas/metrics.schema.ts b/src/metrics-and-logs/metrics/schemas/metrics.schema.ts index f8029e987..630669a21 100644 --- a/src/metrics-and-logs/metrics/schemas/metrics.schema.ts +++ b/src/metrics-and-logs/metrics/schemas/metrics.schema.ts @@ -1,5 +1,5 @@ import { Prop, Schema, SchemaFactory } from "@nestjs/mongoose"; -import { ApiProperty } from "@nestjs/swagger"; +import { ApiProperty, ApiPropertyOptional } from "@nestjs/swagger"; import { Date, Document } from "mongoose"; import { MetricsRecord } from "./metrics-record.schema"; @@ -11,12 +11,12 @@ export type MetricsDocument = Metrics & Document; versionKey: false, }) export class Metrics { - @ApiProperty({ + @ApiPropertyOptional({ description: "The date the metric was recorded", - type: String, + type: Date, }) - @Prop({ type: Date, required: false, default: Date.now }) - date: Date; + @Prop({ type: Date, default: Date.now }) + createdAt?: Date; @ApiProperty({ description: "A list of endpoints with their access log compacted details", From 72084b760c8a46cc00c57db84d648fc898e0ab89 Mon Sep 17 00:00:00 2001 From: junjiequan Date: Mon, 16 Dec 2024 10:22:50 +0100 Subject: [PATCH 5/5] metrics service init part --- .../access-logs/access-logs.service.ts | 1 - .../metrics/metrics.controller.ts | 53 ++++++++++++++----- .../metrics/metrics.module.ts | 2 + .../metrics/metrics.service.ts | 31 +++++------ 4 files changed, 55 insertions(+), 32 deletions(-) diff --git a/src/metrics-and-logs/access-logs/access-logs.service.ts b/src/metrics-and-logs/access-logs/access-logs.service.ts index 974b260b4..a850992b2 100644 --- a/src/metrics-and-logs/access-logs/access-logs.service.ts +++ b/src/metrics-and-logs/access-logs/access-logs.service.ts @@ -7,7 +7,6 @@ export class AccessLogsService { @InjectModel(AccessLog.name) private accessLogModel: Model, ) {} - // Method to log/store metrics async create(accessLogData: AccessLog): Promise { const accessLog = new this.accessLogModel(accessLogData); diff --git a/src/metrics-and-logs/metrics/metrics.controller.ts b/src/metrics-and-logs/metrics/metrics.controller.ts index 1ea8794df..eaf416f4d 100644 --- a/src/metrics-and-logs/metrics/metrics.controller.ts +++ b/src/metrics-and-logs/metrics/metrics.controller.ts @@ -1,20 +1,49 @@ -import { Controller } from "@nestjs/common"; -import { ApiTags } from "@nestjs/swagger"; +import { Controller, Get, Query } from "@nestjs/common"; +import { ApiOperation, ApiQuery, ApiTags } from "@nestjs/swagger"; +import { MetricsService } from "./metrics.service"; @ApiTags("metrics") @Controller("metrics") export class MetricsController { - constructor() {} - logMetrics( - user: string | null, - ip: string | undefined, - userAgent: string | undefined, - endpoint: string, - statusCode: number, - responseTime: number, + constructor(private readonly metricsService: MetricsService) {} + + //TODO: correct checkpolies and api description + @Get("/find") + @ApiOperation({ summary: "Get metrics" }) + @ApiQuery({ + name: "query", + description: `{ "endpoint": { "$regex": "20.500.12269", "$options": "i" } } \n +{ "userId": "example" } \n +{ "$or": [ { "userId": "example" }, { "endpoint": { "$regex": "20.500.12269", "$options": "i" } } ] }`, + required: false, + type: String, + }) + @ApiQuery({ + name: "projection", + description: '{"userId": 1, "endpoint": 1, "createdAt": 1}', + required: false, + type: String, + }) + @ApiQuery({ + name: "options", + description: + "{ limit: 10, skip: 5, sort: { createdAt: -1 }, maxTimeMS: 1000, hint: { createdAt: 1 } }", + required: false, + type: String, + }) + async find( + @Query("query") query: string, + @Query("projection") + projection?: string, + @Query("options") options?: string, ) { - console.log( - `User ID: ${user}, IP: ${ip}, User-Agent: ${userAgent}, Endpoint: ${endpoint}, Status Code: ${statusCode}, Response Time: ${responseTime}ms`, + const parsedQuery = query ? JSON.parse(query) : {}; + const parsedOption = options ? JSON.parse(options) : {}; + const parsedProjection = projection ? JSON.parse(projection) : null; + return this.metricsService.find( + parsedQuery, + parsedProjection, + parsedOption, ); } } diff --git a/src/metrics-and-logs/metrics/metrics.module.ts b/src/metrics-and-logs/metrics/metrics.module.ts index 873a5c306..308bc167b 100644 --- a/src/metrics-and-logs/metrics/metrics.module.ts +++ b/src/metrics-and-logs/metrics/metrics.module.ts @@ -3,6 +3,7 @@ import { MetricsService } from "./metrics.service"; import { MetricsController } from "./metrics.controller"; import { MongooseModule } from "@nestjs/mongoose"; import { Metrics, MetricsSchema } from "./schemas/metrics.schema"; +import { AccessLogsModule } from "../access-logs/access-logs.module"; @Module({ imports: [ @@ -12,6 +13,7 @@ import { Metrics, MetricsSchema } from "./schemas/metrics.schema"; schema: MetricsSchema, }, ]), + AccessLogsModule, ], providers: [MetricsService, MetricsController], exports: [], diff --git a/src/metrics-and-logs/metrics/metrics.service.ts b/src/metrics-and-logs/metrics/metrics.service.ts index 86fdc7966..10d9ae444 100644 --- a/src/metrics-and-logs/metrics/metrics.service.ts +++ b/src/metrics-and-logs/metrics/metrics.service.ts @@ -1,31 +1,24 @@ import { InjectModel } from "@nestjs/mongoose"; import { Metrics, MetricsDocument } from "./schemas/metrics.schema"; -import { Model } from "mongoose"; -import { Logger } from "@nestjs/common"; +import { FilterQuery, Model, ProjectionType, QueryOptions } from "mongoose"; +import { AccessLogsService } from "../access-logs/access-logs.service"; export class MetricsService { - accessLogsService: any; constructor( @InjectModel(Metrics.name) private metricsModel: Model, + private readonly accessLogsService: AccessLogsService, ) {} - - async createMetric(metricData: Partial): Promise { - const metric = new this.metricsModel(metricData); - return metric.save(); - } - - async generateCompactMetrics() { - Logger.log("Starting metrics compaction..."); - const rawLogs = await this.accessLogsService.findLogs({}); - // Compact logic here - const compactedMetrics = this.compactLogs(rawLogs); - await this.createMetric(compactedMetrics); - Logger.log("Metrics compaction completed."); + async create(metricsData: Metrics): Promise { + const metrics = new this.metricsModel(metricsData); + return metrics.save(); } - private compactLogs(logs: any[]): Partial { - // Implement compaction logic - return {}; + async find( + query: FilterQuery, + projection: ProjectionType | null | undefined, + options: QueryOptions | null | undefined, + ): Promise { + return this.metricsModel.find(query, projection, options).exec(); } }