feat(benchmark): add very basic benchmarking

Signed-off-by: david <[email protected]>

packages/benchmark/.mocharc.json

@@ -0,0 +1,10 @@
+{
+  "extension": [
+    "ts"
+  ],
+  "spec": "**/*.test.ts",
+  "require": [
+    "ts-node/register"
+  ],
+  "recursive": true
+}

packages/benchmark/README.md

@@ -0,0 +1,29 @@
+# Package Name
+This is meant to be a template for quickly adding new package libraries.  Replace this readme with relevant information for your package.
+
+## Adding a new package
+1. go into the package folder in this repo.
+2. cp -r template your_package_name
+3. Edit the package.json file renaming the "name" property.
+4. add dependencies as needed.
+
+## Template features
+This template will set you up with typescript, eslint, prettier, some basic scripts, and a main file entry point.
+
+### Scripts
+- build - typescript build and output to dist
+- watch - build watching for changes
+- format - prettier code fixing
+- lint - eslint code fixing
+- fix - eslint and prettier code fixing
+- lint:check - eslint code checking ( no changes )
+- format:check - prettier code checking ( no changes )
+- build:check - run type check without emitting files 
+- check - eslint and prettier and typescript code checking ( no changes )
+- test - run mocha testing
+- test:watch - run mocha testing
+- coverage - see testing coverage
+
+### Adding tests
+Add a `example.test.ts` file to any folder and mocha will find it.
+** note: chai v5 breaks typescript support, so we explicitly use chai 4 **

packages/benchmark/eslint.config.js

@@ -0,0 +1,6 @@
+ // .eslintrc.js in the new package
+module.exports = {
+  root:true,
+  extends: ['@repo/eslint-config/library.js'],
+};
+

packages/benchmark/package.json

@@ -0,0 +1,43 @@
+{
+  "name": "@repo/benchmark",
+  "version": "0.0.1",
+  "description": "",
+  "main": "index.js",
+  "scripts": {
+    "build": "tsc -b",
+    "build:check": "tsc --noEmit",
+    "watch": "tsc -b --watch",
+    "fix": "pnpm format && pnpm lint",
+    "format": "prettier --write src",
+    "format:check": "prettier src --check",
+    "lint": "eslint --fix",
+    "lint:check": "eslint",
+    "check": "pnpm format:check && pnpm lint:check && pnpm build:check",
+    "test": "mocha",
+    "coverage": "nyc mocha",
+    "test:watch": "mocha --watch"
+  },
+  "keywords": [],
+  "author": "",
+  "license": "ISC",
+  "dependencies": {
+  },
+"exports": {
+    ".": "./dist/index.js"
+  },
+  "devDependencies": {
+    "@istanbuljs/nyc-config-typescript": "^1.0.2",
+    "@repo/eslint-config": "workspace:*",
+    "@repo/typescript-config": "workspace:*",
+    "@types/chai": "^4.3.17",
+    "@types/mocha": "^10.0.7",
+    "chai": "^4.5.0",
+    "eslint": "^8.57.0",
+    "mocha": "^10.7.0",
+    "nyc": "^17.0.0",
+    "prettier": "^3.3.3",
+    "source-map-support": "^0.5.21",
+    "ts-node": "^10.9.2",
+    "typescript": "^5.5.4"
+  }
+}

packages/benchmark/src/benchmark.test.ts

@@ -0,0 +1,43 @@
+import { expect } from "chai";
+import { Benchmark } from "./benchmark";
+
+describe("Benchmark", () => {
+  let benchmark: Benchmark;
+
+  beforeEach(() => {
+    benchmark = new Benchmark();
+  });
+
+  it("should start and end a benchmark event correctly", async () => {
+    benchmark.start("testEvent", 0);
+    const duration = benchmark.end("testEvent", 1);
+    expect(duration).to.be.a("number");
+    expect(duration).to.be.greaterThan(0);
+  });
+
+  it("should throw an error if end is called without start", () => {
+    expect(() => benchmark.end("nonExistentEvent")).to.throw(
+      Error,
+      'Benchmark for event "nonExistentEvent" not started. Call start() before end().',
+    );
+  });
+
+  it("should handle multiple events independently", () => {
+    benchmark.start("event1", 0);
+    benchmark.start("event2", 0);
+
+    const duration1 = benchmark.end("event1", 1);
+    expect(duration1).to.be.a("number");
+    expect(duration1).to.be.greaterThan(0);
+
+    const duration2 = benchmark.end("event2", 1);
+    expect(duration2).to.be.a("number");
+    expect(duration2).to.be.greaterThan(0);
+  });
+
+  it("should throw an error if the same event is started twice without ending", () => {
+    benchmark.start("duplicateEvent");
+    expect(() => benchmark.start("duplicateEvent")).to.not.throw();
+    expect(() => benchmark.end("duplicateEvent")).to.not.throw();
+  });
+});

packages/benchmark/src/benchmark.ts

@@ -0,0 +1,113 @@
+/**
+ * A class to benchmark events by tracking their start and end times.
+ */
+export class Benchmark {
+  private events: Map<string, number>;
+
+  /**
+   * Initializes a new instance of the Benchmark class.
+   */
+  constructor() {
+    this.events = new Map();
+  }
+
+  /**
+   * Starts tracking an event by storing its start time.
+   *
+   * @param {string} eventName - The name of the event to start tracking.
+   * @param {number} [now=Date.now()] - The current time in milliseconds. Defaults to the current time.
+   */
+  start(eventName: string, now: number = Date.now()) {
+    this.events.set(eventName, now);
+  }
+
+  /**
+   * Ends tracking an event and calculates its duration.
+   *
+   * @param {string} eventName - The name of the event to end tracking.
+   * @param {number} [now=Date.now()] - The current time in milliseconds. Defaults to the current time.
+   * @returns {number | undefined} The duration of the event in milliseconds, or undefined if the event was not started.
+   * @throws Will throw an error if the event was not started before calling this method.
+   */
+  end(eventName: string, now: number = Date.now()): number | undefined {
+    const startTime = this.events.get(eventName);
+    if (startTime === undefined) {
+      throw new Error(
+        `Benchmark for event "${eventName}" not started. Call start() before end().`,
+      );
+    }
+    const endTime = now;
+    const duration = endTime - startTime;
+    this.events.delete(eventName);
+    return duration;
+  }
+
+  /**
+   * Clears events that are older than the specified age.
+   *
+   * @param {number} age - The age in milliseconds. Events older than this will be cleared.
+   * @param {number} [now=Date.now()] - The current time in milliseconds. Defaults to the current time.
+   */
+  clear(age: number, now: number = Date.now()) {
+    for (const [eventName, startTime] of this.events.entries()) {
+      if (now - startTime > age) {
+        this.events.delete(eventName);
+      }
+    }
+  }
+
+  /**
+   * Provides statistics about the currently tracked events.
+   *
+   * @param {number} [now=Date.now()] - The current time in milliseconds. Defaults to the current time.
+   * @returns {object} An object containing statistics about the events.
+   */
+  getStats(now: number = Date.now()): {
+    total: number;
+    oldest: number | null;
+    newest: number | null;
+    average: number | null;
+    fastest: number | null;
+    slowest: number | null;
+  } {
+    const eventTimes = Array.from(this.events.values());
+    const total = eventTimes.length;
+
+    if (total === 0) {
+      return {
+        total,
+        oldest: null,
+        newest: null,
+        average: null,
+        fastest: null,
+        slowest: null,
+      };
+    }
+
+    let oldest = Number.MAX_VALUE;
+    let newest = Number.MIN_VALUE;
+    let totalAge = 0;
+    let fastest = Number.MAX_VALUE;
+    let slowest = Number.MIN_VALUE;
+
+    for (const time of eventTimes) {
+      const age = now - time;
+      totalAge += age;
+      if (age < fastest) fastest = age;
+      if (age > slowest) slowest = age;
+      if (age < oldest) oldest = age;
+      if (age > newest) newest = age;
+    }
+
+    const average = totalAge / total;
+
+    return {
+      total,
+      oldest,
+      newest,
+      average,
+      fastest,
+      slowest,
+    };
+  }
+}

packages/benchmark/src/index.ts

@@ -0,0 +1 @@
+export * from "./benchmark";

packages/benchmark/tsconfig.json

@@ -0,0 +1,6 @@
+{
+  "extends":"@repo/typescript-config/base.json",
+  "compilerOptions": {
+    "outDir": "./dist"                                   /* Specify an output folder for all emitted files. */
+  }
+}

packages/indexer/package.json

@@ -31,7 +31,8 @@
     "lodash": "^4.17.21",
     "redis": "^4.7.0",
     "superstruct": "^2.0.3-1",
-    "winston": "^3.13.1"
+    "winston": "^3.13.1",
+    "@repo/benchmark": "workspace:*"
   },
   "exports": {
     ".": "./dist/index.js"

packages/indexer/src/data-indexing/service/AcrossIndexerManager.ts

@@ -8,7 +8,10 @@ import { RedisCache } from "../../redis/redisCache";
 import { RetryProvidersFactory } from "../../web3/RetryProvidersFactory";
 import { SpokePoolRepository } from "../../database/SpokePoolRepository";
 import { IndexerQueuesService } from "../../messaging/service";
-import { SpokePoolProcessor } from "../../services/spokePoolProcessor";
+import {
+  SpokePoolProcessor,
+  SpokePoolEventParams,
+} from "../../services/spokePoolProcessor";
 
 import { HubPoolIndexerDataHandler } from "./HubPoolIndexerDataHandler";
 import { SpokePoolIndexerDataHandler } from "./SpokePoolIndexerDataHandler";
@@ -41,10 +44,12 @@ export class AcrossIndexerManager {
     private indexerQueuesService: IndexerQueuesService,
   ) {}
 
-  public async start() {
+  public async start(
+    callback?: (chainId: number, events: SpokePoolEventParams) => void,
+  ) {
     return Promise.all([
       this.startHubPoolIndexer(),
-      this.startSpokePoolIndexers(),
+      this.startSpokePoolIndexers(callback),
     ]);
   }
 
@@ -81,7 +86,9 @@ export class AcrossIndexerManager {
     return this.hubPoolIndexer.start();
   }
 
-  private async startSpokePoolIndexers() {
+  private async startSpokePoolIndexers(
+    callback?: (chainId: number, events: SpokePoolEventParams) => void,
+  ) {
     const spokePoolIndexers = this.config.spokePoolChainsEnabled.map(
       (chainId) => {
         const spokePoolIndexerDataHandler = new SpokePoolIndexerDataHandler(
@@ -93,7 +100,16 @@ export class AcrossIndexerManager {
           this.hubPoolClientFactory,
           this.spokePoolClientFactory,
           this.spokePoolRepository,
-          new SpokePoolProcessor(this.postgres, this.logger, chainId),
+          new SpokePoolProcessor(
+            this.postgres,
+            this.logger,
+            chainId,
+            (events: SpokePoolEventParams) => {
+              if (callback) {
+                callback(chainId, events);
+              }
+            },
+          ),
           this.indexerQueuesService,
         );
         const spokePoolIndexer = new Indexer(