feat: migrate SpokePoolIndexer to new Indexer (#64)

Co-authored-by: Alexandru Matei <[email protected]>
across-protocol · Oct 8, 2024 · 048b8b6 · 048b8b6
1 parent 92c966b
commit 048b8b6
Show file tree

Hide file tree

Showing 18 changed files with 486 additions and 381 deletions.
diff --git a/docker-compose.yml b/docker-compose.yml
@@ -20,6 +20,8 @@ services:
     container_name: redis_cache
     volumes:
       - indexer-redis-volume:/data
+    ports:
+      - 6379:6379
 
   # Indexer Service
   indexer-scraper:

diff --git a/packages/indexer-database/src/entities/evm/ExecutedRelayerRefundRoot.ts b/packages/indexer-database/src/entities/evm/ExecutedRelayerRefundRoot.ts
@@ -52,6 +52,9 @@ export class ExecutedRelayerRefundRoot {
   @Column()
   blockNumber: number;
 
+  @Column()
+  finalised: boolean;
+
   @CreateDateColumn()
   createdAt: Date;
 }
diff --git a/packages/indexer-database/src/entities/evm/FilledV3Relay.ts b/packages/indexer-database/src/entities/evm/FilledV3Relay.ts
@@ -90,6 +90,9 @@ export class FilledV3Relay {
   @Column()
   blockNumber: number;
 
+  @Column()
+  finalised: boolean;
+
   @CreateDateColumn()
   createdAt: Date;
 }
diff --git a/packages/indexer-database/src/entities/evm/RelayedRootBundle.ts b/packages/indexer-database/src/entities/evm/RelayedRootBundle.ts
@@ -39,6 +39,9 @@ export class RelayedRootBundle {
   @Column()
   blockNumber: number;
 
+  @Column()
+  finalised: boolean;
+
   @CreateDateColumn()
   createdAt: Date;
 }
diff --git a/packages/indexer-database/src/entities/evm/RequestedSpeedUpV3Deposit.ts b/packages/indexer-database/src/entities/evm/RequestedSpeedUpV3Deposit.ts
@@ -46,6 +46,9 @@ export class RequestedSpeedUpV3Deposit {
   @Column()
   logIndex: number;
 
+  @Column()
+  finalised: boolean;
+
   @Column()
   blockNumber: number;
 

diff --git a/packages/indexer-database/src/entities/evm/RequestedV3SlowFill.ts b/packages/indexer-database/src/entities/evm/RequestedV3SlowFill.ts
@@ -72,6 +72,9 @@ export class RequestedV3SlowFill {
   @Column()
   blockNumber: number;
 
+  @Column()
+  finalised: boolean;
+
   @CreateDateColumn()
   createdAt: Date;
 }
diff --git a/packages/indexer-database/src/entities/evm/TokensBridged.ts b/packages/indexer-database/src/entities/evm/TokensBridged.ts
@@ -44,6 +44,9 @@ export class TokensBridged {
   @Column()
   blockNumber: number;
 
+  @Column()
+  finalised: boolean;
+
   @CreateDateColumn()
   createdAt: Date;
 }
diff --git a/packages/indexer-database/src/entities/evm/V3FundsDeposited.ts b/packages/indexer-database/src/entities/evm/V3FundsDeposited.ts
@@ -84,6 +84,9 @@ export class V3FundsDeposited {
   @Column()
   blockNumber: number;
 
+  @Column()
+  finalised: boolean;
+
   @CreateDateColumn()
   createdAt: Date;
 }
diff --git a/packages/indexer-database/src/migrations/1728296909794-SpokePoolFinalised.ts b/packages/indexer-database/src/migrations/1728296909794-SpokePoolFinalised.ts
@@ -0,0 +1,53 @@
+import { MigrationInterface, QueryRunner } from "typeorm";
+
+export class SpokePoolFinalised1728296909794 implements MigrationInterface {
+  name = "SpokePoolFinalised1728296909794";
+
+  public async up(queryRunner: QueryRunner): Promise<void> {
+    await queryRunner.query(
+      `ALTER TABLE "evm"."v3_funds_deposited" ADD "finalised" boolean NOT NULL`,
+    );
+    await queryRunner.query(
+      `ALTER TABLE "evm"."filled_v3_relay" ADD "finalised" boolean NOT NULL`,
+    );
+    await queryRunner.query(
+      `ALTER TABLE "evm"."requested_v3_slow_fill" ADD "finalised" boolean NOT NULL`,
+    );
+    await queryRunner.query(
+      `ALTER TABLE "evm"."requested_speed_up_v3_deposit" ADD "finalised" boolean NOT NULL`,
+    );
+    await queryRunner.query(
+      `ALTER TABLE "evm"."relayed_root_bundle" ADD "finalised" boolean NOT NULL`,
+    );
+    await queryRunner.query(
+      `ALTER TABLE "evm"."executed_relayer_refund_root" ADD "finalised" boolean NOT NULL`,
+    );
+    await queryRunner.query(
+      `ALTER TABLE "evm"."tokens_bridged" ADD "finalised" boolean NOT NULL`,
+    );
+  }
+
+  public async down(queryRunner: QueryRunner): Promise<void> {
+    await queryRunner.query(
+      `ALTER TABLE "evm"."tokens_bridged" DROP COLUMN "finalised"`,
+    );
+    await queryRunner.query(
+      `ALTER TABLE "evm"."executed_relayer_refund_root" DROP COLUMN "finalised"`,
+    );
+    await queryRunner.query(
+      `ALTER TABLE "evm"."relayed_root_bundle" DROP COLUMN "finalised"`,
+    );
+    await queryRunner.query(
+      `ALTER TABLE "evm"."requested_speed_up_v3_deposit" DROP COLUMN "finalised"`,
+    );
+    await queryRunner.query(
+      `ALTER TABLE "evm"."requested_v3_slow_fill" DROP COLUMN "finalised"`,
+    );
+    await queryRunner.query(
+      `ALTER TABLE "evm"."filled_v3_relay" DROP COLUMN "finalised"`,
+    );
+    await queryRunner.query(
+      `ALTER TABLE "evm"."v3_funds_deposited" DROP COLUMN "finalised"`,
+    );
+  }
+}
diff --git a/packages/indexer-database/src/utils/BaseRepository.ts b/packages/indexer-database/src/utils/BaseRepository.ts
@@ -37,4 +37,30 @@ export class BaseRepository {
       }
     }
   }
+
+  protected async insertWithFinalisationCheck<Entity extends ObjectLiteral>(
+    entity: EntityTarget<Entity>,
+    data: Partial<Entity>[],
+    uniqueKeys: (keyof Entity)[],
+    lastFinalisedBlock: number,
+  ) {
+    const repository = this.postgres.getRepository(entity);
+    const uniqueKeysAsStrings = uniqueKeys.map((key) => key.toString());
+
+    const savedData = await repository
+      .createQueryBuilder()
+      .insert()
+      .values(data)
+      .orUpdate(Object.keys(data[0] as any), uniqueKeysAsStrings)
+      .returning("*")
+      .execute();
+    await repository
+      .createQueryBuilder()
+      .delete()
+      .where("finalised = false")
+      .andWhere("blockNumber <= :lastFinalisedBlock", { lastFinalisedBlock })
+      .execute();
+
+    return savedData.generatedMaps as Entity[];
+  }
 }
diff --git a/packages/indexer/src/data-indexing/service/Indexer.ts b/packages/indexer/src/data-indexing/service/Indexer.ts
@@ -34,7 +34,8 @@ export class Indexer {
   public async start() {
     while (!this.stopRequested) {
       try {
-        const { latestBlockNumber, blockRange } = await this.getBlockRange();
+        const { latestBlockNumber, blockRange, lastFinalisedBlock } =
+          await this.getBlockRange();
 
         if (!blockRange) {
           this.logger.info({
@@ -44,18 +45,13 @@ export class Indexer {
             dataIdentifier: this.dataHandler.getDataIdentifier(),
           });
         } else {
-          const lastFinalisedBlockInBlockRange =
-            this.getLastFinalisedBlockInBlockRange(
-              latestBlockNumber,
-              blockRange,
-            );
           await this.dataHandler.processBlockRange(
             blockRange,
-            lastFinalisedBlockInBlockRange,
+            lastFinalisedBlock,
           );
           await this.redisCache.set(
             this.getLastFinalisedBlockCacheKey(),
-            lastFinalisedBlockInBlockRange,
+            lastFinalisedBlock,
           );
         }
       } catch (error) {
@@ -83,36 +79,37 @@ export class Indexer {
     this.stopRequested = true;
   }
 
-  private getLastFinalisedBlockInBlockRange(
-    latestBlockNumber: number,
-    blockRange: BlockRange,
-  ) {
-    const lastOnChainFinalisedBlock =
-      latestBlockNumber - this.config.finalisedBlockBufferDistance + 1;
-    const lastFinalisedBlockInBlockRange = Math.min(
-      blockRange.to,
-      lastOnChainFinalisedBlock,
-    );
-
-    return lastFinalisedBlockInBlockRange;
-  }
-
   private async getBlockRange() {
-    const lastBlockFinalised = await this.redisCache.get<number>(
+    const lastBlockFinalisedStored = await this.redisCache.get<number>(
       this.getLastFinalisedBlockCacheKey(),
     );
     const latestBlockNumber = await this.rpcProvider.getBlockNumber();
-    // If the last block finalised is the same as the latest block, no new blocks to process
-    if (latestBlockNumber === lastBlockFinalised) {
-      return { latestBlockNumber, blockRange: undefined };
+    const lastFinalisedBlockOnChain =
+      latestBlockNumber - this.config.finalisedBlockBufferDistance;
+
+    if (lastBlockFinalisedStored === lastFinalisedBlockOnChain) {
+      return {
+        latestBlockNumber,
+        blockRange: undefined,
+        lastFinalisedBlock: lastFinalisedBlockOnChain,
+      };
     }
-    const fromBlock = lastBlockFinalised
-      ? lastBlockFinalised + 1
+    const fromBlock = lastBlockFinalisedStored
+      ? lastBlockFinalisedStored + 1
       : this.dataHandler.getStartIndexingBlockNumber();
     // TODO: hardcoded 200_000, should be a config or removed
     const toBlock = Math.min(fromBlock + 200_000, latestBlockNumber);
     const blockRange: BlockRange = { from: fromBlock, to: toBlock };
-    return { latestBlockNumber, blockRange };
+    const lastFinalisedBlockInBlockRange = Math.min(
+      blockRange.to,
+      lastFinalisedBlockOnChain,
+    );
+
+    return {
+      latestBlockNumber,
+      blockRange,
+      lastFinalisedBlock: lastFinalisedBlockInBlockRange,
+    };
   }
 
   private getLastFinalisedBlockCacheKey() {