mempool#176 Optimise Angor Indexing (#24)

* (feat: angor-optimise): refactors angor indexing * chore: removes redundant code
block-core · Jan 2, 2025 · 3dcf0fc · 3dcf0fc
1 parent 8fa9e03
commit 3dcf0fc
Show file tree

Hide file tree

Showing 4 changed files with 60 additions and 49 deletions.
diff --git a/backend/src/api/database-migration.ts b/backend/src/api/database-migration.ts
@@ -1315,7 +1315,8 @@ class DatabaseMigration {
 
   private getCreateAngorBlocksTableQuery(): string {
     return `CREATE TABLE IF NOT EXISTS angor_blocks (
-      block_height INT PRIMARY KEY,
+      id INT NOT NULL PRIMARY KEY CHECK (id = 1),
+      block_height INT NOT NULL,
       block_hash CHAR(64) NOT NULL,
       indexed_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP
     ) ENGINE=InnoDB DEFAULT CHARSET=utf8;`;

diff --git a/backend/src/indexer.ts b/backend/src/indexer.ts
@@ -229,25 +229,18 @@ class Indexer {
    * Index transactions related to Angor projects.
    */
   public async indexAngorTransactions(): Promise<void> {
+    // Get latest indexed block.
+    const latestIndexedBlockHeight = await AngorBlocksRepository.$getLatestIndexedBlock();
     // All indexed blocks.
-    const indexedBlocks = await BlocksRepository.$getIndexedBlocks();
+    const indexedBlocks = latestIndexedBlockHeight
+      ? await BlocksRepository.$getIndexedBlocksFromHeight(latestIndexedBlockHeight + 1)
+      : await BlocksRepository.$getIndexedBlocks('ASC');
     // Sort indexed blocks by height, the lowest height should be in the beginning.
     // It is necessary to index transactions for project creations before
     // investment transactions.
-    const sortedBlocks = indexedBlocks.sort(
-      (
-        b1: { height: number; hash: string },
-        b2: { height: number; hash: string }
-      ) => b1.height - b2.height
-    );
-
-    for (const indexedBlock of sortedBlocks) {
-      if (indexedBlock.height === 0) {
-        continue;
-      }
 
-      const alreadyIndexed = await AngorBlocksRepository.isBlockIndexed(indexedBlock.height, indexedBlock.hash);
-      if (alreadyIndexed) {
+    for (const indexedBlock of indexedBlocks) {
+      if (indexedBlock.height === 0) {
         continue;
       }
       // Transaction IDs in the block.
@@ -285,7 +278,7 @@ class Indexer {
               });
           });
       }
-      await AngorBlocksRepository.$markBlockAsIndexed(indexedBlock.height, indexedBlock.hash);
+      await AngorBlocksRepository.$updateLatestIndexedBlock(indexedBlock.height, indexedBlock.hash);
     }
 
     logger.info('Indexing Angor transactions completed.');

diff --git a/backend/src/repositories/AngorBlocksRepository.ts b/backend/src/repositories/AngorBlocksRepository.ts
@@ -3,31 +3,38 @@ import logger from "../logger";
 import { RowDataPacket } from "mysql2/typings/mysql";
 
 class AngorBlocksRepository {
-  public async $markBlockAsIndexed(blockHeight: number, blockHash: string): Promise<void> {
+  public async $updateLatestIndexedBlock(blockHeight: number, blockHash: string): Promise<void> {
     try {
       await DB.query(
-        `INSERT INTO angor_blocks (block_height, block_hash) VALUES (?, ?)`, [blockHeight, blockHash])
+        `INSERT INTO angor_blocks (id, block_height, block_hash)
+        VALUES (1, ?, ?)
+        ON DUPLICATE KEY UPDATE
+            block_height = ?,
+            block_hash = ?`, [blockHeight, blockHash, blockHeight, blockHash]
+      );
     } catch (error) {
       logger.err(
-        'Error marking block as indexed. Reason: ' + (error instanceof Error ? error.message : error)
+        'Error updating latest indexed block. Reason: ' + (error instanceof Error ? error.message : error)
       );
     }
   }
 
-  public async isBlockIndexed(blockHeight: number, blockHash: string): Promise<boolean> {
+  public async $getLatestIndexedBlock(): Promise<number | null> {
     try {
-      const result = await DB.query(
-        `SELECT * FROM angor_blocks WHERE block_height = ? AND block_hash = ? LIMIT 1`, [blockHeight, blockHash]
-      );
-
-      const rows = result[0] as RowDataPacket;
+      const [rows]= await DB.query(
+        `SELECT block_height FROM angor_blocks WHERE id = 1`
+      ) as RowDataPacket[][];
 
-      return rows.length > 0;
+      if (rows.length === 0) {
+        return null;
+      } else {
+        return rows[0].block_height as number;
+      }
     } catch (error) {
       logger.err(
-        'Error checking if block is indexed. Reason: ' + (error instanceof Error ? error.message : error)
+        'Error getting latest indexed block. Reason: ' + (error instanceof Error ? error.message : error)
       );
-      return false;
+      throw error;
     }
   }
 }

diff --git a/backend/src/repositories/BlocksRepository.ts b/backend/src/repositories/BlocksRepository.ts
@@ -196,9 +196,9 @@ class BlocksRepository {
 
   /**
    * Save newly indexed data from core coinstatsindex
-   * 
-   * @param utxoSetSize 
-   * @param totalInputAmt 
+   *
+   * @param utxoSetSize
+   * @param totalInputAmt
    */
   public async $updateCoinStatsIndexData(blockHash: string, utxoSetSize: number,
     totalInputAmt: number
@@ -224,9 +224,9 @@ class BlocksRepository {
   /**
    * Update missing fee amounts fields
    *
-   * @param blockHash 
-   * @param feeAmtPercentiles 
-   * @param medianFeeAmt 
+   * @param blockHash
+   * @param feeAmtPercentiles
+   * @param medianFeeAmt
    */
   public async $updateFeeAmounts(blockHash: string, feeAmtPercentiles, medianFeeAmt) : Promise<void> {
     try {
@@ -810,16 +810,26 @@ class BlocksRepository {
   /**
    * Get a list of blocks that have been indexed
    */
-  public async $getIndexedBlocks(): Promise<{ height: number, hash: string }[]> {
+  public async $getIndexedBlocks(order = 'DESC'): Promise<{ height: number, hash: string }[]> {
     try {
-      const [rows] = await DB.query(`SELECT height, hash FROM blocks ORDER BY height DESC`) as RowDataPacket[][];
+      const [rows] = await DB.query(`SELECT height, hash FROM blocks ORDER BY height ${order}`, ) as RowDataPacket[][];
       return rows as { height: number, hash: string }[];
     } catch (e) {
       logger.err('Cannot generate block size and weight history. Reason: ' + (e instanceof Error ? e.message : e));
       throw e;
     }
   }
 
+  public async $getIndexedBlocksFromHeight(height: number): Promise<{ height: number, hash: string}[]> {
+    try {
+      const [rows] = await DB.query(`SELECT height, hash FROM blocks WHERE height >= ? ORDER BY height ASC`, [height]) as RowDataPacket[][];
+      return rows as {height: number, hash: string }[];
+    } catch (error) {
+      logger.err('Cannot generate block size and weight history. Reason: ' + (error instanceof Error ? error.message : error));
+      throw error;
+    }
+  }
+
   /**
    * Get a list of blocks that have not had CPFP data indexed
    */
@@ -949,9 +959,9 @@ class BlocksRepository {
 
   /**
    * Save indexed median fee to avoid recomputing it later
-   * 
-   * @param id 
-   * @param feePercentiles 
+   *
+   * @param id
+   * @param feePercentiles
    */
   public async $saveFeePercentilesForBlockId(id: string, feePercentiles: number[]): Promise<void> {
     try {
@@ -968,9 +978,9 @@ class BlocksRepository {
 
   /**
    * Save indexed effective fee statistics
-   * 
-   * @param id 
-   * @param feeStats 
+   *
+   * @param id
+   * @param feeStats
    */
   public async $saveEffectiveFeeStats(id: string, feeStats: EffectiveFeeStats): Promise<void> {
     try {
@@ -987,7 +997,7 @@ class BlocksRepository {
 
   /**
    * Save coinbase addresses
-   * 
+   *
    * @param id
    * @param addresses
    */
@@ -1006,7 +1016,7 @@ class BlocksRepository {
 
   /**
    * Save pool
-   * 
+   *
    * @param id
    * @param poolId
    */
@@ -1025,8 +1035,8 @@ class BlocksRepository {
 
   /**
    * Save block first seen time
-   * 
-   * @param id 
+   *
+   * @param id
    */
   public async $saveFirstSeenTime(id: string, firstSeen: number): Promise<void> {
     try {
@@ -1044,8 +1054,8 @@ class BlocksRepository {
   /**
    * Convert a mysql row block into a BlockExtended. Note that you
    * must provide the correct field into dbBlk object param
-   * 
-   * @param dbBlk 
+   *
+   * @param dbBlk
    */
   private async formatDbBlockIntoExtendedBlock(dbBlk: DatabaseBlock): Promise<BlockExtended> {
     const blk: Partial<BlockExtended> = {};
@@ -1065,7 +1075,7 @@ class BlocksRepository {
     blk.weight = dbBlk.weight;
     blk.previousblockhash = dbBlk.previousblockhash;
     blk.mediantime = dbBlk.mediantime;
-    
+
     // BlockExtension
     extras.totalFees = dbBlk.totalFees;
     extras.medianFee = dbBlk.medianFee;