From 3dcf0fc8decb4dd1bf045194d8a0c7d7aa3c2f2b Mon Sep 17 00:00:00 2001 From: NostrDev Date: Thu, 2 Jan 2025 14:45:18 +0100 Subject: [PATCH] #176 Optimise Angor Indexing (#24) * (feat: angor-optimise): refactors angor indexing * chore: removes redundant code --- backend/src/api/database-migration.ts | 3 +- backend/src/indexer.ts | 23 +++----- .../src/repositories/AngorBlocksRepository.ts | 31 ++++++----- backend/src/repositories/BlocksRepository.ts | 52 +++++++++++-------- 4 files changed, 60 insertions(+), 49 deletions(-) diff --git a/backend/src/api/database-migration.ts b/backend/src/api/database-migration.ts index fb1b703ea3..4d00b99bfe 100644 --- a/backend/src/api/database-migration.ts +++ b/backend/src/api/database-migration.ts @@ -1315,7 +1315,8 @@ class DatabaseMigration { private getCreateAngorBlocksTableQuery(): string { return `CREATE TABLE IF NOT EXISTS angor_blocks ( - block_height INT PRIMARY KEY, + id INT NOT NULL PRIMARY KEY CHECK (id = 1), + block_height INT NOT NULL, block_hash CHAR(64) NOT NULL, indexed_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP ) ENGINE=InnoDB DEFAULT CHARSET=utf8;`; diff --git a/backend/src/indexer.ts b/backend/src/indexer.ts index 818c94ca89..93f9f096d1 100644 --- a/backend/src/indexer.ts +++ b/backend/src/indexer.ts @@ -229,25 +229,18 @@ class Indexer { * Index transactions related to Angor projects. */ public async indexAngorTransactions(): Promise { + // Get latest indexed block. + const latestIndexedBlockHeight = await AngorBlocksRepository.$getLatestIndexedBlock(); // All indexed blocks. - const indexedBlocks = await BlocksRepository.$getIndexedBlocks(); + const indexedBlocks = latestIndexedBlockHeight + ? await BlocksRepository.$getIndexedBlocksFromHeight(latestIndexedBlockHeight + 1) + : await BlocksRepository.$getIndexedBlocks('ASC'); // Sort indexed blocks by height, the lowest height should be in the beginning. // It is necessary to index transactions for project creations before // investment transactions. - const sortedBlocks = indexedBlocks.sort( - ( - b1: { height: number; hash: string }, - b2: { height: number; hash: string } - ) => b1.height - b2.height - ); - - for (const indexedBlock of sortedBlocks) { - if (indexedBlock.height === 0) { - continue; - } - const alreadyIndexed = await AngorBlocksRepository.isBlockIndexed(indexedBlock.height, indexedBlock.hash); - if (alreadyIndexed) { + for (const indexedBlock of indexedBlocks) { + if (indexedBlock.height === 0) { continue; } // Transaction IDs in the block. @@ -285,7 +278,7 @@ class Indexer { }); }); } - await AngorBlocksRepository.$markBlockAsIndexed(indexedBlock.height, indexedBlock.hash); + await AngorBlocksRepository.$updateLatestIndexedBlock(indexedBlock.height, indexedBlock.hash); } logger.info('Indexing Angor transactions completed.'); diff --git a/backend/src/repositories/AngorBlocksRepository.ts b/backend/src/repositories/AngorBlocksRepository.ts index b9b1a5654d..69a294e96c 100644 --- a/backend/src/repositories/AngorBlocksRepository.ts +++ b/backend/src/repositories/AngorBlocksRepository.ts @@ -3,31 +3,38 @@ import logger from "../logger"; import { RowDataPacket } from "mysql2/typings/mysql"; class AngorBlocksRepository { - public async $markBlockAsIndexed(blockHeight: number, blockHash: string): Promise { + public async $updateLatestIndexedBlock(blockHeight: number, blockHash: string): Promise { try { await DB.query( - `INSERT INTO angor_blocks (block_height, block_hash) VALUES (?, ?)`, [blockHeight, blockHash]) + `INSERT INTO angor_blocks (id, block_height, block_hash) + VALUES (1, ?, ?) + ON DUPLICATE KEY UPDATE + block_height = ?, + block_hash = ?`, [blockHeight, blockHash, blockHeight, blockHash] + ); } catch (error) { logger.err( - 'Error marking block as indexed. Reason: ' + (error instanceof Error ? error.message : error) + 'Error updating latest indexed block. Reason: ' + (error instanceof Error ? error.message : error) ); } } - public async isBlockIndexed(blockHeight: number, blockHash: string): Promise { + public async $getLatestIndexedBlock(): Promise { try { - const result = await DB.query( - `SELECT * FROM angor_blocks WHERE block_height = ? AND block_hash = ? LIMIT 1`, [blockHeight, blockHash] - ); - - const rows = result[0] as RowDataPacket; + const [rows]= await DB.query( + `SELECT block_height FROM angor_blocks WHERE id = 1` + ) as RowDataPacket[][]; - return rows.length > 0; + if (rows.length === 0) { + return null; + } else { + return rows[0].block_height as number; + } } catch (error) { logger.err( - 'Error checking if block is indexed. Reason: ' + (error instanceof Error ? error.message : error) + 'Error getting latest indexed block. Reason: ' + (error instanceof Error ? error.message : error) ); - return false; + throw error; } } } diff --git a/backend/src/repositories/BlocksRepository.ts b/backend/src/repositories/BlocksRepository.ts index 424a668c7d..ef2a93ad42 100644 --- a/backend/src/repositories/BlocksRepository.ts +++ b/backend/src/repositories/BlocksRepository.ts @@ -196,9 +196,9 @@ class BlocksRepository { /** * Save newly indexed data from core coinstatsindex - * - * @param utxoSetSize - * @param totalInputAmt + * + * @param utxoSetSize + * @param totalInputAmt */ public async $updateCoinStatsIndexData(blockHash: string, utxoSetSize: number, totalInputAmt: number @@ -224,9 +224,9 @@ class BlocksRepository { /** * Update missing fee amounts fields * - * @param blockHash - * @param feeAmtPercentiles - * @param medianFeeAmt + * @param blockHash + * @param feeAmtPercentiles + * @param medianFeeAmt */ public async $updateFeeAmounts(blockHash: string, feeAmtPercentiles, medianFeeAmt) : Promise { try { @@ -810,9 +810,9 @@ class BlocksRepository { /** * Get a list of blocks that have been indexed */ - public async $getIndexedBlocks(): Promise<{ height: number, hash: string }[]> { + public async $getIndexedBlocks(order = 'DESC'): Promise<{ height: number, hash: string }[]> { try { - const [rows] = await DB.query(`SELECT height, hash FROM blocks ORDER BY height DESC`) as RowDataPacket[][]; + const [rows] = await DB.query(`SELECT height, hash FROM blocks ORDER BY height ${order}`, ) as RowDataPacket[][]; return rows as { height: number, hash: string }[]; } catch (e) { logger.err('Cannot generate block size and weight history. Reason: ' + (e instanceof Error ? e.message : e)); @@ -820,6 +820,16 @@ class BlocksRepository { } } + public async $getIndexedBlocksFromHeight(height: number): Promise<{ height: number, hash: string}[]> { + try { + const [rows] = await DB.query(`SELECT height, hash FROM blocks WHERE height >= ? ORDER BY height ASC`, [height]) as RowDataPacket[][]; + return rows as {height: number, hash: string }[]; + } catch (error) { + logger.err('Cannot generate block size and weight history. Reason: ' + (error instanceof Error ? error.message : error)); + throw error; + } + } + /** * Get a list of blocks that have not had CPFP data indexed */ @@ -949,9 +959,9 @@ class BlocksRepository { /** * Save indexed median fee to avoid recomputing it later - * - * @param id - * @param feePercentiles + * + * @param id + * @param feePercentiles */ public async $saveFeePercentilesForBlockId(id: string, feePercentiles: number[]): Promise { try { @@ -968,9 +978,9 @@ class BlocksRepository { /** * Save indexed effective fee statistics - * - * @param id - * @param feeStats + * + * @param id + * @param feeStats */ public async $saveEffectiveFeeStats(id: string, feeStats: EffectiveFeeStats): Promise { try { @@ -987,7 +997,7 @@ class BlocksRepository { /** * Save coinbase addresses - * + * * @param id * @param addresses */ @@ -1006,7 +1016,7 @@ class BlocksRepository { /** * Save pool - * + * * @param id * @param poolId */ @@ -1025,8 +1035,8 @@ class BlocksRepository { /** * Save block first seen time - * - * @param id + * + * @param id */ public async $saveFirstSeenTime(id: string, firstSeen: number): Promise { try { @@ -1044,8 +1054,8 @@ class BlocksRepository { /** * Convert a mysql row block into a BlockExtended. Note that you * must provide the correct field into dbBlk object param - * - * @param dbBlk + * + * @param dbBlk */ private async formatDbBlockIntoExtendedBlock(dbBlk: DatabaseBlock): Promise { const blk: Partial = {}; @@ -1065,7 +1075,7 @@ class BlocksRepository { blk.weight = dbBlk.weight; blk.previousblockhash = dbBlk.previousblockhash; blk.mediantime = dbBlk.mediantime; - + // BlockExtension extras.totalFees = dbBlk.totalFees; extras.medianFee = dbBlk.medianFee;