diff --git a/apps/gnosis-pay-rewards-indexer/package.json b/apps/gnosis-pay-rewards-indexer/package.json index 48c20d9..6081ba7 100644 --- a/apps/gnosis-pay-rewards-indexer/package.json +++ b/apps/gnosis-pay-rewards-indexer/package.json @@ -30,6 +30,7 @@ "debug": "^4.3.4", "dotenv": "^16.0.1", "express": "^4.18.2", + "jotai": "^2.2.1", "mongoose": "^7.1.0", "nanoid": "^4.0.1", "numeral": "^2.0.6", diff --git a/apps/gnosis-pay-rewards-indexer/src/addHttpRoutes.ts b/apps/gnosis-pay-rewards-indexer/src/addHttpRoutes.ts index b126d6f..bfd91fd 100644 --- a/apps/gnosis-pay-rewards-indexer/src/addHttpRoutes.ts +++ b/apps/gnosis-pay-rewards-indexer/src/addHttpRoutes.ts @@ -1,30 +1,43 @@ -import { - GnosisPayTransactionFieldsType_Unpopulated, - GnosisPayTransactionFieldsType_Populated, - toWeekDataId, -} from '@karpatkey/gnosis-pay-rewards-sdk'; +import { GnosisPayTransactionFieldsType_Unpopulated, toWeekDataId } from '@karpatkey/gnosis-pay-rewards-sdk'; import { createMongooseLogger, getGnosisPayTransactionModel, getWeekCashbackRewardModel, toDocumentId, + createWeekCashbackRewardDocument, } from '@karpatkey/gnosis-pay-rewards-sdk/mongoose'; import { Response } from 'express'; import { isAddress } from 'viem'; import { buildExpressApp } from './server.js'; +import dayjs from 'dayjs'; +import dayjsUtc from 'dayjs/plugin/utc.js'; +import { IndexerStateAtomType } from './state.js'; + +dayjs.extend(dayjsUtc); export function addHttpRoutes({ expressApp, gnosisPayTransactionModel, weekCashbackRewardModel, + getIndexerState, }: { expressApp: ReturnType; gnosisPayTransactionModel: ReturnType; weekCashbackRewardModel: ReturnType; logger: ReturnType; + getIndexerState: () => IndexerStateAtomType; }) { expressApp.get<'/status'>('/status', (_, res) => { + const { fromBlockNumber, toBlockNumber, latestBlockNumber } = getIndexerState(); + return res.send({ + data: { + indexerState: { + fromBlockNumber: Number(fromBlockNumber), + toBlockNumber: Number(toBlockNumber), + latestBlockNumber: Number(latestBlockNumber), + }, + }, status: 'ok', statusCode: 200, }); @@ -37,19 +50,6 @@ export function addHttpRoutes({ }); }); - expressApp.get<'/pending-rewards'>('/pending-rewards', async (_, res) => { - try { - const spendTransactions = await gnosisPayTransactionModel.find({}).lean(); - return res.json({ - data: spendTransactions, - status: 'ok', - statusCode: 200, - }); - } catch (error) { - return returnInternalServerError(res, error as Error); - } - }); - expressApp.get<'/cashbacks/:safeAddress'>('/cashbacks/:safeAddress', async (req, res) => { try { const safeAddress = req.params.safeAddress; @@ -62,40 +62,48 @@ export function addHttpRoutes({ }); } - const allCashbacks = await weekCashbackRewardModel - .find({ - address: new RegExp(safeAddress, 'i'), - }) - .populate<{ - transactions: GnosisPayTransactionFieldsType_Populated; - }>({ - path: 'transactions', - select: { - _id: 0, - blockNumber: 1, - blockTimestamp: 1, - transactionHash: 1, - spentAmount: 1, - spentAmountUsd: 1, - gnoBalance: 1, - }, - populate: { - path: 'amountToken', - select: { - symbol: 1, - decimals: 1, - name: 1, - }, - transform: (doc, id) => ({ - ...doc, - address: id, - }), - }, - }) - .lean(); + const week = toWeekDataId(dayjs.utc().unix()); + const weekCashbackRewardDocument = await createWeekCashbackRewardDocument({ + address: safeAddress, + populateTransactions: true, + weekCashbackRewardModel, + week, + }); + + // const allCashbacks = await weekCashbackRewardModel + // .find({ + // address: new RegExp(safeAddress, 'i'), + // }) + // .populate<{ + // transactions: GnosisPayTransactionFieldsType_Populated; + // }>({ + // path: 'transactions', + // select: { + // _id: 0, + // blockNumber: 1, + // blockTimestamp: 1, + // transactionHash: 1, + // spentAmount: 1, + // spentAmountUsd: 1, + // gnoBalance: 1, + // }, + // populate: { + // path: 'amountToken', + // select: { + // symbol: 1, + // decimals: 1, + // name: 1, + // }, + // transform: (doc, id) => ({ + // ...doc, + // address: id, + // }), + // }, + // }) + // .lean(); return res.json({ - data: allCashbacks, + data: weekCashbackRewardDocument, status: 'ok', statusCode: 200, _query: { diff --git a/apps/gnosis-pay-rewards-indexer/src/core.ts b/apps/gnosis-pay-rewards-indexer/src/core.ts index 41b9229..c419ff0 100644 --- a/apps/gnosis-pay-rewards-indexer/src/core.ts +++ b/apps/gnosis-pay-rewards-indexer/src/core.ts @@ -8,7 +8,6 @@ import { getLoggerModel, getWeekMetricsSnapshotModel, getBlockModel, - saveBlock, getGnosisPaySafeAddressModel, getWeekCashbackRewardModel, LogLevel, @@ -26,8 +25,10 @@ import { addHttpRoutes } from './addHttpRoutes.js'; import { addSocketComms } from './addSocketComms.js'; import { processRefundLog, processSpendLog } from './processSpendLog.js'; import { getGnosisPayRefundLogs } from './gp/getGnosisPayRefundLogs.js'; +import { atom, createStore } from 'jotai'; +import { IndexerStateAtomType } from './state.js'; -const indexBlockSize = 12n; // 12 blocks is roughly 60 seconds of data +const indexBlockSize = 120n; // 12 blocks is roughly 60 seconds of data export async function startIndexing({ client, @@ -44,97 +45,102 @@ export async function startIndexing({ // Connect to the database const mongooseConnection = await createConnection(MONGODB_URI); - console.log('Migrating Gnosis Pay tokens to database'); - - const gnosisPaySafeAddressModel = getGnosisPaySafeAddressModel(mongooseConnection); - const gnosisPayTransactionModel = getGnosisPayTransactionModel(mongooseConnection); - const weekCashbackRewardModel = getWeekCashbackRewardModel(mongooseConnection); - const weekMetricsSnapshotModel = getWeekMetricsSnapshotModel(mongooseConnection); - const gnosisPayTokenModel = getTokenModel(mongooseConnection); - const loggerModel = getLoggerModel(mongooseConnection); - const blockModel = getBlockModel(mongooseConnection); - const logger = createMongooseLogger(loggerModel); + console.log('Connected to mongodb at', mongooseConnection.connection.host); - const restApiServer = addHttpRoutes({ - expressApp: buildExpressApp(), - gnosisPayTransactionModel, - weekCashbackRewardModel, - logger, - }); + console.log('Migrating Gnosis Pay tokens to database'); - const socketIoServer = addSocketComms({ - socketIoServer: buildSocketIoServer(restApiServer), - gnosisPayTransactionModel, - weekMetricsSnapshotModel, - }); + const mongooseModels = { + gnosisPaySafeAddressModel: getGnosisPaySafeAddressModel(mongooseConnection), + gnosisPayTransactionModel: getGnosisPayTransactionModel(mongooseConnection), + weekCashbackRewardModel: getWeekCashbackRewardModel(mongooseConnection), + weekMetricsSnapshotModel: getWeekMetricsSnapshotModel(mongooseConnection), + gnosisPayTokenModel: getTokenModel(mongooseConnection), + loggerModel: getLoggerModel(mongooseConnection), + blockModel: getBlockModel(mongooseConnection), + }; - restApiServer.listen(HTTP_SERVER_PORT, HTTP_SERVER_HOST); - socketIoServer.listen(SOCKET_IO_SERVER_PORT); + const logger = createMongooseLogger(mongooseModels.loggerModel); console.log('Starting indexing'); // Initialize the latest block - let latestBlock = await client.getBlock({ includeTransactions: false }); - + const latestBlockInitial = await client.getBlock({ includeTransactions: false }); // default value is June 29th, 2024. Otherwise, we fetch the latest block from the indexed pending rewards - let fromBlockNumber = gnosisPayStartBlock; + const fromBlockNumberInitial = gnosisPayStartBlock; + const toBlockNumberInitial = clampToBlockRange(fromBlockNumberInitial, latestBlockInitial.number, indexBlockSize); + + const indexerStateAtom = atom({ + latestBlockNumber: latestBlockInitial.number, + fromBlockNumber: fromBlockNumberInitial, + toBlockNumber: toBlockNumberInitial, + }); + const indexerStateStore = createStore(); + const getIndexerState = () => indexerStateStore.get(indexerStateAtom); if (resumeIndexing === true) { - const [latestGnosisPayTransaction] = await gnosisPayTransactionModel.find().sort({ blockNumber: -1 }).limit(1); + const [latestGnosisPayTransaction] = await mongooseModels.gnosisPayTransactionModel + .find() + .sort({ blockNumber: -1 }) + .limit(1); + if (latestGnosisPayTransaction !== undefined) { - fromBlockNumber = BigInt(latestGnosisPayTransaction.blockNumber) - indexBlockSize; + const fromBlockNumber = BigInt(latestGnosisPayTransaction.blockNumber) - indexBlockSize; + indexerStateStore.set(indexerStateAtom, (prev) => ({ + ...prev, + fromBlockNumber, + })); console.log(`Resuming indexing from #${fromBlockNumber}`); } else { - console.warn(`No pending rewards found, starting from the beginning at #${gnosisPayStartBlock}`); + console.warn(`No pending rewards found, starting from the beginning at #${fromBlockNumberInitial}`); } } else { const session = await mongooseConnection.startSession(); - // Clean up the database await session.withTransaction(async () => { - await gnosisPaySafeAddressModel.deleteMany(); - await gnosisPayTransactionModel.deleteMany(); - await blockModel.deleteMany(); - await weekCashbackRewardModel.deleteMany(); - await weekMetricsSnapshotModel.deleteMany(); - await loggerModel.deleteMany(); - await gnosisPayTokenModel.deleteMany(); + for (const modelName of mongooseConnection.modelNames()) { + await mongooseConnection.model(modelName).deleteMany(); + } }); - await session.commitTransaction(); await session.endSession(); - // Save the Gnosis Pay tokens to the database - await saveGnosisPayTokensToDatabase(gnosisPayTokenModel, gnosisPayTokens); + await saveGnosisPayTokensToDatabase(mongooseModels.gnosisPayTokenModel, gnosisPayTokens); } - let toBlockNumber = clampToBlockRange(fromBlockNumber, latestBlock.number, indexBlockSize); - // Watch for new blocks client.watchBlocks({ includeTransactions: false, onBlock(block) { - latestBlock = block; + indexerStateStore.set(indexerStateAtom, (prev) => ({ + ...prev, + latestBlockNumber: block.number, + })); + }, + }); - saveBlock( - { - number: Number(block.number), - hash: block.hash, - timestamp: Number(block.timestamp), - }, - blockModel - ).catch((e) => { - console.error('Error creating block', e); - }); + const restApiServer = addHttpRoutes({ + expressApp: buildExpressApp(), + gnosisPayTransactionModel: mongooseModels.gnosisPayTransactionModel, + weekCashbackRewardModel: mongooseModels.weekCashbackRewardModel, + logger, + getIndexerState() { + return indexerStateStore.get(indexerStateAtom); }, }); - const shouldFetchLogs = toBlockNumber <= latestBlock.number; + const socketIoServer = addSocketComms({ + socketIoServer: buildSocketIoServer(restApiServer), + gnosisPayTransactionModel: mongooseModels.gnosisPayTransactionModel, + weekMetricsSnapshotModel: mongooseModels.weekMetricsSnapshotModel, + }); - console.log({ fromBlockNumber, toBlockNumber, shouldFetchLogs }); + restApiServer.listen(HTTP_SERVER_PORT, HTTP_SERVER_HOST); + socketIoServer.listen(SOCKET_IO_SERVER_PORT); // Index all the logs until the latest block - while (toBlockNumber <= latestBlock.number) { + while (shouldFetchLogs(getIndexerState)) { + const { fromBlockNumber, toBlockNumber, latestBlockNumber } = getIndexerState(); + try { await logger.logDebug({ message: `Fetching logs from #${fromBlockNumber} to #${toBlockNumber}`, @@ -159,10 +165,10 @@ export async function startIndexing({ await handleBatchLogs({ client, mongooseModels: { - gnosisPayTransactionModel, - weekCashbackRewardModel, - weekMetricsSnapshotModel, - gnosisPaySafeAddressModel, + gnosisPayTransactionModel: mongooseModels.gnosisPayTransactionModel, + weekCashbackRewardModel: mongooseModels.weekCashbackRewardModel, + weekMetricsSnapshotModel: mongooseModels.weekMetricsSnapshotModel, + gnosisPaySafeAddressModel: mongooseModels.gnosisPaySafeAddressModel, }, logs: [...spendLogs, ...refundLogs], logger, @@ -170,12 +176,19 @@ export async function startIndexing({ }); // Move to the next block range - fromBlockNumber += indexBlockSize; - toBlockNumber = clampToBlockRange(fromBlockNumber, latestBlock.number, indexBlockSize); + const nextFromBlockNumber = fromBlockNumber + indexBlockSize; + const nextToBlockNumber = clampToBlockRange(nextFromBlockNumber, latestBlockNumber, indexBlockSize); + + indexerStateStore.set(indexerStateAtom, (prev) => ({ + ...prev, + fromBlockNumber: nextFromBlockNumber, + toBlockNumber: nextToBlockNumber, + })); // Sanity check to make sure we're not going too fast - const distanceToLatestBlock = bigMath.abs(toBlockNumber - latestBlock.number); + const distanceToLatestBlock = bigMath.abs(nextToBlockNumber - latestBlockNumber); console.log({ distanceToLatestBlock }); + // Cooldown for 20 seconds if we're within a distance of 10 blocks if (distanceToLatestBlock < 10n) { const targetBlockNumber = toBlockNumber + indexBlockSize + 3n; @@ -194,6 +207,12 @@ export async function startIndexing({ } } +function shouldFetchLogs(getIndexerState: () => IndexerStateAtomType) { + const { toBlockNumber, latestBlockNumber } = getIndexerState(); + + return toBlockNumber <= latestBlockNumber; +} + async function handleBatchLogs({ client, mongooseModels, diff --git a/apps/gnosis-pay-rewards-indexer/src/state.ts b/apps/gnosis-pay-rewards-indexer/src/state.ts new file mode 100644 index 0000000..0b7b940 --- /dev/null +++ b/apps/gnosis-pay-rewards-indexer/src/state.ts @@ -0,0 +1,5 @@ +export type IndexerStateAtomType = { + latestBlockNumber: bigint; + fromBlockNumber: bigint; + toBlockNumber: bigint; +}; diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 24fad6b..0c1ccd6 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -272,6 +272,9 @@ importers: express: specifier: ^4.18.2 version: 4.19.2 + jotai: + specifier: ^2.2.1 + version: 2.2.1(react@18.2.0) mongoose: specifier: ^7.1.0 version: 7.1.0 @@ -381,7 +384,7 @@ importers: version: 6.9.0 forge-std: specifier: github:foundry-rs/forge-std#v1 - version: github.com/foundry-rs/forge-std/07263d193d621c4b2b0ce8b4d54af58f6957d97d + version: github.com/foundry-rs/forge-std/1714bee72e286e73f76e320d110e0eaf5c4e649d hardhat: specifier: ^2.19.3 version: 2.19.3(ts-node@10.9.2)(typescript@5.5.2) @@ -466,7 +469,7 @@ importers: version: 4.7.5 viem: specifier: ^2.7.16 - version: 2.7.16(typescript@5.3.3)(zod@3.22.4) + version: 2.7.16(typescript@5.5.2)(zod@3.22.4) devDependencies: '@babel/core': specifier: ^7.21.4 @@ -497,7 +500,7 @@ importers: version: 18.15.11 abitype: specifier: ^0.2.5 - version: 0.2.5(typescript@5.3.3)(zod@3.22.4) + version: 0.2.5(typescript@5.5.2)(zod@3.22.4) babel-jest: specifier: ^29.5.0 version: 29.5.0(@babel/core@7.21.4) @@ -7141,9 +7144,9 @@ packages: resolution: {integrity: sha512-alnSxNZKC1ssKrFG5ytluu9kNKGwBifb1xhOyCqwMnm72JksbCEo0UWlNvaeCiYMwhYvMyS++mfxcLAsV/8Gfw==} dependencies: '@noble/hashes': 1.4.0 - '@safe-global/safe-deployments': 1.36.0 + '@safe-global/safe-deployments': 1.37.2 ethereumjs-util: 7.1.5 - ethers: 6.13.0 + ethers: 6.13.2 semver: 7.6.2 web3: 1.10.4 web3-core: 1.10.4 @@ -7182,8 +7185,8 @@ packages: /@safe-global/safe-core-sdk-types@3.0.1: resolution: {integrity: sha512-2AdlK6GJ5YEZXrQwFsHFwQScnNo3OonF3O6KzVeMc0/7OAuOTYBzKq1jzju2Eck6Z8UNPUinlHoF2Zb2pvTKhw==} dependencies: - '@safe-global/safe-deployments': 1.36.0 - ethers: 6.13.0 + '@safe-global/safe-deployments': 1.37.2 + ethers: 6.13.2 web3-core: 1.10.4 web3-utils: 1.10.4 transitivePeerDependencies: @@ -7193,8 +7196,8 @@ packages: - utf-8-validate dev: false - /@safe-global/safe-deployments@1.36.0: - resolution: {integrity: sha512-9MbDJveRR64AbmzjIpuUqmDBDtOZpXpvkyhTUs+5UOPT3WgSO375/ZTO7hZpywP7+EmxnjkGc9EoxjGcC4TAyw==} + /@safe-global/safe-deployments@1.37.2: + resolution: {integrity: sha512-kWRim5vY9W/yNKUUehUQDhCHz7NWzXjhkpMDvvnrrkEn9U461zwCcmJmPVnPrjnaY4dPpJsGZSzUuU4+uxH+vg==} dependencies: semver: 7.6.2 dev: false @@ -9649,20 +9652,6 @@ packages: typescript: 5.5.2 dev: false - /abitype@0.2.5(typescript@5.3.3)(zod@3.22.4): - resolution: {integrity: sha512-t1iiokWYpkrziu4WL2Gb6YdGvaP9ZKs7WnA39TI8TsW2E99GVRgDPW/xOKhzoCdyxOYt550CNYEFluCwGaFHaA==} - engines: {pnpm: '>=7'} - peerDependencies: - typescript: '>=4.7.4' - zod: '>=3.19.1' - peerDependenciesMeta: - zod: - optional: true - dependencies: - typescript: 5.3.3 - zod: 3.22.4 - dev: true - /abitype@0.2.5(typescript@5.5.2)(zod@3.22.4): resolution: {integrity: sha512-t1iiokWYpkrziu4WL2Gb6YdGvaP9ZKs7WnA39TI8TsW2E99GVRgDPW/xOKhzoCdyxOYt550CNYEFluCwGaFHaA==} engines: {pnpm: '>=7'} @@ -9675,7 +9664,6 @@ packages: dependencies: typescript: 5.5.2 zod: 3.22.4 - dev: false /abitype@0.9.8(typescript@5.5.2)(zod@3.22.4): resolution: {integrity: sha512-puLifILdm+8sjyss4S+fsUN09obiT1g2YW6CtcQF+QDzxR0euzgEB29MZujC6zMk2a6SVmtttq1fc6+YFA7WYQ==} @@ -13008,6 +12996,22 @@ packages: - utf-8-validate dev: false + /ethers@6.13.2: + resolution: {integrity: sha512-9VkriTTed+/27BGuY1s0hf441kqwHJ1wtN2edksEtiRvXx+soxRX3iSXTfFqq2+YwrOqbDoTHjIhQnjJRlzKmg==} + engines: {node: '>=14.0.0'} + dependencies: + '@adraffy/ens-normalize': 1.10.1 + '@noble/curves': 1.2.0 + '@noble/hashes': 1.3.2 + '@types/node': 18.15.13 + aes-js: 4.0.0-beta.5 + tslib: 2.4.0 + ws: 8.17.1 + transitivePeerDependencies: + - bufferutil + - utf-8-validate + dev: false + /ethers@6.9.0: resolution: {integrity: sha512-pmfNyQzc2mseLe91FnT2vmNaTt8dDzhxZ/xItAV7uGsF4dI4ek2ufMu3rAkgQETL/TIs0GS5A+U05g9QyWnv3Q==} engines: {node: '>=14.0.0'} @@ -22545,10 +22549,10 @@ packages: use-sync-external-store: 1.2.0(react@18.2.0) dev: false - github.com/foundry-rs/forge-std/07263d193d621c4b2b0ce8b4d54af58f6957d97d: - resolution: {tarball: https://codeload.github.com/foundry-rs/forge-std/tar.gz/07263d193d621c4b2b0ce8b4d54af58f6957d97d} + github.com/foundry-rs/forge-std/1714bee72e286e73f76e320d110e0eaf5c4e649d: + resolution: {tarball: https://codeload.github.com/foundry-rs/forge-std/tar.gz/1714bee72e286e73f76e320d110e0eaf5c4e649d} name: forge-std - version: 1.9.1 + version: 1.9.2 dev: true settings: