From bee57fe3173a4650567fd4f9c548ba4c197187b2 Mon Sep 17 00:00:00 2001 From: vincentwschau <99756290+vincentwschau@users.noreply.github.com> Date: Mon, 14 Oct 2024 16:32:44 -0400 Subject: [PATCH] Improve vault endpoint performance. (#2475) --- .../funding-index-updates-table.test.ts | 34 +++++++ .../src/stores/funding-index-updates-table.ts | 75 +++++++++++++- .../controllers/api/v4/vault-controller.ts | 99 +++++++++++-------- 3 files changed, 164 insertions(+), 44 deletions(-) diff --git a/indexer/packages/postgres/__tests__/stores/funding-index-updates-table.test.ts b/indexer/packages/postgres/__tests__/stores/funding-index-updates-table.test.ts index d42df73764..de7daaa34e 100644 --- a/indexer/packages/postgres/__tests__/stores/funding-index-updates-table.test.ts +++ b/indexer/packages/postgres/__tests__/stores/funding-index-updates-table.test.ts @@ -242,4 +242,38 @@ describe('funding index update store', () => { expect(fundingIndexMap[defaultPerpetualMarket2.id]).toEqual(Big(0)); }, ); + + it('Successfully finds funding index maps for multiple effectiveBeforeOrAtHeights', async () => { + const fundingIndexUpdates2: FundingIndexUpdatesCreateObject = { + ...defaultFundingIndexUpdate, + fundingIndex: '124', + effectiveAtHeight: updatedHeight, + effectiveAt: '1982-05-25T00:00:00.000Z', + eventId: defaultTendermintEventId2, + }; + const fundingIndexUpdates3: FundingIndexUpdatesCreateObject = { + ...defaultFundingIndexUpdate, + eventId: defaultTendermintEventId3, + perpetualId: defaultPerpetualMarket2.id, + }; + await Promise.all([ + FundingIndexUpdatesTable.create(defaultFundingIndexUpdate), + FundingIndexUpdatesTable.create(fundingIndexUpdates2), + FundingIndexUpdatesTable.create(fundingIndexUpdates3), + ]); + + const fundingIndexMaps: {[blockHeight:string]: FundingIndexMap} = await FundingIndexUpdatesTable + .findFundingIndexMaps( + ['3', '6'], + ); + + expect(fundingIndexMaps['3'][defaultFundingIndexUpdate.perpetualId]) + .toEqual(Big(defaultFundingIndexUpdate.fundingIndex)); + expect(fundingIndexMaps['3'][fundingIndexUpdates3.perpetualId]) + .toEqual(Big(fundingIndexUpdates3.fundingIndex)); + expect(fundingIndexMaps['6'][defaultFundingIndexUpdate.perpetualId]) + .toEqual(Big(fundingIndexUpdates2.fundingIndex)); + expect(fundingIndexMaps['6'][fundingIndexUpdates3.perpetualId]) + .toEqual(Big(fundingIndexUpdates3.fundingIndex)); + }); }); diff --git a/indexer/packages/postgres/src/stores/funding-index-updates-table.ts b/indexer/packages/postgres/src/stores/funding-index-updates-table.ts index dce9a47028..8ef55a3537 100644 --- a/indexer/packages/postgres/src/stores/funding-index-updates-table.ts +++ b/indexer/packages/postgres/src/stores/funding-index-updates-table.ts @@ -3,6 +3,7 @@ import _ from 'lodash'; import { QueryBuilder } from 'objection'; import { BUFFER_ENCODING_UTF_8, DEFAULT_POSTGRES_OPTIONS } from '../constants'; +import { knexReadReplica } from '../helpers/knex'; import { setupBaseQuery, verifyAllRequiredFields } from '../helpers/stores-helpers'; import Transaction from '../helpers/transaction'; import { getUuid } from '../helpers/uuid'; @@ -21,6 +22,14 @@ import { } from '../types'; import * as PerpetualMarketTable from './perpetual-market-table'; +// Assuming block time of 1 second, this should be 4 hours of blocks +const FOUR_HOUR_OF_BLOCKS = Big(3600).times(4); +// Type used for querying for funding index maps for multiple effective heights. +interface FundingIndexUpdatesFromDatabaseWithSearchHeight extends FundingIndexUpdatesFromDatabase { + // max effective height being queried for + searchHeight: string, +} + export function uuid( blockHeight: string, eventId: Buffer, @@ -193,8 +202,6 @@ export async function findFundingIndexMap( options, ); - // Assuming block time of 1 second, this should be 4 hours of blocks - const FOUR_HOUR_OF_BLOCKS = Big(3600).times(4); const fundingIndexUpdates: FundingIndexUpdatesFromDatabase[] = await baseQuery .distinctOn(FundingIndexUpdatesColumns.perpetualId) .where(FundingIndexUpdatesColumns.effectiveAtHeight, '<=', effectiveBeforeOrAtHeight) @@ -216,3 +223,67 @@ export async function findFundingIndexMap( initialFundingIndexMap, ); } + +/** + * Finds funding index maps for multiple effective before or at heights. Uses a SQL query unnesting + * an array of effective before or at heights and cross-joining with the funding index updates table + * to find the closest funding index update per effective before or at height. + * @param effectiveBeforeOrAtHeights Heights to get funding index maps for. + * @param options + * @returns Object mapping block heights to the respective funding index maps. + */ +export async function findFundingIndexMaps( + effectiveBeforeOrAtHeights: string[], + options: Options = DEFAULT_POSTGRES_OPTIONS, +): Promise<{[blockHeight: string]: FundingIndexMap}> { + const heightNumbers: number[] = effectiveBeforeOrAtHeights + .map((height: string):number => parseInt(height, 10)) + .filter((parsedHeight: number): boolean => { return !Number.isNaN(parsedHeight); }) + .sort(); + // Get the min height to limit the search to blocks 4 hours or before the min height. + const minHeight: number = heightNumbers[0]; + + const result: { + rows: FundingIndexUpdatesFromDatabaseWithSearchHeight[], + } = await knexReadReplica.getConnection().raw( + ` + SELECT + DISTINCT ON ("perpetualId", "searchHeight") "perpetualId", "searchHeight", + "funding_index_updates".* + FROM + "funding_index_updates", + unnest(ARRAY[${heightNumbers.join(',')}]) AS "searchHeight" + WHERE + "effectiveAtHeight" > ${Big(minHeight).minus(FOUR_HOUR_OF_BLOCKS).toFixed()} AND + "effectiveAtHeight" <= "searchHeight" + ORDER BY + "perpetualId", + "searchHeight", + "effectiveAtHeight" DESC + `, + ) as unknown as { + rows: FundingIndexUpdatesFromDatabaseWithSearchHeight[], + }; + + const perpetualMarkets: PerpetualMarketFromDatabase[] = await PerpetualMarketTable.findAll( + {}, + [], + options, + ); + + const fundingIndexMaps:{[blockHeight: string]: FundingIndexMap} = {}; + for (const height of effectiveBeforeOrAtHeights) { + fundingIndexMaps[height] = _.reduce(perpetualMarkets, + (acc: FundingIndexMap, perpetualMarket: PerpetualMarketFromDatabase): FundingIndexMap => { + acc[perpetualMarket.id] = Big(0); + return acc; + }, + {}, + ); + } + for (const funding of result.rows) { + fundingIndexMaps[funding.searchHeight][funding.perpetualId] = Big(funding.fundingIndex); + } + + return fundingIndexMaps; +} diff --git a/indexer/services/comlink/src/controllers/api/v4/vault-controller.ts b/indexer/services/comlink/src/controllers/api/v4/vault-controller.ts index d5e8ff5c42..9480b096f1 100644 --- a/indexer/services/comlink/src/controllers/api/v4/vault-controller.ts +++ b/indexer/services/comlink/src/controllers/api/v4/vault-controller.ts @@ -375,10 +375,23 @@ async function getVaultPositions( BlockTable.getLatest(), ]); - const latestFundingIndexMap: FundingIndexMap = await FundingIndexUpdatesTable - .findFundingIndexMap( - latestBlock.blockHeight, - ); + const updatedAtHeights: string[] = _(subaccounts).map('updatedAtHeight').uniq().value(); + const [ + latestFundingIndexMap, + fundingIndexMaps, + ]: [ + FundingIndexMap, + {[blockHeight: string]: FundingIndexMap} + ] = await Promise.all([ + FundingIndexUpdatesTable + .findFundingIndexMap( + latestBlock.blockHeight, + ), + FundingIndexUpdatesTable + .findFundingIndexMaps( + updatedAtHeights, + ), + ]); const assetPositionsBySubaccount: { [subaccountId: string]: AssetPositionFromDatabase[] } = _.groupBy( assetPositions, @@ -397,47 +410,49 @@ async function getVaultPositions( const vaultPositionsAndSubaccountId: { position: VaultPosition, subaccountId: string, - }[] = await Promise.all( - subaccounts.map(async (subaccount: SubaccountFromDatabase) => { - const perpetualMarket: PerpetualMarketFromDatabase | undefined = perpetualMarketRefresher - .getPerpetualMarketFromClobPairId(vaultSubaccounts[subaccount.id]); - if (perpetualMarket === undefined) { - throw new Error( - `Vault clob pair id ${vaultSubaccounts[subaccount.id]} does not correspond to a ` + + }[] = subaccounts.map((subaccount: SubaccountFromDatabase) => { + const perpetualMarket: PerpetualMarketFromDatabase | undefined = perpetualMarketRefresher + .getPerpetualMarketFromClobPairId(vaultSubaccounts[subaccount.id]); + if (perpetualMarket === undefined) { + throw new Error( + `Vault clob pair id ${vaultSubaccounts[subaccount.id]} does not correspond to a ` + 'perpetual market.'); - } - const lastUpdatedFundingIndexMap: FundingIndexMap = await FundingIndexUpdatesTable - .findFundingIndexMap( - subaccount.updatedAtHeight, - ); - - const subaccountResponse: SubaccountResponseObject = getSubaccountResponse( - subaccount, - openPerpetualPositionsBySubaccount[subaccount.id] || [], - assetPositionsBySubaccount[subaccount.id] || [], - assets, - markets, - perpetualMarketRefresher.getPerpetualMarketsMap(), - latestBlock.blockHeight, - latestFundingIndexMap, - lastUpdatedFundingIndexMap, + } + const lastUpdatedFundingIndexMap: FundingIndexMap = fundingIndexMaps[ + subaccount.updatedAtHeight + ]; + if (lastUpdatedFundingIndexMap === undefined) { + throw new Error( + `No funding indices could be found for vault with subaccount ${subaccount.id}`, ); + } - return { - position: { - ticker: perpetualMarket.ticker, - assetPosition: subaccountResponse.assetPositions[ - assetIdToAsset[USDC_ASSET_ID].symbol - ], - perpetualPosition: subaccountResponse.openPerpetualPositions[ - perpetualMarket.ticker - ] || undefined, - equity: subaccountResponse.equity, - }, - subaccountId: subaccount.id, - }; - }), - ); + const subaccountResponse: SubaccountResponseObject = getSubaccountResponse( + subaccount, + openPerpetualPositionsBySubaccount[subaccount.id] || [], + assetPositionsBySubaccount[subaccount.id] || [], + assets, + markets, + perpetualMarketRefresher.getPerpetualMarketsMap(), + latestBlock.blockHeight, + latestFundingIndexMap, + lastUpdatedFundingIndexMap, + ); + + return { + position: { + ticker: perpetualMarket.ticker, + assetPosition: subaccountResponse.assetPositions[ + assetIdToAsset[USDC_ASSET_ID].symbol + ], + perpetualPosition: subaccountResponse.openPerpetualPositions[ + perpetualMarket.ticker + ] || undefined, + equity: subaccountResponse.equity, + }, + subaccountId: subaccount.id, + }; + }); return new Map(vaultPositionsAndSubaccountId.map( (obj: { position: VaultPosition, subaccountId: string }) : [string, VaultPosition] => {