From 081aeba0059968c25e646531c88026af012e0c7c Mon Sep 17 00:00:00 2001 From: Enzo Cioppettini Date: Tue, 24 Oct 2023 11:32:12 -0300 Subject: [PATCH] add carp funnel with stake delegation pool tracking --- .../paima-funnel/src/cde/cardanoPool.ts | 49 +++++ .../engine/paima-funnel/src/cde/reading.ts | 4 + .../paima-funnel/src/funnels/carp/funnel.ts | 200 ++++++++++++++++++ packages/engine/paima-funnel/src/index.ts | 2 + .../paima-runtime/src/cde-config/loading.ts | 12 +- packages/engine/paima-runtime/src/types.ts | 36 +++- .../engine/paima-sm/src/cde-cardano-pool.ts | 25 +++ .../engine/paima-sm/src/cde-processing.ts | 3 + .../paima-sdk/paima-db/migrations/down.sql | 1 + packages/paima-sdk/paima-db/migrations/up.sql | 7 + packages/paima-sdk/paima-db/src/index.ts | 2 + .../paima-sdk/paima-db/src/paima-tables.ts | 22 ++ .../cde-cardano-pool-delegation.queries.ts | 69 ++++++ .../src/sql/cde-cardano-pool-delegation.sql | 15 ++ packages/paima-sdk/paima-utils/src/config.ts | 5 + .../paima-sdk/paima-utils/src/constants.ts | 2 + 16 files changed, 450 insertions(+), 4 deletions(-) create mode 100644 packages/engine/paima-funnel/src/cde/cardanoPool.ts create mode 100644 packages/engine/paima-funnel/src/funnels/carp/funnel.ts create mode 100644 packages/engine/paima-sm/src/cde-cardano-pool.ts create mode 100644 packages/paima-sdk/paima-db/src/sql/cde-cardano-pool-delegation.queries.ts create mode 100644 packages/paima-sdk/paima-db/src/sql/cde-cardano-pool-delegation.sql diff --git a/packages/engine/paima-funnel/src/cde/cardanoPool.ts b/packages/engine/paima-funnel/src/cde/cardanoPool.ts new file mode 100644 index 000000000..82eed0b16 --- /dev/null +++ b/packages/engine/paima-funnel/src/cde/cardanoPool.ts @@ -0,0 +1,49 @@ +import { ChainDataExtensionDatumType, DEFAULT_FUNNEL_TIMEOUT, timeout } from '@paima/utils'; +import type { + CdeCardanoPoolDatum, + ChainDataExtensionCardanoDelegation, + ChainDataExtensionDatum, +} from '@paima/runtime'; +import axios from 'axios'; + +interface ApiResult { + credential: string; + pool: string | undefined; + slot: number; +} + +export default async function getCdeData( + url: string, + extension: ChainDataExtensionCardanoDelegation, + fromAbsoluteSlot: number, + toAbsoluteSlot: number, + getBlockNumber: (slot: number) => number +): Promise { + const events = await timeout( + // TODO: replace with the carp client later + axios.post(url, { + pools: extension.pools, + range: { minSlot: fromAbsoluteSlot, maxSlot: toAbsoluteSlot }, + }), + DEFAULT_FUNNEL_TIMEOUT + ); + + return events.data.map(e => eventToCdeDatum(e, extension, getBlockNumber(e.slot))); +} + +function eventToCdeDatum( + event: ApiResult, + extension: ChainDataExtensionCardanoDelegation, + blockNumber: number +): CdeCardanoPoolDatum { + return { + cdeId: extension.cdeId, + cdeDatumType: ChainDataExtensionDatumType.CardanoPool, + blockNumber, + payload: { + address: event.credential, + pool: event.pool, + }, + scheduledPrefix: extension.scheduledPrefix, + }; +} diff --git a/packages/engine/paima-funnel/src/cde/reading.ts b/packages/engine/paima-funnel/src/cde/reading.ts index 9981c1fe7..f0fbd2595 100644 --- a/packages/engine/paima-funnel/src/cde/reading.ts +++ b/packages/engine/paima-funnel/src/cde/reading.ts @@ -49,6 +49,10 @@ async function getSpecificCdeData( return await getCdeGenericData(extension, fromBlock, toBlock); case ChainDataExtensionType.ERC6551Registry: return await getCdeErc6551RegistryData(extension, fromBlock, toBlock); + case ChainDataExtensionType.CardanoPool: + // this is used by the block funnel, which can't get information for this + // extension + return []; default: assertNever(extension); } diff --git a/packages/engine/paima-funnel/src/funnels/carp/funnel.ts b/packages/engine/paima-funnel/src/funnels/carp/funnel.ts new file mode 100644 index 000000000..89313e827 --- /dev/null +++ b/packages/engine/paima-funnel/src/funnels/carp/funnel.ts @@ -0,0 +1,200 @@ +import { + ChainDataExtensionType, + DEFAULT_FUNNEL_TIMEOUT, + doLog, + logError, + timeout, +} from '@paima/utils'; +import { + type ChainData, + type ChainDataExtension, + type ChainDataExtensionCardanoDelegation, + type ChainFunnel, + type PresyncChainData, +} from '@paima/runtime'; +import { composeChainData, groupCdeData } from '../../utils'; +import { BaseFunnel } from '../BaseFunnel'; +import type { FunnelSharedData } from '../BaseFunnel'; +import type { PoolClient } from 'pg'; +import getCdePoolData from '../../cde/cardanoPool'; +import axios from 'axios'; + +type BlockInfo = { + block: { + era: number; + hash: string; + height: number; + epoch: number; + slot: number; + }; +}; + +// hardcoded preview time +const knownTime = 1666656000; + +const confirmationDepth = '10'; + +function timestampToAbsoluteSlot(timestamp: number): number { + const firstSlot = 0; + // map timestamps with a delta, since we are waiting for blocks. + const confirmationTimeDelta = 20 * 10; + + return timestamp - confirmationTimeDelta - knownTime + firstSlot; +} + +export class CarpFunnel extends BaseFunnel implements ChainFunnel { + protected constructor( + sharedData: FunnelSharedData, + dbTx: PoolClient, + private readonly baseFunnel: ChainFunnel, + private readonly carpUrl: string + ) { + super(sharedData, dbTx); + // TODO: replace once TS5 decorators are better supported + this.readData.bind(this); + this.readPresyncData.bind(this); + this.getDbTx.bind(this); + this.bufferedData = null; + } + + private bufferedData: ChainData[] | null; + + public override async readData(blockHeight: number): Promise { + if (!this.bufferedData || this.bufferedData[0].blockNumber != blockHeight) { + const data = await this.baseFunnel.readData(blockHeight); + + if (data.length === 0) { + return data; + } + + this.bufferedData = data; + } + + // there are most likely some slots between the last end of range and the + // first block in the current range, so we need to start from the previous point. + + // TODO: cache this? but it's not in the db afaik, so it can't be done on + // recoverState + const lastTimestamp = await timeout( + this.sharedData.web3.eth.getBlock(blockHeight - 1), + DEFAULT_FUNNEL_TIMEOUT + ); + + let grouped = await readDataInternal( + this.bufferedData, + this.carpUrl, + this.sharedData.extensions, + lastTimestamp.timestamp as number + ); + + const composed = composeChainData(this.bufferedData, grouped); + + this.bufferedData = null; + + return composed; + } + + public override async readPresyncData( + fromBlock: number, + toBlock: number + ): Promise { + // TODO: PresyncChainData doesn't have timestamps, so we need to either + // fetch those here, or add them? + return await this.baseFunnel.readPresyncData(fromBlock, toBlock); + } + + public static async recoverState( + sharedData: FunnelSharedData, + dbTx: PoolClient, + baseFunnel: ChainFunnel, + carpUrl: string + ): Promise { + return new CarpFunnel(sharedData, dbTx, baseFunnel, carpUrl); + } +} + +async function readDataInternal( + data: ChainData[], + carpUrl: string, + extensions: ChainDataExtension[], + lastTimestamp: number +): Promise { + // the lower range is exclusive + const min = timestampToAbsoluteSlot(lastTimestamp); + // the upper range is inclusive + const max = timestampToAbsoluteSlot(Math.max(...data.map(data => data.timestamp))); + + const sleep = (ms: number): Promise => new Promise(resolve => setTimeout(resolve, ms)); + + while (true) { + // TODO: replace with carp client + const stableBlock = await timeout( + axios.post(`${carpUrl}/block/latest`, { + offset: confirmationDepth, + }), + DEFAULT_FUNNEL_TIMEOUT + ); + + if (stableBlock.data.block.slot > max) { + break; + } + + // TODO: is there a more js-like way of doing this? + await sleep(1000); + } + + const blockNumbers = data.reduce( + (dict, data) => { + dict[timestampToAbsoluteSlot(data.timestamp)] = data.blockNumber; + return dict; + }, + {} as { [slot: number]: number } + ); + + const poolEvents = await Promise.all( + extensions + .filter(extension => extension.cdeType === ChainDataExtensionType.CardanoPool) + .map(extension => { + const data = getCdePoolData( + `${carpUrl}/delegation/pool`, + extension as ChainDataExtensionCardanoDelegation, + min, + max, + slot => { + while (true) { + const curr = blockNumbers[slot]; + if (curr) { + return curr; + } + slot += 1; + } + } + ); + return data; + }) + ); + + let grouped = groupCdeData(data[0].blockNumber, data[data.length - 1].blockNumber, poolEvents); + + return grouped; +} + +export async function wrapToCarpFunnel( + chainFunnel: ChainFunnel, + sharedData: FunnelSharedData, + dbTx: PoolClient, + carpUrl: string | undefined +): Promise { + if (!carpUrl) { + return chainFunnel; + } + + try { + const ebp = await CarpFunnel.recoverState(sharedData, dbTx, chainFunnel, carpUrl); + return ebp; + } catch (err) { + doLog('[paima-funnel] Unable to initialize carp events processor:'); + logError(err); + throw new Error('[paima-funnel] Unable to initialize carp events processor'); + } +} diff --git a/packages/engine/paima-funnel/src/index.ts b/packages/engine/paima-funnel/src/index.ts index 3919ccb21..7834073cc 100644 --- a/packages/engine/paima-funnel/src/index.ts +++ b/packages/engine/paima-funnel/src/index.ts @@ -7,6 +7,7 @@ import { wrapToEmulatedBlocksFunnel } from './funnels/emulated/utils.js'; import { BlockFunnel } from './funnels/block/funnel.js'; import type { FunnelSharedData } from './funnels/BaseFunnel.js'; import { FunnelCacheManager } from './funnels/FunnelCache.js'; +import { wrapToCarpFunnel } from './funnels/carp/funnel.js'; export class FunnelFactory implements IFunnelFactory { private constructor(public sharedData: FunnelSharedData) {} @@ -46,6 +47,7 @@ export class FunnelFactory implements IFunnelFactory { // and wrap it with dynamic decorators as needed let chainFunnel: ChainFunnel = await BlockFunnel.recoverState(this.sharedData, dbTx); + chainFunnel = await wrapToCarpFunnel(chainFunnel, this.sharedData, dbTx, ENV.CARP_URL); chainFunnel = await wrapToEmulatedBlocksFunnel( chainFunnel, this.sharedData, diff --git a/packages/engine/paima-runtime/src/cde-config/loading.ts b/packages/engine/paima-runtime/src/cde-config/loading.ts index 8db2d9bda..ee5f4468e 100644 --- a/packages/engine/paima-runtime/src/cde-config/loading.ts +++ b/packages/engine/paima-runtime/src/cde-config/loading.ts @@ -28,13 +28,14 @@ import type { import { CdeBaseConfig, CdeEntryTypeName, + ChainDataExtensionCardanoDelegationConfig, ChainDataExtensionErc20Config, ChainDataExtensionErc20DepositConfig, ChainDataExtensionErc6551RegistryConfig, ChainDataExtensionErc721Config, ChainDataExtensionGenericConfig, } from '../types'; -import type { CdeConfig } from '../types'; +import { CdeConfig } from '../types'; import { loadAbi } from './utils'; import assertNever from 'assert-never'; import fnv from 'fnv-plus'; @@ -86,6 +87,8 @@ export function parseCdeConfigFile(configFileData: string): Static & { + cdeType: ChainDataExtensionType.CardanoPool; + }; + export const CdeConfig = Type.Object({ extensions: Type.Array( Type.Union([ @@ -244,6 +272,7 @@ export const CdeConfig = Type.Object({ ChainDataExtensionErc20DepositConfig, ChainDataExtensionGenericConfig, ChainDataExtensionErc6551RegistryConfig, + ChainDataExtensionCardanoDelegationConfig, ]) ), }); @@ -267,7 +296,8 @@ export type ChainDataExtension = | ChainDataExtensionPaimaErc721 | ChainDataExtensionErc20Deposit | ChainDataExtensionGeneric - | ChainDataExtensionErc6551Registry; + | ChainDataExtensionErc6551Registry + | ChainDataExtensionCardanoDelegation; export interface ChainFunnel { readData: (blockHeight: number) => Promise; diff --git a/packages/engine/paima-sm/src/cde-cardano-pool.ts b/packages/engine/paima-sm/src/cde-cardano-pool.ts new file mode 100644 index 000000000..fba52ca07 --- /dev/null +++ b/packages/engine/paima-sm/src/cde-cardano-pool.ts @@ -0,0 +1,25 @@ +import type { Pool } from 'pg'; + +import { doLog, ENV } from '@paima/utils'; +import type { CdeCardanoPoolDatum } from '@paima/runtime'; +import { createScheduledData, cdeCardanoPoolInsertData } from '@paima/db'; +import type { SQLUpdate } from '@paima/db'; + +export default async function processDatum(cdeDatum: CdeCardanoPoolDatum): Promise { + const cdeId = cdeDatum.cdeId; + const prefix = cdeDatum.scheduledPrefix; + const address = cdeDatum.payload.address; + const pool = cdeDatum.payload.pool; + + const scheduledBlockHeight = Math.max(cdeDatum.blockNumber, ENV.SM_START_BLOCKHEIGHT + 1); + const scheduledInputData = `${prefix}|${address}|${pool}`; + + const updateList: SQLUpdate[] = [ + createScheduledData(scheduledInputData, scheduledBlockHeight), + [ + cdeCardanoPoolInsertData, + { cde_id: cdeId, address: cdeDatum.payload.address, pool: cdeDatum.payload.pool }, + ], + ]; + return updateList; +} diff --git a/packages/engine/paima-sm/src/cde-processing.ts b/packages/engine/paima-sm/src/cde-processing.ts index d5c0f3ba2..2c9da7a6d 100644 --- a/packages/engine/paima-sm/src/cde-processing.ts +++ b/packages/engine/paima-sm/src/cde-processing.ts @@ -9,6 +9,7 @@ import processErc721MintDatum from './cde-erc721-mint'; import processErc20DepositDatum from './cde-erc20-deposit'; import processErc6551RegistryDatum from './cde-erc6551-registry'; import processGenericDatum from './cde-generic'; +import processCardanoDelegationDatum from './cde-cardano-pool'; import type { SQLUpdate } from '@paima/db'; import { getSpecificCdeBlockheight } from '@paima/db'; import assertNever from 'assert-never'; @@ -30,6 +31,8 @@ export async function cdeTransitionFunction( return await processGenericDatum(cdeDatum); case ChainDataExtensionDatumType.ERC6551Registry: return await processErc6551RegistryDatum(cdeDatum); + case ChainDataExtensionDatumType.CardanoPool: + return await processCardanoDelegationDatum(cdeDatum); default: assertNever(cdeDatum); } diff --git a/packages/paima-sdk/paima-db/migrations/down.sql b/packages/paima-sdk/paima-db/migrations/down.sql index 9c30a77a0..32c9f853d 100644 --- a/packages/paima-sdk/paima-db/migrations/down.sql +++ b/packages/paima-sdk/paima-db/migrations/down.sql @@ -10,3 +10,4 @@ DROP TABLE historical_game_inputs; DROP TABLE nonces; DROP TABLE scheduled_data; DROP TABLE block_heights; +DROP TABLE cde_cardano_pool_delegation; diff --git a/packages/paima-sdk/paima-db/migrations/up.sql b/packages/paima-sdk/paima-db/migrations/up.sql index 7345762ad..3e4849fec 100644 --- a/packages/paima-sdk/paima-db/migrations/up.sql +++ b/packages/paima-sdk/paima-db/migrations/up.sql @@ -84,3 +84,10 @@ CREATE TABLE emulated_block_heights ( second_timestamp TEXT NOT NULL, emulated_block_height INTEGER NOT NULL ); + +CREATE TABLE cde_cardano_pool_delegation ( + cde_id INTEGER NOT NULL, + address TEXT NOT NULL, + pool TEXT, + PRIMARY KEY (cde_id, address) +); diff --git a/packages/paima-sdk/paima-db/src/index.ts b/packages/paima-sdk/paima-db/src/index.ts index 00a751896..92ca05534 100644 --- a/packages/paima-sdk/paima-db/src/index.ts +++ b/packages/paima-sdk/paima-db/src/index.ts @@ -29,6 +29,8 @@ export type * from './sql/cde-erc6551-registry.queries'; export * from './sql/emulated.queries'; export type * from './sql/emulated.queries'; export type * from './types'; +export * from './sql/cde-cardano-pool-delegation.queries'; +export type * from './sql/cde-cardano-pool-delegation.queries'; export { tx, diff --git a/packages/paima-sdk/paima-db/src/paima-tables.ts b/packages/paima-sdk/paima-db/src/paima-tables.ts index 04f495348..df92b2864 100644 --- a/packages/paima-sdk/paima-db/src/paima-tables.ts +++ b/packages/paima-sdk/paima-db/src/paima-tables.ts @@ -244,6 +244,27 @@ const TABLE_DATA_CDE_ERC6551_REGISTRY: TableData = { creationQuery: QUERY_CREATE_TABLE_CDE_ERC6551_REGISTRY, }; +const QUERY_CREATE_TABLE_CDE_CARDANO_POOL = ` +CREATE TABLE cde_cardano_pool_delegation ( + cde_id INTEGER NOT NULL, + address TEXT NOT NULL, + pool TEXT, + PRIMARY KEY (cde_id, address) +); +`; + +const TABLE_DATA_CDE_CARDANO_POOL: TableData = { + tableName: 'cde_cardano_pool_delegation', + primaryKeyColumns: ['cde_id', 'address'], + columnData: packTuples([ + ['cde_id', 'integer', 'NO', ''], + ['address', 'text', 'NO', ''], + ['pool', 'text', 'YES', ''], + ]), + serialColumns: [], + creationQuery: QUERY_CREATE_TABLE_CDE_CARDANO_POOL, +}; + const QUERY_CREATE_TABLE_EMULATED = ` CREATE TABLE emulated_block_heights ( deployment_chain_block_height INTEGER PRIMARY KEY, @@ -276,5 +297,6 @@ export const TABLES: TableData[] = [ TABLE_DATA_CDE_ERC20_DEPOSIT, TABLE_DATA_CDE_GENERIC_DATA, TABLE_DATA_CDE_ERC6551_REGISTRY, + TABLE_DATA_CDE_CARDANO_POOL, TABLE_DATA_EMULATED, ]; diff --git a/packages/paima-sdk/paima-db/src/sql/cde-cardano-pool-delegation.queries.ts b/packages/paima-sdk/paima-db/src/sql/cde-cardano-pool-delegation.queries.ts new file mode 100644 index 000000000..aca7374af --- /dev/null +++ b/packages/paima-sdk/paima-db/src/sql/cde-cardano-pool-delegation.queries.ts @@ -0,0 +1,69 @@ +/** Types generated for queries found in "src/sql/cde-cardano-pool-delegation.sql" */ +import { PreparedQuery } from '@pgtyped/runtime'; + +/** 'CdeCardanoPoolGetAddressDelegation' parameters type */ +export interface ICdeCardanoPoolGetAddressDelegationParams { + address: string; +} + +/** 'CdeCardanoPoolGetAddressDelegation' return type */ +export interface ICdeCardanoPoolGetAddressDelegationResult { + address: string; + cde_id: number; + pool: string | null; +} + +/** 'CdeCardanoPoolGetAddressDelegation' query type */ +export interface ICdeCardanoPoolGetAddressDelegationQuery { + params: ICdeCardanoPoolGetAddressDelegationParams; + result: ICdeCardanoPoolGetAddressDelegationResult; +} + +const cdeCardanoPoolGetAddressDelegationIR: any = {"usedParamSet":{"address":true},"params":[{"name":"address","required":true,"transform":{"type":"scalar"},"locs":[{"a":59,"b":67}]}],"statement":"SELECT * FROM cde_cardano_pool_delegation \nWHERE address = :address!"}; + +/** + * Query generated from SQL: + * ``` + * SELECT * FROM cde_cardano_pool_delegation + * WHERE address = :address! + * ``` + */ +export const cdeCardanoPoolGetAddressDelegation = new PreparedQuery(cdeCardanoPoolGetAddressDelegationIR); + + +/** 'CdeCardanoPoolInsertData' parameters type */ +export interface ICdeCardanoPoolInsertDataParams { + address: string; + cde_id: number; + pool: string; +} + +/** 'CdeCardanoPoolInsertData' return type */ +export type ICdeCardanoPoolInsertDataResult = void; + +/** 'CdeCardanoPoolInsertData' query type */ +export interface ICdeCardanoPoolInsertDataQuery { + params: ICdeCardanoPoolInsertDataParams; + result: ICdeCardanoPoolInsertDataResult; +} + +const cdeCardanoPoolInsertDataIR: any = {"usedParamSet":{"cde_id":true,"address":true,"pool":true},"params":[{"name":"cde_id","required":true,"transform":{"type":"scalar"},"locs":[{"a":90,"b":97}]},{"name":"address","required":true,"transform":{"type":"scalar"},"locs":[{"a":104,"b":112}]},{"name":"pool","required":true,"transform":{"type":"scalar"},"locs":[{"a":119,"b":124},{"a":181,"b":186}]}],"statement":"INSERT INTO cde_cardano_pool_delegation(\n cde_id,\n address,\n pool\n) VALUES (\n :cde_id!,\n :address!,\n :pool!\n) ON CONFLICT (cde_id, address) DO\n UPDATE SET pool = :pool!"}; + +/** + * Query generated from SQL: + * ``` + * INSERT INTO cde_cardano_pool_delegation( + * cde_id, + * address, + * pool + * ) VALUES ( + * :cde_id!, + * :address!, + * :pool! + * ) ON CONFLICT (cde_id, address) DO + * UPDATE SET pool = :pool! + * ``` + */ +export const cdeCardanoPoolInsertData = new PreparedQuery(cdeCardanoPoolInsertDataIR); + + diff --git a/packages/paima-sdk/paima-db/src/sql/cde-cardano-pool-delegation.sql b/packages/paima-sdk/paima-db/src/sql/cde-cardano-pool-delegation.sql new file mode 100644 index 000000000..ff1fba4b5 --- /dev/null +++ b/packages/paima-sdk/paima-db/src/sql/cde-cardano-pool-delegation.sql @@ -0,0 +1,15 @@ +/* @name cdeCardanoPoolGetAddressDelegation */ +SELECT * FROM cde_cardano_pool_delegation +WHERE address = :address!; + +/* @name cdeCardanoPoolInsertData */ +INSERT INTO cde_cardano_pool_delegation( + cde_id, + address, + pool +) VALUES ( + :cde_id!, + :address!, + :pool! +) ON CONFLICT (cde_id, address) DO + UPDATE SET pool = :pool!; \ No newline at end of file diff --git a/packages/paima-sdk/paima-utils/src/config.ts b/packages/paima-sdk/paima-utils/src/config.ts index c7a048226..f809a831f 100644 --- a/packages/paima-sdk/paima-utils/src/config.ts +++ b/packages/paima-sdk/paima-utils/src/config.ts @@ -105,4 +105,9 @@ export class ENV { static get BATCHER_URI(): string { return process.env.BATCHER_URI || ''; } + + // TODO: put this in the right place + static get CARP_URL(): string | undefined { + return process.env.CARP_URL; + } } diff --git a/packages/paima-sdk/paima-utils/src/constants.ts b/packages/paima-sdk/paima-utils/src/constants.ts index 62f34a0b0..168aa7f0b 100644 --- a/packages/paima-sdk/paima-utils/src/constants.ts +++ b/packages/paima-sdk/paima-utils/src/constants.ts @@ -16,6 +16,7 @@ export const enum ChainDataExtensionType { ERC20Deposit = 4, Generic = 5, ERC6551Registry = 6, + CardanoPool = 7, } export const enum ChainDataExtensionDatumType { @@ -25,4 +26,5 @@ export const enum ChainDataExtensionDatumType { ERC721Transfer, Generic, ERC6551Registry, + CardanoPool, }