diff --git a/app.ts b/app.ts index 956c06460..6151bbaff 100644 --- a/app.ts +++ b/app.ts @@ -20,25 +20,25 @@ import { ProfilingIntegration } from '@sentry/profiling-node'; import { sentryPlugin } from './app/gql/sentry-apollo-plugin'; import { startWorker } from './worker/worker'; import { startScheduler } from './worker/scheduler'; +import { prisma } from './prisma/prisma-client'; async function startServer() { const app = createExpressApp(); Sentry.init({ dsn: env.SENTRY_DSN, - // tracesSampleRate: 0.005, environment: `multichain-${env.DEPLOYMENT_ENV}`, enabled: env.NODE_ENV === 'production', ignoreErrors: [/.*error: Provide.*chain.*param/], integrations: [ - // new Tracing.Integrations.Apollo(), - // new Tracing.Integrations.GraphQL(), - // new Tracing.Integrations.Prisma({ client: prisma }), + new Sentry.Integrations.Apollo(), + new Sentry.Integrations.GraphQL(), + new Sentry.Integrations.Prisma({ client: prisma }), new Sentry.Integrations.Express({ app }), new Sentry.Integrations.Http({ tracing: true }), new ProfilingIntegration(), ], - tracesSampleRate: 0.2, + tracesSampleRate: 0.005, profilesSampleRate: 0.1, beforeSend(event, hint) { const error = hint.originalException as string; diff --git a/modules/network/arbitrum.ts b/modules/network/arbitrum.ts index 839fb9dcb..55676c40b 100644 --- a/modules/network/arbitrum.ts +++ b/modules/network/arbitrum.ts @@ -52,9 +52,6 @@ const arbitrumNetworkData: NetworkData = { platformId: 'arbitrum-one', excludedTokenAddresses: ['0x6dbf2155b0636cb3fd5359fccefb8a2c02b6cb51'], // plsRDNT, has coingecko entry but no price }, - tokenPrices: { - maxHourlyPriceHistoryNumDays: 100, - }, rpcUrl: env.INFURA_API_KEY && (env.DEPLOYMENT_ENV as DeploymentEnv) === 'main' ? `https://arbitrum-mainnet.infura.io/v3/${env.INFURA_API_KEY}` @@ -266,27 +263,27 @@ export const arbitrumNetworkConfig: NetworkConfig = { }, { name: 'update-liquidity-for-active-pools', - interval: (env.DEPLOYMENT_ENV as DeploymentEnv) === 'canary' ? every(8, 'minutes') : every(4, 'minutes'), + interval: (env.DEPLOYMENT_ENV as DeploymentEnv) === 'canary' ? every(6, 'minutes') : every(2, 'minutes'), }, { name: 'update-pool-apr', - interval: (env.DEPLOYMENT_ENV as DeploymentEnv) === 'canary' ? every(7, 'minutes') : every(5, 'minutes'), + interval: (env.DEPLOYMENT_ENV as DeploymentEnv) === 'canary' ? every(6, 'minutes') : every(2, 'minutes'), }, { name: 'load-on-chain-data-for-pools-with-active-updates', - interval: (env.DEPLOYMENT_ENV as DeploymentEnv) === 'canary' ? every(9, 'minutes') : every(5, 'minutes'), + interval: (env.DEPLOYMENT_ENV as DeploymentEnv) === 'canary' ? every(4, 'minutes') : every(1, 'minutes'), }, { name: 'sync-new-pools-from-subgraph', - interval: (env.DEPLOYMENT_ENV as DeploymentEnv) === 'canary' ? every(12, 'minutes') : every(8, 'minutes'), + interval: (env.DEPLOYMENT_ENV as DeploymentEnv) === 'canary' ? every(6, 'minutes') : every(2, 'minutes'), }, { name: 'sync-tokens-from-pool-tokens', - interval: (env.DEPLOYMENT_ENV as DeploymentEnv) === 'canary' ? every(10, 'minutes') : every(7, 'minutes'), + interval: (env.DEPLOYMENT_ENV as DeploymentEnv) === 'canary' ? every(10, 'minutes') : every(5, 'minutes'), }, { name: 'update-liquidity-24h-ago-for-all-pools', - interval: (env.DEPLOYMENT_ENV as DeploymentEnv) === 'canary' ? every(15, 'minutes') : every(8, 'minutes'), + interval: (env.DEPLOYMENT_ENV as DeploymentEnv) === 'canary' ? every(10, 'minutes') : every(5, 'minutes'), }, { name: 'cache-average-block-time', @@ -294,7 +291,7 @@ export const arbitrumNetworkConfig: NetworkConfig = { }, { name: 'sync-staking-for-pools', - interval: (env.DEPLOYMENT_ENV as DeploymentEnv) === 'canary' ? every(15, 'minutes') : every(10, 'minutes'), + interval: (env.DEPLOYMENT_ENV as DeploymentEnv) === 'canary' ? every(10, 'minutes') : every(5, 'minutes'), }, { name: 'sync-latest-snapshots-for-all-pools', @@ -302,43 +299,41 @@ export const arbitrumNetworkConfig: NetworkConfig = { }, { name: 'update-lifetime-values-for-all-pools', - interval: every(45, 'minutes'), + interval: every(50, 'minutes'), }, { name: 'sync-changed-pools', - interval: (env.DEPLOYMENT_ENV as DeploymentEnv) === 'canary' ? every(2, 'minutes') : every(1, 'minutes'), + interval: (env.DEPLOYMENT_ENV as DeploymentEnv) === 'canary' ? every(2, 'minutes') : every(30, 'seconds'), alarmEvaluationPeriod: (env.DEPLOYMENT_ENV as DeploymentEnv) === 'canary' ? 3 : 1, alarmDatapointsToAlarm: (env.DEPLOYMENT_ENV as DeploymentEnv) === 'canary' ? 3 : 1, }, { name: 'user-sync-wallet-balances-for-all-pools', - interval: (env.DEPLOYMENT_ENV as DeploymentEnv) === 'canary' ? every(29, 'minutes') : every(9, 'minutes'), + interval: (env.DEPLOYMENT_ENV as DeploymentEnv) === 'canary' ? every(5, 'minutes') : every(20, 'seconds'), + alarmEvaluationPeriod: (env.DEPLOYMENT_ENV as DeploymentEnv) === 'canary' ? 3 : 1, + alarmDatapointsToAlarm: (env.DEPLOYMENT_ENV as DeploymentEnv) === 'canary' ? 3 : 1, }, { name: 'user-sync-staked-balances', - interval: (env.DEPLOYMENT_ENV as DeploymentEnv) === 'canary' ? every(31, 'minutes') : every(11, 'minutes'), + interval: (env.DEPLOYMENT_ENV as DeploymentEnv) === 'canary' ? every(5, 'minutes') : every(20, 'seconds'), + alarmEvaluationPeriod: (env.DEPLOYMENT_ENV as DeploymentEnv) === 'canary' ? 3 : 1, + alarmDatapointsToAlarm: (env.DEPLOYMENT_ENV as DeploymentEnv) === 'canary' ? 3 : 1, }, { name: 'sync-coingecko-coinids', interval: every(2, 'hours'), }, - { - name: 'purge-old-tokenprices', - interval: every(1, 'days'), - alarmEvaluationPeriod: 1, - alarmDatapointsToAlarm: 1, - }, { name: 'update-fee-volume-yield-all-pools', - interval: every(75, 'minutes'), + interval: every(1, 'hours'), }, { name: 'sync-vebal-balances', - interval: (env.DEPLOYMENT_ENV as DeploymentEnv) === 'canary' ? every(20, 'minutes') : every(14, 'minutes'), + interval: (env.DEPLOYMENT_ENV as DeploymentEnv) === 'canary' ? every(9, 'minutes') : every(3, 'minutes'), }, { name: 'sync-vebal-totalSupply', - interval: (env.DEPLOYMENT_ENV as DeploymentEnv) === 'canary' ? every(20, 'minutes') : every(16, 'minutes'), + interval: (env.DEPLOYMENT_ENV as DeploymentEnv) === 'canary' ? every(10, 'minutes') : every(5, 'minutes'), }, { name: 'feed-data-to-datastudio', diff --git a/modules/network/avalanche.ts b/modules/network/avalanche.ts index d633625d5..8cf724a68 100644 --- a/modules/network/avalanche.ts +++ b/modules/network/avalanche.ts @@ -52,9 +52,6 @@ const avalancheNetworkData: NetworkData = { platformId: 'avalanche', excludedTokenAddresses: [], }, - tokenPrices: { - maxHourlyPriceHistoryNumDays: 100, - }, rpcUrl: env.INFURA_API_KEY && (env.DEPLOYMENT_ENV as DeploymentEnv) === 'main' ? `https://avalanche-mainnet.infura.io/v3/${env.INFURA_API_KEY}` @@ -260,27 +257,27 @@ export const avalancheNetworkConfig: NetworkConfig = { }, { name: 'update-liquidity-for-active-pools', - interval: (env.DEPLOYMENT_ENV as DeploymentEnv) === 'canary' ? every(8, 'minutes') : every(4, 'minutes'), + interval: (env.DEPLOYMENT_ENV as DeploymentEnv) === 'canary' ? every(6, 'minutes') : every(2, 'minutes'), }, { name: 'update-pool-apr', - interval: (env.DEPLOYMENT_ENV as DeploymentEnv) === 'canary' ? every(7, 'minutes') : every(5, 'minutes'), + interval: (env.DEPLOYMENT_ENV as DeploymentEnv) === 'canary' ? every(6, 'minutes') : every(2, 'minutes'), }, { name: 'load-on-chain-data-for-pools-with-active-updates', - interval: (env.DEPLOYMENT_ENV as DeploymentEnv) === 'canary' ? every(9, 'minutes') : every(5, 'minutes'), + interval: (env.DEPLOYMENT_ENV as DeploymentEnv) === 'canary' ? every(4, 'minutes') : every(1, 'minutes'), }, { name: 'sync-new-pools-from-subgraph', - interval: (env.DEPLOYMENT_ENV as DeploymentEnv) === 'canary' ? every(12, 'minutes') : every(8, 'minutes'), + interval: (env.DEPLOYMENT_ENV as DeploymentEnv) === 'canary' ? every(6, 'minutes') : every(2, 'minutes'), }, { name: 'sync-tokens-from-pool-tokens', - interval: (env.DEPLOYMENT_ENV as DeploymentEnv) === 'canary' ? every(10, 'minutes') : every(7, 'minutes'), + interval: (env.DEPLOYMENT_ENV as DeploymentEnv) === 'canary' ? every(10, 'minutes') : every(5, 'minutes'), }, { name: 'update-liquidity-24h-ago-for-all-pools', - interval: (env.DEPLOYMENT_ENV as DeploymentEnv) === 'canary' ? every(15, 'minutes') : every(8, 'minutes'), + interval: (env.DEPLOYMENT_ENV as DeploymentEnv) === 'canary' ? every(10, 'minutes') : every(5, 'minutes'), }, { name: 'cache-average-block-time', @@ -288,7 +285,7 @@ export const avalancheNetworkConfig: NetworkConfig = { }, { name: 'sync-staking-for-pools', - interval: (env.DEPLOYMENT_ENV as DeploymentEnv) === 'canary' ? every(15, 'minutes') : every(10, 'minutes'), + interval: (env.DEPLOYMENT_ENV as DeploymentEnv) === 'canary' ? every(10, 'minutes') : every(5, 'minutes'), }, { name: 'sync-latest-snapshots-for-all-pools', @@ -296,43 +293,41 @@ export const avalancheNetworkConfig: NetworkConfig = { }, { name: 'update-lifetime-values-for-all-pools', - interval: every(45, 'minutes'), + interval: every(50, 'minutes'), }, { name: 'sync-changed-pools', - interval: (env.DEPLOYMENT_ENV as DeploymentEnv) === 'canary' ? every(2, 'minutes') : every(1, 'minutes'), + interval: (env.DEPLOYMENT_ENV as DeploymentEnv) === 'canary' ? every(2, 'minutes') : every(30, 'seconds'), alarmEvaluationPeriod: (env.DEPLOYMENT_ENV as DeploymentEnv) === 'canary' ? 3 : 1, alarmDatapointsToAlarm: (env.DEPLOYMENT_ENV as DeploymentEnv) === 'canary' ? 3 : 1, }, { name: 'user-sync-wallet-balances-for-all-pools', - interval: (env.DEPLOYMENT_ENV as DeploymentEnv) === 'canary' ? every(29, 'minutes') : every(9, 'minutes'), + interval: (env.DEPLOYMENT_ENV as DeploymentEnv) === 'canary' ? every(5, 'minutes') : every(20, 'seconds'), + alarmEvaluationPeriod: (env.DEPLOYMENT_ENV as DeploymentEnv) === 'canary' ? 3 : 1, + alarmDatapointsToAlarm: (env.DEPLOYMENT_ENV as DeploymentEnv) === 'canary' ? 3 : 1, }, { name: 'user-sync-staked-balances', - interval: (env.DEPLOYMENT_ENV as DeploymentEnv) === 'canary' ? every(31, 'minutes') : every(11, 'minutes'), + interval: (env.DEPLOYMENT_ENV as DeploymentEnv) === 'canary' ? every(5, 'minutes') : every(20, 'seconds'), + alarmEvaluationPeriod: (env.DEPLOYMENT_ENV as DeploymentEnv) === 'canary' ? 3 : 1, + alarmDatapointsToAlarm: (env.DEPLOYMENT_ENV as DeploymentEnv) === 'canary' ? 3 : 1, }, { name: 'sync-coingecko-coinids', interval: every(2, 'hours'), }, - { - name: 'purge-old-tokenprices', - interval: every(1, 'days'), - alarmEvaluationPeriod: 1, - alarmDatapointsToAlarm: 1, - }, { name: 'update-fee-volume-yield-all-pools', - interval: every(75, 'minutes'), + interval: every(1, 'hours'), }, { name: 'sync-vebal-balances', - interval: (env.DEPLOYMENT_ENV as DeploymentEnv) === 'canary' ? every(20, 'minutes') : every(14, 'minutes'), + interval: (env.DEPLOYMENT_ENV as DeploymentEnv) === 'canary' ? every(9, 'minutes') : every(3, 'minutes'), }, { name: 'sync-vebal-totalSupply', - interval: (env.DEPLOYMENT_ENV as DeploymentEnv) === 'canary' ? every(20, 'minutes') : every(16, 'minutes'), + interval: (env.DEPLOYMENT_ENV as DeploymentEnv) === 'canary' ? every(10, 'minutes') : every(5, 'minutes'), }, { name: 'feed-data-to-datastudio', diff --git a/modules/network/base.ts b/modules/network/base.ts index 9bbea516f..f57e9420c 100644 --- a/modules/network/base.ts +++ b/modules/network/base.ts @@ -51,9 +51,6 @@ const baseNetworkData: NetworkData = { platformId: 'base', excludedTokenAddresses: [], }, - tokenPrices: { - maxHourlyPriceHistoryNumDays: 100, - }, rpcUrl: 'https://base.gateway.tenderly.co/7mM7DbBouY1JjnQd9MMDsd', rpcMaxBlockRange: 500, protocolToken: 'bal', @@ -232,12 +229,6 @@ export const baseNetworkConfig: NetworkConfig = { name: 'sync-coingecko-coinids', interval: every(2, 'hours'), }, - { - name: 'purge-old-tokenprices', - interval: every(1, 'days'), - alarmEvaluationPeriod: 1, - alarmDatapointsToAlarm: 1, - }, { name: 'update-fee-volume-yield-all-pools', interval: every(75, 'minutes'), diff --git a/modules/network/fantom.ts b/modules/network/fantom.ts index 6391205c5..ff3a5510c 100644 --- a/modules/network/fantom.ts +++ b/modules/network/fantom.ts @@ -93,9 +93,6 @@ const fantomNetworkData: NetworkData = { '0xb7c2ddb1ebac1056231ef22c1b0a13988537a274', // new tarot ], }, - tokenPrices: { - maxHourlyPriceHistoryNumDays: 100, - }, rpcUrl: (env.DEPLOYMENT_ENV as DeploymentEnv) === 'main' ? `https://rpc.ankr.com/fantom` @@ -438,12 +435,6 @@ export const fantomNetworkConfig: NetworkConfig = { name: 'sync-latest-relic-snapshots', interval: every(1, 'hours'), }, - { - name: 'purge-old-tokenprices', - interval: every(1, 'days'), - alarmEvaluationPeriod: 1, - alarmDatapointsToAlarm: 1, - }, { name: 'sync-coingecko-coinids', interval: every(2, 'hours'), diff --git a/modules/network/gnosis.ts b/modules/network/gnosis.ts index 728bf5a15..8323dd363 100644 --- a/modules/network/gnosis.ts +++ b/modules/network/gnosis.ts @@ -52,9 +52,6 @@ const gnosisNetworkData: NetworkData = { platformId: 'xdai', excludedTokenAddresses: [], }, - tokenPrices: { - maxHourlyPriceHistoryNumDays: 100, - }, rpcUrl: (env.DEPLOYMENT_ENV as DeploymentEnv) === 'main' ? `https://rpc.gnosischain.com` : 'https://gnosis.drpc.org', rpcMaxBlockRange: 2000, @@ -240,12 +237,6 @@ export const gnosisNetworkConfig: NetworkConfig = { name: 'sync-coingecko-coinids', interval: every(2, 'hours'), }, - { - name: 'purge-old-tokenprices', - interval: every(1, 'days'), - alarmEvaluationPeriod: 1, - alarmDatapointsToAlarm: 1, - }, { name: 'update-fee-volume-yield-all-pools', interval: every(75, 'minutes'), diff --git a/modules/network/mainnet.ts b/modules/network/mainnet.ts index b2bfab90f..0fa6025da 100644 --- a/modules/network/mainnet.ts +++ b/modules/network/mainnet.ts @@ -63,13 +63,10 @@ const data: NetworkData = { '0xcd5fe23c85820f7b72d0926fc9b05b43e359b7ee', // weEth, has coingekco entry but no price ], }, - tokenPrices: { - maxHourlyPriceHistoryNumDays: 100, - }, rpcUrl: env.INFURA_API_KEY && (env.DEPLOYMENT_ENV as DeploymentEnv) === 'main' ? `https://mainnet.infura.io/v3/${env.INFURA_API_KEY}` - : 'https://eth.llamarpc.com', + : 'https://rpc.eth.gateway.fm', rpcMaxBlockRange: 700, protocolToken: 'bal', bal: { @@ -468,12 +465,6 @@ export const mainnetNetworkConfig: NetworkConfig = { name: 'sync-coingecko-coinids', interval: every(2, 'hours'), }, - { - name: 'purge-old-tokenprices', - interval: every(1, 'days'), - alarmEvaluationPeriod: 1, - alarmDatapointsToAlarm: 1, - }, { name: 'update-fee-volume-yield-all-pools', interval: every(75, 'minutes'), @@ -490,11 +481,6 @@ export const mainnetNetworkConfig: NetworkConfig = { name: 'sync-vebal-voting-gauges', interval: (env.DEPLOYMENT_ENV as DeploymentEnv) === 'canary' ? every(20, 'minutes') : every(5, 'minutes'), }, - // The following are multichain jobs and should only run once for all chains. - { - name: 'sync-global-coingecko-prices', - interval: (env.DEPLOYMENT_ENV as DeploymentEnv) === 'canary' ? every(10, 'minutes') : every(2, 'minutes'), - }, { name: 'feed-data-to-datastudio', interval: (env.DEPLOYMENT_ENV as DeploymentEnv) === 'canary' ? every(5, 'minutes') : every(5, 'minutes'), @@ -503,5 +489,16 @@ export const mainnetNetworkConfig: NetworkConfig = { name: 'sync-latest-fx-prices', interval: every(10, 'minutes'), }, + // The following are multichain jobs and should only run once for all chains. + { + name: 'sync-global-coingecko-prices', + interval: (env.DEPLOYMENT_ENV as DeploymentEnv) === 'canary' ? every(10, 'minutes') : every(2, 'minutes'), + }, + { + name: 'global-purge-old-tokenprices', + interval: every(1, 'days'), + alarmEvaluationPeriod: 1, + alarmDatapointsToAlarm: 1, + }, ], }; diff --git a/modules/network/network-config-types.ts b/modules/network/network-config-types.ts index 86982b656..9127ac3a4 100644 --- a/modules/network/network-config-types.ts +++ b/modules/network/network-config-types.ts @@ -62,9 +62,6 @@ export interface NetworkData { platformId: string; excludedTokenAddresses: string[]; }; - tokenPrices: { - maxHourlyPriceHistoryNumDays: number; - }; subgraphs: { startDate: string; balancer: string; diff --git a/modules/network/optimism.ts b/modules/network/optimism.ts index d885c656d..34740a0b7 100644 --- a/modules/network/optimism.ts +++ b/modules/network/optimism.ts @@ -53,9 +53,6 @@ const optimismNetworkData: NetworkData = { platformId: 'optimistic-ethereum', excludedTokenAddresses: ['0x97513e975a7fa9072c72c92d8000b0db90b163c5'], //multibeets }, - tokenPrices: { - maxHourlyPriceHistoryNumDays: 100, - }, rpcUrl: env.INFURA_API_KEY ? `https://optimism-mainnet.infura.io/v3/${env.INFURA_API_KEY}` : 'https://mainnet.optimism.io', @@ -322,10 +319,6 @@ export const optimismNetworkConfig: NetworkConfig = { name: 'sync-new-pools-from-subgraph', interval: (env.DEPLOYMENT_ENV as DeploymentEnv) === 'canary' ? every(6, 'minutes') : every(2, 'minutes'), }, - { - name: 'sync-sanity-pool-data', - interval: (env.DEPLOYMENT_ENV as DeploymentEnv) === 'canary' ? every(9, 'minutes') : every(3, 'minutes'), - }, { name: 'sync-tokens-from-pool-tokens', interval: (env.DEPLOYMENT_ENV as DeploymentEnv) === 'canary' ? every(10, 'minutes') : every(5, 'minutes'), @@ -372,12 +365,6 @@ export const optimismNetworkConfig: NetworkConfig = { name: 'sync-coingecko-coinids', interval: every(2, 'hours'), }, - { - name: 'purge-old-tokenprices', - interval: every(1, 'days'), - alarmEvaluationPeriod: 1, - alarmDatapointsToAlarm: 1, - }, { name: 'update-fee-volume-yield-all-pools', interval: every(1, 'hours'), @@ -394,5 +381,9 @@ export const optimismNetworkConfig: NetworkConfig = { name: 'feed-data-to-datastudio', interval: (env.DEPLOYMENT_ENV as DeploymentEnv) === 'canary' ? every(5, 'minutes') : every(5, 'minutes'), }, + { + name: 'sync-sanity-pool-data', + interval: (env.DEPLOYMENT_ENV as DeploymentEnv) === 'canary' ? every(9, 'minutes') : every(3, 'minutes'), + }, ], }; diff --git a/modules/network/polygon.ts b/modules/network/polygon.ts index 398985199..8598d6930 100644 --- a/modules/network/polygon.ts +++ b/modules/network/polygon.ts @@ -52,9 +52,6 @@ const polygonNetworkData: NetworkData = { platformId: 'polygon-pos', excludedTokenAddresses: [], }, - tokenPrices: { - maxHourlyPriceHistoryNumDays: 100, - }, rpcUrl: env.INFURA_API_KEY ? `https://polygon-mainnet.infura.io/v3/${env.INFURA_API_KEY}` : 'https://polygon.sakurarpc.io', @@ -301,27 +298,27 @@ export const polygonNetworkConfig: NetworkConfig = { }, { name: 'update-liquidity-for-active-pools', - interval: (env.DEPLOYMENT_ENV as DeploymentEnv) === 'canary' ? every(8, 'minutes') : every(4, 'minutes'), + interval: (env.DEPLOYMENT_ENV as DeploymentEnv) === 'canary' ? every(6, 'minutes') : every(2, 'minutes'), }, { name: 'update-pool-apr', - interval: (env.DEPLOYMENT_ENV as DeploymentEnv) === 'canary' ? every(7, 'minutes') : every(5, 'minutes'), + interval: (env.DEPLOYMENT_ENV as DeploymentEnv) === 'canary' ? every(6, 'minutes') : every(2, 'minutes'), }, { name: 'load-on-chain-data-for-pools-with-active-updates', - interval: (env.DEPLOYMENT_ENV as DeploymentEnv) === 'canary' ? every(9, 'minutes') : every(5, 'minutes'), + interval: (env.DEPLOYMENT_ENV as DeploymentEnv) === 'canary' ? every(4, 'minutes') : every(1, 'minutes'), }, { name: 'sync-new-pools-from-subgraph', - interval: (env.DEPLOYMENT_ENV as DeploymentEnv) === 'canary' ? every(12, 'minutes') : every(8, 'minutes'), + interval: (env.DEPLOYMENT_ENV as DeploymentEnv) === 'canary' ? every(6, 'minutes') : every(2, 'minutes'), }, { name: 'sync-tokens-from-pool-tokens', - interval: (env.DEPLOYMENT_ENV as DeploymentEnv) === 'canary' ? every(10, 'minutes') : every(7, 'minutes'), + interval: (env.DEPLOYMENT_ENV as DeploymentEnv) === 'canary' ? every(10, 'minutes') : every(5, 'minutes'), }, { name: 'update-liquidity-24h-ago-for-all-pools', - interval: (env.DEPLOYMENT_ENV as DeploymentEnv) === 'canary' ? every(15, 'minutes') : every(8, 'minutes'), + interval: (env.DEPLOYMENT_ENV as DeploymentEnv) === 'canary' ? every(10, 'minutes') : every(5, 'minutes'), }, { name: 'cache-average-block-time', @@ -329,7 +326,7 @@ export const polygonNetworkConfig: NetworkConfig = { }, { name: 'sync-staking-for-pools', - interval: (env.DEPLOYMENT_ENV as DeploymentEnv) === 'canary' ? every(15, 'minutes') : every(10, 'minutes'), + interval: (env.DEPLOYMENT_ENV as DeploymentEnv) === 'canary' ? every(10, 'minutes') : every(5, 'minutes'), }, { name: 'sync-latest-snapshots-for-all-pools', @@ -337,43 +334,41 @@ export const polygonNetworkConfig: NetworkConfig = { }, { name: 'update-lifetime-values-for-all-pools', - interval: every(45, 'minutes'), + interval: every(50, 'minutes'), }, { name: 'sync-changed-pools', - interval: (env.DEPLOYMENT_ENV as DeploymentEnv) === 'canary' ? every(2, 'minutes') : every(1, 'minutes'), + interval: (env.DEPLOYMENT_ENV as DeploymentEnv) === 'canary' ? every(2, 'minutes') : every(30, 'seconds'), alarmEvaluationPeriod: (env.DEPLOYMENT_ENV as DeploymentEnv) === 'canary' ? 3 : 1, alarmDatapointsToAlarm: (env.DEPLOYMENT_ENV as DeploymentEnv) === 'canary' ? 3 : 1, }, { name: 'user-sync-wallet-balances-for-all-pools', - interval: (env.DEPLOYMENT_ENV as DeploymentEnv) === 'canary' ? every(29, 'minutes') : every(9, 'minutes'), + interval: (env.DEPLOYMENT_ENV as DeploymentEnv) === 'canary' ? every(5, 'minutes') : every(20, 'seconds'), + alarmEvaluationPeriod: (env.DEPLOYMENT_ENV as DeploymentEnv) === 'canary' ? 3 : 1, + alarmDatapointsToAlarm: (env.DEPLOYMENT_ENV as DeploymentEnv) === 'canary' ? 3 : 1, }, { name: 'user-sync-staked-balances', - interval: (env.DEPLOYMENT_ENV as DeploymentEnv) === 'canary' ? every(31, 'minutes') : every(11, 'minutes'), + interval: (env.DEPLOYMENT_ENV as DeploymentEnv) === 'canary' ? every(5, 'minutes') : every(20, 'seconds'), + alarmEvaluationPeriod: (env.DEPLOYMENT_ENV as DeploymentEnv) === 'canary' ? 3 : 1, + alarmDatapointsToAlarm: (env.DEPLOYMENT_ENV as DeploymentEnv) === 'canary' ? 3 : 1, }, { name: 'sync-coingecko-coinids', interval: every(2, 'hours'), }, - { - name: 'purge-old-tokenprices', - interval: every(1, 'days'), - alarmEvaluationPeriod: 1, - alarmDatapointsToAlarm: 1, - }, { name: 'update-fee-volume-yield-all-pools', - interval: every(75, 'minutes'), + interval: every(1, 'hours'), }, { name: 'sync-vebal-balances', - interval: (env.DEPLOYMENT_ENV as DeploymentEnv) === 'canary' ? every(20, 'minutes') : every(14, 'minutes'), + interval: (env.DEPLOYMENT_ENV as DeploymentEnv) === 'canary' ? every(9, 'minutes') : every(3, 'minutes'), }, { name: 'sync-vebal-totalSupply', - interval: (env.DEPLOYMENT_ENV as DeploymentEnv) === 'canary' ? every(20, 'minutes') : every(16, 'minutes'), + interval: (env.DEPLOYMENT_ENV as DeploymentEnv) === 'canary' ? every(10, 'minutes') : every(5, 'minutes'), }, { name: 'feed-data-to-datastudio', diff --git a/modules/network/zkevm.ts b/modules/network/zkevm.ts index 09a329daf..b540565dd 100644 --- a/modules/network/zkevm.ts +++ b/modules/network/zkevm.ts @@ -52,9 +52,6 @@ const zkevmNetworkData: NetworkData = { platformId: 'polygon-zkevm', excludedTokenAddresses: [], }, - tokenPrices: { - maxHourlyPriceHistoryNumDays: 100, - }, rpcUrl: env.ALCHEMY_API_KEY && (env.DEPLOYMENT_ENV as DeploymentEnv) === 'main' ? `https://polygonzkevm-mainnet.g.alchemy.com/v2/${env.ALCHEMY_API_KEY}` @@ -264,12 +261,6 @@ export const zkevmNetworkConfig: NetworkConfig = { name: 'sync-coingecko-coinids', interval: every(2, 'hours'), }, - { - name: 'purge-old-tokenprices', - interval: every(1, 'days'), - alarmEvaluationPeriod: 1, - alarmDatapointsToAlarm: 1, - }, { name: 'update-fee-volume-yield-all-pools', interval: every(75, 'minutes'), diff --git a/modules/pool/lib/pool-snapshot.service.ts b/modules/pool/lib/pool-snapshot.service.ts index 8a7e0866d..9ccf126e6 100644 --- a/modules/pool/lib/pool-snapshot.service.ts +++ b/modules/pool/lib/pool-snapshot.service.ts @@ -17,9 +17,7 @@ import { networkContext } from '../../network/network-context.service'; import { TokenHistoricalPrices } from '../../coingecko/coingecko-types'; export class PoolSnapshotService { - constructor( - private readonly coingeckoService: CoingeckoService, - ) {} + constructor(private readonly coingeckoService: CoingeckoService) {} private get balancerSubgraphService() { return networkContext.config.services.balancerSubgraphService; @@ -44,24 +42,6 @@ export class PoolSnapshotService { }); } - public async getSnapshotsForAllPools(chains: Chain[], range: GqlPoolSnapshotDataRange) { - const timestamp = this.getTimestampForRange(range); - - return prisma.prismaPoolSnapshot.findMany({ - where: { - timestamp: { gte: timestamp }, - totalSharesNum: { - gt: 0.000000000001, - }, - pool: { - categories: { none: { category: 'BLACK_LISTED' } }, - }, - chain: { in: chains }, - }, - orderBy: { timestamp: 'asc' }, - }); - } - /* Per default, this method syncs the snapshot from today and from yesterday (daysTosync=2). It is important to also sync the snapshot from yesterday in the cron-job to capture all the changes between when it last ran and midnight. diff --git a/modules/pool/lib/pool-usd-data.service.ts b/modules/pool/lib/pool-usd-data.service.ts index 7125c5d6b..bc4f81952 100644 --- a/modules/pool/lib/pool-usd-data.service.ts +++ b/modules/pool/lib/pool-usd-data.service.ts @@ -279,67 +279,104 @@ export class PoolUsdDataService { where: { chain: this.chain, balanceNum: { gt: 0 } }, }); + const subgraphPoolIds = subgraphPools.map((pool) => pool.id); + + const dbPools = await prisma.prismaPool.findMany({ + where: { id: { in: subgraphPoolIds }, chain: this.chain }, + include: { + dynamicData: true, + snapshots: true, + }, + }); + for (const pool of subgraphPools) { const staked = stakedUsers.find((stakedUser) => stakedUser.poolId === pool.id); + const dbPool = dbPools.find((poolInDb) => poolInDb.id === pool.id); + if (!dbPool) continue; // Limit numbers, as we have seen some pools with skewd values const max = 1e18; const lifetimeVolume = Math.min(parseFloat(pool.totalSwapVolume), max); const lifetimeSwapFees = Math.min(parseFloat(pool.totalSwapFee), max); + const holdersCount = parseInt(pool.holdersCount) + (staked?._count.userAddress || 0); - updates.push( - prisma.prismaPoolDynamicData.update({ - where: { id_chain: { id: pool.id, chain: this.chain } }, - data: { - lifetimeVolume: lifetimeVolume, - lifetimeSwapFees: lifetimeSwapFees, - holdersCount: parseInt(pool.holdersCount) + (staked?._count.userAddress || 0), - swapsCount: parseInt(pool.swapsCount), - }, - }), - ); - - const snapshots = await prisma.prismaPoolSnapshot.findMany({ - where: { poolId: pool.id, chain: this.chain }, - }); - - if (snapshots.length > 0) { - const sharePriceAth = _.orderBy(snapshots, 'sharePrice', 'desc')[0]; - const sharePriceAtl = _.orderBy(snapshots, 'sharePrice', 'asc')[0]; - const totalLiquidityAth = _.orderBy(snapshots, 'totalLiquidity', 'desc')[0]; - const totalLiquidityAtl = _.orderBy(snapshots, 'totalLiquidity', 'asc')[0]; - const volume24hAth = _.orderBy(snapshots, 'volume24h', 'desc')[0]; - const volume24hAtl = _.orderBy(snapshots, 'volume24h', 'asc')[0]; - const fees24hAth = _.orderBy(snapshots, 'fees24h', 'desc')[0]; - const fees24hAtl = _.orderBy(snapshots, 'fees24h', 'asc')[0]; - + if ( + !dbPool.dynamicData || + dbPool.dynamicData.lifetimeVolume !== lifetimeVolume || + dbPool.dynamicData.lifetimeSwapFees !== lifetimeSwapFees || + dbPool.dynamicData.holdersCount !== holdersCount || + dbPool.dynamicData.swapsCount !== parseInt(pool.swapsCount) + ) { updates.push( prisma.prismaPoolDynamicData.update({ where: { id_chain: { id: pool.id, chain: this.chain } }, data: { - sharePriceAth: sharePriceAth.sharePrice, - sharePriceAthTimestamp: sharePriceAth.timestamp, - sharePriceAtl: sharePriceAtl.sharePrice, - sharePriceAtlTimestamp: sharePriceAtl.timestamp, - - totalLiquidityAth: totalLiquidityAth.totalLiquidity, - totalLiquidityAthTimestamp: totalLiquidityAth.timestamp, - totalLiquidityAtl: totalLiquidityAtl.totalLiquidity, - totalLiquidityAtlTimestamp: totalLiquidityAtl.timestamp, - - volume24hAth: volume24hAth.volume24h, - volume24hAthTimestamp: volume24hAth.timestamp, - volume24hAtl: volume24hAtl.volume24h, - volume24hAtlTimestamp: volume24hAtl.timestamp, - - fees24hAth: fees24hAth.fees24h, - fees24hAthTimestamp: fees24hAth.timestamp, - fees24hAtl: fees24hAtl.fees24h, - fees24hAtlTimestamp: fees24hAtl.timestamp, + lifetimeVolume: lifetimeVolume, + lifetimeSwapFees: lifetimeSwapFees, + holdersCount: holdersCount, + swapsCount: parseInt(pool.swapsCount), }, }), ); } + + if (dbPool.snapshots.length > 0) { + const sharePriceAth = _.orderBy(dbPool.snapshots, 'sharePrice', 'desc')[0]; + const sharePriceAtl = _.orderBy(dbPool.snapshots, 'sharePrice', 'asc')[0]; + const totalLiquidityAth = _.orderBy(dbPool.snapshots, 'totalLiquidity', 'desc')[0]; + const totalLiquidityAtl = _.orderBy(dbPool.snapshots, 'totalLiquidity', 'asc')[0]; + const volume24hAth = _.orderBy(dbPool.snapshots, 'volume24h', 'desc')[0]; + const volume24hAtl = _.orderBy(dbPool.snapshots, 'volume24h', 'asc')[0]; + const fees24hAth = _.orderBy(dbPool.snapshots, 'fees24h', 'desc')[0]; + const fees24hAtl = _.orderBy(dbPool.snapshots, 'fees24h', 'asc')[0]; + + if ( + !dbPool.dynamicData || + dbPool.dynamicData.sharePriceAth !== sharePriceAth.sharePrice || + dbPool.dynamicData.sharePriceAthTimestamp !== sharePriceAth.timestamp || + dbPool.dynamicData.sharePriceAtl !== sharePriceAtl.sharePrice || + dbPool.dynamicData.sharePriceAtlTimestamp !== sharePriceAtl.timestamp || + dbPool.dynamicData.totalLiquidityAth !== totalLiquidityAth.totalLiquidity || + dbPool.dynamicData.totalLiquidityAthTimestamp !== totalLiquidityAth.timestamp || + dbPool.dynamicData.totalLiquidityAtl !== totalLiquidityAtl.totalLiquidity || + dbPool.dynamicData.totalLiquidityAtlTimestamp !== totalLiquidityAtl.timestamp || + dbPool.dynamicData.volume24hAth !== volume24hAth.volume24h || + dbPool.dynamicData.volume24hAthTimestamp !== volume24hAth.timestamp || + dbPool.dynamicData.volume24hAtl !== volume24hAtl.volume24h || + dbPool.dynamicData.volume24hAtlTimestamp !== volume24hAtl.timestamp || + dbPool.dynamicData.fees24hAth !== fees24hAth.fees24h || + dbPool.dynamicData.fees24hAthTimestamp !== fees24hAth.timestamp || + dbPool.dynamicData.fees24hAtl !== fees24hAtl.fees24h || + dbPool.dynamicData.fees24hAtlTimestamp !== fees24hAtl.timestamp + ) { + updates.push( + prisma.prismaPoolDynamicData.update({ + where: { id_chain: { id: pool.id, chain: this.chain } }, + data: { + sharePriceAth: sharePriceAth.sharePrice, + sharePriceAthTimestamp: sharePriceAth.timestamp, + sharePriceAtl: sharePriceAtl.sharePrice, + sharePriceAtlTimestamp: sharePriceAtl.timestamp, + + totalLiquidityAth: totalLiquidityAth.totalLiquidity, + totalLiquidityAthTimestamp: totalLiquidityAth.timestamp, + totalLiquidityAtl: totalLiquidityAtl.totalLiquidity, + totalLiquidityAtlTimestamp: totalLiquidityAtl.timestamp, + + volume24hAth: volume24hAth.volume24h, + volume24hAthTimestamp: volume24hAth.timestamp, + volume24hAtl: volume24hAtl.volume24h, + volume24hAtlTimestamp: volume24hAtl.timestamp, + + fees24hAth: fees24hAth.fees24h, + fees24hAthTimestamp: fees24hAth.timestamp, + fees24hAtl: fees24hAtl.fees24h, + fees24hAtlTimestamp: fees24hAtl.timestamp, + }, + }), + ); + } + } } await prismaBulkExecuteOperations(updates); diff --git a/modules/pool/pool.gql b/modules/pool/pool.gql index 2157ec009..f37cbd14a 100644 --- a/modules/pool/pool.gql +++ b/modules/pool/pool.gql @@ -21,7 +21,6 @@ extend type Query { poolGetJoinExits(first: Int, skip: Int, where: GqlPoolJoinExitFilter): [GqlPoolJoinExit!]! poolGetFeaturedPoolGroups(chains: [GqlChain!]): [GqlPoolFeaturedPoolGroup!]! poolGetSnapshots(id: String!, chain: GqlChain, range: GqlPoolSnapshotDataRange!): [GqlPoolSnapshot!]! - poolGetAllPoolsSnapshots(chains: [GqlChain!], range: GqlPoolSnapshotDataRange!): [GqlPoolSnapshot!]! poolGetLinearPools(chains: [GqlChain!]): [GqlPoolLinear!]! poolGetGyroPools(chains: [GqlChain!]): [GqlPoolGyro!]! } diff --git a/modules/pool/pool.resolvers.ts b/modules/pool/pool.resolvers.ts index 750cbb4ef..18a13603a 100644 --- a/modules/pool/pool.resolvers.ts +++ b/modules/pool/pool.resolvers.ts @@ -79,27 +79,6 @@ const balancerResolvers: Resolvers = { holdersCount: `${snapshot.holdersCount}`, })); }, - poolGetAllPoolsSnapshots: async (parent, { chains, range }, context) => { - const currentChain = headerChain(); - if (!chains && currentChain) { - chains = [currentChain]; - } else if (!chains) { - throw new Error('poolGetAllPoolsSnapshots error: Provide "chains" param'); - } - const snapshots = await poolService.getSnapshotsForAllPools(chains, range); - - return snapshots.map((snapshot) => ({ - ...snapshot, - totalLiquidity: `${snapshot.totalLiquidity}`, - sharePrice: `${snapshot.sharePrice}`, - volume24h: `${snapshot.volume24h}`, - fees24h: `${snapshot.fees24h}`, - totalSwapVolume: `${snapshot.totalSwapVolume}`, - totalSwapFee: `${snapshot.totalSwapFee}`, - swapsCount: `${snapshot.swapsCount}`, - holdersCount: `${snapshot.holdersCount}`, - })); - }, poolGetLinearPools: async (parent, { chains }, context) => { const currentChain = headerChain(); if (!chains && currentChain) { diff --git a/modules/pool/pool.service.ts b/modules/pool/pool.service.ts index e5f288e4b..70ea01962 100644 --- a/modules/pool/pool.service.ts +++ b/modules/pool/pool.service.ts @@ -120,10 +120,6 @@ export class PoolService { return this.poolGqlLoaderService.getFeaturedPoolGroups(chains); } - public async getSnapshotsForAllPools(chains: Chain[], range: GqlPoolSnapshotDataRange) { - return this.poolSnapshotService.getSnapshotsForAllPools(chains, range); - } - public async getSnapshotsForPool(poolId: string, chain: Chain, range: GqlPoolSnapshotDataRange) { return this.poolSnapshotService.getSnapshotsForPool(poolId, chain, range); } diff --git a/modules/token/lib/token-price.service.ts b/modules/token/lib/token-price.service.ts index 1d5b77fa9..792132e72 100644 --- a/modules/token/lib/token-price.service.ts +++ b/modules/token/lib/token-price.service.ts @@ -255,21 +255,16 @@ export class TokenPriceService { } } - public async purgeOldTokenPrices(): Promise { - const purgeBeforeTimestamp = moment() - .startOf('day') - .subtract(networkContext.data.tokenPrices.maxHourlyPriceHistoryNumDays, 'days') - .utc() - .unix(); + public async purgeOldTokenPricesForAllChains(): Promise { + const purgeBeforeTimestamp = moment().startOf('day').subtract(180, 'days').utc().unix(); const oldPrices = await prisma.prismaTokenPrice.findMany({ where: { - chain: networkContext.chain, timestamp: { lt: purgeBeforeTimestamp }, }, }); // returns all non midnight prices - const tobeDeleted = oldPrices.filter((tokenPrice) => tokenPrice.timestamp % secondsPerDay !== 0); + const tobeDeleted = _.uniq(oldPrices.filter((tokenPrice) => tokenPrice.timestamp % secondsPerDay !== 0)); //apparently prisma has a limitation on delete const chunks = _.chunk(tobeDeleted, 1000); @@ -277,7 +272,6 @@ export class TokenPriceService { for (const chunk of chunks) { await prisma.prismaTokenPrice.deleteMany({ where: { - chain: networkContext.chain, timestamp: { in: chunk.map((tokenPrice) => tokenPrice.timestamp) }, }, }); diff --git a/modules/token/token.service.ts b/modules/token/token.service.ts index 21aff4d98..8b2258e30 100644 --- a/modules/token/token.service.ts +++ b/modules/token/token.service.ts @@ -216,8 +216,8 @@ export class TokenService { return this.tokenPriceService.getHistoricalTokenPrices(); } - public async purgeOldTokenPrices(): Promise { - return this.tokenPriceService.purgeOldTokenPrices(); + public async purgeOldTokenPricesForAllChains() { + return this.tokenPriceService.purgeOldTokenPricesForAllChains(); } public async deleteTokenPrice(args: MutationTokenDeletePriceArgs) { diff --git a/worker/job-handlers.ts b/worker/job-handlers.ts index 8329d69cd..aa91a476a 100644 --- a/worker/job-handlers.ts +++ b/worker/job-handlers.ts @@ -251,8 +251,14 @@ export function configureWorkerRoutes(app: Express) { case 'sync-latest-relic-snapshots': await runIfNotAlreadyRunning(job.name, chainId, () => userService.syncUserRelicSnapshots(), res, next); break; - case 'purge-old-tokenprices': - await runIfNotAlreadyRunning(job.name, chainId, () => tokenService.purgeOldTokenPrices(), res, next); + case 'global-purge-old-tokenprices': + await runIfNotAlreadyRunning( + job.name, + chainId, + () => tokenService.purgeOldTokenPricesForAllChains(), + res, + next, + ); break; case 'sync-coingecko-coinids': await runIfNotAlreadyRunning(job.name, chainId, () => tokenService.syncCoingeckoIds(), res, next); diff --git a/worker/worker.ts b/worker/worker.ts index c5da70337..375a8095a 100644 --- a/worker/worker.ts +++ b/worker/worker.ts @@ -3,6 +3,7 @@ import * as Sentry from '@sentry/node'; import { ProfilingIntegration } from '@sentry/profiling-node'; import { env } from '../app/env'; import { configureWorkerRoutes } from './job-handlers'; +import { prisma } from '../prisma/prisma-client'; export async function startWorker() { const app = express(); @@ -12,11 +13,14 @@ export async function startWorker() { environment: `multichain-worker-${env.DEPLOYMENT_ENV}`, enabled: env.NODE_ENV === 'production', integrations: [ - // new Tracing.Integrations.Express({ app }), + new Sentry.Integrations.Apollo(), + new Sentry.Integrations.GraphQL(), + new Sentry.Integrations.Prisma({ client: prisma }), + new Sentry.Integrations.Express({ app }), new Sentry.Integrations.Http({ tracing: true }), new ProfilingIntegration(), ], - tracesSampleRate: 0.2, + tracesSampleRate: 0.005, profilesSampleRate: 0.1, });