pool tokens etc.

config/sepolia.ts

@@ -36,7 +36,9 @@ export default <NetworkData>{
         platformId: 'ethereum',
         excludedTokenAddresses: [],
     },
-    rpcUrl: env.INFURA_API_KEY
+    rpcUrl: env.GROVE_CITY ?
+        `https://sepolia.rpc.grove.city/v1/${env.GROVE_CITY}`
+        : env.INFURA_API_KEY
         ? `https://sepolia.infura.io/v3/${env.INFURA_API_KEY}`
         : 'https://gateway.tenderly.co/public/sepolia',
     rpcMaxBlockRange: 700,

modules/actions/jobs_actions/sync_pools.test.ts

@@ -1,9 +1,9 @@
 import { syncPools } from './sync_pools';
 import { PrismaPool } from '@prisma/client';
-import { poolsStore } from '@modules/stores/pools_store';
+import { poolsStore, tokensStore } from '@modules/stores';
 import { getClient, V3SubgraphClient } from '@modules/subgraphs/balancer-v3';
 import { PoolFragment } from '@modules/subgraphs/balancer-v3/generated/types';
-import { fetchPoolHeaders } from '@modules/sources/contracts';
+import { fetchErc20Headers } from '@modules/sources/contracts';
 import { getViemClient, ViemClient } from '@modules/sources/viem-client';
 
 jest.mock('@modules/stores/pools_store');
@@ -15,40 +15,42 @@ jest.mock('@modules/subgraphs/balancer-v3', () => ({
 }));
 jest.mock('@modules/sources/contracts', () => ({
     ...jest.requireActual('@modules/sources/contracts'),
-    fetchPoolHeaders: jest.fn(),
+    fetchErc20Headers: jest.fn()
 }));
 
 describe('syncPools', () => {
     let store: jest.Mocked<typeof poolsStore>;
+    let tStore: jest.Mocked<typeof tokensStore>;
     let subgraphClient: jest.Mocked<V3SubgraphClient>;
     let viemClient: jest.Mocked<ViemClient>;
 
     // Mock the dependencies
     beforeEach(() => {
         store = jest.mocked(poolsStore);
+        tStore = jest.mocked(tokensStore);
         subgraphClient = getClient('') as jest.Mocked<V3SubgraphClient>;
         viemClient = jest.mocked(getViemClient('SEPOLIA'));
 
         store.getPools.mockResolvedValue([{ id: '1' }] as PrismaPool[]);
         subgraphClient.Pools.mockResolvedValueOnce({ pools: [{ id: '1' }, { id: '2' }] as PoolFragment[] });
-        (fetchPoolHeaders as jest.Mock).mockResolvedValueOnce({ '2': { name: 'name', symbol: 'symbol' } });
+        (fetchErc20Headers as jest.Mock).mockResolvedValueOnce({ '2': { name: 'name', symbol: 'symbol' } });
     });
 
     it('should fetch pools from subgraph', async () => {
-        await syncPools(store, subgraphClient, viemClient, 'SEPOLIA');
+        await syncPools(store, tStore, subgraphClient, viemClient, 'SEPOLIA');
 
         expect(subgraphClient.Pools).toHaveBeenCalled();
     });
 
     it('should fetch additional data from contracts for missing pools', async () => {
-        await syncPools(store, subgraphClient, viemClient, 'SEPOLIA');
+        await syncPools(store, tStore,subgraphClient, viemClient, 'SEPOLIA');
 
-        expect(fetchPoolHeaders).toHaveBeenCalledWith(['2'], expect.anything());
+        expect(fetchErc20Headers).toHaveBeenCalledWith(['2'], expect.anything());
     });
 
     it('should store missing pools in the database', async () => {
-        await syncPools(store, subgraphClient, viemClient, 'SEPOLIA');
+        await syncPools(store, tStore, subgraphClient, viemClient, 'SEPOLIA');
 
-        expect(store.createPools).toHaveBeenCalledWith([expect.objectContaining({ id: '2' })]);
+        expect(store.createPools).toHaveBeenCalledWith([{ pool: expect.objectContaining({ id: '2' }) }]);
     });
 });

modules/actions/jobs_actions/sync_pools.ts

@@ -1,22 +1,25 @@
 import { Chain } from '@prisma/client';
-import { poolsStore, PoolsStore } from '@modules/stores/pools_store';
+import { PoolsStore, TokensStore } from '@modules/stores';
 import { V3SubgraphClient } from '@modules/subgraphs/balancer-v3';
 import { poolTransformer } from '@modules/sources/transformers/pool-transformer';
-import { fetchPoolHeaders } from '@modules/sources/contracts';
+import { fetchErc20Headers, fetchPoolTokens } from '@modules/sources/contracts';
 import type { ViemClient } from 'modules/sources/types'
 
 /**
  * Makes sure that all pools are in the database
  *
  * @param store
+ * @param tokensStore
  * @param subgraphClient
  * @param chain
  * @returns syncedPools - the pools that were synced
  */
 export async function syncPools(
-    store: PoolsStore = poolsStore,
+    store: PoolsStore,
+    tokensStore: TokensStore,
     subgraphClient: V3SubgraphClient,
     viemClient: ViemClient,
+    vaultAddress: string,
     chain = 'SEPOLIA' as Chain,
 ) {
     // Fetch pools from subgraph
@@ -29,9 +32,26 @@ export async function syncPools(
     const missingPools = subgraphPoolIds.filter((id) => !dbPoolIds.has(id));
 
     // Fetch additional data from contracts required in the DB for missing pools
-    const contractData = await fetchPoolHeaders(missingPools as `0x${string}`[], viemClient);
+    const contractData = await fetchErc20Headers(missingPools as `0x${string}`[], viemClient);
+    const poolTokens = await fetchPoolTokens(vaultAddress, missingPools, viemClient);
 
-    // Transform for the database
+    // Check if we need to get token information from the contracts as well
+    const tokenAddresses = Object.values(poolTokens).flatMap((poolTokens) => poolTokens.map((token) => token.address));
+    const dbTokens = await tokensStore.getTokens({ addresses: tokenAddresses, chains: [chain] });
+    const missingTokens = tokenAddresses.filter((address) => !dbTokens.some((token) => token.address === address));
+    const tokenData = await fetchErc20Headers(missingTokens as `0x${string}`[], viemClient)
+
+    // Store pool tokens and BPT in the tokens table
+    try {
+        const allTokens = Object.entries({ ...tokenData, ...contractData })
+          .map(([address, token]) => ({ ...token, address }));
+
+        await tokensStore.createTokens(allTokens.map((data) => ({ ...data, chain })));
+    } catch (e) {
+        console.error('Error creating tokens', e);
+    }
+
+    // Transform pool data for the database
     const transformedPools = missingPools
         .map((id) => {
             const subgraphPool = subgraphPools.find((pool) => pool.id === id);
@@ -40,10 +60,14 @@ export async function syncPools(
                 // That won't happen, but TS doesn't know that
                 return null;
             }
-            return poolTransformer(subgraphPool, data, chain);
+            return {
+                pool: poolTransformer(subgraphPool, data, chain),
+                poolTokens: poolTokens[id]
+            }
         })
         .filter((pool) => pool !== null) as Parameters<PoolsStore['createPools']>[0];
 
+    console.log(transformedPools)
     // Store missing pools in the database
     const syncedPools = await store.createPools(transformedPools);
 

modules/controllers/jobs_controller.ts

@@ -20,6 +20,7 @@ export function JobsController(tracer?: any) {
             const chain = chainIdToChain[chainId];
             const {
                 subgraphs: { balancerV3 },
+                balancer: { v3: { vaultAddress } },
             } = config[chain];
 
             // Guard against unconfigured chains
@@ -33,7 +34,7 @@ export function JobsController(tracer?: any) {
             // TODO: add syncing v2 pools as well by splitting the poolService into separate
             // actions with extracted configuration
 
-            return actions.syncPools(stores.poolsStore, subgraphClient, viemClient, chain);
+            return actions.syncPools(stores.poolsStore, stores.tokensStore, subgraphClient, viemClient, vaultAddress, chain);
         },
     };
 }

modules/sources/contracts/fetch_pool_headers.ts → modules/sources/contracts/fetch-erc20-headers.ts

@@ -1,10 +1,15 @@
-import { createPublicClient, parseAbi } from 'viem';
+import { parseAbi } from 'viem';
+import type { ViemClient } from '../types';
 
-const poolAbi = ['function name() view returns (string)', 'function symbol() view returns (string)'];
+const poolAbi = [
+    'function name() view returns (string)',
+    'function symbol() view returns (string)',
+    'function decimals() view returns (uint8)'
+];
 
 const abi = parseAbi(poolAbi);
 
-export async function fetchPoolHeaders(addresses: `0x${string}`[], client: ReturnType<typeof createPublicClient>) {
+export async function fetchErc20Headers(addresses: `0x${string}`[], client: ViemClient) {
     const contracts = addresses
         .map((address) => [
             {
@@ -17,6 +22,11 @@ export async function fetchPoolHeaders(addresses: `0x${string}`[], client: Retur
                 abi,
                 functionName: 'symbol',
             },
+            {
+                address,
+                abi,
+                functionName: 'decimals',
+            },
         ])
         .flat();
 
@@ -28,15 +38,16 @@ export async function fetchPoolHeaders(addresses: `0x${string}`[], client: Retur
             return result.result as string;
         }
         // Handle the error
-        return 'undefined';
+        return undefined;
     });
 
     return Object.fromEntries(
         addresses.map((address, i) => [
             address,
             {
-                name: parsedResults[i * 2],
-                symbol: parsedResults[i * 2 + 1],
+                name: String(parsedResults[i * 3]),
+                symbol: String(parsedResults[i * 3 + 1]),
+                decimals: Number(parsedResults[i * 3 + 2]),
             },
         ]),
     );

modules/sources/contracts/fetch-pool-tokens.ts

@@ -0,0 +1,62 @@
+import { PoolTokenCreate } from '@modules/stores/pools_store';
+import { parseAbi } from 'viem';
+import { ViemClient } from '../types';
+
+type PoolTokenInfo = {
+  tokens: `0x${string}`[],
+  tokenTypes: number[],
+  balancesRaw: bigint[],
+  decimalScalingFactors: bigint[],
+  rateProviders: `0x${string}`[],
+};
+
+const abi = parseAbi([
+    'function getPoolTokenInfo(address pool) view returns (address[] tokens, uint8[] tokenTypes, uint[] balancesRaw, uint[] decimalScalingFactors, address[] rateProviders)'
+]);
+
+const poolTokensTransformer = (
+    poolAddress: string,
+    poolTokenInfo: PoolTokenInfo,
+): PoolTokenCreate[] => poolTokenInfo.tokens.map((address, i) => ({
+    id: `${poolAddress}-${address}`,
+    address,
+    index: i,
+    priceRateProvider: poolTokenInfo.rateProviders[i],
+    exemptFromProtocolYieldFee: false, // where do we have this information onchain?
+    nestedPoolId: null, // TODO: might be easier to do that in SQL on read
+}));
+
+export async function fetchPoolTokens(vault: string, pools: string[], client: ViemClient) {
+    const contracts = pools
+        .map((pool) => [
+            {
+                address: vault as `0x${string}`,
+                abi,
+                args: [pool as `0x${string}`],
+                functionName: 'getPoolTokenInfo',
+            },
+        ])
+        .flat();
+
+    const results = await client.multicall({ contracts });
+
+    // Parse the results
+    const parsedResults = results.map((result, i) => {
+        if (result.status === 'success' && result.result !== undefined) {
+            // parse the result here using the abi
+            const poolTokens = poolTokensTransformer(pools[i], {
+                tokens: result.result[0],
+                tokenTypes: result.result[1],
+                balancesRaw: result.result[2],
+                decimalScalingFactors: result.result[3],
+                rateProviders: result.result[4],
+            } as PoolTokenInfo)
+
+            return [pools[i], poolTokens];
+        }
+        // Handle the error
+        return undefined;
+    }).filter((result): result is [string, PoolTokenCreate[]] => result !== undefined);
+
+    return Object.fromEntries(parsedResults);
+}

modules/sources/contracts/index.ts

@@ -1 +1,2 @@
-export * from './fetch_pool_headers';
+export * from './fetch-erc20-headers';
+export * from './fetch-pool-tokens';