Agent: warn and wait if subgraph is behind the chain head

packages/indexer-agent/src/commands/start.ts

@@ -37,6 +37,9 @@ import { displayZodParsingError } from './error-handling'
 // eslint-disable-next-line  @typescript-eslint/no-explicit-any
 export type AgentOptions = { [key: string]: any } & Argv['argv']
 
+const DEFAULT_SUBGRAPH_MAX_BLOCK_DISTANCE = 0
+const DEFAULT_SUBGRAPH_FRESHNESS_SLEEP_MILLISECONDS = 5_000
+
 export const start = {
   command: 'start',
   describe: 'Start the agent',
@@ -157,6 +160,20 @@ export const start = {
         required: true,
         group: 'Protocol',
       })
+      .option('subgraph-max-block-distance', {
+        description:
+          'How many blocks subgraphs are allowed to stay behind chain head',
+        type: 'number',
+        default: DEFAULT_SUBGRAPH_MAX_BLOCK_DISTANCE,
+        group: 'Protocol',
+      })
+      .option('subgraph-freshness-sleep-milliseconds', {
+        description:
+          'How long to wait before retrying subgraph query if it is not fresh',
+        type: 'number',
+        default: DEFAULT_SUBGRAPH_FRESHNESS_SLEEP_MILLISECONDS,
+        group: 'Protocol',
+      })
       .option('default-allocation-amount', {
         description:
           'Default amount of GRT to allocate to a subgraph deployment',
@@ -298,6 +315,7 @@ export const start = {
 
 export async function createNetworkSpecification(
   argv: AgentOptions,
+  logger: Logger,
 ): Promise<spec.NetworkSpecification> {
   const gateway = {
     url: argv.gatewayEndpoint,
@@ -333,6 +351,8 @@ export async function createNetworkSpecification(
   }
 
   const subgraphs = {
+    maxBlockDistance: argv.subgraphMaxBlockDistance,
+    freshnessSleepMilliseconds: argv.subgraphFreshnessSleepMilliseconds,
     networkSubgraph: {
       deployment: argv.networkSubgraphDeployment,
       url: argv.networkSubgraphEndpoint,
@@ -359,6 +379,36 @@ export async function createNetworkSpecification(
   const chainId = await fetchChainId(networkProvider.url)
   const networkIdentifier = resolveChainId(chainId)
 
+  // Warn about inappropriate max block distance for subgraph threshold checks for given networks.
+  if (networkIdentifier.startsWith('eip155:42161')) {
+    // Arbitrum-One and Arbitrum-Goerli
+    if (subgraphs.maxBlockDistance <= DEFAULT_SUBGRAPH_MAX_BLOCK_DISTANCE) {
+      logger.warn(
+        `Consider increasing 'subgraph-max-block-distance' for Arbitrum networks`,
+        {
+          problem:
+            'A low subgraph freshness threshold might cause the Agent to discard too many subgraph queries in fast-paced networks.',
+          hint: `Increase the 'subgraph-max-block-distance' parameter to a value that accomodates for block and indexing speeds.`,
+          configuredValue: subgraphs.maxBlockDistance,
+        },
+      )
+    }
+    if (
+      subgraphs.freshnessSleepMilliseconds <=
+      DEFAULT_SUBGRAPH_FRESHNESS_SLEEP_MILLISECONDS
+    ) {
+      logger.warn(
+        `Consider increasing 'subgraph-freshness-sleep-milliseconds' for Arbitrum networks`,
+        {
+          problem:
+            'A short subgraph freshness wait time might be insufficient for the subgraph to sync with fast-paced networks.',
+          hint: `Increase the 'subgraph-freshness-sleep-milliseconds' parameter to a value that accomodates for block and indexing speeds.`,
+          configuredValue: subgraphs.freshnessSleepMilliseconds,
+        },
+      )
+    }
+  }
+
   try {
     return spec.NetworkSpecification.parse({
       networkIdentifier,

packages/indexer-agent/src/index.ts

@@ -60,7 +60,7 @@ async function processArgumentsAndRun(args: AgentOptions): Promise<void> {
     await run(args, specifications, logger)
   } else {
     reviewArgumentsForWarnings(args, logger)
-    const specification = await createNetworkSpecification(args)
+    const specification = await createNetworkSpecification(args, logger)
     await run(args, [specification], logger)
   }
 }

packages/indexer-agent/src/syncing-server.ts

@@ -82,7 +82,7 @@ export const createSyncingServer = async ({
 
       let result
       try {
-        result = await networkSubgraph.query(parsedQuery, variables)
+        result = await networkSubgraph.checkedQuery(parsedQuery, variables)
       } catch (err) {
         logger.error(err)
         return res.status(400).send({ error: err.message })

packages/indexer-common/package.json

@@ -25,6 +25,7 @@
     "@graphprotocol/common-ts": "2.0.3",
     "@graphprotocol/cost-model": "0.1.16",
     "@thi.ng/heaps": "1.2.38",
+    "@types/lodash.clonedeep": "^4.5.7",
     "@types/lodash.intersection": "^4.4.7",
     "@types/lodash.xor": "^4.5.7",
     "@urql/core": "2.4.4",
@@ -39,6 +40,7 @@
     "graphql": "16.3.0",
     "graphql-tag": "2.12.6",
     "jayson": "3.6.6",
+    "lodash.clonedeep": "^4.5.0",
     "lodash.groupby": "^4.6.0",
     "lodash.intersection": "^4.4.0",
     "lodash.isequal": "^4.5.0",

packages/indexer-common/src/__tests__/network-specification-files/invalid-negative-max-block-distance.yml

@@ -0,0 +1,35 @@
+networkIdentifier: mainnet
+gateway:
+  url: http://gateway
+indexerOptions:
+  address: "0x4e8a4C63Df58bf59Fef513aB67a76319a9faf448"
+  mnemonic: word ivory whale diesel slab pelican voyage oxygen chat find tobacco sport
+  url: http://indexer
+  geoCoordinates: [25.1, -71.2]
+  restakeRewards: true
+  rebateClaimThreshold: 400
+  rebateClaimBatchThreshold: 5000
+  rebateClaimMaxBatchSize: 10
+  poiDisputeMonitoring: false
+  poiDisputableEpochs: 5
+  defaultAllocationAmount: 0.05
+  voucherRedemptionThreshold: 2
+  voucherRedemptionBatchThreshold: 2000
+  voucherRedemptionMaxBatchSize: 15
+  allocationManagementMode: "auto"
+  autoAllocationMinBatchSize: 20
+transactionMonitoring:
+  gasIncreaseTimeout: 10
+  gasIncreaseFactor: 10
+  baseFeePerGasMax: 10
+  maxTransactionAttempts: 10
+subgraphs:
+  maxBlockDistance: -10
+  networkSubgraph:
+    deployment: QmPK1s3pNYLi9ERiq3BDxKa4XosgWwFRQUydHUtz4YgpqB
+  epochSubgraph:
+    url: http://subgraph
+networkProvider:
+  url: http://provider
+dai:
+  contractAddress: "0x4e8a4C63Df58bf59Fef513aB67a76319a9faf448"

packages/indexer-common/src/__tests__/network-specification.test.ts

@@ -70,6 +70,11 @@ describe('Failed deserialization', () => {
       path: ['dai', 'contractAddress'],
       message: 'Invalid contract address',
     },
+    {
+      file: 'invalid-negative-max-block-distance.yml',
+      path: ['subgraphs', 'maxBlockDistance'],
+      message: 'Number must be greater than or equal to 0',
+    },
   ]
 
   test.each(failedTests)(