common: add SubgraphFreshnesschecker to Epoch and Network Subgraphs

packages/indexer-common/src/epoch-subgraph.ts

@@ -3,12 +3,17 @@ import { DocumentNode, print } from 'graphql'
 import { CombinedError } from '@urql/core'
 import { QueryResult } from './network-subgraph'
 import { Logger } from '@graphprotocol/common-ts'
-
+import { SubgraphFreshnessChecker } from './subgraphs'
 export class EpochSubgraph {
   endpointClient: AxiosInstance
+  freshnessChecker: SubgraphFreshnessChecker
   logger: Logger
 
-  constructor(endpoint: string, logger: Logger) {
+  constructor(
+    endpoint: string,
+    freshnessChecker: SubgraphFreshnessChecker,
+    logger: Logger,
+  ) {
     this.endpointClient = axios.create({
       baseURL: endpoint,
       headers: { 'content-type': 'application/json' },
@@ -19,9 +24,19 @@ export class EpochSubgraph {
       // Don't transform responses
       transformResponse: (data) => data,
     })
+    this.freshnessChecker = freshnessChecker
     this.logger = logger
   }
 
+  // eslint-disable-next-line @typescript-eslint/no-explicit-any
+  async checkedQuery<Data = any>(
+    query: DocumentNode,
+    // eslint-disable-next-line @typescript-eslint/no-explicit-any
+    variables?: Record<string, any>,
+  ): Promise<QueryResult<Data>> {
+    return this.freshnessChecker.checkedQuery(this, query, variables)
+  }
+
   // eslint-disable-next-line @typescript-eslint/no-explicit-any
   async query<Data = any>(
     query: DocumentNode,

packages/indexer-common/src/indexer-management/__tests__/helpers.test.ts

@@ -16,7 +16,7 @@ import {
 import { defineQueryFeeModels, specification as spec } from '../../index'
 import { networkIsL1, networkIsL2 } from '../types'
 import { fetchIndexingRules, upsertIndexingRule } from '../rules'
-import { SubgraphIdentifierType } from '../../subgraphs'
+import { SubgraphFreshnessChecker, SubgraphIdentifierType } from '../../subgraphs'
 import { ActionManager } from '../actions'
 import { actionFilterToWhereOptions, ActionStatus, ActionType } from '../../actions'
 import { literal, Op, Sequelize } from 'sequelize'
@@ -34,6 +34,7 @@ import {
   SubgraphDeployment,
   getTestProvider,
 } from '@graphprotocol/indexer-common'
+import { mockLogger, mockProvider } from '../../__tests__/subgraph.test'
 import { BigNumber, ethers, utils } from 'ethers'
 
 // Make global Jest variable available
@@ -72,14 +73,27 @@ const setupMonitor = async () => {
   })
   ethereum = getTestProvider('goerli')
   contracts = await connectContracts(ethereum, 5)
+
+  const subgraphFreshnessChecker = new SubgraphFreshnessChecker(
+    'Test Subgraph',
+    mockProvider,
+    10,
+    10,
+    mockLogger,
+    1,
+  )
+
   networkSubgraph = await NetworkSubgraph.create({
     logger,
     endpoint:
       'https://api.thegraph.com/subgraphs/name/graphprotocol/graph-network-goerli',
     deployment: undefined,
+    subgraphFreshnessChecker,
   })
+
   epochSubgraph = new EpochSubgraph(
     'https://api.thegraph.com/subgraphs/name/graphprotocol/goerli-epoch-block-oracle',
+    subgraphFreshnessChecker,
     logger,
   )
   graphNode = new GraphNode(

packages/indexer-common/src/network-subgraph.ts

@@ -4,6 +4,7 @@ import { DocumentNode, print } from 'graphql'
 import { OperationResult, CombinedError } from '@urql/core'
 import { BlockPointer, IndexingError } from './types'
 import { GraphNode } from './graph-node'
+import { SubgraphFreshnessChecker } from './subgraphs'
 
 export interface NetworkSubgraphCreateOptions {
   logger: Logger
@@ -12,6 +13,7 @@ export interface NetworkSubgraphCreateOptions {
     graphNode: GraphNode
     deployment: SubgraphDeploymentID
   }
+  subgraphFreshnessChecker: SubgraphFreshnessChecker
 }
 
 interface DeploymentStatus {
@@ -30,6 +32,7 @@ interface NetworkSubgraphOptions {
     status: Eventual<DeploymentStatus>
     graphNode: GraphNode
   }
+  subgraphFreshnessChecker: SubgraphFreshnessChecker
 }
 
 export type QueryResult<Data> = Pick<
@@ -39,7 +42,7 @@ export type QueryResult<Data> = Pick<
 
 export class NetworkSubgraph {
   logger: Logger
-
+  freshnessChecker: SubgraphFreshnessChecker
   endpointClient?: AxiosInstance
 
   public readonly deployment?: {
@@ -50,6 +53,7 @@ export class NetworkSubgraph {
 
   private constructor(options: NetworkSubgraphOptions) {
     this.logger = options.logger
+    this.freshnessChecker = options.subgraphFreshnessChecker
 
     if (options.endpoint) {
       this.endpointClient = axios.create({
@@ -83,6 +87,7 @@ export class NetworkSubgraph {
     logger: parentLogger,
     endpoint,
     deployment,
+    subgraphFreshnessChecker,
   }: NetworkSubgraphCreateOptions): Promise<NetworkSubgraph> {
     // Either an endpoint or a deployment needs to be provided; the CLI
     // validation should already guarantee that but we're asserting this again
@@ -122,6 +127,7 @@ export class NetworkSubgraph {
       logger,
       endpoint,
       deployment: deploymentInfo,
+      subgraphFreshnessChecker,
     })
 
     // If we don't have a network subgraph endpoint configured, we
@@ -160,6 +166,15 @@ export class NetworkSubgraph {
     }
   }
 
+  // eslint-disable-next-line @typescript-eslint/no-explicit-any
+  async checkedQuery<Data = any>(
+    query: DocumentNode,
+    // eslint-disable-next-line @typescript-eslint/no-explicit-any
+    variables?: Record<string, any>,
+  ): Promise<QueryResult<Data>> {
+    return this.freshnessChecker.checkedQuery(this, query, variables)
+  }
+
   // eslint-disable-next-line @typescript-eslint/no-explicit-any
   async query<Data = any>(
     query: DocumentNode,

packages/indexer-common/src/network.ts

@@ -18,6 +18,7 @@ import {
   EpochSubgraph,
   NetworkMonitor,
   AllocationReceiptCollector,
+  SubgraphFreshnessChecker,
 } from '.'
 import { BigNumber, providers, Wallet } from 'ethers'
 import { strict as assert } from 'assert'
@@ -81,9 +82,29 @@ export class Network {
       protocolNetwork: specification.networkIdentifier,
     })
 
+    // * -----------------------------------------------------------------------
+    // * Network Provider
+    // * -----------------------------------------------------------------------
+    const networkProvider = await Network.provider(
+      logger,
+      metrics,
+      specification.networkIdentifier,
+      specification.networkProvider.url,
+      specification.networkProvider.pollingInterval,
+    )
+
     // * -----------------------------------------------------------------------
     // * Network Subgraph
     // * -----------------------------------------------------------------------
+    const networkSubgraphFreshnessChecker = new SubgraphFreshnessChecker(
+      'Network Subgraph',
+      networkProvider,
+      specification.subgraphs.maxBlockDistance,
+      specification.subgraphs.freshnessSleepMilliseconds,
+      logger.child({ component: 'FreshnessChecker' }),
+      Infinity,
+    )
+
     const networkSubgraphDeploymentId = specification.subgraphs.networkSubgraph.deployment
       ? new SubgraphDeploymentID(specification.subgraphs.networkSubgraph.deployment)
       : undefined
@@ -98,19 +119,9 @@ export class Network {
               deployment: networkSubgraphDeploymentId,
             }
           : undefined,
+      subgraphFreshnessChecker: networkSubgraphFreshnessChecker,
     })
 
-    // * -----------------------------------------------------------------------
-    // * Network Provider
-    // * -----------------------------------------------------------------------
-    const networkProvider = await Network.provider(
-      logger,
-      metrics,
-      specification.networkIdentifier,
-      specification.networkProvider.url,
-      specification.networkProvider.pollingInterval,
-    )
-
     // * -----------------------------------------------------------------------
     // * Contracts
     // * -----------------------------------------------------------------------
@@ -138,13 +149,23 @@ export class Network {
     // * -----------------------------------------------------------------------
     // * Epoch Subgraph
     // * -----------------------------------------------------------------------
+    const epochSubgraphFreshnessChecker = new SubgraphFreshnessChecker(
+      'Epoch Subgraph',
+      networkProvider,
+      specification.subgraphs.maxBlockDistance,
+      specification.subgraphs.freshnessSleepMilliseconds,
+      logger.child({ component: 'FreshnessChecker' }),
+      Infinity,
+    )
+
     const epochSubgraph = new EpochSubgraph(
       /* eslint-disable-next-line @typescript-eslint/no-non-null-assertion --
        * Accept the non-null `url` property of the Epoch Subgraph, as it has
        * already been validated during parsing. Once indexing is supported,
        * initialize it in the same way as the NetworkSubgraph
        */
       specification.subgraphs.epochSubgraph.url!,
+      epochSubgraphFreshnessChecker,
       logger.child({ component: 'EpochSubgraph' }),
     )