Try-catch update functions in cache-update loops. (#1042)

indexer/packages/postgres/src/loops/asset-refresher.ts

@@ -1,21 +1,25 @@
 import {
-  stats, delay, logger, NodeEnv,
+  stats,
+  logger,
+  NodeEnv,
 } from '@dydxprotocol-indexer/base';
 
 import config from '../config';
 import * as AssetTable from '../stores/asset-table';
 import { AssetFromDatabase, AssetsMap, Options } from '../types';
+import { startUpdateLoop } from './loopHelper';
 
 let idToAsset: AssetsMap = {};
 
 /**
  * Refresh loop to cache the list of all assets from the database in-memory.
  */
 export async function start(): Promise<void> {
-  for (;;) {
-    await updateAssets();
-    await delay(config.ASSET_REFRESHER_INTERVAL_MS);
-  }
+  await startUpdateLoop(
+    updateAssets,
+    config.ASSET_REFRESHER_INTERVAL_MS,
+    'updateAssets',
+  );
 }
 
 /**

indexer/packages/postgres/src/loops/liquidity-tier-refresher.ts

@@ -1,21 +1,25 @@
 import {
-  NodeEnv, delay, logger, stats,
+  NodeEnv,
+  logger,
+  stats,
 } from '@dydxprotocol-indexer/base';
 
 import config from '../config';
 import * as LiquidityTiersTable from '../stores/liquidity-tiers-table';
 import { LiquidityTiersFromDatabase, LiquidityTiersMap, Options } from '../types';
+import { startUpdateLoop } from './loopHelper';
 
 let idToLiquidityTier: LiquidityTiersMap = {};
 
 /**
  * Refresh loop to cache the list of all liquidity tiers from the database in-memory.
  */
 export async function start(): Promise<void> {
-  for (; ;) {
-    await updateLiquidityTiers();
-    await delay(config.LIQUIDITY_TIER_REFRESHER_INTERVAL_MS);
-  }
+  await startUpdateLoop(
+    updateLiquidityTiers,
+    config.LIQUIDITY_TIER_REFRESHER_INTERVAL_MS,
+    'updateLiquidityTiers',
+  );
 }
 
 /**

indexer/packages/postgres/src/loops/loopHelper.ts

@@ -0,0 +1,20 @@
+import { delay, logger } from '@dydxprotocol-indexer/base';
+
+export async function startUpdateLoop(
+  updateFunction: () => Promise<void>,
+  delayMs: number,
+  name: string,
+): Promise<void> {
+  for (;;) {
+    try {
+      await updateFunction();
+    } catch (error) {
+      logger.error({
+        at: name,
+        message: 'Failed to run update',
+        error,
+      });
+    }
+    await delay(delayMs);
+  }
+}

indexer/packages/postgres/src/loops/perpetual-market-refresher.ts

@@ -1,11 +1,15 @@
-import { stats, delay, NodeEnv } from '@dydxprotocol-indexer/base';
+import {
+  stats,
+  NodeEnv,
+} from '@dydxprotocol-indexer/base';
 import _ from 'lodash';
 
 import config from '../config';
 import * as PerpetualMarketTable from '../stores/perpetual-market-table';
 import {
   Options, PerpetualMarketColumns, PerpetualMarketFromDatabase, PerpetualMarketsMap,
 } from '../types';
+import { startUpdateLoop } from './loopHelper';
 
 let idToPerpetualMarket: Record<string, PerpetualMarketFromDatabase> = {};
 
@@ -16,10 +20,11 @@ let idToPerpetualMarket: Record<string, PerpetualMarketFromDatabase> = {};
  * Refresh loop to cache the list of all perpetual markets from the database in-memory.
  */
 export async function start(): Promise<void> {
-  for (;;) {
-    await updatePerpetualMarkets();
-    await delay(config.PERPETUAL_MARKETS_REFRESHER_INTERVAL_MS);
-  }
+  await startUpdateLoop(
+    updatePerpetualMarkets,
+    config.PERPETUAL_MARKETS_REFRESHER_INTERVAL_MS,
+    'updatePerpetualMarkets',
+  );
 }
 
 /**