From 087e93951d734088ac894fd86329fa546f29a569 Mon Sep 17 00:00:00 2001
From: vincentwschau <99756290+vincentwschau@users.noreply.github.com>
Date: Mon, 5 Feb 2024 15:08:21 -0500
Subject: [PATCH] Try-catch update functions in cache-update loops. (#1042)

---
 .../postgres/src/loops/asset-refresher.ts     | 14 ++++++++-----
 .../src/loops/liquidity-tier-refresher.ts     | 14 ++++++++-----
 .../packages/postgres/src/loops/loopHelper.ts | 20 +++++++++++++++++++
 .../src/loops/perpetual-market-refresher.ts   | 15 +++++++++-----
 4 files changed, 48 insertions(+), 15 deletions(-)
 create mode 100644 indexer/packages/postgres/src/loops/loopHelper.ts

diff --git a/indexer/packages/postgres/src/loops/asset-refresher.ts b/indexer/packages/postgres/src/loops/asset-refresher.ts
index 5ab433df09..70a283ee2a 100644
--- a/indexer/packages/postgres/src/loops/asset-refresher.ts
+++ b/indexer/packages/postgres/src/loops/asset-refresher.ts
@@ -1,10 +1,13 @@
 import {
-  stats, delay, logger, NodeEnv,
+  stats,
+  logger,
+  NodeEnv,
 } from '@dydxprotocol-indexer/base';
 
 import config from '../config';
 import * as AssetTable from '../stores/asset-table';
 import { AssetFromDatabase, AssetsMap, Options } from '../types';
+import { startUpdateLoop } from './loopHelper';
 
 let idToAsset: AssetsMap = {};
 
@@ -12,10 +15,11 @@ let idToAsset: AssetsMap = {};
  * Refresh loop to cache the list of all assets from the database in-memory.
  */
 export async function start(): Promise<void> {
-  for (;;) {
-    await updateAssets();
-    await delay(config.ASSET_REFRESHER_INTERVAL_MS);
-  }
+  await startUpdateLoop(
+    updateAssets,
+    config.ASSET_REFRESHER_INTERVAL_MS,
+    'updateAssets',
+  );
 }
 
 /**
diff --git a/indexer/packages/postgres/src/loops/liquidity-tier-refresher.ts b/indexer/packages/postgres/src/loops/liquidity-tier-refresher.ts
index 0722004db9..3c7c2b74b8 100644
--- a/indexer/packages/postgres/src/loops/liquidity-tier-refresher.ts
+++ b/indexer/packages/postgres/src/loops/liquidity-tier-refresher.ts
@@ -1,10 +1,13 @@
 import {
-  NodeEnv, delay, logger, stats,
+  NodeEnv,
+  logger,
+  stats,
 } from '@dydxprotocol-indexer/base';
 
 import config from '../config';
 import * as LiquidityTiersTable from '../stores/liquidity-tiers-table';
 import { LiquidityTiersFromDatabase, LiquidityTiersMap, Options } from '../types';
+import { startUpdateLoop } from './loopHelper';
 
 let idToLiquidityTier: LiquidityTiersMap = {};
 
@@ -12,10 +15,11 @@ let idToLiquidityTier: LiquidityTiersMap = {};
  * Refresh loop to cache the list of all liquidity tiers from the database in-memory.
  */
 export async function start(): Promise<void> {
-  for (; ;) {
-    await updateLiquidityTiers();
-    await delay(config.LIQUIDITY_TIER_REFRESHER_INTERVAL_MS);
-  }
+  await startUpdateLoop(
+    updateLiquidityTiers,
+    config.LIQUIDITY_TIER_REFRESHER_INTERVAL_MS,
+    'updateLiquidityTiers',
+  );
 }
 
 /**
diff --git a/indexer/packages/postgres/src/loops/loopHelper.ts b/indexer/packages/postgres/src/loops/loopHelper.ts
new file mode 100644
index 0000000000..ec213f8adb
--- /dev/null
+++ b/indexer/packages/postgres/src/loops/loopHelper.ts
@@ -0,0 +1,20 @@
+import { delay, logger } from '@dydxprotocol-indexer/base';
+
+export async function startUpdateLoop(
+  updateFunction: () => Promise<void>,
+  delayMs: number,
+  name: string,
+): Promise<void> {
+  for (;;) {
+    try {
+      await updateFunction();
+    } catch (error) {
+      logger.error({
+        at: name,
+        message: 'Failed to run update',
+        error,
+      });
+    }
+    await delay(delayMs);
+  }
+}
diff --git a/indexer/packages/postgres/src/loops/perpetual-market-refresher.ts b/indexer/packages/postgres/src/loops/perpetual-market-refresher.ts
index ccb74bb5f4..80cc4ef2ea 100644
--- a/indexer/packages/postgres/src/loops/perpetual-market-refresher.ts
+++ b/indexer/packages/postgres/src/loops/perpetual-market-refresher.ts
@@ -1,4 +1,7 @@
-import { stats, delay, NodeEnv } from '@dydxprotocol-indexer/base';
+import {
+  stats,
+  NodeEnv,
+} from '@dydxprotocol-indexer/base';
 import _ from 'lodash';
 
 import config from '../config';
@@ -6,6 +9,7 @@ import * as PerpetualMarketTable from '../stores/perpetual-market-table';
 import {
   Options, PerpetualMarketColumns, PerpetualMarketFromDatabase, PerpetualMarketsMap,
 } from '../types';
+import { startUpdateLoop } from './loopHelper';
 
 let idToPerpetualMarket: Record<string, PerpetualMarketFromDatabase> = {};
 
@@ -16,10 +20,11 @@ let idToPerpetualMarket: Record<string, PerpetualMarketFromDatabase> = {};
  * Refresh loop to cache the list of all perpetual markets from the database in-memory.
  */
 export async function start(): Promise<void> {
-  for (;;) {
-    await updatePerpetualMarkets();
-    await delay(config.PERPETUAL_MARKETS_REFRESHER_INTERVAL_MS);
-  }
+  await startUpdateLoop(
+    updatePerpetualMarkets,
+    config.PERPETUAL_MARKETS_REFRESHER_INTERVAL_MS,
+    'updatePerpetualMarkets',
+  );
 }
 
 /**