Introduce flag to ignore invalid FundingEvent where the perpetualId d…

…oesn't exist (#1093)
dydxprotocol · Feb 24, 2024 · 157c95a · 157c95a
1 parent c1d7948
commit 157c95a
Show file tree

Hide file tree

Showing 6 changed files with 113 additions and 6 deletions.
diff --git a/indexer/services/ender/__tests__/handlers/funding-handler.test.ts b/indexer/services/ender/__tests__/handlers/funding-handler.test.ts
@@ -26,6 +26,7 @@ import { FundingHandler } from '../../src/handlers/funding-handler';
 import {
   defaultFundingRateEvent,
   defaultFundingUpdateSampleEvent,
+  defaultFundingUpdateSampleEventWithAdditionalMarket,
   defaultHeight,
   defaultPreviousHeight,
   defaultTime,
@@ -41,6 +42,7 @@ import Big from 'big.js';
 import { redisClient } from '../../src/helpers/redis/redis-controller';
 import { bigIntToBytes } from '@dydxprotocol-indexer/v4-proto-parser';
 import { createPostgresFunctions } from '../../src/helpers/postgres/postgres-functions';
+import config from '../../src/config';
 
 describe('fundingHandler', () => {
   beforeAll(async () => {
@@ -66,6 +68,7 @@ describe('fundingHandler', () => {
     await dbHelpers.clearData();
     jest.clearAllMocks();
     await redis.deleteAllAsync(redisClient);
+    config.IGNORE_NONEXISTENT_PERPETUAL_MARKET = false;
   });
 
   afterAll(async () => {
@@ -167,6 +170,48 @@ describe('fundingHandler', () => {
     );
   });
 
+  it('successfully processes and clears cache for a new funding rate with non-existent market', async () => {
+    config.IGNORE_NONEXISTENT_PERPETUAL_MARKET = true;
+    const kafkaMessage: KafkaMessage = createKafkaMessageFromFundingEvents({
+      fundingEvents: [defaultFundingUpdateSampleEventWithAdditionalMarket],
+      height: defaultHeight,
+      time: defaultTime,
+    });
+
+    await onMessage(kafkaMessage);
+
+    await expectNextFundingRate(
+      'BTC-USD',
+      new Big(protocolTranslations.funding8HourValuePpmTo1HourRate(
+        defaultFundingUpdateSampleEvent.updates[0].fundingValuePpm,
+      )),
+    );
+
+    const kafkaMessage2: KafkaMessage = createKafkaMessageFromFundingEvents({
+      fundingEvents: [defaultFundingRateEvent],
+      height: 4,
+      time: defaultTime,
+    });
+
+    await onMessage(kafkaMessage2);
+    await expectNextFundingRate(
+      'BTC-USD',
+      undefined,
+    );
+    const fundingIndices: FundingIndexUpdatesFromDatabase[] = await
+    FundingIndexUpdatesTable.findAll({}, [], {});
+
+    expect(fundingIndices.length).toEqual(1);
+    expect(fundingIndices[0]).toEqual(expect.objectContaining({
+      perpetualId: '0',
+      rate: '0.00000125',
+      oraclePrice: '10000',
+      fundingIndex: '0.1',
+    }));
+    expect(stats.gauge).toHaveBeenCalledWith('ender.funding_index_update_event', 0.1, { ticker: 'BTC-USD' });
+    expect(stats.gauge).toHaveBeenCalledWith('ender.funding_index_update', 0.1, { ticker: 'BTC-USD' });
+  });
+
   it('successfully processes and clears cache for a new funding rate', async () => {
     const kafkaMessage: KafkaMessage = createKafkaMessageFromFundingEvents({
       fundingEvents: [defaultFundingUpdateSampleEvent],

diff --git a/indexer/services/ender/__tests__/helpers/constants.ts b/indexer/services/ender/__tests__/helpers/constants.ts
@@ -72,6 +72,22 @@ export const defaultFundingUpdateSampleEvent: FundingEventMessage = {
   ],
 };
 
+export const defaultFundingUpdateSampleEventWithAdditionalMarket: FundingEventMessage = {
+  type: FundingEventV1_Type.TYPE_PREMIUM_SAMPLE,
+  updates: [
+    {
+      perpetualId: 0,
+      fundingValuePpm: 10,
+      fundingIndex: bigIntToBytes(BigInt(0)),
+    },
+    {
+      perpetualId: 99999,
+      fundingValuePpm: 10,
+      fundingIndex: bigIntToBytes(BigInt(0)),
+    },
+  ],
+};
+
 export const defaultFundingRateEvent: FundingEventMessage = {
   type: FundingEventV1_Type.TYPE_FUNDING_RATE_AND_INDEX,
   updates: [

diff --git a/indexer/services/ender/__tests__/validators/funding-validator.test.ts b/indexer/services/ender/__tests__/validators/funding-validator.test.ts
@@ -21,6 +21,7 @@ import {
 } from '../helpers/indexer-proto-helpers';
 import { expectDidntLogError, expectLoggedParseMessageError } from '../helpers/validator-helpers';
 import { bigIntToBytes } from '@dydxprotocol-indexer/v4-proto-parser';
+import config from '../../src/config';
 
 describe('funding-validator', () => {
   beforeEach(async () => {
@@ -32,6 +33,7 @@ describe('funding-validator', () => {
   afterEach(async () => {
     await dbHelpers.clearData();
     jest.clearAllMocks();
+    config.IGNORE_NONEXISTENT_PERPETUAL_MARKET = false;
   });
 
   describe('validate', () => {
@@ -49,6 +51,35 @@ describe('funding-validator', () => {
       expectDidntLogError();
     });
 
+    it('does not throw error if IGNORE_NONEXISTENT_PERPETUAL_MARKET is true', () => {
+      config.IGNORE_NONEXISTENT_PERPETUAL_MARKET = true;
+      const event: FundingEventV1 = {
+        type: FundingEventV1_Type.TYPE_FUNDING_RATE_AND_INDEX,
+        updates: [
+          {
+            perpetualId: 10,
+            fundingValuePpm: 10,
+            fundingIndex: bigIntToBytes(BigInt(0)),
+          },
+        ],
+      } as FundingEventV1;
+      const validator: FundingValidator = new FundingValidator(
+        event,
+        createBlock(event),
+        0,
+      );
+
+      const errMsg: string = 'Invalid FundingEvent, perpetualId does not exist';
+      expect(() => validator.validate()).not.toThrow(new ParseMessageError(errMsg));
+      expect(logger.error).toHaveBeenCalledWith({
+        at: `${FundingValidator.name}#validate`,
+        message: errMsg,
+        blockHeight: defaultHeight,
+        // eslint-disable-next-line @typescript-eslint/no-explicit-any
+        event,
+      });
+    });
+
     it.each([
       // Base Validation Errors
       [

diff --git a/indexer/services/ender/src/config.ts b/indexer/services/ender/src/config.ts
@@ -23,6 +23,9 @@ export const configSchema = {
   SEND_WEBSOCKET_MESSAGES: parseBoolean({
     default: true,
   }),
+  IGNORE_NONEXISTENT_PERPETUAL_MARKET: parseBoolean({
+    default: false,
+  }),
 };
 
 export default parseSchema(configSchema);
diff --git a/indexer/services/ender/src/scripts/handlers/dydx_funding_handler.sql b/indexer/services/ender/src/scripts/handlers/dydx_funding_handler.sql
@@ -38,7 +38,8 @@ BEGIN
         perpetual_market_id = (funding_update->'perpetualId')::bigint;
         SELECT * INTO perpetual_market_record FROM perpetual_markets WHERE "id" = perpetual_market_id;
         IF NOT FOUND THEN
-            errors_response = array_append(errors_response, 'Received FundingUpdate with unknown perpetualId.');
+            errors_response = array_append(errors_response, '"Received FundingUpdate with unknown perpetualId."'::jsonb);
+            CONTINUE;
         END IF;
 
         perpetual_markets_response = jsonb_set(perpetual_markets_response, ARRAY[(perpetual_market_record."id")::text], dydx_to_jsonb(perpetual_market_record));
@@ -81,7 +82,7 @@ BEGIN
                 funding_update_response = jsonb_set(funding_update_response, ARRAY[(funding_index_updates_record."perpetualId")::text], dydx_to_jsonb(funding_index_updates_record));
 
             ELSE
-                errors_response = array_append(errors_response, 'Received unknown FundingEvent type.');
+                errors_response = array_append(errors_response, '"Received unknown FundingEvent type."'::jsonb);
                 CONTINUE;
         END CASE;
 

diff --git a/indexer/services/ender/src/validators/funding-validator.ts b/indexer/services/ender/src/validators/funding-validator.ts
@@ -1,3 +1,4 @@
+import { logger } from '@dydxprotocol-indexer/base';
 import { perpetualMarketRefresher } from '@dydxprotocol-indexer/postgres';
 import {
   FundingEventV1,
@@ -6,6 +7,7 @@ import {
   FundingUpdateV1,
 } from '@dydxprotocol-indexer/v4-protos';
 
+import config from '../config';
 import { FundingHandler } from '../handlers/funding-handler';
 import { Handler } from '../handlers/handler';
 import { FundingEventMessage } from '../lib/types';
@@ -28,10 +30,19 @@ export class FundingValidator extends Validator<FundingEventV1> {
         perpetualMarketRefresher.getPerpetualMarketFromId(
           perpetualId.toString(),
         ) === undefined) {
-        return this.logAndThrowParseMessageError(
-          'Invalid FundingEvent, perpetualId does not exist',
-          { event: this.event },
-        );
+        if (config.IGNORE_NONEXISTENT_PERPETUAL_MARKET) {
+          logger.error({
+            at: `${this.constructor.name}#validate`,
+            message: 'Invalid FundingEvent, perpetualId does not exist',
+            blockHeight: this.block.height,
+            event: this.event,
+          });
+        } else {
+          return this.logAndThrowParseMessageError(
+            'Invalid FundingEvent, perpetualId does not exist',
+            { event: this.event },
+          );
+        }
       }
     });
   }