From fb7248e61dba08cf341b02b79e919338c66063b3 Mon Sep 17 00:00:00 2001 From: Tate Date: Wed, 30 Oct 2024 03:08:04 +0000 Subject: [PATCH 1/8] fix multi entpoint deploy issues --- packages/node-core/CHANGELOG.md | 3 ++ .../indexer/connectionPool.service.spec.ts | 4 ++ .../src/indexer/connectionPool.service.ts | 6 +-- .../connectionPoolState.manager.spec.ts | 2 + .../indexer/connectionPoolState.manager.ts | 52 ++++++++++++------- 5 files changed, 44 insertions(+), 23 deletions(-) diff --git a/packages/node-core/CHANGELOG.md b/packages/node-core/CHANGELOG.md index a87b5f8220..e407547887 100644 --- a/packages/node-core/CHANGELOG.md +++ b/packages/node-core/CHANGELOG.md @@ -6,6 +6,9 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] +### Fixed +- When configuring multiple endpoints, poor network conditions may lead to block crawling delays. (#2572) + ## [14.1.6] - 2024-10-21 ### Fixed - Issues with setting a large block range for bypass blocks (#2566) diff --git a/packages/node-core/src/indexer/connectionPool.service.spec.ts b/packages/node-core/src/indexer/connectionPool.service.spec.ts index adfd6a1b64..5d08981d2a 100644 --- a/packages/node-core/src/indexer/connectionPool.service.spec.ts +++ b/packages/node-core/src/indexer/connectionPool.service.spec.ts @@ -127,4 +127,8 @@ describe('ConnectionPoolService', () => { expect(handleApiDisconnectsSpy).toHaveBeenCalledTimes(1); }, 15000); }); + + // describe('The endpoint with rate limiting.', () => { + // it('should handle all nodes being rate limited', async () => {}); + // }); }); diff --git a/packages/node-core/src/indexer/connectionPool.service.ts b/packages/node-core/src/indexer/connectionPool.service.ts index dacde9be3f..df464ca155 100644 --- a/packages/node-core/src/indexer/connectionPool.service.ts +++ b/packages/node-core/src/indexer/connectionPool.service.ts @@ -104,9 +104,9 @@ export class ConnectionPoolService { endpoint: string; @@ -22,6 +24,7 @@ export interface ConnectionPoolItem { backoffDelay: number; failureCount: number; rateLimited: boolean; + rateLimitDelay: number; failed: boolean; lastRequestTime: number; connected: boolean; @@ -72,6 +75,7 @@ export class ConnectionPoolStateManager { + async setRecoverTimeout(endpoint: string, delay: number): Promise { // Make sure there is no existing timeout await this.clearTimeout(endpoint); this.pool[endpoint].timeoutId = setTimeout(() => { this.pool[endpoint].backoffDelay = 0; // Reset backoff delay only if there are no consecutive errors this.pool[endpoint].rateLimited = false; + this.pool[endpoint].rateLimitDelay = 0; this.pool[endpoint].failed = false; this.pool[endpoint].timeoutId = undefined; // Clear the timeout ID @@ -247,35 +252,42 @@ export class ConnectionPoolStateManager): number { + // Exponential backoff using failure count, Start with RETRY_DELAY and double on each failure, MAX_RETRY_DELAY is the maximum delay + return Math.min(RETRY_DELAY * Math.pow(2, poolItem.failureCount - 1), MAX_RETRY_DELAY); } private calculatePerformanceScore(responseTime: number, failureCount: number): number { From 3377d86ade874a27e460f8c35a7aedf0eefb5fae Mon Sep 17 00:00:00 2001 From: Tate Date: Wed, 30 Oct 2024 09:46:54 +0000 Subject: [PATCH 2/8] unit test --- .../indexer/connectionPool.service.spec.ts | 30 ++++++- .../connectionPoolState.manager.spec.ts | 89 +++++++++---------- 2 files changed, 66 insertions(+), 53 deletions(-) diff --git a/packages/node-core/src/indexer/connectionPool.service.spec.ts b/packages/node-core/src/indexer/connectionPool.service.spec.ts index 5d08981d2a..47f93af60a 100644 --- a/packages/node-core/src/indexer/connectionPool.service.spec.ts +++ b/packages/node-core/src/indexer/connectionPool.service.spec.ts @@ -2,7 +2,7 @@ // SPDX-License-Identifier: GPL-3.0 import {delay} from '@subql/common'; -import {ApiErrorType, ConnectionPoolStateManager, IApiConnectionSpecific, NodeConfig} from '..'; +import {ApiErrorType, ConnectionPoolStateManager, getLogger, IApiConnectionSpecific, NodeConfig} from '..'; import {ConnectionPoolService} from './connectionPool.service'; async function waitFor(conditionFn: () => boolean, timeout = 50000, interval = 100): Promise { @@ -128,7 +128,29 @@ describe('ConnectionPoolService', () => { }, 15000); }); - // describe('The endpoint with rate limiting.', () => { - // it('should handle all nodes being rate limited', async () => {}); - // }); + describe('Rate limit endpoint delay 20s', () => { + it('call delay', async () => { + const logger = getLogger('connection-pool'); + const consoleSpy = jest.spyOn(logger, 'info'); + + await connectionPoolService.addToConnections(mockApiConnection, TEST_URL); + await connectionPoolService.addToConnections(mockApiConnection, `${TEST_URL}/2`); + await connectionPoolService.handleApiError(TEST_URL, { + name: 'timeout', + errorType: ApiErrorType.Timeout, + message: 'timeout error', + }); + await connectionPoolService.handleApiError(`${TEST_URL}/2`, { + name: 'DefaultError', + errorType: ApiErrorType.Default, + message: 'Default error', + }); + await (connectionPoolService as any).flushResultCache(); + + await connectionPoolService.api.fetchBlocks([34365]); + + expect(consoleSpy).toHaveBeenCalledWith('throtling on ratelimited endpoint 20s'); + consoleSpy.mockRestore(); + }, 30000); + }); }); diff --git a/packages/node-core/src/indexer/connectionPoolState.manager.spec.ts b/packages/node-core/src/indexer/connectionPoolState.manager.spec.ts index 0481128e42..710304a0c1 100644 --- a/packages/node-core/src/indexer/connectionPoolState.manager.spec.ts +++ b/packages/node-core/src/indexer/connectionPoolState.manager.spec.ts @@ -1,7 +1,8 @@ // Copyright 2020-2024 SubQuery Pte Ltd authors & contributors // SPDX-License-Identifier: GPL-3.0 -import {ConnectionPoolStateManager} from './connectionPoolState.manager'; +import {ApiErrorType} from '../api.connection.error'; +import {ConnectionPoolItem, ConnectionPoolStateManager} from './connectionPoolState.manager'; describe('ConnectionPoolStateManager', function () { let connectionPoolStateManager: ConnectionPoolStateManager; @@ -12,62 +13,54 @@ describe('ConnectionPoolStateManager', function () { connectionPoolStateManager = new ConnectionPoolStateManager(); }); + afterEach(async function () { + await connectionPoolStateManager.onApplicationShutdown(); + }); + it('chooses primary endpoint first', async function () { - (connectionPoolStateManager as any).pool[EXAMPLE_ENDPOINT1] = { - primary: true, - performanceScore: 100, - failureCount: 0, - endpoint: '', - backoffDelay: 0, - rateLimited: false, - failed: false, - connected: true, - lastRequestTime: 0, - }; - - (connectionPoolStateManager as any).pool[EXAMPLE_ENDPOINT2] = { - primary: false, - performanceScore: 100, - failureCount: 0, - endpoint: '', - backoffDelay: 0, - rateLimited: false, - failed: false, - connected: true, - lastRequestTime: 0, - }; + await connectionPoolStateManager.addToConnections(EXAMPLE_ENDPOINT1, true); + await connectionPoolStateManager.addToConnections(EXAMPLE_ENDPOINT2, false); expect(await connectionPoolStateManager.getNextConnectedEndpoint()).toEqual(EXAMPLE_ENDPOINT1); }); it('does not choose primary endpoint if failed', async function () { - (connectionPoolStateManager as any).pool[EXAMPLE_ENDPOINT1] = { - primary: true, - performanceScore: 100, - failureCount: 0, - endpoint: '', - backoffDelay: 0, - rateLimited: false, - failed: false, - connected: false, - lastRequestTime: 0, - }; - - (connectionPoolStateManager as any).pool[EXAMPLE_ENDPOINT2] = { - primary: false, - performanceScore: 100, - failureCount: 0, - endpoint: '', - backoffDelay: 0, - rateLimited: false, - failed: false, - connected: true, - lastRequestTime: 0, - }; + await connectionPoolStateManager.addToConnections(EXAMPLE_ENDPOINT1, true); + await connectionPoolStateManager.addToConnections(EXAMPLE_ENDPOINT2, false); + + await connectionPoolStateManager.handleApiError(EXAMPLE_ENDPOINT1, ApiErrorType.Default); expect(await connectionPoolStateManager.getNextConnectedEndpoint()).toEqual(EXAMPLE_ENDPOINT2); }); + it('All endpoints backoff; select a rateLimited endpoint. reason: ApiErrorType.Timeout', async function () { + await connectionPoolStateManager.addToConnections(EXAMPLE_ENDPOINT1, false); + await connectionPoolStateManager.addToConnections(EXAMPLE_ENDPOINT2, false); + + await connectionPoolStateManager.handleApiError(EXAMPLE_ENDPOINT1, ApiErrorType.Default); + await connectionPoolStateManager.handleApiError(EXAMPLE_ENDPOINT2, ApiErrorType.Timeout); + + const nextEndpoint = await connectionPoolStateManager.getNextConnectedEndpoint(); + const endpointInfo = (connectionPoolStateManager as any).pool[EXAMPLE_ENDPOINT2] as ConnectionPoolItem; + expect(nextEndpoint).toEqual(EXAMPLE_ENDPOINT2); + expect(endpointInfo.rateLimited).toBe(true); + expect((connectionPoolStateManager as any).pool[EXAMPLE_ENDPOINT2].rateLimitDelay).toBe(20 * 1000); + }); + + it('All endpoints backoff; select a rateLimited endpoint. reason: ApiErrorType.RateLimit', async function () { + await connectionPoolStateManager.addToConnections(EXAMPLE_ENDPOINT1, false); + await connectionPoolStateManager.addToConnections(EXAMPLE_ENDPOINT2, false); + + await connectionPoolStateManager.handleApiError(EXAMPLE_ENDPOINT1, ApiErrorType.Default); + await connectionPoolStateManager.handleApiError(EXAMPLE_ENDPOINT2, ApiErrorType.RateLimit); + + const nextEndpoint = await connectionPoolStateManager.getNextConnectedEndpoint(); + const endpointInfo = (connectionPoolStateManager as any).pool[EXAMPLE_ENDPOINT2] as ConnectionPoolItem; + expect(nextEndpoint).toEqual(EXAMPLE_ENDPOINT2); + expect(endpointInfo.rateLimited).toBe(true); + expect((connectionPoolStateManager as any).pool[EXAMPLE_ENDPOINT2].rateLimitDelay).toBe(20 * 1000); + }); + it('can calculate performance score for response time of zero', function () { const score = (connectionPoolStateManager as any).calculatePerformanceScore(0, 0); expect(score).not.toBeNaN(); @@ -78,6 +71,4 @@ describe('ConnectionPoolStateManager', function () { const score2 = (connectionPoolStateManager as any).calculatePerformanceScore(2, 0); expect(score1).toBeGreaterThan(score2); }); - - // it('handleApiError', async function () {}); }); From 915e8a4810e0ea09a3de57632e35c85f2ee11b4b Mon Sep 17 00:00:00 2001 From: Tate Date: Mon, 4 Nov 2024 06:29:52 +0000 Subject: [PATCH 3/8] backoffDelay rule --- .../connectionPoolState.manager.spec.ts | 26 +++++++++++++++++++ .../indexer/connectionPoolState.manager.ts | 14 +++++++--- 2 files changed, 36 insertions(+), 4 deletions(-) diff --git a/packages/node-core/src/indexer/connectionPoolState.manager.spec.ts b/packages/node-core/src/indexer/connectionPoolState.manager.spec.ts index 710304a0c1..5edb6662e9 100644 --- a/packages/node-core/src/indexer/connectionPoolState.manager.spec.ts +++ b/packages/node-core/src/indexer/connectionPoolState.manager.spec.ts @@ -71,4 +71,30 @@ describe('ConnectionPoolStateManager', function () { const score2 = (connectionPoolStateManager as any).calculatePerformanceScore(2, 0); expect(score1).toBeGreaterThan(score2); }); + + it('backoff delay rule', function () { + const delay1 = (connectionPoolStateManager as any).calculateNextDelay({failureCount: 0}); + expect(delay1).toBe(0); + + const delay2 = (connectionPoolStateManager as any).calculateNextDelay({failureCount: 1}); + expect(delay2).toBe(10000); + + const delay3 = (connectionPoolStateManager as any).calculateNextDelay({failureCount: 2}); + expect(delay3).toBe(20000); + + const delay4 = (connectionPoolStateManager as any).calculateNextDelay({failureCount: 3}); + expect(delay4).toBe(40000); + + const delay5 = (connectionPoolStateManager as any).calculateNextDelay({failureCount: 4}); + expect(delay5).toBe(80000); + + const delay6 = (connectionPoolStateManager as any).calculateNextDelay({failureCount: 5}); + expect(delay6).toBe(160000); + + const delay7 = (connectionPoolStateManager as any).calculateNextDelay({failureCount: 6}); + expect(delay7).toBe(320000); + + const delayMax = (connectionPoolStateManager as any).calculateNextDelay({failureCount: 7}); + expect(delayMax).toBe(320000); + }); }); diff --git a/packages/node-core/src/indexer/connectionPoolState.manager.ts b/packages/node-core/src/indexer/connectionPoolState.manager.ts index 6547cc527b..81947238c1 100644 --- a/packages/node-core/src/indexer/connectionPoolState.manager.ts +++ b/packages/node-core/src/indexer/connectionPoolState.manager.ts @@ -10,8 +10,6 @@ import {getLogger} from '../logger'; import {exitWithError} from '../process'; import {errorTypeToScoreAdjustment} from './connectionPool.service'; -const RETRY_DELAY = 60 * 1000; -const MAX_RETRY_DELAY = 60 * RETRY_DELAY; const MAX_FAILURES = 5; const RESPONSE_TIME_WEIGHT = 0.7; const FAILURE_WEIGHT = 0.3; @@ -286,8 +284,16 @@ export class ConnectionPoolStateManager): number { - // Exponential backoff using failure count, Start with RETRY_DELAY and double on each failure, MAX_RETRY_DELAY is the maximum delay - return Math.min(RETRY_DELAY * Math.pow(2, poolItem.failureCount - 1), MAX_RETRY_DELAY); + const delayRule = [10, 20, 40, 80, 160, 320]; + let delayTime = 0; + if (poolItem.failureCount < 1) { + delayTime = 0; + } else if (poolItem.failureCount >= 1 && poolItem.failureCount <= delayRule.length) { + delayTime = delayRule[poolItem.failureCount - 1]; + } else { + delayTime = delayRule[delayRule.length - 1]; + } + return delayTime * 1000; } private calculatePerformanceScore(responseTime: number, failureCount: number): number { From 8c2e54f0c64723b9bd463502520387cbe3aa4348 Mon Sep 17 00:00:00 2001 From: Tate Date: Tue, 5 Nov 2024 02:25:14 +0000 Subject: [PATCH 4/8] remove rateLimitDelay --- .../indexer/connectionPool.service.spec.ts | 26 ------------------- .../src/indexer/connectionPool.service.ts | 7 ----- .../indexer/connectionPoolState.manager.ts | 4 --- 3 files changed, 37 deletions(-) diff --git a/packages/node-core/src/indexer/connectionPool.service.spec.ts b/packages/node-core/src/indexer/connectionPool.service.spec.ts index 47f93af60a..09d535851d 100644 --- a/packages/node-core/src/indexer/connectionPool.service.spec.ts +++ b/packages/node-core/src/indexer/connectionPool.service.spec.ts @@ -127,30 +127,4 @@ describe('ConnectionPoolService', () => { expect(handleApiDisconnectsSpy).toHaveBeenCalledTimes(1); }, 15000); }); - - describe('Rate limit endpoint delay 20s', () => { - it('call delay', async () => { - const logger = getLogger('connection-pool'); - const consoleSpy = jest.spyOn(logger, 'info'); - - await connectionPoolService.addToConnections(mockApiConnection, TEST_URL); - await connectionPoolService.addToConnections(mockApiConnection, `${TEST_URL}/2`); - await connectionPoolService.handleApiError(TEST_URL, { - name: 'timeout', - errorType: ApiErrorType.Timeout, - message: 'timeout error', - }); - await connectionPoolService.handleApiError(`${TEST_URL}/2`, { - name: 'DefaultError', - errorType: ApiErrorType.Default, - message: 'Default error', - }); - await (connectionPoolService as any).flushResultCache(); - - await connectionPoolService.api.fetchBlocks([34365]); - - expect(consoleSpy).toHaveBeenCalledWith('throtling on ratelimited endpoint 20s'); - consoleSpy.mockRestore(); - }, 30000); - }); }); diff --git a/packages/node-core/src/indexer/connectionPool.service.ts b/packages/node-core/src/indexer/connectionPool.service.ts index df464ca155..43cb698c64 100644 --- a/packages/node-core/src/indexer/connectionPool.service.ts +++ b/packages/node-core/src/indexer/connectionPool.service.ts @@ -102,13 +102,6 @@ export class ConnectionPoolService => { try { - // Check if the endpoint is rate-limited - if (await this.poolStateManager.getFieldValue(endpoint, 'rateLimited')) { - const rateLimitDelay = await this.poolStateManager.getFieldValue(endpoint, 'rateLimitDelay'); - logger.info(`throtling on ratelimited endpoint ${rateLimitDelay / 1000}s`); - await delay(rateLimitDelay / 1000); - } - const start = Date.now(); const result = await target.fetchBlocks(heights, ...args); const end = Date.now(); diff --git a/packages/node-core/src/indexer/connectionPoolState.manager.ts b/packages/node-core/src/indexer/connectionPoolState.manager.ts index 81947238c1..4fbcda4e95 100644 --- a/packages/node-core/src/indexer/connectionPoolState.manager.ts +++ b/packages/node-core/src/indexer/connectionPoolState.manager.ts @@ -22,7 +22,6 @@ export interface ConnectionPoolItem { backoffDelay: number; failureCount: number; rateLimited: boolean; - rateLimitDelay: number; failed: boolean; lastRequestTime: number; connected: boolean; @@ -73,7 +72,6 @@ export class ConnectionPoolStateManager { this.pool[endpoint].backoffDelay = 0; // Reset backoff delay only if there are no consecutive errors this.pool[endpoint].rateLimited = false; - this.pool[endpoint].rateLimitDelay = 0; this.pool[endpoint].failed = false; this.pool[endpoint].timeoutId = undefined; // Clear the timeout ID @@ -259,7 +256,6 @@ export class ConnectionPoolStateManager Date: Wed, 6 Nov 2024 02:33:19 +0000 Subject: [PATCH 5/8] Revert "backoffDelay rule" This reverts commit 915e8a4810e0ea09a3de57632e35c85f2ee11b4b. --- .../connectionPoolState.manager.spec.ts | 26 ------------------- .../indexer/connectionPoolState.manager.ts | 14 +++------- 2 files changed, 4 insertions(+), 36 deletions(-) diff --git a/packages/node-core/src/indexer/connectionPoolState.manager.spec.ts b/packages/node-core/src/indexer/connectionPoolState.manager.spec.ts index 5edb6662e9..710304a0c1 100644 --- a/packages/node-core/src/indexer/connectionPoolState.manager.spec.ts +++ b/packages/node-core/src/indexer/connectionPoolState.manager.spec.ts @@ -71,30 +71,4 @@ describe('ConnectionPoolStateManager', function () { const score2 = (connectionPoolStateManager as any).calculatePerformanceScore(2, 0); expect(score1).toBeGreaterThan(score2); }); - - it('backoff delay rule', function () { - const delay1 = (connectionPoolStateManager as any).calculateNextDelay({failureCount: 0}); - expect(delay1).toBe(0); - - const delay2 = (connectionPoolStateManager as any).calculateNextDelay({failureCount: 1}); - expect(delay2).toBe(10000); - - const delay3 = (connectionPoolStateManager as any).calculateNextDelay({failureCount: 2}); - expect(delay3).toBe(20000); - - const delay4 = (connectionPoolStateManager as any).calculateNextDelay({failureCount: 3}); - expect(delay4).toBe(40000); - - const delay5 = (connectionPoolStateManager as any).calculateNextDelay({failureCount: 4}); - expect(delay5).toBe(80000); - - const delay6 = (connectionPoolStateManager as any).calculateNextDelay({failureCount: 5}); - expect(delay6).toBe(160000); - - const delay7 = (connectionPoolStateManager as any).calculateNextDelay({failureCount: 6}); - expect(delay7).toBe(320000); - - const delayMax = (connectionPoolStateManager as any).calculateNextDelay({failureCount: 7}); - expect(delayMax).toBe(320000); - }); }); diff --git a/packages/node-core/src/indexer/connectionPoolState.manager.ts b/packages/node-core/src/indexer/connectionPoolState.manager.ts index 4fbcda4e95..963d9ddce7 100644 --- a/packages/node-core/src/indexer/connectionPoolState.manager.ts +++ b/packages/node-core/src/indexer/connectionPoolState.manager.ts @@ -10,6 +10,8 @@ import {getLogger} from '../logger'; import {exitWithError} from '../process'; import {errorTypeToScoreAdjustment} from './connectionPool.service'; +const RETRY_DELAY = 60 * 1000; +const MAX_RETRY_DELAY = 60 * RETRY_DELAY; const MAX_FAILURES = 5; const RESPONSE_TIME_WEIGHT = 0.7; const FAILURE_WEIGHT = 0.3; @@ -280,16 +282,8 @@ export class ConnectionPoolStateManager): number { - const delayRule = [10, 20, 40, 80, 160, 320]; - let delayTime = 0; - if (poolItem.failureCount < 1) { - delayTime = 0; - } else if (poolItem.failureCount >= 1 && poolItem.failureCount <= delayRule.length) { - delayTime = delayRule[poolItem.failureCount - 1]; - } else { - delayTime = delayRule[delayRule.length - 1]; - } - return delayTime * 1000; + // Exponential backoff using failure count, Start with RETRY_DELAY and double on each failure, MAX_RETRY_DELAY is the maximum delay + return Math.min(RETRY_DELAY * Math.pow(2, poolItem.failureCount - 1), MAX_RETRY_DELAY); } private calculatePerformanceScore(responseTime: number, failureCount: number): number { From 89cd7804d0a7312504d8d156660231bc1d5008ab Mon Sep 17 00:00:00 2001 From: Tate Date: Wed, 6 Nov 2024 02:33:40 +0000 Subject: [PATCH 6/8] Revert "remove rateLimitDelay" This reverts commit 8c2e54f0c64723b9bd463502520387cbe3aa4348. --- .../indexer/connectionPool.service.spec.ts | 26 +++++++++++++++++++ .../src/indexer/connectionPool.service.ts | 7 +++++ .../indexer/connectionPoolState.manager.ts | 4 +++ 3 files changed, 37 insertions(+) diff --git a/packages/node-core/src/indexer/connectionPool.service.spec.ts b/packages/node-core/src/indexer/connectionPool.service.spec.ts index 09d535851d..47f93af60a 100644 --- a/packages/node-core/src/indexer/connectionPool.service.spec.ts +++ b/packages/node-core/src/indexer/connectionPool.service.spec.ts @@ -127,4 +127,30 @@ describe('ConnectionPoolService', () => { expect(handleApiDisconnectsSpy).toHaveBeenCalledTimes(1); }, 15000); }); + + describe('Rate limit endpoint delay 20s', () => { + it('call delay', async () => { + const logger = getLogger('connection-pool'); + const consoleSpy = jest.spyOn(logger, 'info'); + + await connectionPoolService.addToConnections(mockApiConnection, TEST_URL); + await connectionPoolService.addToConnections(mockApiConnection, `${TEST_URL}/2`); + await connectionPoolService.handleApiError(TEST_URL, { + name: 'timeout', + errorType: ApiErrorType.Timeout, + message: 'timeout error', + }); + await connectionPoolService.handleApiError(`${TEST_URL}/2`, { + name: 'DefaultError', + errorType: ApiErrorType.Default, + message: 'Default error', + }); + await (connectionPoolService as any).flushResultCache(); + + await connectionPoolService.api.fetchBlocks([34365]); + + expect(consoleSpy).toHaveBeenCalledWith('throtling on ratelimited endpoint 20s'); + consoleSpy.mockRestore(); + }, 30000); + }); }); diff --git a/packages/node-core/src/indexer/connectionPool.service.ts b/packages/node-core/src/indexer/connectionPool.service.ts index 43cb698c64..df464ca155 100644 --- a/packages/node-core/src/indexer/connectionPool.service.ts +++ b/packages/node-core/src/indexer/connectionPool.service.ts @@ -102,6 +102,13 @@ export class ConnectionPoolService => { try { + // Check if the endpoint is rate-limited + if (await this.poolStateManager.getFieldValue(endpoint, 'rateLimited')) { + const rateLimitDelay = await this.poolStateManager.getFieldValue(endpoint, 'rateLimitDelay'); + logger.info(`throtling on ratelimited endpoint ${rateLimitDelay / 1000}s`); + await delay(rateLimitDelay / 1000); + } + const start = Date.now(); const result = await target.fetchBlocks(heights, ...args); const end = Date.now(); diff --git a/packages/node-core/src/indexer/connectionPoolState.manager.ts b/packages/node-core/src/indexer/connectionPoolState.manager.ts index 963d9ddce7..6547cc527b 100644 --- a/packages/node-core/src/indexer/connectionPoolState.manager.ts +++ b/packages/node-core/src/indexer/connectionPoolState.manager.ts @@ -24,6 +24,7 @@ export interface ConnectionPoolItem { backoffDelay: number; failureCount: number; rateLimited: boolean; + rateLimitDelay: number; failed: boolean; lastRequestTime: number; connected: boolean; @@ -74,6 +75,7 @@ export class ConnectionPoolStateManager { this.pool[endpoint].backoffDelay = 0; // Reset backoff delay only if there are no consecutive errors this.pool[endpoint].rateLimited = false; + this.pool[endpoint].rateLimitDelay = 0; this.pool[endpoint].failed = false; this.pool[endpoint].timeoutId = undefined; // Clear the timeout ID @@ -258,6 +261,7 @@ export class ConnectionPoolStateManager Date: Wed, 6 Nov 2024 07:18:14 +0000 Subject: [PATCH 7/8] rateLimited --- .../src/indexer/connectionPool.service.spec.ts | 4 ++-- .../node-core/src/indexer/connectionPool.service.ts | 5 ++--- .../src/indexer/connectionPoolState.manager.ts | 11 +++-------- 3 files changed, 7 insertions(+), 13 deletions(-) diff --git a/packages/node-core/src/indexer/connectionPool.service.spec.ts b/packages/node-core/src/indexer/connectionPool.service.spec.ts index 47f93af60a..be89dccbf8 100644 --- a/packages/node-core/src/indexer/connectionPool.service.spec.ts +++ b/packages/node-core/src/indexer/connectionPool.service.spec.ts @@ -128,7 +128,7 @@ describe('ConnectionPoolService', () => { }, 15000); }); - describe('Rate limit endpoint delay 20s', () => { + describe('Rate limit endpoint delay', () => { it('call delay', async () => { const logger = getLogger('connection-pool'); const consoleSpy = jest.spyOn(logger, 'info'); @@ -149,7 +149,7 @@ describe('ConnectionPoolService', () => { await connectionPoolService.api.fetchBlocks([34365]); - expect(consoleSpy).toHaveBeenCalledWith('throtling on ratelimited endpoint 20s'); + expect(consoleSpy).toHaveBeenCalledWith('throtling on ratelimited endpoint 10s'); consoleSpy.mockRestore(); }, 30000); }); diff --git a/packages/node-core/src/indexer/connectionPool.service.ts b/packages/node-core/src/indexer/connectionPool.service.ts index df464ca155..d9eebae592 100644 --- a/packages/node-core/src/indexer/connectionPool.service.ts +++ b/packages/node-core/src/indexer/connectionPool.service.ts @@ -104,9 +104,8 @@ export class ConnectionPoolService { endpoint: string; @@ -24,7 +23,6 @@ export interface ConnectionPoolItem { backoffDelay: number; failureCount: number; rateLimited: boolean; - rateLimitDelay: number; failed: boolean; lastRequestTime: number; connected: boolean; @@ -75,7 +73,6 @@ export class ConnectionPoolStateManager { this.pool[endpoint].backoffDelay = 0; // Reset backoff delay only if there are no consecutive errors - this.pool[endpoint].rateLimited = false; - this.pool[endpoint].rateLimitDelay = 0; + // this.pool[endpoint].rateLimited = false; // Do not reset rateLimited status this.pool[endpoint].failed = false; this.pool[endpoint].timeoutId = undefined; // Clear the timeout ID @@ -261,7 +257,6 @@ export class ConnectionPoolStateManager Date: Tue, 12 Nov 2024 01:30:15 +0000 Subject: [PATCH 8/8] fix test --- .../node-core/src/indexer/connectionPoolState.manager.spec.ts | 2 -- 1 file changed, 2 deletions(-) diff --git a/packages/node-core/src/indexer/connectionPoolState.manager.spec.ts b/packages/node-core/src/indexer/connectionPoolState.manager.spec.ts index 710304a0c1..b476605385 100644 --- a/packages/node-core/src/indexer/connectionPoolState.manager.spec.ts +++ b/packages/node-core/src/indexer/connectionPoolState.manager.spec.ts @@ -44,7 +44,6 @@ describe('ConnectionPoolStateManager', function () { const endpointInfo = (connectionPoolStateManager as any).pool[EXAMPLE_ENDPOINT2] as ConnectionPoolItem; expect(nextEndpoint).toEqual(EXAMPLE_ENDPOINT2); expect(endpointInfo.rateLimited).toBe(true); - expect((connectionPoolStateManager as any).pool[EXAMPLE_ENDPOINT2].rateLimitDelay).toBe(20 * 1000); }); it('All endpoints backoff; select a rateLimited endpoint. reason: ApiErrorType.RateLimit', async function () { @@ -58,7 +57,6 @@ describe('ConnectionPoolStateManager', function () { const endpointInfo = (connectionPoolStateManager as any).pool[EXAMPLE_ENDPOINT2] as ConnectionPoolItem; expect(nextEndpoint).toEqual(EXAMPLE_ENDPOINT2); expect(endpointInfo.rateLimited).toBe(true); - expect((connectionPoolStateManager as any).pool[EXAMPLE_ENDPOINT2].rateLimitDelay).toBe(20 * 1000); }); it('can calculate performance score for response time of zero', function () {