From 1300dd57cf5dd9a282ec89830249b7994eda56d2 Mon Sep 17 00:00:00 2001 From: Tom Kirkpatrick Date: Mon, 19 Feb 2024 18:39:35 +0000 Subject: [PATCH] Add maxHeightDelta setting to exclude stale data --- README.md | 21 +++++++++-------- config/custom-environment-variables.json | 3 ++- config/default.json | 3 ++- src/lib/DataProviderManager.ts | 29 +++++++++++++++++++++++- src/server.tsx | 2 ++ test/DataProviderManager.test.ts | 1 - 6 files changed, 45 insertions(+), 14 deletions(-) diff --git a/README.md b/README.md index 0aa2520..320eaca 100644 --- a/README.md +++ b/README.md @@ -68,16 +68,17 @@ Here are the available configuration options: ### Application settings -| Config Key | Description | Default Value | Environment Variable | -| ------------------------ | --------------------------------------------------------------------------------------------------------- | ----------------------- | -------------------- | -| `server.port` | The port on which the server runs | `3000` | `PORT` | -| `server.baseUrl` | The base url port on which the server is accessible | `http://localhost:3000` | `BASE_URL` | -| `settings.logLevel` | The log level to use for the application | `debug` | `LOGLEVEL` | -| `settings.timeout` | Timeout to use when fetching data (ms) | `5000` | `TIMEOUT` | -| `settings.feeMultiplier` | The multiplier to apply to the fee estimates | `1` | `FEE_MULTIPLIER` | -| `settings.feeMinimum` | The minimum fee (sat/vB) to use for fee estimates if we could not determine from a configured data source | `2` | `FEE_MINIMUM` | -| `cache.stdTTL` | The standard time to live in seconds for every generated cache element | `15` | `CACHE_STDTTL` | -| `cache.checkperiod` | The period in seconds, used for the automatic delete check interval | `20` | `CACHE_CHECKPERIOD` | +| Config Key | Description | Default Value | Environment Variable | +| ------------------------- | --------------------------------------------------------------------------------------------------------- | ----------------------- | -------------------- | +| `server.port` | The port on which the server runs | `3000` | `PORT` | +| `server.baseUrl` | The base url port on which the server is accessible | `http://localhost:3000` | `BASE_URL` | +| `settings.logLevel` | The log level to use for the application | `debug` | `LOGLEVEL` | +| `settings.timeout` | Timeout to use when fetching data (ms) | `5000` | `TIMEOUT` | +| `settings.feeMultiplier` | The multiplier to apply to the fee estimates | `1` | `FEE_MULTIPLIER` | +| `settings.feeMinimum` | The minimum fee (sat/vB) to use for fee estimates if we could not determine from a configured data source | `2` | `FEE_MINIMUM` | +| `settings.maxHeightDelta` | The maximum height difference between the current block and the block for which we have fee estimates | `3` | `MAX_HEIGHT_DELTA` | +| `cache.stdTTL` | The standard time to live in seconds for every generated cache element | `15` | `CACHE_STDTTL` | +| `cache.checkperiod` | The period in seconds, used for the automatic delete check interval | `20` | `CACHE_CHECKPERIOD` | ### Mempool settings diff --git a/config/custom-environment-variables.json b/config/custom-environment-variables.json index b721057..e632b48 100644 --- a/config/custom-environment-variables.json +++ b/config/custom-environment-variables.json @@ -7,7 +7,8 @@ "loglevel": "LOGLEVEL", "timeout": "TIMEOUT", "feeMultiplier": "FEE_MULTIPLIER", - "feeMinimum": "FEE_MINIMUM" + "feeMinimum": "FEE_MINIMUM", + "maxHeightDelta": "MAX_HEIGHT_DELTA" }, "mempool": { "baseUrl": "MEMPOOL_BASE_URL", diff --git a/config/default.json b/config/default.json index a7beb90..88bffe9 100644 --- a/config/default.json +++ b/config/default.json @@ -7,7 +7,8 @@ "loglevel": "debug", "timeout": 5000, "feeMultiplier": 1, - "feeMinimum": 2 + "feeMinimum": 2, + "maxHeightDelta": 3 }, "mempool": { "baseUrl": "https://mempool.space", diff --git a/src/lib/DataProviderManager.ts b/src/lib/DataProviderManager.ts index acd731a..976286a 100644 --- a/src/lib/DataProviderManager.ts +++ b/src/lib/DataProviderManager.ts @@ -7,16 +7,19 @@ const log = logger(LOGLEVEL); export class DataProviderManager { private providers: Provider[] = []; private cache: NodeCache; + private maxHeightDelta: number; private feeMultiplier: number; private feeMinimum: number; private cacheKey: string = "data"; constructor( cacheConfig: CacheConfig, + maxHeightDelta: number = 1, feeMultiplier: number = 1, feeMinimum: number = 1, ) { this.cache = new NodeCache(cacheConfig); + this.maxHeightDelta = maxHeightDelta; this.feeMultiplier = feeMultiplier; this.feeMinimum = feeMinimum; } @@ -43,7 +46,7 @@ export class DataProviderManager { return data; } - const dataPoints = await this.getSortedDataPoints(); + const dataPoints = await this.getRelevantDataPoints(); const blockHeight = dataPoints[0].blockHeight; const blockHash = dataPoints[0].blockHash; const feeEstimates = this.mergeFeeEstimates(dataPoints); @@ -120,6 +123,30 @@ export class DataProviderManager { return dataPoints; } + /** + * Gets relevant data points based on the height difference threshold. + * + * @returns A promise that resolves to an array of relevant data points. + */ + private async getRelevantDataPoints(): Promise { + // Get sorted data points from all providers + const dataPoints = await this.getSortedDataPoints(); + + // Filter out providers that don't meet the relevancy threshold criteria + return dataPoints.filter((dp) => { + const isRelevant = + dataPoints[0].blockHeight - dp.blockHeight <= this.maxHeightDelta; + + if (!isRelevant) { + console.warn({ + msg: `Data point from block ${dp.blockHeight} was filtered out due to relevancy threshold.`, + }); + } + + return isRelevant; + }); + } + /** * Merges fee estimates from multiple data points. * diff --git a/src/server.tsx b/src/server.tsx index ae23ff7..badc158 100644 --- a/src/server.tsx +++ b/src/server.tsx @@ -35,6 +35,7 @@ const LOGLEVEL = config.get("settings.loglevel"); const TIMEOUT = config.get("settings.timeout"); const FEE_MULTIPLIER = config.get("settings.feeMultiplier"); const FEE_MINIMUM = config.get("settings.feeMinimum"); +const MAX_HEIGHT_DELTA = config.get("settings.maxHeightDelta"); const CACHE_STDTTL = config.get("cache.stdTTL"); const CACHE_CHECKPERIOD = config.get("cache.checkperiod"); @@ -49,6 +50,7 @@ const service = new DataProviderManager( stdTTL: CACHE_STDTTL, checkperiod: CACHE_CHECKPERIOD, }, + MAX_HEIGHT_DELTA, FEE_MULTIPLIER, FEE_MINIMUM, ); diff --git a/test/DataProviderManager.test.ts b/test/DataProviderManager.test.ts index bb8743c..e91338f 100644 --- a/test/DataProviderManager.test.ts +++ b/test/DataProviderManager.test.ts @@ -76,6 +76,5 @@ test("should merge fee estimates from multiple providers correctly", async () => "2": 20000, "3": 5000, "5": 3000, - "10": 1000, }); });