From 3810eb9f42836efad31c122311c21425d19480ab Mon Sep 17 00:00:00 2001 From: Tom Kirkpatrick Date: Thu, 8 Feb 2024 09:15:41 +0000 Subject: [PATCH] Add support for bitcoind --- .gitignore | 5 +- README.md | 49 ++- bun.lockb | Bin 4086 -> 5553 bytes config/custom-environment-variables.json | 10 +- config/default.json | 12 +- config/test.json | 7 + package.json | 1 + src/custom.d.ts | 53 +-- src/server.tsx | 407 +++++++++++++++-------- test/README.md | 32 ++ test/fixtures/bitcoin-cli | 1 + test/fixtures/bitcoin.conf | 11 + test/fixtures/docker-compose.yml | 17 + test/fixtures/init.sh | 92 +++++ 14 files changed, 505 insertions(+), 192 deletions(-) create mode 100644 config/test.json create mode 100644 test/README.md create mode 100755 test/fixtures/bitcoin-cli create mode 100644 test/fixtures/bitcoin.conf create mode 100644 test/fixtures/docker-compose.yml create mode 100755 test/fixtures/init.sh diff --git a/.gitignore b/.gitignore index 7807a4f..88b6dec 100644 --- a/.gitignore +++ b/.gitignore @@ -17,4 +17,7 @@ npm-debug.log .DS_Store # Ignore IntelliJ IDEA settings -.idea/ \ No newline at end of file +.idea/ + +# Ignore local config files +config/local.* \ No newline at end of file diff --git a/README.md b/README.md index 55b0e70..6977d9a 100644 --- a/README.md +++ b/README.md @@ -62,38 +62,33 @@ In addition, you may need to adjust the settings of your Mempool or Esplora inst ## Configuration Options -This project uses the [`config`](https://www.npmjs.com/package/config) package for configuration. The configuration options are stored in `default.json` and `custom-environment-variables.json` in the `config` directory. +This project uses the [`config`](https://www.npmjs.com/package/config) package for configuration. The default configuration options are stored in [`config/default.json`](./config/default.json) and [`config/custom-environment-variables.json`](./config/custom-environment-variables.json). You can override these options by creating a `config/local.json` file or environment specific configuration files. Here are the available configuration options: -- `server.port`: The port on which the server runs. Default is `3000` -- `server.baseUrl`: The base url port on which the server is accessible. Default is `http://localhost:3000` -- `esplora.baseUrl`: The base URL of the Esplora API instance to connect to. Default is `https://blockstream.info` -- `esplora.fallbacekBaseUrl`: The base URL of the Esplora API instance to fallback to if the primary instance is unavailable. -- `mempool.baseUrl`: The base URL of the Mempool instance to connect to. Default is `https://mempool.space` -- `mempool.fallbacekBaseUrl`: The base URL of the Mempool instance to fallback to if the primary instance is unavailable. -- `mempool.depth`: The number of blocks to use for mempool-based fee estimates. Default is `6`. Valid options are `1`, `3`, and `6` -- `settings.feeMultiplier`: The multiplier to apply to the fee estimates. Default is `1` (a conservative approach to ensure that the fee estimates are always slightly higher than the raw estimates) -- `cache.stdTTL`: The standard time to live in seconds for every generated cache element. Default is `15` -- `cache.checkperiod`: The period in seconds, used for the automatic delete check interval. Default is `20` - -In addition to configuring the application through the config files, you can also override these options by setting the corresponding environment variables: - -- `PORT`: Overrides `server.port` -- `BASE_URL`: Overrides `server.baseUrl` -- `ESPLORA_BASE_URL`: Overrides `esplora.baseUrl` -- `ESPLORA_FALLBACK_BASE_URL`: Overrides `esplora.fallbackBaseUrl` -- `MEMPOOL_BASE_URL`: Overrides `mempool.baseUrl` -- `MEMPOOL_FALLBACK_BASE_URL`: Overrides `mempool.fallbackBaseUrl` -- `MEMPOOL_DEPTH`: Overrides `mempool.depth` -- `FEE_MULTIPLIER`: Overrides `settings.feeMultiplier` -- `CACHE_STDTTL`: Overrides `cache.stdTTL` -- `CACHE_CHECKPERIOD`: Overrides `cache.checkperiod` +| Config Key | Description | Default Value | Environment Variable | +| --- | --- | --- | --- | +| `server.port` | The port on which the server runs | `3000` | `PORT` | +| `server.baseUrl` | The base url port on which the server is accessible | `http://localhost:3000` | `BASE_URL` | +| `esplora.baseUrl` | The base URL of the Esplora API instance to connect to | `https://blockstream.info` | `ESPLORA_BASE_URL` | +| `esplora.fallbacekBaseUrl` | The base URL of the Esplora API instance to fallback to if the primary instance is unavailable | - | `ESPLORA_FALLBACK_BASE_URL` | +| `mempool.baseUrl` | The base URL of the Mempool instance to connect to | `https://mempool.space` | `MEMPOOL_BASE_URL` | +| `mempool.fallbacekBaseUrl` | The base URL of the Mempool instance to fallback to if the primary instance is unavailable | - | `MEMPOOL_FALLBACK_BASE_URL` | +| `mempool.depth` | The number of blocks to use for mempool-based fee estimates | `6` | `MEMPOOL_DEPTH` | +| `bitcoind.baseUrl` | The base URL of the bitcoind instance to connect to | `http://localhost:8332` | `BITCOIND_BASE_URL` | +| `bitcoind.username` | The username to use for authenticating with the bitcoind instance | - | `BITCOIND_USERNAME` | +| `bitcoind.password` | The password to use for authenticating with the bitcoind instance | - | `BITCOIND_PASSWORD` | +| `bitcoind.confTargets` | The block targets to use for history-based fee estimates | `[1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 144, 504, 1008]` | `BITCOIND_CONF_TARGETS` | +| `settings.timeout` | Timeout to use when fetching data (ms) | `5000` | `TIMEOUT` | +| `settings.feeMultiplier` | The multiplier to apply to the fee estimates | `1` | `FEE_MULTIPLIER` | +| `settings.feeMinimum` | The minimum fee (sat/vB) to use for fee estimates if we could not determine from a configured data source | `2` | `FEE_MINIMUM` | +| `cache.stdTTL` | The standard time to live in seconds for every generated cache element | `15` | `CACHE_STDTTL` | +| `cache.checkperiod` | The period in seconds, used for the automatic delete check interval | `20` | `CACHE_CHECKPERIOD` | For example, to run the server on port 4000 and connect to a local Mempool instance, you can start the server like this: ```bash -PORT=4000 MEMPOOL_BASE_URL=localhost npm start +PORT=4000 MEMPOOL_BASE_URL=localhost bun start ``` ## Development @@ -126,8 +121,8 @@ Please ensure that Docker is installed and running on your machine before runnin You can build a Docker image from the source and run it with the following scripts: -- `docker:build`: Builds a Docker image of the project. You can run this script with `npm run docker:build`. -- `docker:run`: Runs the Docker image. You can run this script with `npm run docker:run`. +- `docker:build`: Builds a Docker image of the project. You can run this script with `bun run docker:build`. +- `docker:run`: Runs the Docker image. You can run this script with `bun run docker:run`. For example, to build and run the Docker image, you can use the following commands: diff --git a/bun.lockb b/bun.lockb index 8038b4686444886a16c9d00e36ba6ee16e718eb3..f28fb66dcc3e22d5c67743f5562e2420c3e53665 100755 GIT binary patch delta 1606 zcmbVMX;4#F6n-xR$R2h9iy??nz^o9%A_JwRYN-N02vsSz1%$-PVxq`0f>RY3u{PXR zkxpevQ5I1{rZ}Q3jtXN7GqlsO6vY+?l&!YTq%frgdoIbN{OGTqnfvZN_xtX-=bU%X znG=|TK9vL*#?M;v0~4MKpZ{%ne7EdG^N`w{T2XAVM3$^w=x)C%bSIXDXGj*fs(Dt$ z`t>1*(nLeWMn){?98e6`J$+JX1i^%Ln{94@nGL$X9dHbT!3?9TdY?mW&G9PI*VFiI zbwRE6dyPT5=Zxs<$S`Kk*1qY)np?=1+)nOA8ZwG`9&9V%JPqZMD)IrEBBRK^Pyj88 z3_}`Pl+yu{Aff>W8orl+qnptCB!g^4%|LpArf5;l0pNrZNWfhddn6<`VwQ6zAQ7t& z?-is2kPtu^aKau2YXzAGBpeVo*oDvtrxip7LB0+M=6l1y_XLrSsLO5*8nE*O$m%5a zgM}3kJYyYO0>cE?rsGuOeLf6IRxCj<|HG0wj4r0dWj$#uyRp9U_GP|DxiaWAJ61-q zq2;GepLw?|)oGG8^Fs?OFJ*dLrA}|yar_PPZP)#(;|2vcX!D+h^!f{va4|_N@t43i zN3`1^88IBWsLLU)1ZSTTx-L~<{){bdo;cj6+4s`u8*|b4+Zldv<7s!V*^?dvPkQUz zo|-Dp-M{i)%CLr5+Ws)@asKE9E$g;v*!9cUho68U>J}`MIdN5< zrQ&OM7axUpwa#4D&X+iS;dATQuCmUAwv%~x)HmL`Ssa~VI@r_EuMRbD3pr8~e_%15 zTeZqC&NwCX?sS*L{*iN4f!_(VFUBWFl4hsI$}jz#+i_@A@hVPTwVuIWx;d9<3`?p{ zZWNZA`uEl`Ts}Tk64_lu_x#0VAxi%B7;0ff=%tq|h5Bc2ZvJt{1}5)n&-MAhw%@Of zX#CeU99(>$sXgDoY)5J1bs6tuE!28#`bID@)I9fcD1j*Iop7xm?YIM$s7&CeK?z-y zP#N*OULV71Nz6kH0=2BvO`R3|>-~sw3J-n6+UmXZ z$|~3i?98wY=I~J;D_rCNTljC`l5vEA6O40#S)sOftjAP8J9sQe#on+E(=mmjVyjIy zs>B&-*&-+@0-a^Yuc5jO)j(*ZOcExMqepBNnTT9DJK_5Q$E$?uKt;2egXJzWel$`p zkx3L=2!iTGRJ);(#}qP&TrLTwx)s%s011^SC34w+oCySnuq<~pD0Zr8!5Jl7!( zN1d&dqt56eCz{n{Q@z~LB*zzL2?Q~?v78Sz`m`KPcDndbZkl-~S4j$2??}bc)vDMU z?-);nDtMKo5_#}rycJs{`pm*;eL-%9PMV#Qo?%<&>re~NkC(3lBl$Y9H5vqG*I^QD o;z?``U%qVPaCW*TO@n;+LXmYTwi*>nSF2(iBcui{g$Gpq4}3Sh+W-In delta 928 zcmdm}{Y`#?o@VZy)rmJ&eOKY0X&8OV^;24gm4czk_329`R`WW}*b}Og#?Jr-T$4Yt z2v5x75l-TQ2tY_#28IULiJj`QCP2O*kfRQyd4V)gf&r+qL5PWgfpg+pBU zJsABbuVl36>;_630i~HHZ)7xQ+&fv5$)553BrGaKz0|kKMU?4O(kx^Tb4af#zkSIG?9F=CA{FOz1vJ5MaDoB!O z)te8nsxeKr3#CM$zEt_5T z=H_^2#Hy@h1_p$5y$vvU940E~g8~BNO%Nbg3}_TXR&jnFF#H*mfd+!2 z6v)IUrbVnm2Ia|&+~%8aaPMN|S^-T}d$=Yi@Y%?LtoZjI0zhm~D7}Cx;o+XVjn7I9 zBm|0PkPj`mfdmjFfTWZr+wiOKfutuug{SaLuH#pmyn^3vvVedIqw8c>0be^;paPI@ zG?D~4Ui~ci&%~$&G6D#|Nfszm{U-fvhtO*!CPqU&13g0v28Me)5Eb`$fUXpo>?5e7 z#Q{w0AP@)3CpM)Pb_#|H#hF#9`Dr>pR$@+OdR~4S*W?v~uA6@fnlet#6Q0axpjVP! fl~kIiTT)q&T0FT>NP4oCh!mTl1xS3dAomOaQvB0d diff --git a/config/custom-environment-variables.json b/config/custom-environment-variables.json index bfd14e1..53a24fc 100644 --- a/config/custom-environment-variables.json +++ b/config/custom-environment-variables.json @@ -4,7 +4,9 @@ "port": "PORT" }, "settings": { - "feeMultiplier": "FEE_MULTIPLIER" + "timeout": "TIMEOUT", + "feeMultiplier": "FEE_MULTIPLIER", + "feeMinimum" : "FEE_MINIMUM" }, "mempool": { "baseUrl": "MEMPOOL_BASE_URL", @@ -15,6 +17,12 @@ "baseUrl": "ESPLORA_BASE_URL", "fallbackBaseUrl": "ESPLORA_FALLBACK_BASE_URL" }, + "bitcoind": { + "baseUrl": "BITCOIND_BASE_URL", + "username": "BITCOIND_USERNAME", + "password": "BITCOIND_PASSWORD", + "confTargets": "BITCOIND_CONF_TARGETS" + }, "cache": { "stdTTL": "CACHE_STD_TTL", "checkperiod": "CACHE_CHECKPERIOD" diff --git a/config/default.json b/config/default.json index 01aa45f..08b7274 100644 --- a/config/default.json +++ b/config/default.json @@ -4,7 +4,9 @@ "port": 3000 }, "settings": { - "feeMultiplier": 1 + "timeout": 5000, + "feeMultiplier": 1, + "feeMinimum": 2 }, "mempool": { "baseUrl": "https://mempool.space", @@ -15,8 +17,14 @@ "baseUrl": "https://blockstream.info", "fallbackBaseUrl": null }, + "bitcoind": { + "baseUrl": null, + "username": null, + "password": null, + "confTargets": [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 144, 504, 1008] + }, "cache": { "stdTTL": 15, "checkperiod": 20 } -} \ No newline at end of file +} diff --git a/config/test.json b/config/test.json new file mode 100644 index 0000000..2f4ffa5 --- /dev/null +++ b/config/test.json @@ -0,0 +1,7 @@ +{ + "bitcoind": { + "baseUrl": "http://127.0.0.1:18445", + "username": "user", + "password": "pass" + } +} \ No newline at end of file diff --git a/package.json b/package.json index c840a4d..3043a3c 100644 --- a/package.json +++ b/package.json @@ -19,6 +19,7 @@ "author": "Tom Kirkpatrick (https://twitter.com/mrfelton)", "license": "MIT", "dependencies": { + "bitcoind-rpc": "0.9.1", "config": "3.3.9", "hono": "3.11.12", "node-cache": "5.1.2" diff --git a/src/custom.d.ts b/src/custom.d.ts index 5228cbb..696d5b4 100644 --- a/src/custom.d.ts +++ b/src/custom.d.ts @@ -1,33 +1,48 @@ +// MempoolFeeEstimates represents the fee estimates for different transaction speeds. type MempoolFeeEstimates = { - [key: string]: number | undefined; - fastestFee: number; - halfHourFee: number; - hourFee: number; - economyFee: number; - minimumFee: number; -}; - -type EsploraFeeEstimates = { - [key: number]: number; + [key: string]: number; // dynamic keys with number as value (sat/vb) + fastestFee: number; // fee for the fastest transaction speed (sat/vb) + halfHourFee: number; // fee for half an hour transaction speed (sat/vb) + hourFee: number; // fee for an hour transaction speed (sat/vb) + economyFee: number; // fee for economy transaction speed (sat/vb) + minimumFee: number; // minimum relay fee (sat/vb) }; +// FeeByBlockTarget represents the fee by block target. type FeeByBlockTarget = { - [key: string]: number; + [key: string]: number; // fees by confirmation target }; +// Estimates represents the current block hash and fee by block target. type Estimates = { - current_block_hash: string | null; - fee_by_block_target: FeeByBlockTarget; + current_block_hash: string | null; // current block hash + fee_by_block_target: FeeByBlockTarget; // fee by block target (in sat/kb) }; +// BlockTargetMapping represents the mapping of block targets. type BlockTargetMapping = { - [key: number]: string; + [key: number]: string; // dynamic numeric keys with string as value }; +// SiteData represents the data of a site. interface SiteData { - title: string, - subtitle: string, - children?: any -} + title: string, // title of the site + subtitle: string, // subtitle of the site + children?: any // children of the site (optional) +}; + +// ExpectedResponseType represents the expected response type for an http request. +type ExpectedResponseType = 'json' | 'text'; // can be either 'json' or 'text' -type ExpectedResponseType = 'json' | 'text'; \ No newline at end of file +// BatchRequest represents a bitcoind batch request response. +interface BitcoindRpcBatchResponse { + result?: EstimateSmartFeeResponse; + error?: any; +}; + +// EstimateSmartFeeResponse represents the response of the estimatesmarttee method. +interface EstimateSmartFeeResponse { + feerate?: number, // estimate fee rate in BTC/kB (only present if no errors were encountered) + errors?: [string], // errors encountered during processing (if there are any) + blocks?: number // block number where estimate was found +}; diff --git a/src/server.tsx b/src/server.tsx index 62a7aa0..91c8ff2 100644 --- a/src/server.tsx +++ b/src/server.tsx @@ -6,50 +6,80 @@ import { cors } from 'hono/cors' import { serveStatic } from 'hono/bun' import config from 'config' import NodeCache from 'node-cache'; +import RpcClient from 'bitcoind-rpc' // Get application configuration values from the config package. -const port = config.get('server.port'); -const baseUrl = config.get('server.baseUrl'); -const esploraBaseUrl = config.get('esplora.baseUrl'); -const esploraFallbackBaseUrl = config.get('esplora.fallbackBaseUrl'); -const mempoolBaseUrl = config.get('mempool.baseUrl'); -const mempoolFallbackBaseUrl = config.get('mempool.fallbackBaseUrl'); -const mempoolDepth = config.get('mempool.depth'); -const feeMultiplier = config.get('settings.feeMultiplier'); -const stdTTL = config.get('cache.stdTTL'); -const checkperiod = config.get('cache.checkperiod'); - -// Set the timeout for fetch requests. -const TIMEOUT: number = 2500; +const PORT = config.get('server.port'); +const BASE_URL = config.get('server.baseUrl'); + +const ESPLORA_BASE_URL = config.get('esplora.baseUrl'); +const ESPLORA_FALLBACK_BASE_URL = config.get('esplora.fallbackBaseUrl'); + +const MEMPOOL_BASE_URL = config.get('mempool.baseUrl'); +const MEMPOOL_FALLBACK_BASE_URL = config.get('mempool.fallbackBaseUrl'); +const MEMPOOL_DEPTH = config.get('mempool.depth'); + +const BITCOIND_BASE_URL = config.get('bitcoind.baseUrl'); +const BITCOIND_USERNAME = config.get('bitcoind.username'); +const BITCOIND_PASSWORD = config.get('bitcoind.password'); +const BITCOIND_CONF_TARGETS = config.get('bitcoind.confTargets'); + +const TIMEOUT = config.get('settings.timeout'); +const FEE_MULTIPLIER = config.get('settings.feeMultiplier'); +const FEE_MINIMUM = config.get('settings.feeMinimum'); +const CACHE_STDTTL = config.get('cache.stdTTL'); +const CACHE_CHECKPERIOD = config.get('cache.checkperiod'); + // Log the configuration values. -console.info('---'); -console.info(`Using port: ${port}`); -console.info(`Using base URL: ${baseUrl}`); -console.info(`Using Esplora base URL: ${esploraBaseUrl}`); -console.info(`Using Esplora fallback base URL: ${esploraFallbackBaseUrl}`); -console.info(`Using Mempool base URL: ${mempoolBaseUrl}`); -console.info(`Using Mempool fallback base URL: ${mempoolFallbackBaseUrl}`); -console.info(`Using Mempool estimation depth: ${mempoolDepth}`); -console.info(`Using fee multiplier: ${feeMultiplier}`); -console.info(`Using cache stdTTL: ${stdTTL}`); -console.info(`Using cache checkperiod: ${checkperiod}`); -console.info('---'); +console.info(JSON.stringify({ + mempoolSettings: { + baseUrl: MEMPOOL_BASE_URL, + fallbackBaseUrl: MEMPOOL_FALLBACK_BASE_URL, + estimationDepth: MEMPOOL_DEPTH + }, + esploraSettings: { + baseUrl: ESPLORA_BASE_URL, + fallbackBaseUrl: ESPLORA_FALLBACK_BASE_URL + }, + bitcoindSettings: { + baseUrl: BITCOIND_BASE_URL, + username: BITCOIND_USERNAME, + password: '******' + }, + appSettings: { + serverPort: PORT, + baseUrl: BASE_URL, + feeMinimum: FEE_MINIMUM, + feeMultiplier: FEE_MULTIPLIER, + timeout: TIMEOUT, + cacheStdTTL: CACHE_STDTTL, + cacheCheckPeriod: CACHE_CHECKPERIOD + } +})); // Constants -const MEMPOOL_TIP_HASH_URL = mempoolBaseUrl && `${mempoolBaseUrl}/api/blocks/tip/hash`; -const ESPLORA_TIP_HASH_URL = esploraBaseUrl && `${esploraBaseUrl}/api/blocks/tip/hash`; -const MEMPOOL_FEES_URL = mempoolBaseUrl && `${mempoolBaseUrl}/api/v1/fees/recommended`; -const ESPLORA_FEE_ESTIMATES_URL = esploraBaseUrl && `${esploraBaseUrl}/api/fee-estimates`; +const MEMPOOL_TIP_HASH_URL = MEMPOOL_BASE_URL && `${MEMPOOL_BASE_URL}/api/blocks/tip/hash`; +const ESPLORA_TIP_HASH_URL = ESPLORA_BASE_URL && `${ESPLORA_BASE_URL}/api/blocks/tip/hash`; +const MEMPOOL_FEES_URL = MEMPOOL_BASE_URL && `${MEMPOOL_BASE_URL}/api/v1/fees/recommended`; +const ESPLORA_FEE_ESTIMATES_URL = ESPLORA_BASE_URL && `${ESPLORA_BASE_URL}/api/fee-estimates`; -const MEMPOOL_TIP_HASH_URL_FALLBACK = mempoolFallbackBaseUrl && `${mempoolFallbackBaseUrl}/api/blocks/tip/hash`; -const ESPLORA_TIP_HASH_URL_FALLBACK = esploraFallbackBaseUrl && `${esploraFallbackBaseUrl}/api/blocks/tip/hash`; -const MEMPOOL_FEES_URL_FALLBACK = mempoolFallbackBaseUrl && `${mempoolFallbackBaseUrl}/api/v1/fees/recommended`; -const ESPLORA_FEE_ESTIMATES_URL_FALLBACK = esploraFallbackBaseUrl && `${esploraFallbackBaseUrl}/api/fee-estimates`; +const MEMPOOL_TIP_HASH_URL_FALLBACK = MEMPOOL_FALLBACK_BASE_URL && `${MEMPOOL_FALLBACK_BASE_URL}/api/blocks/tip/hash`; +const ESPLORA_TIP_HASH_URL_FALLBACK = ESPLORA_FALLBACK_BASE_URL && `${ESPLORA_FALLBACK_BASE_URL}/api/blocks/tip/hash`; +const MEMPOOL_FEES_URL_FALLBACK = MEMPOOL_FALLBACK_BASE_URL && `${MEMPOOL_FALLBACK_BASE_URL}/api/v1/fees/recommended`; +const ESPLORA_FEE_ESTIMATES_URL_FALLBACK = ESPLORA_FALLBACK_BASE_URL && `${ESPLORA_FALLBACK_BASE_URL}/api/fee-estimates`; // Initialize the cache. -const cache = new NodeCache({ stdTTL: stdTTL, checkperiod: checkperiod }); const CACHE_KEY = 'estimates'; +const cache = new NodeCache({ stdTTL: CACHE_STDTTL, checkperiod: CACHE_CHECKPERIOD }); + + +/** + * Helper function to extract value from a fulfilled promise. + */ +function getValueFromFulfilledPromise(result: PromiseSettledResult) { + return result && result.status === "fulfilled" && result.value ? result.value : null; +} // NOTE: fetch signal abortcontroller does not work on Bun. // See https://github.com/oven-sh/bun/issues/2489 @@ -63,9 +93,8 @@ async function fetchWithTimeout(url: string, timeout: number = TIMEOUT): Promise return Promise.race([fetchPromise, timeoutPromise]) as Promise; } - /** - * Fetches data from the given URL and returns the response as a string or object. + * Fetches data from the given URL and validates and processes the response. */ async function fetchAndProcess(url: string, expectedResponseType: ExpectedResponseType): Promise { const response = await fetchWithTimeout(url, TIMEOUT); @@ -90,7 +119,7 @@ async function fetchAndProcess(url: string, expectedResponseType: ExpectedRespon } /** - * Fetches data from the given URL and returns the response as a string or object. + * Fetches data from the given URL with a timeout, fallback, and error handling. */ async function fetchAndHandle(url: string, expectedResponseType: ExpectedResponseType, fallbackUrl?: string): Promise { try { @@ -113,29 +142,6 @@ async function fetchAndHandle(url: string, expectedResponseType: ExpectedRespons } } - -// Initialize the Express app. -const app = new Hono(); -console.info(`Fee Estimates available at ${baseUrl}/v1/fee-estimates`); - -// Add a health/ready endpoint. -app.get('/health/ready', async (c) => { - return c.text('OK'); -}); - -// Add a health/live endpoint. -app.get('/health/live', async (c) => { - return c.text('OK'); -}); - -// Add middleware. -app.use('*', logger()) -app.use('*', etag()) -app.use('*', cors({ - origin: '*', -})) -app.use('/static/*', serveStatic({ root: './' })) - /** * Fetches mempool fees. */ @@ -147,8 +153,8 @@ async function fetchMempoolFees() : Promise { const res = await Promise.allSettled(tasks); console.debug('Fetched mempool fees', res); - let res0 = res[0] && getValueFromFulfilledPromise(res[0]); - let res1 = res[1] && getValueFromFulfilledPromise(res[1]); + let res0 = getValueFromFulfilledPromise(res[0]); + let res1 = getValueFromFulfilledPromise(res[1]); // If all of the response properties are 1, then the response is an error (probably the mempool data is not available). const isRes0Invalid = !res0 || (Object.values(res0).every((value) => value === 1)); @@ -164,7 +170,7 @@ async function fetchMempoolFees() : Promise { /** * Fetches esplora fees. */ -async function fetchEsploraFees() : Promise { +async function fetchEsploraFees() : Promise { const tasks = [ ESPLORA_FEE_ESTIMATES_URL && fetchAndHandle(ESPLORA_FEE_ESTIMATES_URL, 'json'), ESPLORA_FEE_ESTIMATES_URL_FALLBACK && fetchAndHandle(ESPLORA_FEE_ESTIMATES_URL_FALLBACK, 'json'), @@ -172,10 +178,84 @@ async function fetchEsploraFees() : Promise { const res = await Promise.allSettled(tasks); console.debug('Fetched esplora fees', res); - let res0 = res[0] && getValueFromFulfilledPromise(res[0]); - let res1 = res[1] && getValueFromFulfilledPromise(res[1]); + let res0 = getValueFromFulfilledPromise(res[0]); + let res1 = getValueFromFulfilledPromise(res[1]); - return res0 || res1; + return res0 || res1 || null; +} + +/** + * Fetches bitcoind fees. + */ +async function fetchBitcoindFees() : Promise { + if (!BITCOIND_BASE_URL) { + return null; + } + + return new Promise((resolve, _) => { + var result : FeeByBlockTarget = {}; + + // Define the targets for which to fetch fee estimates. + const targets = BITCOIND_CONF_TARGETS; + + // Extract protocol, host, port from bitcoindBaseUrl. + var { protocol, hostname: host, port } = new URL(BITCOIND_BASE_URL); + + // Strip the trailing colon from the protocol. + protocol = protocol.replace(/.$/, '') + + var config = { + protocol, + host, + port, + user: BITCOIND_USERNAME, + pass: BITCOIND_PASSWORD, + }; + + var rpc = new RpcClient(config); + + function batchCall() { + targets.forEach(function (target) { + rpc.estimatesmartfee(target); + }); + } + + rpc.batch(batchCall, (err: Error | null, response: BitcoindRpcBatchResponse[]) => { + if (err) { + console.error('Unable to fetch fee estimates from bitcoind', err); + resolve(null); + } else { + targets.forEach((target, i) => { + var feeRate = response[i].result?.feerate; + if (feeRate) { + console.debug(`Raw bitcoind estimate for target ${target}: ${feeRate} BTC`); + + // convert the returned value to satoshis, as it's currently returned in BTC. + const satPerKB = feeRate * 100000000; + + console.debug(`Converted bitcoind estimate for target ${target}: ${satPerKB} sat/vb`); + result[target] = applyFeeMultiplier(satPerKB); + } else { + console.error(`Failed to fetch fee estimate from bitcoind for confirmation target ${target}`, response[i].result?.errors); + } + }); + + resolve(result); + } + }); + }); +} + +function processEstimates(estimates: FeeByBlockTarget, applyMultiplier = true, convert = false) : FeeByBlockTarget { + for (const [blockTarget, fee] of Object.entries(estimates) as [string, number][]) { + if (applyMultiplier) { + estimates[blockTarget] = applyFeeMultiplier(fee); + } + if (convert) { + estimates[blockTarget] = Math.ceil(fee * 1000); + } + } + return estimates; } /** @@ -188,10 +268,10 @@ async function fetchBlocksTipHash() : Promise { ].filter(Boolean); const res = await Promise.allSettled(tasks); - let res0 = res[0] && getValueFromFulfilledPromise(res[0]); - let res1 = res[1] && getValueFromFulfilledPromise(res[1]); + let res0 = getValueFromFulfilledPromise(res[0]); + let res1 = getValueFromFulfilledPromise(res[1]); - return res0 || res1; + return res0 || res1 || null; } /** @@ -200,40 +280,38 @@ async function fetchBlocksTipHash() : Promise { async function getEstimates() : Promise { let estimates: Estimates | undefined = cache.get(CACHE_KEY); - if (!estimates) { - - const tasks = [ - await fetchMempoolFees(), - await fetchEsploraFees(), - await fetchBlocksTipHash(), - ]; - const [result1, result2, result3] = await Promise.allSettled(tasks); - const mempoolFeeEstimates = getValueFromFulfilledPromise(result1); - const esploraFeeEstimates = getValueFromFulfilledPromise(result2); - const blocksTipHash = getValueFromFulfilledPromise(result3); - - const feeByBlockTarget = calculateFees(mempoolFeeEstimates, esploraFeeEstimates); - - estimates = { - current_block_hash: blocksTipHash, - fee_by_block_target: feeByBlockTarget - }; - - cache.set(CACHE_KEY, estimates); + if (estimates) { + console.info('Got estimates (from cache)', estimates); + return estimates; } - console.debug('Got estimates', estimates); + const tasks = [ + await fetchMempoolFees(), + await fetchEsploraFees(), + await fetchBitcoindFees(), + await fetchBlocksTipHash(), + ]; + const [result1, result2, result3, result4] = await Promise.allSettled(tasks); + const mempoolFeeEstimates = getValueFromFulfilledPromise(result1); + const esploraFeeEstimates = getValueFromFulfilledPromise(result2); + const bitcoindFeeEstimates = getValueFromFulfilledPromise(result3); + const blocksTipHash = getValueFromFulfilledPromise(result4); + + estimates = { + current_block_hash: blocksTipHash, + fee_by_block_target: calculateFees(mempoolFeeEstimates, esploraFeeEstimates, bitcoindFeeEstimates), + }; + + cache.set(CACHE_KEY, estimates); + + console.info('Got estimates', estimates); return estimates; } /** - * Helper function to extract value from a fulfilled promise. + * Get the fee estimates that are above the desired mempool depth. */ -function getValueFromFulfilledPromise(result: PromiseSettledResult) { - return result.status === "fulfilled" && result.value ? result.value : null; -} - -function calculateMempoolFees(mempoolFeeEstimates: MempoolFeeEstimates | null | undefined): FeeByBlockTarget { +function extractMempoolFees(mempoolFeeEstimates: MempoolFeeEstimates): FeeByBlockTarget { const feeByBlockTarget: FeeByBlockTarget = {}; if (mempoolFeeEstimates) { @@ -242,71 +320,90 @@ function calculateMempoolFees(mempoolFeeEstimates: MempoolFeeEstimates | null | 3: 'halfHourFee', 6: 'hourFee' }; - for (let i = 1; i <= mempoolDepth; i++) { + for (let i = 1; i <= MEMPOOL_DEPTH; i++) { const feeProperty = blockTargetMapping[i]; - if (feeProperty && mempoolFeeEstimates[feeProperty] !== undefined) { - const adjustedFee = Math.round(mempoolFeeEstimates[feeProperty]! * 1000 * feeMultiplier); - feeByBlockTarget[i] = adjustedFee; + if (feeProperty && mempoolFeeEstimates[feeProperty]) { + feeByBlockTarget[i] = mempoolFeeEstimates[feeProperty]; } } } + return feeByBlockTarget; } -function calculateMinMempoolFee(feeByBlockTarget: FeeByBlockTarget) { +/** + * Gets the lowest fee from the feeByBlockTarget object. + */ +function getLowestFee(feeByBlockTarget: FeeByBlockTarget) : number | null { const values = Object.values(feeByBlockTarget); - return values.length > 0 ? Math.min(...values) : undefined; + return values.length > 0 ? Math.min(...values) : null; } -function calculateEsploraFees(esploraFeeEstimates: EsploraFeeEstimates | null | undefined): FeeByBlockTarget { - const feeByBlockTarget: FeeByBlockTarget = {}; - if (esploraFeeEstimates) { - for (const [blockTarget, fee] of Object.entries(esploraFeeEstimates)) { - const adjustedFee = Math.round(fee * 1000 * feeMultiplier); - feeByBlockTarget[blockTarget] = adjustedFee; +/** + * Applies the fee multiplier to the given estimate. + */ +function applyFeeMultiplier(fee: number) : number { + return fee * FEE_MULTIPLIER; +} + +/** + * Filters the estimates to remove any that are lower than the desired minimum fee. + */ +function filterEstimates(feeByBlockTarget: FeeByBlockTarget, minFee: number): FeeByBlockTarget { + const result: FeeByBlockTarget = {}; + for (const [blockTarget, fee] of Object.entries(feeByBlockTarget)) { + if (fee >= minFee) { + result[blockTarget] = fee; } } - return feeByBlockTarget; + return result; } -function filterEstimates(feeByBlockTarget: FeeByBlockTarget, minFee: number | undefined): FeeByBlockTarget { - const filteredEstimates: FeeByBlockTarget = {}; - let lastAddedFee: number | null = null; - - for (const key of Object.keys(feeByBlockTarget)) { - const fee = feeByBlockTarget[key]; - if (lastAddedFee && fee >= lastAddedFee) continue; - if (minFee && fee < minFee) continue; - - filteredEstimates[key] = fee; - - lastAddedFee = fee; +/** + * Appends estimates to the feeByBlockTarget object. + */ +function addFeeEstimates(feeByBlockTarget: FeeByBlockTarget, feeEstimates: FeeByBlockTarget) { + const lowestFee = getLowestFee(feeByBlockTarget) + for (const [blockTarget, fee] of Object.entries(feeEstimates)) { + if (!lowestFee || fee < lowestFee) { + feeByBlockTarget[blockTarget] = fee; + } } - - return filteredEstimates; } -function calculateFees(mempoolFeeEstimates: MempoolFeeEstimates | null | undefined, esploraFeeEstimates: EsploraFeeEstimates | null | undefined) { +/** + * Calculates the fees. + */ +function calculateFees(mempoolFeeEstimates: MempoolFeeEstimates, esploraFeeEstimates: FeeByBlockTarget, bitcoindFeeEstimates: FeeByBlockTarget) { let feeByBlockTarget: FeeByBlockTarget = {}; - // Get the minimum fee. If the mempool fee estimates are not available, use a default value of 5 sat/vbyte as a safety net. - const minFee = (mempoolFeeEstimates?.minimumFee ?? 5) * 1000; - // Get the mempool fee estimates. - feeByBlockTarget = calculateMempoolFees(mempoolFeeEstimates); - const minMempoolFee = calculateMinMempoolFee(feeByBlockTarget); + if (mempoolFeeEstimates) { + let estimates = extractMempoolFees(mempoolFeeEstimates); + estimates = processEstimates(estimates, true, true); + addFeeEstimates(feeByBlockTarget, estimates); + } - // Add the esplora fee estimates. - const esploraFeeEstimatesAdjusted = calculateEsploraFees(esploraFeeEstimates); + // Add the bitcoind fee estimates. + if (bitcoindFeeEstimates) { + const estimates = processEstimates(bitcoindFeeEstimates, true, false); + addFeeEstimates(feeByBlockTarget, estimates); + } - for (const [blockTarget, fee] of Object.entries(esploraFeeEstimatesAdjusted)) { - if (!minMempoolFee || fee < minMempoolFee) { - feeByBlockTarget[blockTarget] = fee; - } + // Add the esplora fee estimates. + if (esploraFeeEstimates) { + const estimates = processEstimates(esploraFeeEstimates, true, true); + addFeeEstimates(feeByBlockTarget, estimates); } - // Filter the estimates. - feeByBlockTarget = filterEstimates(feeByBlockTarget, minFee); + // Get the minimum fee. If the mempool fee estimates are not available, use a default value of FEE_MINIMUM sat/vbyte as a safety net. + const minFee = (mempoolFeeEstimates?.minimumFee ?? FEE_MINIMUM) * 1000; + console.debug('Using minimum fee:', minFee); + + // Return fees filterd to remove any that are lower than the determined minimum fee. + if (minFee) { + return filterEstimates(feeByBlockTarget, minFee); + } return feeByBlockTarget; } @@ -348,7 +445,7 @@ const Content = (props: { siteData: SiteData; estimates: Estimates }) => ( 
-        curl -L -X GET '{baseUrl}/v1/fee-estimates'
+        curl -L -X GET '{BASE_URL}/v1/fee-estimates'
       
@@ -363,17 +460,43 @@ const Content = (props: { siteData: SiteData; estimates: Estimates }) => (
   
 );
 
+// Define the app.
+
+// Initialize the Express app.
+const app = new Hono();
+console.info(`Fee Estimates available at ${BASE_URL}/v1/fee-estimates`);
+
+// Add a health/ready endpoint.
+app.get('/health/ready', async (c) => {
+  return c.text('OK');
+});
+
+// Add a health/live endpoint.
+app.get('/health/live', async (c) => {
+  return c.text('OK');
+});
+
+// Add middleware.
+app.use('*', logger())
+app.use('*', etag())
+app.use('*', cors({
+  origin: '*',
+}))
+app.use('/static/*', serveStatic({ root: './' }))
+
+// Define the routes.
+
 /**
  * Returns the current fee estimates for the Bitcoin network, rendered as HTML.
  */
 app.get('/', async (c) => {
-  let estimates : Estimates | undefined;
+  let estimates : Estimates;
 
   try {
     estimates = await getEstimates();
 
     // Set cache headers.
-    c.res.headers.set('Cache-Control', `public, max-age=${stdTTL}`)
+    c.res.headers.set('Cache-Control', `public, max-age=${CACHE_STDTTL}`)
 
   } catch (error) {
     console.error(error);
@@ -402,7 +525,7 @@ app.get('/v1/fee-estimates', async (c) => {
     let estimates = await getEstimates();
 
     // Set cache headers.
-    c.res.headers.set('Cache-Control', `public, max-age=${stdTTL}`)
+    c.res.headers.set('Cache-Control', `public, max-age=${CACHE_STDTTL}`)
 
     // Return the estimates.
     return c.json(estimates);
@@ -414,7 +537,7 @@ app.get('/v1/fee-estimates', async (c) => {
 });
 
 export default {
-  port,
+  port: PORT,
   fetch: app.fetch,
 }
 
diff --git a/test/README.md b/test/README.md
new file mode 100644
index 0000000..1c4b993
--- /dev/null
+++ b/test/README.md
@@ -0,0 +1,32 @@
+# Testing
+
+Start the docker stack with the following command:
+
+```bash
+cd test/fixtures && docker-compose up
+```
+
+To fill the mempool, exec onto the container and run the `init.sh` script:
+
+```bash
+docker exec bitcoin-blended-fee-estimator-bitcoind-1 bash
+/init.sh
+```
+
+Run the app with test config:
+
+```bash
+NODE_ENV=test bun run dev
+```
+
+Run commands against the running docker stack:
+
+```bash
+./test/fixtures/bitcoin-cli estimatesmartfee 2
+```
+
+Stop and cleanup the docker stack:
+
+```bash
+docker-compose down --volumes
+```
\ No newline at end of file
diff --git a/test/fixtures/bitcoin-cli b/test/fixtures/bitcoin-cli
new file mode 100755
index 0000000..a016383
--- /dev/null
+++ b/test/fixtures/bitcoin-cli
@@ -0,0 +1 @@
+docker exec bitcoin-blended-fee-estimator-bitcoind-1 bitcoin-cli -rpcport=18445 -rpcuser=user -rpcpassword=pass -regtest "$@"
\ No newline at end of file
diff --git a/test/fixtures/bitcoin.conf b/test/fixtures/bitcoin.conf
new file mode 100644
index 0000000..7199f86
--- /dev/null
+++ b/test/fixtures/bitcoin.conf
@@ -0,0 +1,11 @@
+regtest=1
+
+[regtest]
+
+rpcuser=user
+rpcpassword=pass
+rpcport=18445
+rpcbind=0.0.0.0:18445
+rpcallowip=0.0.0.0/0
+whitelist=0.0.0.0/0
+wallet=default
diff --git a/test/fixtures/docker-compose.yml b/test/fixtures/docker-compose.yml
new file mode 100644
index 0000000..0e01391
--- /dev/null
+++ b/test/fixtures/docker-compose.yml
@@ -0,0 +1,17 @@
+version: "3.9"
+
+services:
+
+  bitcoind:
+    image: us-east1-docker.pkg.dev/zap-strike-infrastructure/zap-container-registry/bitcoind:25.1.0
+    restart: unless-stopped
+    ports:
+      - "18445:18445"
+      - "18444:18444"
+    volumes:
+      - "bitcoind:/bitcoin/.bitcoin"
+      - "./bitcoin.conf:/bitcoin/.bitcoin/bitcoin.conf"
+      - "./init.sh:/init.sh"
+
+volumes:
+    bitcoind:
diff --git a/test/fixtures/init.sh b/test/fixtures/init.sh
new file mode 100755
index 0000000..32736d8
--- /dev/null
+++ b/test/fixtures/init.sh
@@ -0,0 +1,92 @@
+#!/bin/bash
+
+DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
+
+waitFor() {
+  until $@; do
+    >&2 echo "$@ unavailable - waiting..."
+    sleep 1
+  done
+}
+
+# bitcoin-cli() {
+#   $DIR/bitcoin-cli $@
+# }
+
+createBitcoindWallet() {
+  bitcoin-cli createwallet default || bitcoin-cli loadwallet default || true
+}
+
+waitForNodes() {
+  waitFor bitcoin-cli getnetworkinfo
+}
+
+mineBlocks() {
+  ADDRESS=$1
+  AMOUNT=${2:-1}
+  echo Mining $AMOUNT blocks to $ADDRESS...
+  bitcoin-cli generatetoaddress $AMOUNT $ADDRESS
+  sleep 0.5 # waiting for blocks to be propagated
+}
+
+generateAddresses() {
+  BITCOIN_ADDRESS=$(bitcoin-cli getnewaddress)
+  echo BITCOIN_ADDRESS: $BITCOIN_ADDRESS
+}
+
+initBitcoinChain() {
+  mineBlocks $BITCOIN_ADDRESS 500
+}
+
+# Function to send transaction
+send_transaction() {
+    address="$1"
+    amount="$2"
+    fee_rate="$3"
+    txid=$(bitcoin-cli -regtest -named sendtoaddress "$address" amount=$amount fee_rate=$fee_rate)
+    echo "Transaction $txid sent to $address with fee rate $fee_rate sat/vB"
+}
+
+init() {
+    # Generate addresses to send transactions to
+    num_addresses=100
+    addresses=()
+    for ((i=0; i