From e31f01619afc9f172d54bbae918f02faecb2101e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jos=C3=A9=20Luis=20Landabaso=20D=C3=ADaz?= Date: Fri, 28 Jun 2024 14:45:14 +0200 Subject: [PATCH] feat: improve transaction ordering and expose compareTxOrder - Improved transaction ordering to ensure correct order for transactions in the same block that depend on each other - Exposed compareTxOrder function for external transaction sorting - Added TxoMap to map transactions outputs with corresponding indexed descriptors - Enhanced getDescriptor performance by leveraging TxoMap - Updated deriveDataFactory and DiscoveryFactory to handle transaction ordering and TxoMap - Updated tests to cover new functionalities --- package-lock.json | 4 +- package.json | 2 +- src/deriveData.ts | 234 +++++++++++++++++++++---------- src/discovery.ts | 147 ++++++++++--------- src/index.ts | 3 + src/types.ts | 30 ++++ test/integration/regtest.test.ts | 62 +++++--- 7 files changed, 310 insertions(+), 172 deletions(-) diff --git a/package-lock.json b/package-lock.json index 2f1b3ae..ccecbb2 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,12 +1,12 @@ { "name": "@bitcoinerlab/discovery", - "version": "1.2.2", + "version": "1.2.3", "lockfileVersion": 2, "requires": true, "packages": { "": { "name": "@bitcoinerlab/discovery", - "version": "1.2.2", + "version": "1.2.3", "license": "MIT", "dependencies": { "@bitcoinerlab/descriptors": "^2.1.0", diff --git a/package.json b/package.json index 5ffce9e..6264047 100644 --- a/package.json +++ b/package.json @@ -2,7 +2,7 @@ "name": "@bitcoinerlab/discovery", "description": "A TypeScript library for retrieving Bitcoin funds from ranged descriptors, leveraging @bitcoinerlab/explorer for standardized access to multiple blockchain explorers.", "homepage": "https://github.com/bitcoinerlab/discovery", - "version": "1.2.2", + "version": "1.2.3", "author": "Jose-Luis Landabaso", "license": "MIT", "prettier": "@bitcoinerlab/configs/prettierConfig.json", diff --git a/src/deriveData.ts b/src/deriveData.ts index 8514f64..3ccf5e8 100644 --- a/src/deriveData.ts +++ b/src/deriveData.ts @@ -3,7 +3,7 @@ import memoizee from 'memoizee'; import { memoizeOneWithShallowArraysCheck } from './memoizers'; -import { shallowEqualArrays } from 'shallow-equal'; +import { shallowEqualArrays, shallowEqualObjects } from 'shallow-equal'; import { NetworkId, OutputData, @@ -17,7 +17,9 @@ import { TxAttribution, TxId, TxData, - Stxo + Stxo, + TxWithOrder, + TxoMap } from './types'; import { Transaction, Network } from 'bitcoinjs-lib'; import { DescriptorsFactory } from '@bitcoinerlab/descriptors'; @@ -60,6 +62,64 @@ export function deriveDataFactory({ descriptorsCacheSize: number; outputsPerDescriptorCacheSize: number; }) { + /** + * Compares two transactions based on their blockHeight and input dependencies. + * Can be used as callback in Array.sort function to sort from old to new. + * + * @param txWithOrderA - The first transaction data to compare. + * @param txWithOrderB - The second transaction data to compare. + * + * + * txWithOrderA and txWithOrderB should contain the `blockHeight` (use 0 if + * in the mempool) and either `tx` (`Transaction` type) or `txHex` (the + * hexadecimal representation of the transaction) + * + * @returns < 0 if txWithOrderA is older than txWithOrderB, > 01 if txWithOrderA is newer than txWithOrderB, and 0 if undecided. + */ + function compareTxOrder( + txWithOrderA: TA, + txWithOrderB: TB + ): number { + // txWithOrderA is in mempool and txWithOrderB no, so txWithOrderA is newer + if (txWithOrderA.blockHeight === 0 && txWithOrderB.blockHeight !== 0) + return 1; + // txWithOrderB is in mempool and txWithOrderA no, so txWithOrderB is newer + if (txWithOrderB.blockHeight === 0 && txWithOrderA.blockHeight !== 0) + return -1; + // If blockHeight is different and none are in the mempool, sort by blockHeight + if (txWithOrderA.blockHeight !== txWithOrderB.blockHeight) + return txWithOrderA.blockHeight - txWithOrderB.blockHeight; + + // If blockHeight is the same, check input dependencies + let txA = txWithOrderA.tx; + let txIdA: string | undefined; + if (!txA) { + if (!txWithOrderA.txHex) throw new Error('Pass tx or txHex'); + const { tx, txId } = transactionFromHex(txWithOrderA.txHex); + txA = tx; + txIdA = txId; + } + let txB = txWithOrderB.tx; + let txIdB: string | undefined; + if (!txB) { + if (!txWithOrderB.txHex) throw new Error('Pass tx or txHex'); + const { tx, txId } = transactionFromHex(txWithOrderB.txHex); + txB = tx; + txIdB = txId; + } + //getHash is slow, try to avoid it if we can use a cached getId: + const txHashB = txIdB ? hex2RevBuf(txIdB) : txB.getHash(); + // txA is newer because it uses an input from txB + for (const Ainput of txA.ins) if (Ainput.hash.equals(txHashB)) return 1; + + //getHash is slow, try to avoid it if we can use a cached getId: + const txHashA = txIdA ? hex2RevBuf(txIdA) : txA.getHash(); + // txB is newer because it uses an input from txA + for (const Binput of txB.ins) if (Binput.hash.equals(txHashA)) return -1; + + return 0; // Cannot decide, keep the original order + } + const deriveScriptPubKeyFactory = memoizee( (networkId: NetworkId) => memoizee( @@ -93,20 +153,25 @@ export function deriveDataFactory({ networkId: NetworkId, descriptor: Descriptor, index: DescriptorIndex, - txDataArray: Array, + txWithOrderArray: Array, txStatus: TxStatus - ): { utxos: Array; stxos: Array } => { + ): { + utxos: Array; + stxos: Array; + txoMap: TxoMap; + } => { const scriptPubKey = deriveScriptPubKey(networkId, descriptor, index); + const txoMap: TxoMap = {}; //All prev outputs (spent or unspent) sent to this output descriptor: const allPrevOutputs: Utxo[] = []; - //all outputs in txDataArray which have been spent. + //all outputs in txWithOrderArray which have been spent. //May be outputs NOT snt to thil output descriptor: const spendingTxIdByOutput: Record = {}; //Means: Utxo was spent in txId - //Note that txDataArray cannot be assumed to be in correct order. See: + //Note that txWithOrderArray cannot be assumed to be in correct order if + //in the same block and if one does not depend on the other. See: //https://github.com/Blockstream/esplora/issues/165#issuecomment-1584471718 - //TODO: but we should guarantee same order always so use txId as second order criteria? - probably not needed? - for (const txData of txDataArray) { + for (const txData of txWithOrderArray) { if ( txStatus === TxStatus.ALL || (txStatus === TxStatus.IRREVERSIBLE && txData.irreversible) || @@ -117,32 +182,24 @@ export function deriveDataFactory({ throw new Error( `txHex not yet retrieved for an element of ${descriptor}, ${index}` ); - const tx = transactionFromHex(txHex); - const txId = tx.getId(); - - for (let vin = 0; vin < tx.ins.length; vin++) { - const input = tx.ins[vin]; - if (!input) - throw new Error(`Error: invalid input for ${txId}:${vin}`); - //Note we create a new Buffer since reverse() mutates the Buffer - const prevTxId = Buffer.from(input.hash).reverse().toString('hex'); + const { tx, txId } = transactionFromHex(txHex); + + for (const [vin, input] of tx.ins.entries()) { + const prevTxId = buf2RevHex(input.hash); const prevVout = input.index; const prevUtxo: Utxo = `${prevTxId}:${prevVout}`; spendingTxIdByOutput[prevUtxo] = `${txId}:${vin}`; //prevUtxo was spent by txId in input vin } - for (let vout = 0; vout < tx.outs.length; vout++) { - const outputScript = tx.outs[vout]?.script; - if (!outputScript) - throw new Error(`Error: invalid output script for ${txId}:${vout}`); - if (outputScript.equals(scriptPubKey)) { - const outputKey: Utxo = `${txId}:${vout}`; - allPrevOutputs.push(outputKey); + for (const [vout, output] of tx.outs.entries()) { + if (output.script.equals(scriptPubKey)) { + const txo = `${txId}:${vout}`; + allPrevOutputs.push(txo); + txoMap[txo] = `${descriptor}~${index}`; } } } } - // UTXOs are those in allPrevOutputs that have not been spent const utxos = allPrevOutputs.filter( output => !Object.keys(spendingTxIdByOutput).includes(output) @@ -151,7 +208,7 @@ export function deriveDataFactory({ .filter(output => Object.keys(spendingTxIdByOutput).includes(output)) .map(txo => `${txo}:${spendingTxIdByOutput[txo]}`); - return { utxos, stxos }; + return { utxos, stxos, txoMap }; }; const deriveUtxosAndBalanceByOutputFactory = memoizee( @@ -164,34 +221,37 @@ export function deriveDataFactory({ (index: DescriptorIndex) => { // Create one function per each expression x index x txStatus // coreDeriveTxosByOutput shares all params wrt the parent - // function except for additional param txDataArray. - // As soon as txDataArray in coreDeriveTxosByOutput changes, + // function except for additional param txWithOrderArray. + // As soon as txWithOrderArray in coreDeriveTxosByOutput changes, // it will resets its memory. const deriveTxosByOutput = memoizee(coreDeriveTxosByOutput, { max: 1 }); let lastUtxos: Array | null = null; let lastStxos: Array | null = null; + let lastTxoMap: TxoMap | null = null; let lastBalance: number; return memoizee( ( txMap: Record, descriptorMap: Record ) => { - const txDataArray = deriveTxDataArray( + const txWithOrderArray = deriveTxDataArray( txMap, descriptorMap, descriptor, index ); - let { utxos, stxos } = deriveTxosByOutput( + let { utxos, stxos, txoMap } = deriveTxosByOutput( networkId, descriptor, index, - txDataArray, + txWithOrderArray, txStatus ); let balance: number; + if (lastTxoMap && shallowEqualObjects(lastTxoMap, txoMap)) + txoMap = lastTxoMap; if (lastStxos && shallowEqualArrays(lastStxos, stxos)) stxos = lastStxos; if (lastUtxos && shallowEqualArrays(lastUtxos, utxos)) { @@ -199,10 +259,11 @@ export function deriveDataFactory({ balance = lastBalance; } else balance = coreDeriveUtxosBalance(txMap, utxos); + lastTxoMap = txoMap; lastUtxos = utxos; lastStxos = stxos; lastBalance = balance; - return { stxos, utxos, balance }; + return { txoMap, stxos, utxos, balance }; }, { max: 1 } ); @@ -249,8 +310,8 @@ export function deriveDataFactory({ ) => { const range = deriveUsedRange(descriptorMap[descriptor]); const txIds = range[index]?.txIds || []; - const txDataArray = coreDeriveTxDataArray(txIds, txMap); - return txDataArray; + const txWithOrderArray = coreDeriveTxDataArray(txIds, txMap); + return txWithOrderArray; } ); }, @@ -296,11 +357,10 @@ export function deriveDataFactory({ return txHistory.map(txData => { const { txHex, irreversible, blockHeight } = txData; if (!txHex) throw new Error(`Error: txHex not found`); - const tx = transactionFromHex(txHex); - const txId = tx.getId(); + const { tx, txId } = transactionFromHex(txHex); const ins = tx.ins.map(input => { - const prevTxId = Buffer.from(input.hash).reverse().toString('hex'); + const prevTxId = buf2RevHex(input.hash); const prevVout = input.index; const prevTxo: Utxo = `${prevTxId}:${prevVout}`; const ownedPrevTxo: Utxo | false = txoSet.has(prevTxo) @@ -309,7 +369,7 @@ export function deriveDataFactory({ if (ownedPrevTxo) { const prevTxHex = txMap[prevTxId]?.txHex; if (!prevTxHex) throw new Error(`txHex not set for ${prevTxId}`); - const prevTx = transactionFromHex(prevTxHex); + const { tx: prevTx } = transactionFromHex(prevTxHex); const value = prevTx.outs[prevVout]?.value; if (value === undefined) throw new Error(`value should exist for ${prevTxId}:${prevVout}`); @@ -395,16 +455,17 @@ export function deriveDataFactory({ (txStatus === TxStatus.CONFIRMED && txData.blockHeight !== 0) ); + const sortedHistory = txHistory.sort(compareTxOrder); if (withAttributions) return deriveAttributions( - txHistory, + sortedHistory, networkId, txMap, descriptorMap, descriptor, txStatus ); - else return txHistory; + else return sortedHistory; } ); }, @@ -476,20 +537,11 @@ export function deriveDataFactory({ //since we have txs belonging to different expressions let's try to order //them from old to new (blockHeight ascending order). //Note that we cannot guarantee to keep correct order to txs - //that belong to the same blockHeight - //TODO: but we should guarantee same order always so use txId as second order criteria? - probably not needed? - const sortedHistory = dedupedHistory.sort((txDataA, txDataB) => { - if (txDataA.blockHeight === 0 && txDataB.blockHeight === 0) { - return 0; // Both are in mempool, keep their relative order unchanged - } - if (txDataA.blockHeight === 0) { - return 1; // txDataA is in mempool, so it should come after txDataB - } - if (txDataB.blockHeight === 0) { - return -1; // txDataB is in mempool, so it should come after txDataA - } - return txDataA.blockHeight - txDataB.blockHeight; // Regular ascending order sort - }); + //that belong to the same blockHeight except when one of the txs depends + //on the other. + //See https://github.com/Blockstream/esplora/issues/165#issuecomment-1584471718 + const sortedHistory = dedupedHistory.sort(compareTxOrder); + if (withAttributions) return deriveAttributions( sortedHistory, @@ -551,9 +603,10 @@ export function deriveDataFactory({ txMap: Record, descriptorOrDescriptors: Array | Descriptor, txStatus: TxStatus - ): { utxos: Array; stxos: Array } => { + ): { utxos: Array; stxos: Array; txoMap: TxoMap } => { const utxos: Utxo[] = []; const stxos: Stxo[] = []; + const txoMap: TxoMap = {}; const descriptorArray = Array.isArray(descriptorOrDescriptors) ? descriptorOrDescriptors : [descriptorOrDescriptors]; @@ -563,24 +616,28 @@ export function deriveDataFactory({ .sort() //Sort it to be deterministic .forEach(indexStr => { const index = indexStr === 'non-ranged' ? indexStr : Number(indexStr); - const { utxos: utxosByO, stxos: stxosByO } = - deriveUtxosAndBalanceByOutput( - networkId, - txMap, - descriptorMap, - descriptor, - index, - txStatus - ); + const { + utxos: utxosByO, + stxos: stxosByO, + txoMap: txoMapByO + } = deriveUtxosAndBalanceByOutput( + networkId, + txMap, + descriptorMap, + descriptor, + index, + txStatus + ); utxos.push(...utxosByO); stxos.push(...stxosByO); + Object.assign(txoMap, txoMapByO); }); } //Deduplicate in case of expression: Array with duplicated //descriptorOrDescriptors const dedupedUtxos = [...new Set(utxos)]; const dedupedStxos = [...new Set(stxos)]; - return { utxos: dedupedUtxos, stxos: dedupedStxos }; + return { utxos: dedupedUtxos, stxos: dedupedStxos, txoMap }; }; //unbound memoizee wrt TxStatus is fine since it has a small Search Space @@ -593,6 +650,7 @@ export function deriveDataFactory({ (txStatus: TxStatus) => memoizee( (descriptorOrDescriptors: Array | Descriptor) => { + let lastTxoMap: TxoMap | null = null; let lastUtxos: Array | null = null; let lastStxos: Array | null = null; let lastBalance: number; @@ -601,7 +659,7 @@ export function deriveDataFactory({ txMap: Record, descriptorMap: Record ) => { - let { utxos, stxos } = coreDeriveTxos( + let { utxos, stxos, txoMap } = coreDeriveTxos( networkId, descriptorMap, txMap, @@ -609,6 +667,8 @@ export function deriveDataFactory({ txStatus ); let balance: number; + if (lastTxoMap && shallowEqualObjects(lastTxoMap, txoMap)) + txoMap = lastTxoMap; if (lastStxos && shallowEqualArrays(lastStxos, stxos)) stxos = lastStxos; if (lastUtxos && shallowEqualArrays(lastUtxos, utxos)) { @@ -616,10 +676,11 @@ export function deriveDataFactory({ balance = lastBalance; } else balance = coreDeriveUtxosBalance(txMap, utxos); + lastTxoMap = txoMap; lastUtxos = utxos; lastStxos = stxos; lastBalance = balance; - return { stxos, utxos, balance }; + return { stxos, utxos, txoMap, balance }; }, { max: 1 } ); @@ -643,10 +704,36 @@ export function deriveDataFactory({ descriptorMap ); - const transactionFromHex = memoizee(Transaction.fromHex, { - primitive: true, - max: 1000 - }); + const transactionFromHex = memoizee( + (txHex: string) => { + const tx = Transaction.fromHex(txHex); + const txId = tx.getId(); + return { tx, txId }; + }, + { + primitive: true, + max: 1000 + } + ); + const hex2RevBuf = memoizee( + (idOrHash: string) => { + return Buffer.from(idOrHash).reverse(); + }, + { + primitive: true, + max: 1000 + } + ); + const buf2RevHex = memoizee( + (idOrHash: Buffer) => { + //Note we create a new Buffer since reverse() mutates the Buffer + return Buffer.from(idOrHash).reverse().toString('hex'); + }, + { + primitive: true, + max: 1000 + } + ); const coreDeriveUtxosBalance = ( txMap: Record, @@ -671,7 +758,7 @@ export function deriveDataFactory({ throw new Error(`txData not saved for ${txId}, vout:${vout} - ${utxo}`); const txHex = txData.txHex; if (!txHex) throw new Error(`txHex not yet retrieved for ${txId}`); - const tx = transactionFromHex(txHex); + const { tx } = transactionFromHex(txHex); const output = tx.outs[vout]; if (!output) throw new Error(`Error: invalid output for ${txId}:${vout}`); const outputValue = output.value; // value in satoshis @@ -804,6 +891,7 @@ export function deriveDataFactory({ deriveAccountDescriptors, deriveHistoryByOutput, deriveHistory, - transactionFromHex + transactionFromHex, + compareTxOrder }; } diff --git a/src/discovery.ts b/src/discovery.ts index 7e6dc0b..92823c9 100644 --- a/src/discovery.ts +++ b/src/discovery.ts @@ -34,7 +34,9 @@ import { TxStatus, Stxo, TxHex, - TxAttribution + TxAttribution, + TxWithOrder, + TxoMap } from './types'; const now = () => Math.floor(Date.now() / 1000); @@ -831,6 +833,10 @@ export function DiscoveryFactory( * scriptPubKeys and the total balance of these UTXOs. * It also returns previous UTXOs that had been * eventually spent as stxos: Array + * Finally, it returns `txoMap`. `txoMap` maps all the txos (unspent or spent + * outputs) with their corresponding `indexedDescriptor: IndexedDescriptor` + * (see {@link IndexedDescriptor IndexedDescriptor}) + * */ getUtxosAndBalance({ descriptor, @@ -840,6 +846,7 @@ export function DiscoveryFactory( }: OutputCriteria): { utxos: Array; stxos: Array; + txoMap: TxoMap; balance: number; } { this.#ensureFetched({ @@ -1022,13 +1029,13 @@ export function DiscoveryFactory( const descriptorMap = this.#discoveryData[networkId].descriptorMap; const txMap = this.#discoveryData[networkId].txMap; - let txDataArray: Array = []; + let txWithOrderArray: Array = []; if ( descriptor && (typeof index !== 'undefined' || !descriptor.includes('*')) ) { const internalIndex = typeof index === 'number' ? index : 'non-ranged'; - txDataArray = this.#derivers.deriveHistoryByOutput( + txWithOrderArray = this.#derivers.deriveHistoryByOutput( withAttributions, networkId, txMap, @@ -1038,7 +1045,7 @@ export function DiscoveryFactory( txStatus ); } else - txDataArray = this.#derivers.deriveHistory( + txWithOrderArray = this.#derivers.deriveHistory( withAttributions, networkId, txMap, @@ -1047,7 +1054,7 @@ export function DiscoveryFactory( txStatus ); - return txDataArray; + return txWithOrderArray; } /** @@ -1091,8 +1098,11 @@ export function DiscoveryFactory( * Retrieves the transaction data as a bitcoinjs-lib * {@link https://github.com/bitcoinjs/bitcoinjs-lib/blob/master/ts_src/transaction.ts Transaction} * object given the transaction - * ID (TxId) or a Unspent Transaction Output (Utxo). The transaction data is obtained by first getting - * the transaction hexadecimal representation using getTxHex() method. + * ID (TxId) or a Unspent Transaction Output (Utxo) or the hexadecimal + * representation of the transaction (it will then use memoization). + * The transaction data is obtained by first getting + * the transaction hexadecimal representation using getTxHex() method + * (unless the txHex was passed). * * Use this method for quick access to the Transaction object, which avoids the * need to parse the transaction hexadecimal representation (txHex). @@ -1104,22 +1114,49 @@ export function DiscoveryFactory( */ getTransaction({ txId, + txHex, utxo }: { /** * The transaction ID. */ txId?: TxId; + /** + * The transaction txHex. + */ + txHex?: TxId; /** * The UTXO. */ utxo?: Utxo; }): Transaction { - const txHex = this.getTxHex({ - ...(utxo ? { utxo } : {}), - ...(txId ? { txId } : {}) - }); - return this.#derivers.transactionFromHex(txHex); + if (!txHex) + txHex = this.getTxHex({ + ...(utxo ? { utxo } : {}), + ...(txId ? { txId } : {}) + }); + return this.#derivers.transactionFromHex(txHex).tx; + } + + /** + * Compares two transactions based on their blockHeight and input dependencies. + * Can be used as callback in Array.sort function to sort from old to new. + * + * @param txWithOrderA - The first transaction data to compare. + * @param txWithOrderB - The second transaction data to compare. + * + * txWithOrderA and txWithOrderB should contain the `blockHeight` (use 0 if + * in the mempool) and either `tx` (`Transaction` type) or `txHex` (the + * hexadecimal representation of the transaction) + * + * @returns < 0 if txWithOrderA is older than txWithOrderB, > 0 if + * txWithOrderA is newer than txWithOrderB, and 0 if undecided. + */ + compareTxOrder( + txWithOrderA: TA, + txWithOrderB: TB + ): number { + return this.#derivers.compareTxOrder(txWithOrderA, txWithOrderB); } /** @@ -1129,8 +1166,6 @@ export function DiscoveryFactory( * txo can be in any of these formats: `${txId}:${vout}` or * using its extended form: `${txId}:${vout}:${recipientTxId}:${recipientVin}` * - * This query can be quite slow so use wisely. - * * Returns the descriptor (and index if ranged) or undefined if not found. */ getDescriptor({ @@ -1153,7 +1188,10 @@ export function DiscoveryFactory( const networkId = getNetworkId(network); if (!txo) throw new Error('Pass either txo or utxo'); const split = txo.split(':'); - if (split.length !== 2) throw new Error(`Error: invalid txo: ${txo}`); + if (utxo && split.length !== 2) + throw new Error(`Error: invalid utxo: ${utxo}`); + if (!utxo && split.length !== 3) + throw new Error(`Error: invalid txo: ${txo}`); const txId = split[0]; if (!txId) throw new Error(`Error: invalid txo: ${txo}`); const strVout = split[1]; @@ -1161,13 +1199,7 @@ export function DiscoveryFactory( const vout = parseInt(strVout); if (vout.toString() !== strVout) throw new Error(`Error: invalid txo: ${txo}`); - //const txHex = this.#discoveryData[networkId].txMap[txId]?.txHex; - //if (!txHex) - // throw new Error( - // `Error: txHex not found for ${txo} while looking for its descriptor.` - // ); - const descriptorMap = this.#discoveryData[networkId].descriptorMap; const descriptors = this.#derivers.deriveUsedDescriptors( this.#discoveryData, networkId @@ -1178,58 +1210,25 @@ export function DiscoveryFactory( index?: number; } | undefined; - descriptors.forEach(descriptor => { - const range = - descriptorMap[descriptor]?.range || - ({} as Record); + const { txoMap } = this.getUtxosAndBalance({ descriptors }); + const indexedDescriptor = txoMap[txo]; + if (indexedDescriptor) { + const splitTxo = (str: string): [string, string] => { + const lastIndex = str.lastIndexOf('~'); + if (lastIndex === -1) + throw new Error(`Separator '~' not found in string`); + return [str.slice(0, lastIndex), str.slice(lastIndex + 1)]; + }; + const [descriptor, internalIndex] = splitTxo(indexedDescriptor); - Object.keys(range).forEach(indexStr => { - const isRanged = indexStr !== 'non-ranged'; - const index = isRanged && Number(indexStr); + output = { + descriptor, + ...(internalIndex === 'non-ranged' + ? {} + : { index: Number(internalIndex) }) + }; + } - if (!txo) throw new Error('txo not defined'); - const { utxos, stxos } = this.getUtxosAndBalance({ - descriptor, - ...(isRanged ? { index: Number(indexStr) } : {}) - }); - if (utxo) { - if (utxos.includes(txo)) { - if (output) - throw new Error( - `output {${descriptor}, ${index}} is already represented by {${output.descriptor}, ${output.index}} .` - ); - output = { - descriptor, - ...(isRanged ? { index: Number(indexStr) } : {}) - }; - } - } else { - //Descriptor txos (Unspent txos and Spent txos). Note that - //stxos have this format: `${txId}:${vout}:${recipientTxId}:${recipientVin}` - //so normalize to Utxo format: - const txoSet = new Set([ - ...utxos, - ...stxos.map(stxo => { - const [txId, voutStr] = stxo.split(':'); - if (txId === undefined || voutStr === undefined) { - throw new Error(`Undefined txId or vout for STXO: ${stxo}`); - } - return `${txId}:${voutStr}`; - }) - ]); - if (txoSet.has(txo)) { - if (output) - throw new Error( - `output {${descriptor}, ${index}} is already represented by {${output.descriptor}, ${output.index}} .` - ); - output = { - descriptor, - ...(isRanged ? { index: Number(indexStr) } : {}) - }; - } - } - }); - }); return output; } @@ -1261,8 +1260,7 @@ export function DiscoveryFactory( }): Promise { const DETECTION_INTERVAL = 3000; const DETECT_RETRY_MAX = 20; - const tx = this.#derivers.transactionFromHex(txHex); - const txId = tx.getId(); + const { txId } = this.#derivers.transactionFromHex(txHex); await explorer.push(txHex); @@ -1330,8 +1328,7 @@ export function DiscoveryFactory( const txHex = txData.txHex; if (!txHex) throw new Error('txData must contain complete txHex information'); - const tx = this.#derivers.transactionFromHex(txHex); - const txId = tx.getId(); + const { tx, txId } = this.#derivers.transactionFromHex(txHex); const networkId = getNetworkId(network); this.#discoveryData = produce(this.#discoveryData, discoveryData => { diff --git a/src/index.ts b/src/index.ts index 219b403..6995014 100644 --- a/src/index.ts +++ b/src/index.ts @@ -3,10 +3,13 @@ import { DiscoveryFactory, DiscoveryInstance } from './discovery'; export { DiscoveryFactory, DiscoveryInstance }; export { + TxWithOrder, OutputCriteria, TxStatus, Account, + TxoMap, Utxo, Stxo, + IndexedDescriptor, TxAttribution } from './types'; diff --git a/src/types.ts b/src/types.ts index 219c6c3..bee4158 100644 --- a/src/types.ts +++ b/src/types.ts @@ -1,3 +1,4 @@ +import type { Transaction } from 'bitcoinjs-lib'; /** * Versions the structure of the data model. This variable should to be * changed when any of the types below change. @@ -84,6 +85,29 @@ export enum TxStatus { /** CONFIRMED with at least 1 confirmation */ CONFIRMED = 'CONFIRMED' } +/** + * A string representing an indexed descriptor for ranged descriptors or a + * descriptor followed by a separator and the keyword "non-ranged". + * + * An `IndexedDescriptor` is a descriptor representation what must correspond to + * a single output. + * + * - If it is ranged, then add an integer after the separaror (a + * tilde "\~"). + * - It it is non-ranged, add the string "non-ranged" after the tilde "\~". + * + * Examples: + * pkh([73c5da0a/44'/1'/0']tpubDC5FSnBiZDMmhiuCmWAYsLwgLYrrT9rAqvTySfuCCrgsWz8wxMXUS9Tb9iVMvcRbvFcAHGkMD5Kx8koh4GquNGNTfohfk7pgjhaPCdXpoba/0/*)\~12 + * pkh([73c5da0a/44'/1'/0']tpubDC5FSnBiZDMmhiuCmWAYsLwgLYrrT9rAqvTySfuCCrgsWz8wxMXUS9Tb9iVMvcRbvFcAHGkMD5Kx8koh4GquNGNTfohfk7pgjhaPCdXpoba)\~non-ranged + */ +export type IndexedDescriptor = string; +/** + * a Txo is represented in a similar manner as a Utxo, that is, + * prevtxId:vout. Hovewer, we use a different type name to denote we're dealing + * here with tx outputs that may have been spent or not + */ +type Txo = string; +export type TxoMap = Record; /** * Type definition for Transaction ID. @@ -103,6 +127,12 @@ export type Utxo = string; //`${txId}:${vout}` */ export type Stxo = string; //`${txId}:${vout}:${recipientTxId}:${recipientVin}` +export type TxWithOrder = { + blockHeight: number; + tx?: Transaction; + txHex?: string; +}; + /** * Type definition for Transaction Information. */ diff --git a/test/integration/regtest.test.ts b/test/integration/regtest.test.ts index 4aa8bc1..9ad4022 100644 --- a/test/integration/regtest.test.ts +++ b/test/integration/regtest.test.ts @@ -32,7 +32,8 @@ import { Account, TxStatus, Utxo, - Stxo + Stxo, + TxoMap } from '../../dist'; type DescriptorIndex = number | 'non-ranged'; const ESPLORA_CATCHUP_TIME = 5000; @@ -330,11 +331,13 @@ describe('Discovery on regtest', () => { let balanceDefault: number; let utxosDefault: Array; let stxosDefault: Array; + let txoMapDefault: TxoMap; test(`getUtxosAndBalance default status for ${descriptor} using ${discoverer.name} after ${totalMined} blocks`, () => { ({ balance: balanceDefault, utxos: utxosDefault, - stxos: stxosDefault + stxos: stxosDefault, + txoMap: txoMapDefault } = discoverer.discovery!.getUtxosAndBalance({ descriptor })); @@ -348,15 +351,19 @@ describe('Discovery on regtest', () => { ).toEqual({ balance: balanceDefault, utxos: utxosDefault, - stxos: stxosDefault + stxos: stxosDefault, + txoMap: txoMapDefault }); }); test(`getUtxosAndBalance ALL for ${descriptor} using ${discoverer.name} after ${totalMined} blocks`, () => { - const { balance: balanceAll, utxos: utxosAll } = - discoverer.discovery!.getUtxosAndBalance({ - descriptor, - txStatus: TxStatus.ALL - }); + const { + balance: balanceAll, + utxos: utxosAll, + txoMap: txoMapAll + } = discoverer.discovery!.getUtxosAndBalance({ + descriptor, + txStatus: TxStatus.ALL + }); expect(balanceAll).toEqual(totalBalance); expect(balanceAll).toEqual(balanceDefault); expect(utxosAll.length).toEqual(totalUtxosCount); @@ -365,14 +372,22 @@ describe('Discovery on regtest', () => { new Discovery({ imported: discoverer.discovery!.export() }).getUtxosAndBalance({ descriptor, txStatus: TxStatus.ALL }) - ).toEqual({ balance: balanceAll, utxos: utxosAll, stxos: [] }); + ).toEqual({ + balance: balanceAll, + utxos: utxosAll, + stxos: [], + txoMap: txoMapAll + }); }); test(`getUtxosAndBalance CONFIRMED for ${descriptor} using ${discoverer.name} after ${totalMined} blocks`, () => { - const { balance: balanceConfirmed, utxos: utxosConfirmed } = - discoverer.discovery!.getUtxosAndBalance({ - descriptor, - txStatus: TxStatus.CONFIRMED - }); + const { + balance: balanceConfirmed, + utxos: utxosConfirmed, + txoMap: txoMapConfirmed + } = discoverer.discovery!.getUtxosAndBalance({ + descriptor, + txStatus: TxStatus.CONFIRMED + }); expect(balanceConfirmed).toEqual(totalMined > 0 ? totalBalance : 0); expect(utxosConfirmed.length).toEqual( totalMined > 0 ? totalUtxosCount : 0 @@ -388,15 +403,19 @@ describe('Discovery on regtest', () => { ).toEqual({ balance: balanceConfirmed, utxos: utxosConfirmed, - stxos: [] + stxos: [], + txoMap: txoMapConfirmed }); }); test(`getUtxosAndBalance IRREVERSIBLE for ${descriptor} using ${discoverer.name} after ${totalMined} blocks`, () => { - const { balance: balanceIrreversible, utxos: utxosIrreversible } = - discoverer.discovery!.getUtxosAndBalance({ - descriptor, - txStatus: TxStatus.IRREVERSIBLE - }); + const { + balance: balanceIrreversible, + utxos: utxosIrreversible, + txoMap: txoMapIrreversible + } = discoverer.discovery!.getUtxosAndBalance({ + descriptor, + txStatus: TxStatus.IRREVERSIBLE + }); expect(balanceIrreversible).toEqual( totalMined >= irrevConfThresh ? totalBalance : 0 ); @@ -414,7 +433,8 @@ describe('Discovery on regtest', () => { ).toEqual({ balance: balanceIrreversible, utxos: utxosIrreversible, - stxos: [] + stxos: [], + txoMap: txoMapIrreversible }); }); }