From 071e3624ab8ccd196e1a61cc93ed33f562580fdc Mon Sep 17 00:00:00 2001 From: Kristaps Kaupe Date: Fri, 17 Nov 2023 16:11:24 +0200 Subject: [PATCH] WIP for minimum tx feerate setting for makers --- scripts/obwatch/ob-watcher.py | 1 + scripts/yg-privacyenhanced.py | 4 ++- src/jmclient/blockchaininterface.py | 23 ++++++++++------- src/jmclient/configure.py | 5 ++++ src/jmclient/maker.py | 14 ++++++++++- src/jmclient/support.py | 37 +++++++++++++++++++--------- src/jmclient/yieldgenerator.py | 5 ++-- src/jmdaemon/message_channel.py | 11 ++++++--- src/jmdaemon/orderbookwatch.py | 12 ++++++--- test/jmbitcoin/test_tx_signing.py | 1 + test/jmclient/commontest.py | 2 +- test/jmclient/taker_test_data.py | 32 +++++++++++++++--------- test/jmclient/test_yieldgenerator.py | 3 +++ 13 files changed, 103 insertions(+), 47 deletions(-) diff --git a/scripts/obwatch/ob-watcher.py b/scripts/obwatch/ob-watcher.py index 72a46738d..60da71d5a 100755 --- a/scripts/obwatch/ob-watcher.py +++ b/scripts/obwatch/ob-watcher.py @@ -115,6 +115,7 @@ def create_offerbook_table_heading(btc_unit, rel_unit): col.format('txfee', 'Miner Fee Contribution / ' + btc_unit), col.format('minsize', 'Minimum Size / ' + btc_unit), col.format('maxsize', 'Maximum Size / ' + btc_unit), + col.format('minimum_tx_fee_rate', 'Minimum feerate (sat/vB)'), col.format('bondvalue', 'Bond value / ' + btc_unit + '' + bond_exponent + '') ]) + ' ' return tableheading diff --git a/scripts/yg-privacyenhanced.py b/scripts/yg-privacyenhanced.py index 4c4e8404b..4cebd6602 100755 --- a/scripts/yg-privacyenhanced.py +++ b/scripts/yg-privacyenhanced.py @@ -90,7 +90,9 @@ def create_my_orders(self): 'minsize': randomize_minsize, 'maxsize': randomize_maxsize, 'txfee': randomize_txfee, - 'cjfee': str(randomize_cjfee)} + 'cjfee': str(randomize_cjfee), + # TODO: add some randomization factor here? + 'minimum_tx_fee_rate': self.minimum_tx_fee_rate} # sanity check assert order['minsize'] >= jm_single().DUST_THRESHOLD diff --git a/src/jmclient/blockchaininterface.py b/src/jmclient/blockchaininterface.py index 663dce800..0eec3d9db 100644 --- a/src/jmclient/blockchaininterface.py +++ b/src/jmclient/blockchaininterface.py @@ -42,9 +42,9 @@ def pushtx(self, txbin: bytes) -> bool: @abstractmethod def query_utxo_set(self, - txouts: Union[Tuple[bytes, int], List[Tuple[bytes, int]]], + txouts: Union[Tuple[bytes, int], Iterable[Tuple[bytes, int]]], includeconfs: bool = False, - include_mempool: bool = True) -> List[Optional[dict]]: + include_mempool: bool = True) -> List[dict]: """If txout is either (a) a single utxo in (txidbin, n) form, or a list of the same, returns, as a list for each txout item, the result of gettxout from the bitcoind rpc for those utxos; @@ -239,7 +239,7 @@ def _fee_per_kb_has_been_manually_set(self, tx_fees: int) -> bool: """ return tx_fees > 1000 - def estimate_fee_per_kb(self, tx_fees: int) -> int: + def estimate_fee_per_kb(self, tx_fees: int, randomize: bool = True) -> int: """ The argument tx_fees may be either a number of blocks target, for estimation of feerate by Core, or a number of satoshis per kilo-vbyte (see `_fee_per_kb_has_been_manually_set` for @@ -255,7 +255,11 @@ def estimate_fee_per_kb(self, tx_fees: int) -> int: # default to use if fees cannot be estimated fallback_fee = 10000 - tx_fees_factor = abs(jm_single().config.getfloat('POLICY', 'tx_fees_factor')) + if randomize: + tx_fees_factor = abs(jm_single().config.getfloat( + 'POLICY', 'tx_fees_factor')) + else: + tx_fees_factor = 0 mempoolminfee_in_sat = self._get_mempool_min_fee() # in case of error @@ -543,11 +547,12 @@ def pushtx(self, txbin: bytes) -> bool: return True def query_utxo_set(self, - txouts: Union[Tuple[bytes, int], List[Tuple[bytes, int]]], + txouts: Union[Tuple[bytes, int], Iterable[Tuple[bytes, int]]], includeconfs: bool = False, - include_mempool: bool = True) -> List[Optional[dict]]: - if not isinstance(txouts, list): + include_mempool: bool = True) -> List[dict]: + if isinstance(txouts, tuple): txouts = [txouts] + assert isinstance(txouts, Iterable) result = [] for txo in txouts: txo_hex = bintohex(txo[0]) @@ -802,9 +807,9 @@ def __init__(self, jsonRpc: JsonRpc, wallet_name: str) -> None: self.shutdown_signal = False self.destn_addr = self._rpc("getnewaddress", []) - def estimate_fee_per_kb(self, tx_fees: int) -> int: + def estimate_fee_per_kb(self, tx_fees: int, randomize: bool = True) -> int: if not self.absurd_fees: - return super().estimate_fee_per_kb(tx_fees) + return super().estimate_fee_per_kb(tx_fees, randomize) else: return jm_single().config.getint("POLICY", "absurd_fee_per_kb") + 100 diff --git a/src/jmclient/configure.py b/src/jmclient/configure.py index 14482ff8d..c39889559 100644 --- a/src/jmclient/configure.py +++ b/src/jmclient/configure.py @@ -519,6 +519,11 @@ def jm_single() -> AttributeDict: # [fraction, 0-1] / variance around all offer sizes. Ex: 500k minsize, 0.1 var = 450k-550k size_factor = 0.1 +# +minimum_tx_fee_rate = 1000 + +gaplimit = 6 + [SNICKER] # Any other value than 'true' will be treated as False, # and no SNICKER actions will be enabled in that case: diff --git a/src/jmclient/maker.py b/src/jmclient/maker.py index 43c0ddc06..a6586a3a7 100644 --- a/src/jmclient/maker.py +++ b/src/jmclient/maker.py @@ -15,7 +15,7 @@ jlog = get_log() class Maker(object): - def __init__(self, wallet_service): + def __init__(self, wallet_service: WalletService): self.active_orders = {} assert isinstance(wallet_service, WalletService) self.wallet_service = wallet_service @@ -28,6 +28,8 @@ def __init__(self, wallet_service): # not-enough-coins: self.sync_wait_loop.start(2.0, now=False) self.aborted = False + self.minimum_tx_fee_rate = jm_single().config.getint( + "YIELDGENERATOR", "minimum_tx_fee_rate") def try_to_create_my_orders(self): """Because wallet syncing is not synchronous(!), @@ -187,6 +189,16 @@ def verify_unsigned_tx(self, tx, offerinfo): """ tx_utxo_set = set((x.prevout.hash[::-1], x.prevout.n) for x in tx.vin) + if self.minimum_tx_fee_rate > 1000: + tx_inp_data = jm_single().bc_interface.query_utxo_set(tx_utxo_set) + total_input_value_sum = 0 + for utxo in tx_inp_data: + total_input_value_sum = total_input_value_sum + utxo["value"] + tx_fee = total_input_value_sum - btc.tx_total_outputs_value(tx) + tx_fee_rate = tx_fee / btc.tx_vsize(tx) * 1000 + if tx_fee_rate < self.minimum_tx_fee_rate: + return (False, "tx feerate below configured minimum tx feerate") + utxos = offerinfo["utxos"] cjaddr = offerinfo["cjaddr"] cjaddr_script = btc.CCoinAddress(cjaddr).to_scriptPubKey() diff --git a/src/jmclient/support.py b/src/jmclient/support.py index e8641a095..1d289e1eb 100644 --- a/src/jmclient/support.py +++ b/src/jmclient/support.py @@ -1,13 +1,15 @@ -from functools import reduce import random -from jmbase.support import get_log from decimal import Decimal -from .configure import get_bondless_makers_allowance - +from functools import reduce from math import exp +from typing import List, Optional, Tuple + +from jmbase.support import get_log +from .configure import get_bondless_makers_allowance, jm_single + ORDER_KEYS = ['counterparty', 'oid', 'ordertype', 'minsize', 'maxsize', 'txfee', - 'cjfee'] + 'cjfee', 'minimum_tx_fee_rate'] log = get_log() @@ -247,19 +249,28 @@ def check_max_fee(fee): return check_max_fee -def choose_orders(offers, cj_amount, n, chooseOrdersBy, ignored_makers=None, - pick=False, allowed_types=["sw0reloffer", "sw0absoffer"], - max_cj_fee=(1, float('inf'))): +def choose_orders(offers: List[dict], + cj_amount, + num_counterparties: int, + chooseOrdersBy, + ignored_makers: Optional[List[str]] = None, + pick: bool = False, + allowed_types: List[str] = ["sw0reloffer", "sw0absoffer"], + max_cj_fee: Tuple = (1, float('inf'))) -> Tuple[Optional[dict], int]: is_within_max_limits = _get_is_within_max_limits( max_cj_fee[0], max_cj_fee[1], cj_amount) if ignored_makers is None: ignored_makers = [] + fee_per_kb = jm_single().bc_interface.estimate_fee_per_kb( + jm_single().config.getint("POLICY", "tx_fees"), randomize=False) #Filter ignored makers and inappropriate amounts orders = [o for o in offers if o['counterparty'] not in ignored_makers] orders = [o for o in orders if o['minsize'] < cj_amount] orders = [o for o in orders if o['maxsize'] > cj_amount] #Filter those not using wished-for offertypes orders = [o for o in orders if o["ordertype"] in allowed_types] + #Filter those not accepting our tx feerate + orders = [o for o in orders if o["minimum_tx_fee_rate"] <= fee_per_kb] orders_fees = [] for o in orders: @@ -268,10 +279,11 @@ def choose_orders(offers, cj_amount, n, chooseOrdersBy, ignored_makers=None, orders_fees.append((o, fee)) counterparties = set(o['counterparty'] for o, f in orders_fees) - if n > len(counterparties): + if num_counterparties > len(counterparties): log.warn(('ERROR not enough liquidity in the orderbook n=%d ' 'suitable-counterparties=%d amount=%d totalorders=%d') % - (n, len(counterparties), cj_amount, len(orders_fees))) + (num_counterparties, len(counterparties), cj_amount, + len(orders_fees))) # TODO handle not enough liquidity better, maybe an Exception return None, 0 """ @@ -294,8 +306,9 @@ def choose_orders(offers, cj_amount, n, chooseOrdersBy, ignored_makers=None, ])) total_cj_fee = 0 chosen_orders = [] - for i in range(n): - chosen_order, chosen_fee = chooseOrdersBy(orders_fees, n) + for i in range(num_counterparties): + chosen_order, chosen_fee = chooseOrdersBy(orders_fees, + num_counterparties) # remove all orders from that same counterparty # only needed if offers are manually picked orders_fees = [o diff --git a/src/jmclient/yieldgenerator.py b/src/jmclient/yieldgenerator.py index 4f3c6e594..a05920368 100644 --- a/src/jmclient/yieldgenerator.py +++ b/src/jmclient/yieldgenerator.py @@ -89,8 +89,6 @@ def __init__(self, wallet_service, offerconfig): self.size_factor = offerconfig super().__init__(wallet_service) - - def create_my_orders(self): mix_balance = self.get_available_mixdepths() if len([b for m, b in mix_balance.items() if b > 0]) == 0: @@ -113,7 +111,8 @@ def create_my_orders(self): 'maxsize': mix_balance[max_mix] - max( jm_single().DUST_THRESHOLD, self.txfee_contribution), 'txfee': self.txfee_contribution, - 'cjfee': f} + 'cjfee': f, + 'minimum_tx_fee_rate': self.minimum_tx_fee_rate} # sanity check assert order['minsize'] >= 0 diff --git a/src/jmdaemon/message_channel.py b/src/jmdaemon/message_channel.py index 3734ccea9..a35546471 100644 --- a/src/jmdaemon/message_channel.py +++ b/src/jmdaemon/message_channel.py @@ -293,7 +293,8 @@ def announce_orders(self, orderlist, nick, fidelity_bond_proof_msg, new_mc): privmsg, on a specific mc. Fidelity bonds can only be announced over privmsg, nick must be nonNone """ - order_keys = ['oid', 'minsize', 'maxsize', 'txfee', 'cjfee'] + order_keys = ['oid', 'minsize', 'maxsize', 'txfee', 'cjfee', + 'minimum_tx_fee_rate'] orderlines = [] for order in orderlist: orderlines.append(COMMAND_PREFIX + order['ordertype'] + \ @@ -545,7 +546,7 @@ def on_nick_change_trigger(self, new_nick): self.on_nick_change(new_nick) def on_order_seen_trigger(self, mc, counterparty, oid, ordertype, minsize, - maxsize, txfee, cjfee): + maxsize, txfee, cjfee, minimum_tx_fee_rate): """This is the entry point into private messaging. Hence, it fixes for the rest of the conversation, which message channel the bots are going to communicate over @@ -564,7 +565,7 @@ def on_order_seen_trigger(self, mc, counterparty, oid, ordertype, minsize, self.active_channels[counterparty] = mc if self.on_order_seen: self.on_order_seen(counterparty, oid, ordertype, minsize, maxsize, - txfee, cjfee) + txfee, cjfee, minimum_tx_fee_rate) # orderbook watcher commands def register_orderbookwatch_callbacks(self, @@ -785,9 +786,11 @@ def check_for_orders(self, nick, _chunks): maxsize = _chunks[3] txfee = _chunks[4] cjfee = _chunks[5] + minimum_tx_fee_rate = _chunks[6] if len(_chunks) > 6 else 0 if self.on_order_seen: self.on_order_seen(self, counterparty, oid, ordertype, - minsize, maxsize, txfee, cjfee) + minsize, maxsize, txfee, cjfee, + minimum_tx_fee_rate) except IndexError as e: log.debug(e) log.debug('index error parsing chunks, possibly malformed ' diff --git a/src/jmdaemon/orderbookwatch.py b/src/jmdaemon/orderbookwatch.py index 796675945..63e448ab4 100644 --- a/src/jmdaemon/orderbookwatch.py +++ b/src/jmdaemon/orderbookwatch.py @@ -40,7 +40,8 @@ def set_msgchan(self, msgchan): self.dblock.acquire(True) self.db.execute("CREATE TABLE orderbook(counterparty TEXT, " "oid INTEGER, ordertype TEXT, minsize INTEGER, " - "maxsize INTEGER, txfee INTEGER, cjfee TEXT);") + "maxsize INTEGER, txfee INTEGER, cjfee TEXT, " + "minimum_tx_fee_rate INTEGER);") self.db.execute("CREATE TABLE fidelitybonds(counterparty TEXT, " "takernick TEXT, proof TEXT);"); finally: @@ -65,8 +66,10 @@ def on_set_topic(newtopic): print('=' * 60) joinmarket_alert[0] = alert - def on_order_seen(self, counterparty, oid, ordertype, minsize, maxsize, - txfee, cjfee): + def on_order_seen(self, counterparty: str, oid: int, ordertype: str, + minsize: int, maxsize: int, + txfee: int, cjfee: Integral, + minimum_tx_fee_rate: int) -> None: try: self.dblock.acquire(True) if int(oid) < 0 or int(oid) > sys.maxsize: @@ -114,7 +117,8 @@ def on_order_seen(self, counterparty, oid, ordertype, minsize, maxsize, self.db.execute( 'INSERT INTO orderbook VALUES(?, ?, ?, ?, ?, ?, ?);', (counterparty, oid, ordertype, minsize, maxsize, txfee, - str(Decimal(cjfee)))) # any parseable Decimal is a valid cjfee + str(Decimal(cjfee)), # any parseable Decimal is a valid cjfee + minimum_tx_fee_rate)) except InvalidOperation: log.debug("Got invalid cjfee: " + str(cjfee) + " from " + counterparty) except Exception as e: diff --git a/test/jmbitcoin/test_tx_signing.py b/test/jmbitcoin/test_tx_signing.py index ebe73eff5..36c6f7bee 100644 --- a/test/jmbitcoin/test_tx_signing.py +++ b/test/jmbitcoin/test_tx_signing.py @@ -85,6 +85,7 @@ def test_sign_standard_txs(addrtype): txin = btc.CTxIn(btc.COutPoint(txid[::-1], vout)) txout = btc.CTxOut(amount_less_fee, target_scriptPubKey) tx = btc.CMutableTransaction([txin], [txout]) + assert btc.tx_total_outputs_value(tx) == amount_less_fee # Calculate the signature hash for the transaction. This is then signed by the # private key that controls the UTXO being spent here at this txin_index. diff --git a/test/jmclient/commontest.py b/test/jmclient/commontest.py index c383e7535..b1c1afaa5 100644 --- a/test/jmclient/commontest.py +++ b/test/jmclient/commontest.py @@ -170,7 +170,7 @@ def query_utxo_set(self, result.append(result_dict) return result - def estimate_fee_per_kb(self, tx_fees: int) -> int: + def estimate_fee_per_kb(self, tx_fees: int, randomize: bool = True) -> int: return 30000 diff --git a/test/jmclient/taker_test_data.py b/test/jmclient/taker_test_data.py index ec65a7754..7a9b700a5 100644 --- a/test/jmclient/taker_test_data.py +++ b/test/jmclient/taker_test_data.py @@ -1,16 +1,24 @@ #orderbook -t_orderbook = [{u'counterparty': u'J6FA1Gj7Ln4vSGne', u'ordertype': u'sw0reloffer', u'oid': 0, - u'minsize': 7500000, u'txfee': 1000, u'maxsize': 599972700, u'cjfee': u'0.0002'}, - {u'counterparty': u'J6CFffuuewjG44UJ', u'ordertype': u'sw0reloffer', u'oid': 0, - u'minsize': 7500000, u'txfee': 1000, u'maxsize': 599972700, u'cjfee': u'0.0002'}, - {u'counterparty': u'J65z23xdjxJjC7er', u'ordertype': u'sw0reloffer', u'oid': 0, - u'minsize': 7500000, u'txfee': 1000, u'maxsize': 599972700, u'cjfee': u'0.0002'}, - {u'counterparty': u'J64Ghp5PXCdY9H3t', u'ordertype': u'sw0reloffer', u'oid': 0, - u'minsize': 7500000, u'txfee': 1000, u'maxsize': 599972700, u'cjfee': u'0.0002'}, - {u'counterparty': u'J659UPUSLLjHJpaB', u'ordertype': u'sw0reloffer', u'oid': 0, - u'minsize': 7500000, u'txfee': 1000, u'maxsize': 599972700, u'cjfee': u'0.0002'}, - {u'counterparty': u'J6cBx1FwUVh9zzoO', u'ordertype': u'sw0reloffer', u'oid': 0, - u'minsize': 7500000, u'txfee': 1000, u'maxsize': 599972700, u'cjfee': u'0.0002'}] +t_orderbook = [ + {u'counterparty': u'J6FA1Gj7Ln4vSGne', u'ordertype': u'sw0reloffer', + u'oid': 0, u'minsize': 7500000, u'txfee': 1000, u'maxsize': 599972700, + u'cjfee': u'0.0002', u'minimum_tx_fee_rate': 0}, + {u'counterparty': u'J6CFffuuewjG44UJ', u'ordertype': u'sw0reloffer', + u'oid': 0, u'minsize': 7500000, u'txfee': 1000, u'maxsize': 599972700, + u'cjfee': u'0.0002', u'minimum_tx_fee_rate': 0}, + {u'counterparty': u'J65z23xdjxJjC7er', u'ordertype': u'sw0reloffer', + u'oid': 0, u'minsize': 7500000, u'txfee': 1000, u'maxsize': 599972700, + u'cjfee': u'0.0002', u'minimum_tx_fee_rate': 0}, + {u'counterparty': u'J64Ghp5PXCdY9H3t', u'ordertype': u'sw0reloffer', + u'oid': 0, u'minsize': 7500000, u'txfee': 1000, u'maxsize': 599972700, + u'cjfee': u'0.0002', u'minimum_tx_fee_rate': 0}, + {u'counterparty': u'J659UPUSLLjHJpaB', u'ordertype': u'sw0reloffer', + u'oid': 0, u'minsize': 7500000, u'txfee': 1000, u'maxsize': 599972700, + u'cjfee': u'0.0002', u'minimum_tx_fee_rate': 0}, + {u'counterparty': u'J6cBx1FwUVh9zzoO', u'ordertype': u'sw0reloffer', + u'oid': 0, u'minsize': 7500000, u'txfee': 1000, u'maxsize': 599972700, + u'cjfee': u'0.0002', u'minimum_tx_fee_rate': 0} +] t_dest_addr = "mvw1NazKDRbeNufFANqpYNAANafsMC2zVU" diff --git a/test/jmclient/test_yieldgenerator.py b/test/jmclient/test_yieldgenerator.py index 54d3f9ccb..fb354536a 100644 --- a/test/jmclient/test_yieldgenerator.py +++ b/test/jmclient/test_yieldgenerator.py @@ -81,6 +81,7 @@ def test_abs_fee(self): self.assertEqual(yg.create_my_orders(), [ {'oid': 0, 'ordertype': 'swabsoffer', + 'minimum_tx_fee_rate': 1000, 'minsize': 100000, 'maxsize': 1999000, 'txfee': 1000, @@ -94,6 +95,7 @@ def test_rel_fee(self): self.assertEqual(yg.create_my_orders(), [ {'oid': 0, 'ordertype': 'sw0reloffer', + 'minimum_tx_fee_rate': 1000, 'minsize': 15000, 'maxsize': 1999000, 'txfee': 1000, @@ -107,6 +109,7 @@ def test_dust_threshold(self): self.assertEqual(yg.create_my_orders(), [ {'oid': 0, 'ordertype': 'swabsoffer', + 'minimum_tx_fee_rate': 1000, 'minsize': 100000, 'maxsize': 1999000, 'txfee': 10,