WIP for minimum tx feerate setting for makers

scripts/obwatch/ob-watcher.py

@@ -115,6 +115,7 @@ def create_offerbook_table_heading(btc_unit, rel_unit):
                 col.format('txfee', 'Miner Fee Contribution / ' + btc_unit),
                 col.format('minsize', 'Minimum Size / ' + btc_unit),
                 col.format('maxsize', 'Maximum Size / ' + btc_unit),
+                col.format('minimum_tx_fee_rate', 'Minimum feerate (sat/vB)'),
                 col.format('bondvalue', 'Bond value / ' + btc_unit + '<sup>' + bond_exponent + '</sup>')
             ]) + ' </tr>'
     return tableheading

scripts/yg-privacyenhanced.py

@@ -90,7 +90,9 @@ def create_my_orders(self):
                  'minsize': randomize_minsize,
                  'maxsize': randomize_maxsize,
                  'txfee': randomize_txfee,
-                 'cjfee': str(randomize_cjfee)}
+                 'cjfee': str(randomize_cjfee),
+                 # TODO: add some randomization factor here?
+                 'minimum_tx_fee_rate': self.minimum_tx_fee_rate}
 
         # sanity check
         assert order['minsize'] >= jm_single().DUST_THRESHOLD

src/jmclient/blockchaininterface.py

@@ -42,9 +42,9 @@ def pushtx(self, txbin: bytes) -> bool:
 
     @abstractmethod
     def query_utxo_set(self,
-                       txouts: Union[Tuple[bytes, int], List[Tuple[bytes, int]]],
+                       txouts: Union[Tuple[bytes, int], Iterable[Tuple[bytes, int]]],
                        includeconfs: bool = False,
-                       include_mempool: bool = True) -> List[Optional[dict]]:
+                       include_mempool: bool = True) -> List[dict]:
         """If txout is either (a) a single utxo in (txidbin, n) form,
         or a list of the same, returns, as a list for each txout item,
         the result of gettxout from the bitcoind rpc for those utxos;
@@ -239,7 +239,7 @@ def _fee_per_kb_has_been_manually_set(self, tx_fees: int) -> bool:
         """
         return tx_fees > 1000
 
-    def estimate_fee_per_kb(self, tx_fees: int) -> int:
+    def estimate_fee_per_kb(self, tx_fees: int, randomize: bool = True) -> int:
         """ The argument tx_fees may be either a number of blocks target,
         for estimation of feerate by Core, or a number of satoshis
         per kilo-vbyte (see `_fee_per_kb_has_been_manually_set` for
@@ -255,7 +255,11 @@ def estimate_fee_per_kb(self, tx_fees: int) -> int:
         # default to use if fees cannot be estimated
         fallback_fee = 10000
 
-        tx_fees_factor = abs(jm_single().config.getfloat('POLICY', 'tx_fees_factor'))
+        if randomize:
+            tx_fees_factor = abs(jm_single().config.getfloat(
+                'POLICY', 'tx_fees_factor'))
+        else:
+            tx_fees_factor = 0
 
         mempoolminfee_in_sat = self._get_mempool_min_fee()
         # in case of error
@@ -543,11 +547,12 @@ def pushtx(self, txbin: bytes) -> bool:
         return True
 
     def query_utxo_set(self,
-                       txouts: Union[Tuple[bytes, int], List[Tuple[bytes, int]]],
+                       txouts: Union[Tuple[bytes, int], Iterable[Tuple[bytes, int]]],
                        includeconfs: bool = False,
-                       include_mempool: bool = True) -> List[Optional[dict]]:
-        if not isinstance(txouts, list):
+                       include_mempool: bool = True) -> List[dict]:
+        if isinstance(txouts, tuple):
             txouts = [txouts]
+        assert isinstance(txouts, Iterable)
         result = []
         for txo in txouts:
             txo_hex = bintohex(txo[0])
@@ -802,9 +807,9 @@ def __init__(self, jsonRpc: JsonRpc, wallet_name: str) -> None:
         self.shutdown_signal = False
         self.destn_addr = self._rpc("getnewaddress", [])
 
-    def estimate_fee_per_kb(self, tx_fees: int) -> int:
+    def estimate_fee_per_kb(self, tx_fees: int, randomize: bool = True) -> int:
         if not self.absurd_fees:
-            return super().estimate_fee_per_kb(tx_fees)
+            return super().estimate_fee_per_kb(tx_fees, randomize)
         else:
             return jm_single().config.getint("POLICY",
                                              "absurd_fee_per_kb") + 100

src/jmclient/configure.py

@@ -519,6 +519,11 @@ def jm_single() -> AttributeDict:
 # [fraction, 0-1] / variance around all offer sizes. Ex: 500k minsize, 0.1 var = 450k-550k
 size_factor = 0.1
 
+#
+minimum_tx_fee_rate = 1000
+
+gaplimit = 6
+
 [SNICKER]
 # Any other value than 'true' will be treated as False,
 # and no SNICKER actions will be enabled in that case:

src/jmclient/maker.py

@@ -15,7 +15,7 @@
 jlog = get_log()
 
 class Maker(object):
-    def __init__(self, wallet_service):
+    def __init__(self, wallet_service: WalletService):
         self.active_orders = {}
         assert isinstance(wallet_service, WalletService)
         self.wallet_service = wallet_service
@@ -28,6 +28,8 @@ def __init__(self, wallet_service):
         # not-enough-coins:
         self.sync_wait_loop.start(2.0, now=False)
         self.aborted = False
+        self.minimum_tx_fee_rate = jm_single().config.getint(
+            "YIELDGENERATOR", "minimum_tx_fee_rate")
 
     def try_to_create_my_orders(self):
         """Because wallet syncing is not synchronous(!),
@@ -187,6 +189,16 @@ def verify_unsigned_tx(self, tx, offerinfo):
         """
         tx_utxo_set = set((x.prevout.hash[::-1], x.prevout.n) for x in tx.vin)
 
+        if self.minimum_tx_fee_rate > 1000:
+            tx_inp_data = jm_single().bc_interface.query_utxo_set(tx_utxo_set)
+            total_input_value_sum = 0
+            for utxo in tx_inp_data:
+                total_input_value_sum = total_input_value_sum + utxo["value"]
+            tx_fee = total_input_value_sum - btc.tx_total_outputs_value(tx)
+            tx_fee_rate = tx_fee / btc.tx_vsize(tx) * 1000
+            if tx_fee_rate < self.minimum_tx_fee_rate:
+                return (False, "tx feerate below configured minimum tx feerate")
+
         utxos = offerinfo["utxos"]
         cjaddr = offerinfo["cjaddr"]
         cjaddr_script = btc.CCoinAddress(cjaddr).to_scriptPubKey()

src/jmclient/support.py

@@ -1,13 +1,15 @@
-from functools import reduce
 import random
-from jmbase.support import get_log
 from decimal import Decimal
-from .configure import get_bondless_makers_allowance
-
+from functools import reduce
 from math import exp
+from typing import List, Optional, Tuple
+
+from jmbase.support import get_log
+from .configure import get_bondless_makers_allowance, jm_single
+
 
 ORDER_KEYS = ['counterparty', 'oid', 'ordertype', 'minsize', 'maxsize', 'txfee',
-              'cjfee']
+              'cjfee', 'minimum_tx_fee_rate']
 
 log = get_log()
 
@@ -247,19 +249,28 @@ def check_max_fee(fee):
     return check_max_fee
 
 
-def choose_orders(offers, cj_amount, n, chooseOrdersBy, ignored_makers=None,
-                  pick=False, allowed_types=["sw0reloffer", "sw0absoffer"],
-                  max_cj_fee=(1, float('inf'))):
+def choose_orders(offers: List[dict],
+                  cj_amount,
+                  num_counterparties: int,
+                  chooseOrdersBy,
+                  ignored_makers: Optional[List[str]] = None,
+                  pick: bool = False,
+                  allowed_types: List[str] = ["sw0reloffer", "sw0absoffer"],
+                  max_cj_fee: Tuple = (1, float('inf'))) -> Tuple[Optional[dict], int]:
     is_within_max_limits = _get_is_within_max_limits(
         max_cj_fee[0], max_cj_fee[1], cj_amount)
     if ignored_makers is None:
         ignored_makers = []
+    fee_per_kb = jm_single().bc_interface.estimate_fee_per_kb(
+        jm_single().config.getint("POLICY", "tx_fees"), randomize=False)
     #Filter ignored makers and inappropriate amounts
     orders = [o for o in offers if o['counterparty'] not in ignored_makers]
     orders = [o for o in orders if o['minsize'] < cj_amount]
     orders = [o for o in orders if o['maxsize'] > cj_amount]
     #Filter those not using wished-for offertypes
     orders = [o for o in orders if o["ordertype"] in allowed_types]
+    #Filter those not accepting our tx feerate
+    orders = [o for o in orders if o["minimum_tx_fee_rate"] <= fee_per_kb]
 
     orders_fees = []
     for o in orders:
@@ -268,10 +279,11 @@ def choose_orders(offers, cj_amount, n, chooseOrdersBy, ignored_makers=None,
             orders_fees.append((o, fee))
 
     counterparties = set(o['counterparty'] for o, f in orders_fees)
-    if n > len(counterparties):
+    if num_counterparties > len(counterparties):
         log.warn(('ERROR not enough liquidity in the orderbook n=%d '
                    'suitable-counterparties=%d amount=%d totalorders=%d') %
-                  (n, len(counterparties), cj_amount, len(orders_fees)))
+                  (num_counterparties, len(counterparties), cj_amount,
+                   len(orders_fees)))
         # TODO handle not enough liquidity better, maybe an Exception
         return None, 0
     """
@@ -294,8 +306,9 @@ def choose_orders(offers, cj_amount, n, chooseOrdersBy, ignored_makers=None,
                                                    ]))
     total_cj_fee = 0
     chosen_orders = []
-    for i in range(n):
-        chosen_order, chosen_fee = chooseOrdersBy(orders_fees, n)
+    for i in range(num_counterparties):
+        chosen_order, chosen_fee = chooseOrdersBy(orders_fees,
+                                                  num_counterparties)
         # remove all orders from that same counterparty
         # only needed if offers are manually picked
         orders_fees = [o

src/jmclient/yieldgenerator.py

@@ -89,8 +89,6 @@ def __init__(self, wallet_service, offerconfig):
             self.size_factor = offerconfig
         super().__init__(wallet_service)
 
-
-
     def create_my_orders(self):
         mix_balance = self.get_available_mixdepths()
         if len([b for m, b in mix_balance.items() if b > 0]) == 0:
@@ -113,7 +111,8 @@ def create_my_orders(self):
                  'maxsize': mix_balance[max_mix] - max(
                      jm_single().DUST_THRESHOLD, self.txfee_contribution),
                  'txfee': self.txfee_contribution,
-                 'cjfee': f}
+                 'cjfee': f,
+                 'minimum_tx_fee_rate': self.minimum_tx_fee_rate}
 
         # sanity check
         assert order['minsize'] >= 0

src/jmdaemon/message_channel.py

@@ -293,7 +293,8 @@ def announce_orders(self, orderlist, nick, fidelity_bond_proof_msg, new_mc):
         privmsg, on a specific mc.
         Fidelity bonds can only be announced over privmsg, nick must be nonNone
         """
-        order_keys = ['oid', 'minsize', 'maxsize', 'txfee', 'cjfee']
+        order_keys = ['oid', 'minsize', 'maxsize', 'txfee', 'cjfee',
+                      'minimum_tx_fee_rate']
         orderlines = []
         for order in orderlist:
             orderlines.append(COMMAND_PREFIX + order['ordertype'] + \
@@ -545,7 +546,7 @@ def on_nick_change_trigger(self, new_nick):
             self.on_nick_change(new_nick)
 
     def on_order_seen_trigger(self, mc, counterparty, oid, ordertype, minsize,
-                              maxsize, txfee, cjfee):
+                              maxsize, txfee, cjfee, minimum_tx_fee_rate):
         """This is the entry point into private messaging.
         Hence, it fixes for the rest of the conversation, which
         message channel the bots are going to communicate over
@@ -564,7 +565,7 @@ def on_order_seen_trigger(self, mc, counterparty, oid, ordertype, minsize,
         self.active_channels[counterparty] = mc
         if self.on_order_seen:
             self.on_order_seen(counterparty, oid, ordertype, minsize, maxsize,
-                               txfee, cjfee)
+                               txfee, cjfee, minimum_tx_fee_rate)
 
     # orderbook watcher commands
     def register_orderbookwatch_callbacks(self,
@@ -785,9 +786,11 @@ def check_for_orders(self, nick, _chunks):
                 maxsize = _chunks[3]
                 txfee = _chunks[4]
                 cjfee = _chunks[5]
+                minimum_tx_fee_rate = _chunks[6] if len(_chunks) > 6 else 0
                 if self.on_order_seen:
                     self.on_order_seen(self, counterparty, oid, ordertype,
-                                       minsize, maxsize, txfee, cjfee)
+                                       minsize, maxsize, txfee, cjfee,
+                                       minimum_tx_fee_rate)
             except IndexError as e:
                 log.debug(e)
                 log.debug('index error parsing chunks, possibly malformed '

src/jmdaemon/orderbookwatch.py

@@ -40,7 +40,8 @@ def set_msgchan(self, msgchan):
             self.dblock.acquire(True)
             self.db.execute("CREATE TABLE orderbook(counterparty TEXT, "
                             "oid INTEGER, ordertype TEXT, minsize INTEGER, "
-                            "maxsize INTEGER, txfee INTEGER, cjfee TEXT);")
+                            "maxsize INTEGER, txfee INTEGER, cjfee TEXT, "
+                            "minimum_tx_fee_rate INTEGER);")
             self.db.execute("CREATE TABLE fidelitybonds(counterparty TEXT, "
                 "takernick TEXT, proof TEXT);");
         finally:
@@ -65,8 +66,10 @@ def on_set_topic(newtopic):
                 print('=' * 60)
                 joinmarket_alert[0] = alert
 
-    def on_order_seen(self, counterparty, oid, ordertype, minsize, maxsize,
-                      txfee, cjfee):
+    def on_order_seen(self, counterparty: str, oid: int, ordertype: str,
+                      minsize: int, maxsize: int,
+                      txfee: int, cjfee: Integral,
+                      minimum_tx_fee_rate: int) -> None:
         try:
             self.dblock.acquire(True)
             if int(oid) < 0 or int(oid) > sys.maxsize:
@@ -114,7 +117,8 @@ def on_order_seen(self, counterparty, oid, ordertype, minsize, maxsize,
             self.db.execute(
                 'INSERT INTO orderbook VALUES(?, ?, ?, ?, ?, ?, ?);',
                 (counterparty, oid, ordertype, minsize, maxsize, txfee,
-                 str(Decimal(cjfee))))  # any parseable Decimal is a valid cjfee
+                 str(Decimal(cjfee)),   # any parseable Decimal is a valid cjfee
+                 minimum_tx_fee_rate))
         except InvalidOperation:
             log.debug("Got invalid cjfee: " + str(cjfee) + " from " + counterparty)
         except Exception as e:

test/jmbitcoin/test_tx_signing.py

@@ -85,6 +85,7 @@ def test_sign_standard_txs(addrtype):
     txin = btc.CTxIn(btc.COutPoint(txid[::-1], vout))
     txout = btc.CTxOut(amount_less_fee, target_scriptPubKey)
     tx = btc.CMutableTransaction([txin], [txout])
+    assert btc.tx_total_outputs_value(tx) == amount_less_fee
 
     # Calculate the signature hash for the transaction. This is then signed by the
     # private key that controls the UTXO being spent here at this txin_index.

test/jmclient/commontest.py

@@ -170,7 +170,7 @@ def query_utxo_set(self,
             result.append(result_dict)        
         return result
 
-    def estimate_fee_per_kb(self, tx_fees: int) -> int:
+    def estimate_fee_per_kb(self, tx_fees: int, randomize: bool = True) -> int:
         return 30000
 
 

test/jmclient/taker_test_data.py

@@ -1,16 +1,24 @@
 #orderbook
-t_orderbook = [{u'counterparty': u'J6FA1Gj7Ln4vSGne', u'ordertype': u'sw0reloffer', u'oid': 0,
-  u'minsize': 7500000, u'txfee': 1000, u'maxsize': 599972700, u'cjfee': u'0.0002'},
- {u'counterparty': u'J6CFffuuewjG44UJ', u'ordertype': u'sw0reloffer', u'oid': 0,
-  u'minsize': 7500000, u'txfee': 1000, u'maxsize': 599972700, u'cjfee': u'0.0002'},
- {u'counterparty': u'J65z23xdjxJjC7er', u'ordertype': u'sw0reloffer', u'oid': 0,
-  u'minsize': 7500000, u'txfee': 1000, u'maxsize': 599972700, u'cjfee': u'0.0002'},
- {u'counterparty': u'J64Ghp5PXCdY9H3t', u'ordertype': u'sw0reloffer', u'oid': 0,
-  u'minsize': 7500000, u'txfee': 1000, u'maxsize': 599972700, u'cjfee': u'0.0002'},
- {u'counterparty': u'J659UPUSLLjHJpaB', u'ordertype': u'sw0reloffer', u'oid': 0,
-  u'minsize': 7500000, u'txfee': 1000, u'maxsize': 599972700, u'cjfee': u'0.0002'},
- {u'counterparty': u'J6cBx1FwUVh9zzoO', u'ordertype': u'sw0reloffer', u'oid': 0,
-  u'minsize': 7500000, u'txfee': 1000, u'maxsize': 599972700, u'cjfee': u'0.0002'}]
+t_orderbook = [
+    {u'counterparty': u'J6FA1Gj7Ln4vSGne', u'ordertype': u'sw0reloffer',
+     u'oid': 0, u'minsize': 7500000, u'txfee': 1000, u'maxsize': 599972700,
+     u'cjfee': u'0.0002', u'minimum_tx_fee_rate': 0},
+    {u'counterparty': u'J6CFffuuewjG44UJ', u'ordertype': u'sw0reloffer',
+     u'oid': 0, u'minsize': 7500000, u'txfee': 1000, u'maxsize': 599972700,
+     u'cjfee': u'0.0002', u'minimum_tx_fee_rate': 0},
+    {u'counterparty': u'J65z23xdjxJjC7er', u'ordertype': u'sw0reloffer',
+     u'oid': 0, u'minsize': 7500000, u'txfee': 1000, u'maxsize': 599972700,
+     u'cjfee': u'0.0002', u'minimum_tx_fee_rate': 0},
+    {u'counterparty': u'J64Ghp5PXCdY9H3t', u'ordertype': u'sw0reloffer',
+     u'oid': 0, u'minsize': 7500000, u'txfee': 1000, u'maxsize': 599972700,
+     u'cjfee': u'0.0002', u'minimum_tx_fee_rate': 0},
+    {u'counterparty': u'J659UPUSLLjHJpaB', u'ordertype': u'sw0reloffer',
+     u'oid': 0, u'minsize': 7500000, u'txfee': 1000, u'maxsize': 599972700,
+     u'cjfee': u'0.0002', u'minimum_tx_fee_rate': 0},
+    {u'counterparty': u'J6cBx1FwUVh9zzoO', u'ordertype': u'sw0reloffer',
+     u'oid': 0, u'minsize': 7500000, u'txfee': 1000, u'maxsize': 599972700,
+     u'cjfee': u'0.0002', u'minimum_tx_fee_rate': 0}
+]
 
 t_dest_addr = "mvw1NazKDRbeNufFANqpYNAANafsMC2zVU"
 

test/jmclient/test_yieldgenerator.py

@@ -81,6 +81,7 @@ def test_abs_fee(self):
         self.assertEqual(yg.create_my_orders(), [
           {'oid': 0,
            'ordertype': 'swabsoffer',
+           'minimum_tx_fee_rate': 1000,
            'minsize': 100000,
            'maxsize': 1999000,
            'txfee': 1000,
@@ -94,6 +95,7 @@ def test_rel_fee(self):
         self.assertEqual(yg.create_my_orders(), [
           {'oid': 0,
            'ordertype': 'sw0reloffer',
+           'minimum_tx_fee_rate': 1000,
            'minsize': 15000,
            'maxsize': 1999000,
            'txfee': 1000,
@@ -107,6 +109,7 @@ def test_dust_threshold(self):
         self.assertEqual(yg.create_my_orders(), [
           {'oid': 0,
            'ordertype': 'swabsoffer',
+           'minimum_tx_fee_rate': 1000,
            'minsize': 100000,
            'maxsize': 1999000,
            'txfee': 10,