WIP for minimum tx feerate setting for makers

scripts/obwatch/ob-watcher.py

@@ -115,6 +115,7 @@ def create_offerbook_table_heading(btc_unit, rel_unit):
                 col.format('txfee', 'Miner Fee Contribution / ' + btc_unit),
                 col.format('minsize', 'Minimum Size / ' + btc_unit),
                 col.format('maxsize', 'Maximum Size / ' + btc_unit),
+                col.format('minimum_tx_fee_rate', 'Minimum feerate (sat/vB)'),
                 col.format('bondvalue', 'Bond value / ' + btc_unit + '<sup>' + bond_exponent + '</sup>')
             ]) + ' </tr>'
     return tableheading

scripts/yg-privacyenhanced.py

@@ -90,7 +90,9 @@ def create_my_orders(self):
                  'minsize': randomize_minsize,
                  'maxsize': randomize_maxsize,
                  'txfee': randomize_txfee,
-                 'cjfee': str(randomize_cjfee)}
+                 'cjfee': str(randomize_cjfee),
+                 # TODO: add some randomization factor here?
+                 'minimum_tx_fee_rate': self.minimum_tx_fee_rate}
 
         # sanity check
         assert order['minsize'] >= jm_single().DUST_THRESHOLD

src/jmclient/blockchaininterface.py

@@ -42,9 +42,9 @@ def pushtx(self, txbin: bytes) -> bool:
 
     @abstractmethod
     def query_utxo_set(self,
-                       txouts: Union[Tuple[bytes, int], List[Tuple[bytes, int]]],
+                       txouts: Union[Tuple[bytes, int], Iterable[Tuple[bytes, int]]],
                        includeconfs: bool = False,
-                       include_mempool: bool = True) -> List[Optional[dict]]:
+                       include_mempool: bool = True) -> List[dict]:
         """If txout is either (a) a single utxo in (txidbin, n) form,
         or a list of the same, returns, as a list for each txout item,
         the result of gettxout from the bitcoind rpc for those utxos;
@@ -239,7 +239,7 @@ def _fee_per_kb_has_been_manually_set(self, tx_fees: int) -> bool:
         """
         return tx_fees > 1000
 
-    def estimate_fee_per_kb(self, tx_fees: int) -> int:
+    def estimate_fee_per_kb(self, tx_fees: int, randomize: bool = True) -> int:
         """ The argument tx_fees may be either a number of blocks target,
         for estimation of feerate by Core, or a number of satoshis
         per kilo-vbyte (see `_fee_per_kb_has_been_manually_set` for
@@ -255,7 +255,11 @@ def estimate_fee_per_kb(self, tx_fees: int) -> int:
         # default to use if fees cannot be estimated
         fallback_fee = 10000
 
-        tx_fees_factor = abs(jm_single().config.getfloat('POLICY', 'tx_fees_factor'))
+        if randomize:
+            tx_fees_factor = abs(jm_single().config.getfloat(
+                'POLICY', 'tx_fees_factor'))
+        else:
+            tx_fees_factor = 0
 
         mempoolminfee_in_sat = self._get_mempool_min_fee()
         # in case of error
@@ -543,11 +547,12 @@ def pushtx(self, txbin: bytes) -> bool:
         return True
 
     def query_utxo_set(self,
-                       txouts: Union[Tuple[bytes, int], List[Tuple[bytes, int]]],
+                       txouts: Union[Tuple[bytes, int], Iterable[Tuple[bytes, int]]],
                        includeconfs: bool = False,
-                       include_mempool: bool = True) -> List[Optional[dict]]:
-        if not isinstance(txouts, list):
+                       include_mempool: bool = True) -> List[dict]:
+        if isinstance(txouts, tuple):
             txouts = [txouts]
+        assert isinstance(txouts, Iterable)
         result = []
         for txo in txouts:
             txo_hex = bintohex(txo[0])
@@ -802,9 +807,9 @@ def __init__(self, jsonRpc: JsonRpc, wallet_name: str) -> None:
         self.shutdown_signal = False
         self.destn_addr = self._rpc("getnewaddress", [])
 
-    def estimate_fee_per_kb(self, tx_fees: int) -> int:
+    def estimate_fee_per_kb(self, tx_fees: int, randomize: bool = True) -> int:
         if not self.absurd_fees:
-            return super().estimate_fee_per_kb(tx_fees)
+            return super().estimate_fee_per_kb(tx_fees, randomize)
         else:
             return jm_single().config.getint("POLICY",
                                              "absurd_fee_per_kb") + 100

src/jmclient/configure.py

@@ -519,6 +519,11 @@ def jm_single() -> AttributeDict:
 # [fraction, 0-1] / variance around all offer sizes. Ex: 500k minsize, 0.1 var = 450k-550k
 size_factor = 0.1
 
+#
+minimum_tx_fee_rate = 1000
+
+gaplimit = 6
+
 [SNICKER]
 # Any other value than 'true' will be treated as False,
 # and no SNICKER actions will be enabled in that case:

src/jmclient/maker.py

@@ -15,7 +15,7 @@
 jlog = get_log()
 
 class Maker(object):
-    def __init__(self, wallet_service):
+    def __init__(self, wallet_service: WalletService):
         self.active_orders = {}
         assert isinstance(wallet_service, WalletService)
         self.wallet_service = wallet_service
@@ -28,6 +28,8 @@ def __init__(self, wallet_service):
         # not-enough-coins:
         self.sync_wait_loop.start(2.0, now=False)
         self.aborted = False
+        self.minimum_tx_fee_rate = jm_single().config.getint(
+            "YIELDGENERATOR", "minimum_tx_fee_rate")
 
     def try_to_create_my_orders(self):
         """Because wallet syncing is not synchronous(!),
@@ -187,6 +189,16 @@ def verify_unsigned_tx(self, tx, offerinfo):
         """
         tx_utxo_set = set((x.prevout.hash[::-1], x.prevout.n) for x in tx.vin)
 
+        if self.minimum_tx_fee_rate > 1000:
+            tx_inp_data = jm_single().bc_interface.query_utxo_set(tx_utxo_set)
+            total_input_value_sum = 0
+            for utxo in tx_inp_data:
+                total_input_value_sum = total_input_value_sum + utxo["value"]
+            tx_fee = total_input_value_sum - btc.tx_total_outputs_value(tx)
+            tx_fee_rate = tx_fee / btc.tx_vsize(tx) * 1000
+            if tx_fee_rate < self.minimum_tx_fee_rate:
+                return (False, "tx feerate below configured minimum tx feerate")
+
         utxos = offerinfo["utxos"]
         cjaddr = offerinfo["cjaddr"]
         cjaddr_script = btc.CCoinAddress(cjaddr).to_scriptPubKey()

src/jmclient/support.py

@@ -1,13 +1,15 @@
-from functools import reduce
 import random
-from jmbase.support import get_log
 from decimal import Decimal
-from .configure import get_bondless_makers_allowance
-
+from functools import reduce
 from math import exp
+from typing import Callable, List, Optional, Tuple
+
+from jmbase.support import get_log
+from .configure import get_bondless_makers_allowance, jm_single
+
 
 ORDER_KEYS = ['counterparty', 'oid', 'ordertype', 'minsize', 'maxsize', 'txfee',
-              'cjfee']
+              'cjfee', 'minimum_tx_fee_rate']
 
 log = get_log()
 
@@ -45,7 +47,7 @@ def rand_exp_array(lamda, n):
     return [random.expovariate(1.0 / lamda) for _ in range(n)]
 
 
-def rand_weighted_choice(n, p_arr):
+def rand_weighted_choice(n: int, p_arr: list) -> int:
     """
     Choose a value in 0..n-1
     with the choice weighted by the probabilities
@@ -177,7 +179,7 @@ def calc_cj_fee(ordertype, cjfee, cj_amount):
     return real_cjfee
 
 
-def weighted_order_choose(orders, n):
+def weighted_order_choose(orders: List[dict], n: int) -> dict:
     """
     Algorithm for choosing the weighting function
     it is an exponential
@@ -208,18 +210,18 @@ def weighted_order_choose(orders, n):
     return orders[chosen_order_index]
 
 
-def random_under_max_order_choose(orders, n):
+def random_under_max_order_choose(orders: List[dict], n: int) -> dict:
     # orders are already pre-filtered for max_cj_fee
     return random.choice(orders)
 
 
-def cheapest_order_choose(orders, n):
+def cheapest_order_choose(orders: List[dict], n: int) -> dict:
     """
     Return the cheapest order from the orders.
     """
     return orders[0]
 
-def fidelity_bond_weighted_order_choose(orders, n):
+def fidelity_bond_weighted_order_choose(orders: List[dict], n: int) -> dict:
     """
     choose orders based on fidelity bond for improved sybil resistance
 
@@ -247,19 +249,28 @@ def check_max_fee(fee):
     return check_max_fee
 
 
-def choose_orders(offers, cj_amount, n, chooseOrdersBy, ignored_makers=None,
-                  pick=False, allowed_types=["sw0reloffer", "sw0absoffer"],
-                  max_cj_fee=(1, float('inf'))):
+def choose_orders(offers: List[dict],
+                  cj_amount: int,
+                  num_counterparties: int,
+                  chooseOrdersBy: Callable[[List[dict], int], dict],
+                  ignored_makers: Optional[List[str]] = None,
+                  pick: bool = False,
+                  allowed_types: List[str] = ["sw0reloffer", "sw0absoffer"],
+                  max_cj_fee: Tuple = (1, float('inf'))) -> Tuple[Optional[dict], int]:
     is_within_max_limits = _get_is_within_max_limits(
         max_cj_fee[0], max_cj_fee[1], cj_amount)
     if ignored_makers is None:
         ignored_makers = []
+    fee_per_kb = jm_single().bc_interface.estimate_fee_per_kb(
+        jm_single().config.getint("POLICY", "tx_fees"), randomize=False)
     #Filter ignored makers and inappropriate amounts
     orders = [o for o in offers if o['counterparty'] not in ignored_makers]
     orders = [o for o in orders if o['minsize'] < cj_amount]
     orders = [o for o in orders if o['maxsize'] > cj_amount]
     #Filter those not using wished-for offertypes
     orders = [o for o in orders if o["ordertype"] in allowed_types]
+    #Filter those not accepting our tx feerate
+    orders = [o for o in orders if o["minimum_tx_fee_rate"] <= fee_per_kb]
 
     orders_fees = []
     for o in orders:
@@ -268,10 +279,11 @@ def choose_orders(offers, cj_amount, n, chooseOrdersBy, ignored_makers=None,
             orders_fees.append((o, fee))
 
     counterparties = set(o['counterparty'] for o, f in orders_fees)
-    if n > len(counterparties):
+    if num_counterparties > len(counterparties):
         log.warn(('ERROR not enough liquidity in the orderbook n=%d '
                    'suitable-counterparties=%d amount=%d totalorders=%d') %
-                  (n, len(counterparties), cj_amount, len(orders_fees)))
+                  (num_counterparties, len(counterparties), cj_amount,
+                   len(orders_fees)))
         # TODO handle not enough liquidity better, maybe an Exception
         return None, 0
     """
@@ -294,8 +306,9 @@ def choose_orders(offers, cj_amount, n, chooseOrdersBy, ignored_makers=None,
                                                    ]))
     total_cj_fee = 0
     chosen_orders = []
-    for i in range(n):
-        chosen_order, chosen_fee = chooseOrdersBy(orders_fees, n)
+    for i in range(num_counterparties):
+        chosen_order, chosen_fee = chooseOrdersBy(orders_fees,
+                                                  num_counterparties)
         # remove all orders from that same counterparty
         # only needed if offers are manually picked
         orders_fees = [o
@@ -308,14 +321,14 @@ def choose_orders(offers, cj_amount, n, chooseOrdersBy, ignored_makers=None,
     return result, total_cj_fee
 
 
-def choose_sweep_orders(offers,
-                        total_input_value,
-                        total_txfee,
-                        n,
-                        chooseOrdersBy,
-                        ignored_makers=None,
-                        allowed_types=['sw0reloffer', 'sw0absoffer'],
-                        max_cj_fee=(1, float('inf'))):
+def choose_sweep_orders(offers: List[dict],
+                        total_input_value: int,
+                        total_txfee: int,
+                        num_counterparties: int,
+                        chooseOrdersBy: Callable[[List[dict], int], dict],
+                        ignored_makers: Optional[List[str]] = None,
+                        allowed_types: List[str] = ['sw0reloffer', 'sw0absoffer'],
+                        max_cj_fee: Tuple = (1, float('inf'))) -> Tuple[Optional[dict], int, int]:
     """
     choose an order given that we want to be left with no change
     i.e. sweep an entire group of utxos
@@ -332,7 +345,7 @@ def choose_sweep_orders(offers,
     if ignored_makers is None:
         ignored_makers = []
 
-    def calc_zero_change_cj_amount(ordercombo):
+    def calc_zero_change_cj_amount(ordercombo: List[dict]) -> Tuple[int, int]:
         sumabsfee = 0
         sumrelfee = Decimal('0')
         sumtxfee_contribution = 0
@@ -352,14 +365,18 @@ def calc_zero_change_cj_amount(ordercombo):
         cjamount = int(cjamount.quantize(Decimal(1)))
         return cjamount, int(sumabsfee + sumrelfee * cjamount)
 
+    fee_per_kb = jm_single().bc_interface.estimate_fee_per_kb(
+        jm_single().config.getint("POLICY", "tx_fees"), randomize=False)
     log.debug('choosing sweep orders for total_input_value = ' + str(
-        total_input_value) + ' n=' + str(n))
+        total_input_value) + ' n=' + str(num_counterparties))
     offers = [o for o in offers if o["ordertype"] in allowed_types]
     #Filter ignored makers and inappropriate amounts
     offers = [o for o in offers if o['counterparty'] not in ignored_makers]
     offers = [o for o in offers if o['minsize'] < total_input_value]
     # while we do not know the exact cj value yet, we can approximate a ceiling:
     offers = [o for o in offers if o['maxsize'] > (total_input_value - total_txfee)]
+    #Filter those not accepting our tx feerate
+    offers = [o for o in offers if o["minimum_tx_fee_rate"] <= fee_per_kb]
 
     log.debug('orderlist = \n' + '\n'.join([str(o) for o in offers]))
     orders_fees = [(o, calc_cj_fee(o['ordertype'], o['cjfee'],
@@ -376,13 +393,14 @@ def calc_zero_change_cj_amount(ordercombo):
              if is_within_max_limits(v[1])).values(),
         key=feekey)
     chosen_orders = []
-    while len(chosen_orders) < n:
-        for i in range(n - len(chosen_orders)):
-            if len(orders_fees) < n - len(chosen_orders):
+    while len(chosen_orders) < num_counterparties:
+        for i in range(num_counterparties - len(chosen_orders)):
+            if len(orders_fees) < num_counterparties - len(chosen_orders):
                 log.debug('ERROR not enough liquidity in the orderbook')
                 # TODO handle not enough liquidity better, maybe an Exception
                 return None, 0, 0
-            chosen_order, chosen_fee = chooseOrdersBy(orders_fees, n)
+            chosen_order, chosen_fee = chooseOrdersBy(orders_fees,
+                                                      num_counterparties)
             log.debug('chosen = ' + str(chosen_order))
             # remove all orders from that same counterparty
             orders_fees = [

src/jmclient/yieldgenerator.py

@@ -89,8 +89,6 @@ def __init__(self, wallet_service, offerconfig):
             self.size_factor = offerconfig
         super().__init__(wallet_service)
 
-
-
     def create_my_orders(self):
         mix_balance = self.get_available_mixdepths()
         if len([b for m, b in mix_balance.items() if b > 0]) == 0:
@@ -113,7 +111,8 @@ def create_my_orders(self):
                  'maxsize': mix_balance[max_mix] - max(
                      jm_single().DUST_THRESHOLD, self.txfee_contribution),
                  'txfee': self.txfee_contribution,
-                 'cjfee': f}
+                 'cjfee': f,
+                 'minimum_tx_fee_rate': self.minimum_tx_fee_rate}
 
         # sanity check
         assert order['minsize'] >= 0

src/jmdaemon/message_channel.py

@@ -293,7 +293,8 @@ def announce_orders(self, orderlist, nick, fidelity_bond_proof_msg, new_mc):
         privmsg, on a specific mc.
         Fidelity bonds can only be announced over privmsg, nick must be nonNone
         """
-        order_keys = ['oid', 'minsize', 'maxsize', 'txfee', 'cjfee']
+        order_keys = ['oid', 'minsize', 'maxsize', 'txfee', 'cjfee',
+                      'minimum_tx_fee_rate']
         orderlines = []
         for order in orderlist:
             orderlines.append(COMMAND_PREFIX + order['ordertype'] + \
@@ -545,7 +546,7 @@ def on_nick_change_trigger(self, new_nick):
             self.on_nick_change(new_nick)
 
     def on_order_seen_trigger(self, mc, counterparty, oid, ordertype, minsize,
-                              maxsize, txfee, cjfee):
+                              maxsize, txfee, cjfee, minimum_tx_fee_rate):
         """This is the entry point into private messaging.
         Hence, it fixes for the rest of the conversation, which
         message channel the bots are going to communicate over
@@ -564,7 +565,7 @@ def on_order_seen_trigger(self, mc, counterparty, oid, ordertype, minsize,
         self.active_channels[counterparty] = mc
         if self.on_order_seen:
             self.on_order_seen(counterparty, oid, ordertype, minsize, maxsize,
-                               txfee, cjfee)
+                               txfee, cjfee, minimum_tx_fee_rate)
 
     # orderbook watcher commands
     def register_orderbookwatch_callbacks(self,
@@ -785,9 +786,11 @@ def check_for_orders(self, nick, _chunks):
                 maxsize = _chunks[3]
                 txfee = _chunks[4]
                 cjfee = _chunks[5]
+                minimum_tx_fee_rate = _chunks[6] if len(_chunks) > 6 else 0
                 if self.on_order_seen:
                     self.on_order_seen(self, counterparty, oid, ordertype,
-                                       minsize, maxsize, txfee, cjfee)
+                                       minsize, maxsize, txfee, cjfee,
+                                       minimum_tx_fee_rate)
             except IndexError as e:
                 log.debug(e)
                 log.debug('index error parsing chunks, possibly malformed '