Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Implement minimum tx feerate setting for makers #1602

Draft
wants to merge 2 commits into
base: master
Choose a base branch
from
Draft
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions scripts/obwatch/ob-watcher.py
Original file line number Diff line number Diff line change
Expand Up @@ -115,6 +115,7 @@ def create_offerbook_table_heading(btc_unit, rel_unit):
col.format('txfee', 'Miner Fee Contribution / ' + btc_unit),
col.format('minsize', 'Minimum Size / ' + btc_unit),
col.format('maxsize', 'Maximum Size / ' + btc_unit),
col.format('minimum_tx_fee_rate', 'Minimum feerate (sat/vB)'),
col.format('bondvalue', 'Bond value / ' + btc_unit + '<sup>' + bond_exponent + '</sup>')
]) + ' </tr>'
return tableheading
Expand Down
4 changes: 3 additions & 1 deletion scripts/yg-privacyenhanced.py
Original file line number Diff line number Diff line change
Expand Up @@ -90,7 +90,9 @@ def create_my_orders(self):
'minsize': randomize_minsize,
'maxsize': randomize_maxsize,
'txfee': randomize_txfee,
'cjfee': str(randomize_cjfee)}
'cjfee': str(randomize_cjfee),
# TODO: add some randomization factor here?
'minimum_tx_fee_rate': self.minimum_tx_fee_rate}

# sanity check
assert order['minsize'] >= jm_single().DUST_THRESHOLD
Expand Down
23 changes: 14 additions & 9 deletions src/jmclient/blockchaininterface.py
Original file line number Diff line number Diff line change
Expand Up @@ -42,9 +42,9 @@ def pushtx(self, txbin: bytes) -> bool:

@abstractmethod
def query_utxo_set(self,
txouts: Union[Tuple[bytes, int], List[Tuple[bytes, int]]],
txouts: Union[Tuple[bytes, int], Iterable[Tuple[bytes, int]]],
includeconfs: bool = False,
include_mempool: bool = True) -> List[Optional[dict]]:
include_mempool: bool = True) -> List[dict]:
"""If txout is either (a) a single utxo in (txidbin, n) form,
or a list of the same, returns, as a list for each txout item,
the result of gettxout from the bitcoind rpc for those utxos;
Expand Down Expand Up @@ -239,7 +239,7 @@ def _fee_per_kb_has_been_manually_set(self, tx_fees: int) -> bool:
"""
return tx_fees > 1000

def estimate_fee_per_kb(self, tx_fees: int) -> int:
def estimate_fee_per_kb(self, tx_fees: int, randomize: bool = True) -> int:
""" The argument tx_fees may be either a number of blocks target,
for estimation of feerate by Core, or a number of satoshis
per kilo-vbyte (see `_fee_per_kb_has_been_manually_set` for
Expand All @@ -255,7 +255,11 @@ def estimate_fee_per_kb(self, tx_fees: int) -> int:
# default to use if fees cannot be estimated
fallback_fee = 10000

tx_fees_factor = abs(jm_single().config.getfloat('POLICY', 'tx_fees_factor'))
if randomize:
tx_fees_factor = abs(jm_single().config.getfloat(
'POLICY', 'tx_fees_factor'))
else:
tx_fees_factor = 0

mempoolminfee_in_sat = self._get_mempool_min_fee()
# in case of error
Expand Down Expand Up @@ -543,11 +547,12 @@ def pushtx(self, txbin: bytes) -> bool:
return True

def query_utxo_set(self,
txouts: Union[Tuple[bytes, int], List[Tuple[bytes, int]]],
txouts: Union[Tuple[bytes, int], Iterable[Tuple[bytes, int]]],
includeconfs: bool = False,
include_mempool: bool = True) -> List[Optional[dict]]:
if not isinstance(txouts, list):
include_mempool: bool = True) -> List[dict]:
if isinstance(txouts, tuple):
txouts = [txouts]
assert isinstance(txouts, Iterable)
result = []
for txo in txouts:
txo_hex = bintohex(txo[0])
Expand Down Expand Up @@ -802,9 +807,9 @@ def __init__(self, jsonRpc: JsonRpc, wallet_name: str) -> None:
self.shutdown_signal = False
self.destn_addr = self._rpc("getnewaddress", [])

def estimate_fee_per_kb(self, tx_fees: int) -> int:
def estimate_fee_per_kb(self, tx_fees: int, randomize: bool = True) -> int:
if not self.absurd_fees:
return super().estimate_fee_per_kb(tx_fees)
return super().estimate_fee_per_kb(tx_fees, randomize)
else:
return jm_single().config.getint("POLICY",
"absurd_fee_per_kb") + 100
Expand Down
5 changes: 5 additions & 0 deletions src/jmclient/configure.py
Original file line number Diff line number Diff line change
Expand Up @@ -519,6 +519,11 @@ def jm_single() -> AttributeDict:
# [fraction, 0-1] / variance around all offer sizes. Ex: 500k minsize, 0.1 var = 450k-550k
size_factor = 0.1

#
minimum_tx_fee_rate = 1000
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Could you share some thinking behind this number? (I mean, apart from the obvious :) ).

Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Nothing apart from the obvious - it's default minimum tx relay fee.

Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Right. This might be related to point 3/ in my list of suggestions here.

You want this var to be continuously updated, at least as transactions are confirmed, but much more, also when new blocks come in, probably. I agree with you that we want that kind of frequency of update.

I didn't think about this in detail earlier today but ... YGs advertise their offers when they start, and also advertise updates in the pit on transaction confirmation or "arrival" (sometimes) but when the taker requests !orderbook they're going to privmsg their offer at that point. The problem is that that announcement-in-private (the response to the Taker's !orderbook) is handled entirely within the daemon, as per the comment to src.jmdaemon.daemon_protocol.JMDaemonServerProtocol.on_orderbook_requested, this is dealt with by the daemon assuming the offerlist is up to date.

If we wanted to make that a dynamic update, we'll have to add more code to inject updates to the offerlist, e.g. at a regular interval, or perhaps even, every time an orderbook request is received.

There's also a question of whether we want the starting value (default?) to be 1000, or maybe have a starting value be undefined, forcing it to be calculated or something like that.

Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

You mean that makers should never sign transactions that are below local mempoolminfee and thus will not be seen in local mempool, which means maker either will see it only after longer time when it is confirmed or will make full-RBF replacement without even knowing it?

If we cannot avoid dynamic offer updates after each block, then: 1) no point to not implement also block confimation target for this setting too, I avoided it to get MVP ready faster, 2) it is related to #1042, same problems holding it back needs to be solved.

Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

If we cannot avoid dynamic offer updates after each block

I think we can, but, it might be the case that we need some extra code for client-daemon communication, whether we choose to update immediately when the taker sends us the !orderbook request, or, say, once per block. In either case, it is more dynamic than what we do now: we only update offer details when there is a trigger for on_tx_unconfirmed or on_tx_confirmed.

This "extra code" is an extra communication type specified in jmbase.commands.py and then extra functions to send/receive message in jmdaemon.daemon_protocol.py and jmclient.client_protocol.py. If we don't do that, then as mentioned above, the daemon just reads the current offer from an internally stored offerlist variable, and doesn't know that anything changed in jmclient.

Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

You mean that makers should never sign transactions that are below local mempoolminfee and thus will not be seen in local mempool, which means maker either will see it only after longer time when it is confirmed or will make full-RBF replacement without even knowing it?

I didn't specifically mean that, but sure, that is a scenario where it could cause problems. I just don't understand why it would make sense to have the exact threshold of failure (1sat/vb) to propagate, as the default.

Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Ok, another reason is that values below 1000 for tx feerates are treated as block confirmation targets not sats per kvB by estimate_fee_per_kb().


gaplimit = 6

[SNICKER]
# Any other value than 'true' will be treated as False,
# and no SNICKER actions will be enabled in that case:
Expand Down
14 changes: 13 additions & 1 deletion src/jmclient/maker.py
Original file line number Diff line number Diff line change
Expand Up @@ -15,7 +15,7 @@
jlog = get_log()

class Maker(object):
def __init__(self, wallet_service):
def __init__(self, wallet_service: WalletService):
self.active_orders = {}
assert isinstance(wallet_service, WalletService)
self.wallet_service = wallet_service
Expand All @@ -28,6 +28,8 @@ def __init__(self, wallet_service):
# not-enough-coins:
self.sync_wait_loop.start(2.0, now=False)
self.aborted = False
self.minimum_tx_fee_rate = jm_single().config.getint(
"YIELDGENERATOR", "minimum_tx_fee_rate")

def try_to_create_my_orders(self):
"""Because wallet syncing is not synchronous(!),
Expand Down Expand Up @@ -187,6 +189,16 @@ def verify_unsigned_tx(self, tx, offerinfo):
"""
tx_utxo_set = set((x.prevout.hash[::-1], x.prevout.n) for x in tx.vin)

if self.minimum_tx_fee_rate > 1000:
AdamISZ marked this conversation as resolved.
Show resolved Hide resolved
tx_inp_data = jm_single().bc_interface.query_utxo_set(tx_utxo_set)
total_input_value_sum = 0
for utxo in tx_inp_data:
total_input_value_sum = total_input_value_sum + utxo["value"]
tx_fee = total_input_value_sum - btc.tx_total_outputs_value(tx)
tx_fee_rate = tx_fee / btc.tx_vsize(tx) * 1000
if tx_fee_rate < self.minimum_tx_fee_rate:
return (False, "tx feerate below configured minimum tx feerate")

utxos = offerinfo["utxos"]
cjaddr = offerinfo["cjaddr"]
cjaddr_script = btc.CCoinAddress(cjaddr).to_scriptPubKey()
Expand Down
90 changes: 59 additions & 31 deletions src/jmclient/support.py
Original file line number Diff line number Diff line change
@@ -1,13 +1,15 @@
from functools import reduce
import random
from jmbase.support import get_log
from decimal import Decimal
from .configure import get_bondless_makers_allowance

from functools import reduce
from math import exp
from typing import Callable, List, Optional, Tuple

from jmbase.support import get_log
from .configure import get_bondless_makers_allowance, jm_single


ORDER_KEYS = ['counterparty', 'oid', 'ordertype', 'minsize', 'maxsize', 'txfee',
'cjfee']
'cjfee', 'minimum_tx_fee_rate']

log = get_log()

Expand Down Expand Up @@ -45,7 +47,7 @@ def rand_exp_array(lamda, n):
return [random.expovariate(1.0 / lamda) for _ in range(n)]


def rand_weighted_choice(n, p_arr):
def rand_weighted_choice(n: int, p_arr: list) -> int:
"""
Choose a value in 0..n-1
with the choice weighted by the probabilities
Expand Down Expand Up @@ -177,7 +179,7 @@ def calc_cj_fee(ordertype, cjfee, cj_amount):
return real_cjfee


def weighted_order_choose(orders, n):
def weighted_order_choose(orders: List[dict], n: int) -> dict:
"""
Algorithm for choosing the weighting function
it is an exponential
Expand Down Expand Up @@ -208,18 +210,18 @@ def weighted_order_choose(orders, n):
return orders[chosen_order_index]


def random_under_max_order_choose(orders, n):
def random_under_max_order_choose(orders: List[dict], n: int) -> dict:
# orders are already pre-filtered for max_cj_fee
return random.choice(orders)


def cheapest_order_choose(orders, n):
def cheapest_order_choose(orders: List[dict], n: int) -> dict:
"""
Return the cheapest order from the orders.
"""
return orders[0]

def fidelity_bond_weighted_order_choose(orders, n):
def fidelity_bond_weighted_order_choose(orders: List[dict], n: int) -> dict:
"""
choose orders based on fidelity bond for improved sybil resistance

Expand Down Expand Up @@ -247,19 +249,31 @@ def check_max_fee(fee):
return check_max_fee


def choose_orders(offers, cj_amount, n, chooseOrdersBy, ignored_makers=None,
pick=False, allowed_types=["sw0reloffer", "sw0absoffer"],
max_cj_fee=(1, float('inf'))):
def choose_orders(offers: List[dict],
cj_amount: int,
num_counterparties: int,
chooseOrdersBy: Callable[[List[dict], int], dict],
ignored_makers: Optional[List[str]] = None,
pick: bool = False,
allowed_types: List[str] = ["sw0reloffer", "sw0absoffer"],
max_cj_fee: Tuple = (1, float('inf'))) -> Tuple[Optional[dict], int]:
is_within_max_limits = _get_is_within_max_limits(
max_cj_fee[0], max_cj_fee[1], cj_amount)
if ignored_makers is None:
ignored_makers = []
fee_per_kb = jm_single().bc_interface.estimate_fee_per_kb(
jm_single().config.getint("POLICY", "tx_fees"), randomize=False)
#Filter ignored makers and inappropriate amounts
orders = [o for o in offers if o['counterparty'] not in ignored_makers]
orders = [o for o in orders if o['minsize'] < cj_amount]
orders = [o for o in orders if o['maxsize'] > cj_amount]
#Filter those not using wished-for offertypes
orders = [o for o in orders if o["ordertype"] in allowed_types]
# provide a message to the user to let them know that there are offers not
# available due to too-low feerate:
orders_fee_too_low = [o for o in orders if o["minimum_tx_fee_rate"] > fee_per_kb]
#Filter those not accepting our tx feerate
AdamISZ marked this conversation as resolved.
Show resolved Hide resolved
orders = [o for o in orders if o["minimum_tx_fee_rate"] <= fee_per_kb]

orders_fees = []
for o in orders:
Expand All @@ -268,10 +282,18 @@ def choose_orders(offers, cj_amount, n, chooseOrdersBy, ignored_makers=None,
orders_fees.append((o, fee))

counterparties = set(o['counterparty'] for o, f in orders_fees)
if n > len(counterparties):
if num_counterparties > len(counterparties):
log.warn(('ERROR not enough liquidity in the orderbook n=%d '
'suitable-counterparties=%d amount=%d totalorders=%d') %
(n, len(counterparties), cj_amount, len(orders_fees)))
(num_counterparties, len(counterparties), cj_amount,
len(orders_fees)))
if len(orders_fee_too_low) > 0:
log.warn(f"Note that {len(orders_fee_too_low)} counterparty "
"offer(s) were rejected because they demanded too "
"high a transaction fee in range from "
"{min(orders_fee_too_low, key=lambda f: 'minimum_tx_fee_rate')} to "
"{max(orders_fee_too_low, key=lambda f: 'minimum_tx_fee_rate')}.")
log.debug(str(orders_fee_too_low))
# TODO handle not enough liquidity better, maybe an Exception
return None, 0
"""
Expand All @@ -294,8 +316,9 @@ def choose_orders(offers, cj_amount, n, chooseOrdersBy, ignored_makers=None,
]))
total_cj_fee = 0
chosen_orders = []
for i in range(n):
chosen_order, chosen_fee = chooseOrdersBy(orders_fees, n)
for i in range(num_counterparties):
chosen_order, chosen_fee = chooseOrdersBy(orders_fees,
num_counterparties)
# remove all orders from that same counterparty
# only needed if offers are manually picked
orders_fees = [o
Expand All @@ -308,14 +331,14 @@ def choose_orders(offers, cj_amount, n, chooseOrdersBy, ignored_makers=None,
return result, total_cj_fee


def choose_sweep_orders(offers,
total_input_value,
total_txfee,
n,
chooseOrdersBy,
ignored_makers=None,
allowed_types=['sw0reloffer', 'sw0absoffer'],
max_cj_fee=(1, float('inf'))):
def choose_sweep_orders(offers: List[dict],
total_input_value: int,
total_txfee: int,
num_counterparties: int,
chooseOrdersBy: Callable[[List[dict], int], dict],
ignored_makers: Optional[List[str]] = None,
allowed_types: List[str] = ['sw0reloffer', 'sw0absoffer'],
max_cj_fee: Tuple = (1, float('inf'))) -> Tuple[Optional[dict], int, int]:
"""
choose an order given that we want to be left with no change
i.e. sweep an entire group of utxos
Expand All @@ -332,7 +355,7 @@ def choose_sweep_orders(offers,
if ignored_makers is None:
ignored_makers = []

def calc_zero_change_cj_amount(ordercombo):
def calc_zero_change_cj_amount(ordercombo: List[dict]) -> Tuple[int, int]:
sumabsfee = 0
sumrelfee = Decimal('0')
sumtxfee_contribution = 0
Expand All @@ -352,14 +375,18 @@ def calc_zero_change_cj_amount(ordercombo):
cjamount = int(cjamount.quantize(Decimal(1)))
return cjamount, int(sumabsfee + sumrelfee * cjamount)

fee_per_kb = jm_single().bc_interface.estimate_fee_per_kb(
jm_single().config.getint("POLICY", "tx_fees"), randomize=False)
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Is there an argument for throwing in a 20% tolerance here, i.e. if 2 participants are trying to use roughly the same fee estimate (e.g. 10 blocks target), and just happen to be slightly different, then let's make it more likely that they'll succeed to negotiate?

log.debug('choosing sweep orders for total_input_value = ' + str(
total_input_value) + ' n=' + str(n))
total_input_value) + ' n=' + str(num_counterparties))
offers = [o for o in offers if o["ordertype"] in allowed_types]
#Filter ignored makers and inappropriate amounts
offers = [o for o in offers if o['counterparty'] not in ignored_makers]
offers = [o for o in offers if o['minsize'] < total_input_value]
# while we do not know the exact cj value yet, we can approximate a ceiling:
offers = [o for o in offers if o['maxsize'] > (total_input_value - total_txfee)]
#Filter those not accepting our tx feerate
offers = [o for o in offers if o["minimum_tx_fee_rate"] <= fee_per_kb]

log.debug('orderlist = \n' + '\n'.join([str(o) for o in offers]))
orders_fees = [(o, calc_cj_fee(o['ordertype'], o['cjfee'],
Expand All @@ -376,13 +403,14 @@ def calc_zero_change_cj_amount(ordercombo):
if is_within_max_limits(v[1])).values(),
key=feekey)
chosen_orders = []
while len(chosen_orders) < n:
for i in range(n - len(chosen_orders)):
if len(orders_fees) < n - len(chosen_orders):
while len(chosen_orders) < num_counterparties:
for i in range(num_counterparties - len(chosen_orders)):
if len(orders_fees) < num_counterparties - len(chosen_orders):
log.debug('ERROR not enough liquidity in the orderbook')
# TODO handle not enough liquidity better, maybe an Exception
return None, 0, 0
chosen_order, chosen_fee = chooseOrdersBy(orders_fees, n)
chosen_order, chosen_fee = chooseOrdersBy(orders_fees,
num_counterparties)
log.debug('chosen = ' + str(chosen_order))
# remove all orders from that same counterparty
orders_fees = [
Expand Down
5 changes: 2 additions & 3 deletions src/jmclient/yieldgenerator.py
Original file line number Diff line number Diff line change
Expand Up @@ -89,8 +89,6 @@ def __init__(self, wallet_service, offerconfig):
self.size_factor = offerconfig
super().__init__(wallet_service)



def create_my_orders(self):
mix_balance = self.get_available_mixdepths()
if len([b for m, b in mix_balance.items() if b > 0]) == 0:
Expand All @@ -113,7 +111,8 @@ def create_my_orders(self):
'maxsize': mix_balance[max_mix] - max(
jm_single().DUST_THRESHOLD, self.txfee_contribution),
'txfee': self.txfee_contribution,
'cjfee': f}
'cjfee': f,
'minimum_tx_fee_rate': self.minimum_tx_fee_rate}

# sanity check
assert order['minsize'] >= 0
Expand Down
11 changes: 7 additions & 4 deletions src/jmdaemon/message_channel.py
Original file line number Diff line number Diff line change
Expand Up @@ -293,7 +293,8 @@ def announce_orders(self, orderlist, nick, fidelity_bond_proof_msg, new_mc):
privmsg, on a specific mc.
Fidelity bonds can only be announced over privmsg, nick must be nonNone
"""
order_keys = ['oid', 'minsize', 'maxsize', 'txfee', 'cjfee']
order_keys = ['oid', 'minsize', 'maxsize', 'txfee', 'cjfee',
'minimum_tx_fee_rate']
orderlines = []
for order in orderlist:
orderlines.append(COMMAND_PREFIX + order['ordertype'] + \
Expand Down Expand Up @@ -545,7 +546,7 @@ def on_nick_change_trigger(self, new_nick):
self.on_nick_change(new_nick)

def on_order_seen_trigger(self, mc, counterparty, oid, ordertype, minsize,
maxsize, txfee, cjfee):
maxsize, txfee, cjfee, minimum_tx_fee_rate):
"""This is the entry point into private messaging.
Hence, it fixes for the rest of the conversation, which
message channel the bots are going to communicate over
Expand All @@ -564,7 +565,7 @@ def on_order_seen_trigger(self, mc, counterparty, oid, ordertype, minsize,
self.active_channels[counterparty] = mc
if self.on_order_seen:
self.on_order_seen(counterparty, oid, ordertype, minsize, maxsize,
txfee, cjfee)
txfee, cjfee, minimum_tx_fee_rate)

# orderbook watcher commands
def register_orderbookwatch_callbacks(self,
Expand Down Expand Up @@ -785,9 +786,11 @@ def check_for_orders(self, nick, _chunks):
maxsize = _chunks[3]
txfee = _chunks[4]
cjfee = _chunks[5]
minimum_tx_fee_rate = _chunks[6] if len(_chunks) > 6 else 0
if self.on_order_seen:
self.on_order_seen(self, counterparty, oid, ordertype,
minsize, maxsize, txfee, cjfee)
minsize, maxsize, txfee, cjfee,
minimum_tx_fee_rate)
except IndexError as e:
log.debug(e)
log.debug('index error parsing chunks, possibly malformed '
Expand Down
Loading