Skip to content

Commit

Permalink
Fix merging of data frames for payments (#453)
Browse files Browse the repository at this point in the history 
This PR changes which columns are expected from dune results and
restricts columns before merging.

This is necessary since multiple queries now contain a `solver_name`
field. In merging, the filed was renamed to `solver_name_x`,
`solver_name_y`, ..., resulting in uncaught exceptions.

This can be tested by running `python -m src.fetch.transfer_file --start
2024-11-26 --post-tx --dry-run --min-transfer-amount-wei
"1000000000000000" --min-transfer-amount-cow-atoms "1000000000000000000"
--ignore-slippage` with suitable environments set up for Gnosis and
Arbitrum One.

---------

Co-authored-by: Haris Angelidakis <[email protected]>
  • Loading branch information
@fhenneke @harisang
fhenneke and harisang authored Dec 12, 2024
1 parent 471a097 commit 9305c01
Show file tree
Hide file tree
Showing 2 changed files with 52 additions and 52 deletions.
37 changes: 18 additions & 19 deletions src/fetch/payouts.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -36,10 +36,9 @@
}
SLIPPAGE_COLUMNS = {
"solver",
"solver_name",
"eth_slippage_wei",
}
REWARD_TARGET_COLUMNS = {"solver", "reward_target", "pool_address"}
REWARD_TARGET_COLUMNS = {"solver", "solver_name", "reward_target", "pool_address"}
SERVICE_FEE_COLUMNS = {"solver", "service_fee"}
ADDITIONAL_PAYMENT_COLUMNS = {"buffer_accounting_target", "reward_token_address"}

Expand Down Expand Up @@ -416,9 +415,12 @@ def construct_payout_dataframe(

# 3. Merge the three dataframes (joining on solver)
merged_df = (
payment_df.merge(slippage_df, on=join_column, how="left")
.merge(reward_target_df, on=join_column, how="left")
.merge(service_fee_df, on=join_column, how="left")
payment_df[list(PAYMENT_COLUMNS)]
.merge(slippage_df[list(SLIPPAGE_COLUMNS)], on=join_column, how="left")
.merge(
reward_target_df[list(REWARD_TARGET_COLUMNS)], on=join_column, how="left"
)
.merge(service_fee_df[list(SERVICE_FEE_COLUMNS)], on=join_column, how="left")
)

# 4. Add slippage from fees to slippage
Expand Down Expand Up @@ -510,11 +512,15 @@ def construct_payouts(
quote_rewards_df, batch_rewards_df, on="solver", how="outer"
).fillna(0)

service_fee_df = pandas.DataFrame(dune.get_service_fee_status())
service_fee_df["service_fee"] = [
service_fee_flag * config.reward_config.service_fee_factor
for service_fee_flag in service_fee_df["service_fee"]
]
service_fee_dune = dune.get_service_fee_status()
if service_fee_dune:
service_fee_df = pandas.DataFrame(service_fee_dune)
service_fee_df["service_fee"] = [
service_fee_flag * config.reward_config.service_fee_factor
for service_fee_flag in service_fee_df["service_fee"]
]
else:
service_fee_df = DataFrame(columns=["solver", "service_fee"])

vouches = dune.get_vouches()
if vouches:
Expand All @@ -525,15 +531,8 @@ def construct_payouts(
)
# construct slippage df
if ignore_slippage_flag or (not config.buffer_accounting_config.include_slippage):
slippage_df_temp = pandas.merge(
merged_df[["solver"]],
reward_target_df[["solver", "solver_name"]],
on="solver",
how="inner",
)
slippage_df = slippage_df_temp.assign(
eth_slippage_wei=[0] * slippage_df_temp.shape[0]
)
slippage_df = merged_df[["solver"]].copy()
slippage_df["eth_slippage_wei"] = [0] * slippage_df.shape[0]
else:
slippage_df = pandas.DataFrame(dune.get_period_slippage())
# TODO - After CIP-20 phased in, adapt query to return `solver` like all the others
Expand Down
67 changes: 34 additions & 33 deletions tests/unit/test_payouts.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -153,11 +153,9 @@ def test_validate_df_columns(self):
"quote_reward_cow": [],
}
)
legit_slippages = DataFrame(
{"solver": [], "solver_name": [], "eth_slippage_wei": []}
)
legit_slippages = DataFrame({"solver": [], "eth_slippage_wei": []})
legit_reward_targets = DataFrame(
{"solver": [], "reward_target": [], "pool_address": []}
{"solver": [], "solver_name": [], "reward_target": [], "pool_address": []}
)
legit_service_fees = DataFrame({"solver": [], "service_fee": []})

Expand Down Expand Up @@ -221,14 +219,14 @@ def test_construct_payouts(self):
"solver": self.solvers[:3],
# Note that one of the solvers did not appear,
# in this list (we are testing the left join)
"solver_name": ["S_1", "S_2", "S_3"],
"eth_slippage_wei": [1, 0, -1],
}
)

reward_targets = DataFrame(
{
"solver": self.solvers,
"solver_name": ["S_1", "S_2", "S_3", "S_4"],
"reward_target": self.reward_targets,
"pool_address": self.pool_addresses,
}
Expand All @@ -247,69 +245,72 @@ def test_construct_payouts(self):
)
expected = DataFrame(
{
"solver": self.solvers,
"num_quotes": self.num_quotes,
"primary_reward_eth": [600000000000000.0, 1.2e16, -1e16, 0.0],
"protocol_fee_eth": [
1000000000000000.0,
2000000000000000.0,
0.0,
0.0,
"buffer_accounting_target": [
"0x0000000000000000000000000000000000000005",
str(self.solvers[1]),
str(self.solvers[2]),
str(self.solvers[3]),
],
"eth_slippage_wei": [2000000000000001.0, 4000000000000000.0, -1.0, 0.0],
"network_fee_eth": [
2000000000000000.0,
4000000000000000.0,
0.0,
0.0,
],
"pool_address": [
str(self.config.reward_config.cow_bonding_pool),
"0x0000000000000000000000000000000000000010",
"0x0000000000000000000000000000000000000011",
"0x0000000000000000000000000000000000000012",
],
"primary_reward_cow": [
600000000000000000.0,
12000000000000000000.0,
-10000000000000000000.0,
0.0,
],
"primary_reward_eth": [600000000000000.0, 1.2e16, -1e16, 0.0],
"protocol_fee_eth": [
1000000000000000.0,
2000000000000000.0,
0.0,
0.0,
],
"quote_reward_cow": [
0.00000,
0.00000,
6000000000000000000.00000,
12000000000000000000.00000,
],
"solver_name": ["S_1", "S_2", "S_3", None],
"eth_slippage_wei": [2000000000000001.0, 4000000000000000.0, -1.0, 0.0],
"reward_target": [
"0x0000000000000000000000000000000000000005",
"0x0000000000000000000000000000000000000006",
"0x0000000000000000000000000000000000000007",
"0x0000000000000000000000000000000000000008",
],
"pool_address": [
str(self.config.reward_config.cow_bonding_pool),
"0x0000000000000000000000000000000000000010",
"0x0000000000000000000000000000000000000011",
"0x0000000000000000000000000000000000000012",
"reward_token_address": [
str(self.config.reward_config.reward_token_address),
str(self.config.reward_config.reward_token_address),
str(self.config.reward_config.reward_token_address),
str(self.config.reward_config.reward_token_address),
],
"service_fee": [
Fraction(0, 100),
Fraction(0, 100),
Fraction(0, 100),
Fraction(15, 100),
],
"buffer_accounting_target": [
"0x0000000000000000000000000000000000000005",
str(self.solvers[1]),
str(self.solvers[2]),
str(self.solvers[3]),
],
"reward_token_address": [
str(self.config.reward_config.reward_token_address),
str(self.config.reward_config.reward_token_address),
str(self.config.reward_config.reward_token_address),
str(self.config.reward_config.reward_token_address),
],
"solver": self.solvers,
"solver_name": ["S_1", "S_2", "S_3", "S_4"],
}
)

self.assertIsNone(pandas.testing.assert_frame_equal(expected, result))
self.assertIsNone(
pandas.testing.assert_frame_equal(
expected, result.reindex(sorted(result.columns), axis=1)
)
)

def test_prepare_transfers(self):
# Need Example of every possible scenario
Expand Down

0 comments on commit 9305c01

Please sign in to comment.