From f8aefd59b8d9870d529f885f1ecb9696b7e3744d Mon Sep 17 00:00:00 2001 From: Felix Henneke Date: Mon, 4 Nov 2024 11:13:02 +0100 Subject: [PATCH 01/19] config: fix mypy error --- src/config.py | 207 ++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 207 insertions(+) create mode 100644 src/config.py diff --git a/src/config.py b/src/config.py new file mode 100644 index 00000000..5ca58ccd --- /dev/null +++ b/src/config.py @@ -0,0 +1,207 @@ +"""Config for solver accounting.""" + +import os +from dataclasses import dataclass +from fractions import Fraction +from pathlib import Path + +from eth_typing.evm import ChecksumAddress +from dotenv import load_dotenv +from dune_client.types import Address +from gnosis.eth.ethereum_network import EthereumNetwork +from web3 import Web3 + +load_dotenv() + + +@dataclass(frozen=True) +class RewardConfig: + """Configuration for reward mechanism.""" + + reward_token_address: Address + cow_bonding_pool: Address + batch_reward_cap_upper: int + batch_reward_cap_lower: int + quote_reward_cow: int + quote_reward_cap_native: int + service_fee_factor: Fraction + + +@dataclass(frozen=True) +class ProtocolFeeConfig: + """Configuration for protocol and parner fees.""" + + protocol_fee_safe: Address + partner_fee_cut: float + partner_fee_reduced_cut: float + reduced_cut_address: str + + +@dataclass(frozen=True) +class BufferAccountingConfig: + """Configuration for buffer accounting.""" + + include_slippage: bool + + +@dataclass(frozen=True) +class OrderbookConfig: + """Configuration for orderbook fetcher""" + + prod_db_url: str + barn_db_url: str + + +@dataclass(frozen=True) +class DuneConfig: + """Configuration for DuneFetcher.""" + + dune_api_key: str + dune_blockchain: str + + +@dataclass(frozen=True) +class NodeConfig: + """Configuration for web3 node.""" + + node_url: str + + +@dataclass(frozen=True) +class PaymentConfig: + """Configuration of payment.""" + + # pylint: disable=too-many-instance-attributes + + network: EthereumNetwork + cow_token_address: Address + payment_safe_address: ChecksumAddress + signing_key: str | None + safe_queue_url: str + csv_airdrop_url: str + verification_docs_url: str + weth_address: ChecksumAddress + + +@dataclass(frozen=True) +class IOConfig: + """Configuration of input and output.""" + + log_config_file: Path + project_root_dir: Path + query_dir: Path + csv_output_dir: Path + dashboard_dir: Path + slack_channel: str | None + slack_token: str | None + + +@dataclass(frozen=True) +class AccountingConfig: + """Full configuration for solver accounting.""" + + # pylint: disable=too-many-instance-attributes + + payment_config: PaymentConfig + orderbook_config: OrderbookConfig + dune_config: DuneConfig + node_config: NodeConfig + reward_config: RewardConfig + protocol_fee_config: ProtocolFeeConfig + buffer_accounting_config: BufferAccountingConfig + io_config: IOConfig + + +def get_accounting_config(network: str) -> AccountingConfig: + """Get config for specified network.""" + + # pylint: disable=too-many-locals + + project_root_dir = Path(__file__).parent.parent + file_out_dir = project_root_dir / Path("out") + log_config_file = project_root_dir / Path("logging.conf") + query_dir = project_root_dir / Path("queries") + dashboard_dir = project_root_dir / Path("dashboards/solver-rewards-accounting") + + docs_url = "https://www.notion.so/cownation/Solver-Payouts-3dfee64eb3d449ed8157a652cc817a8c" + + payment_safe_address = Web3.to_checksum_address( + os.environ.get("SAFE_ADDRESS", "0xA03be496e67Ec29bC62F01a428683D7F9c204930") + ) + # Found this exposed infura key on https://rpc.info/ + infura_key = os.environ.get("INFURA_KEY", "9aa3d95b3bc440fa88ea12eaa4456161") + + node_url = f"https://{network}.infura.io/v3/{infura_key}" + + short_name = { + "mainnet": "eth", + "rinkeby": "rin", + "gnosis": "gno", + "goerli": "gor", + }[network] + + csv_app_hash = "Qme49gESuwpSvwANmEqo34yfCkzyQehooJ5yL7aHmKJnpZ" + safe_url = f"https://app.safe.global/{short_name}:{payment_safe_address}" + airdrop_url = ( + f"{safe_url}/apps?appUrl=https://cloudflare-ipfs.com/ipfs/{csv_app_hash}/" + ) + safe_queue_url = f"{safe_url}/transactions/queue" + + dune_api_key = os.environ["DUNE_API_KEY"] + signing_key = os.getenv("PROPOSER_PK") + if signing_key == "": + signing_key = None + result = AccountingConfig( + payment_config=PaymentConfig( + network=EthereumNetwork.MAINNET, + cow_token_address=Address("0xDEf1CA1fb7FBcDC777520aa7f396b4E015F497aB"), + payment_safe_address=Web3.to_checksum_address( + "0xA03be496e67Ec29bC62F01a428683D7F9c204930" + ), + signing_key=signing_key, + safe_queue_url=safe_queue_url, + csv_airdrop_url=airdrop_url, + verification_docs_url=docs_url, + weth_address=Web3.to_checksum_address( + "0xc02aaa39b223fe8d0a0e5c4f27ead9083c756cc2" + ), + ), + orderbook_config=OrderbookConfig( + prod_db_url=os.environ["PROD_DB_URL"], + barn_db_url=os.environ["BARN_DB_URL"], + ), + dune_config=DuneConfig(dune_api_key=dune_api_key, dune_blockchain="ethereum"), + node_config=NodeConfig(node_url=node_url), + reward_config=RewardConfig( + reward_token_address=Address("0xDEf1CA1fb7FBcDC777520aa7f396b4E015F497aB"), + batch_reward_cap_upper=12 * 10**15, + batch_reward_cap_lower=10 * 10**15, + quote_reward_cow=6 * 10**18, + quote_reward_cap_native=6 * 10**14, + service_fee_factor=Fraction(15, 100), + cow_bonding_pool=Address("0x5d4020b9261f01b6f8a45db929704b0ad6f5e9e6"), + ), + protocol_fee_config=ProtocolFeeConfig( + protocol_fee_safe=Address("0xB64963f95215FDe6510657e719bd832BB8bb941B"), + partner_fee_cut=0.15, + partner_fee_reduced_cut=0.10, + reduced_cut_address="0x63695Eee2c3141BDE314C5a6f89B98E62808d716", + ), + buffer_accounting_config=BufferAccountingConfig(include_slippage=True), + io_config=IOConfig( + project_root_dir=project_root_dir, + log_config_file=log_config_file, + query_dir=query_dir, + csv_output_dir=file_out_dir, + dashboard_dir=dashboard_dir, + slack_channel=os.getenv("SLACK_CHANNEL", None), + slack_token=os.getenv("SLACK_TOKEN", None), + ), + ) + + return result + + +config = get_accounting_config(os.environ["NETWORK"]) + +web3 = Web3(Web3.HTTPProvider(config.node_config.node_url)) From 62bbac588d3ee25aaa4e4cad67f9363eaddb03a3 Mon Sep 17 00:00:00 2001 From: Felix Henneke Date: Mon, 4 Nov 2024 11:13:33 +0100 Subject: [PATCH 02/19] transfer_file: fix lint --- src/abis/load.py | 9 +++----- src/fetch/payouts.py | 43 +++++++++++++++++++---------------- src/fetch/prices.py | 4 ++-- src/fetch/token_list.py | 4 ++-- src/fetch/transfer_file.py | 44 ++++++++++++++++++------------------ src/logger.py | 5 ++-- src/models/token.py | 4 ++-- src/multisend.py | 5 ++-- src/pg_client.py | 17 ++++++++++---- src/utils/query_file.py | 6 ++--- src/utils/token_details.py | 4 ++-- tests/unit/test_models.py | 16 ++++++------- tests/unit/test_multisend.py | 4 ++-- tests/unit/test_payouts.py | 12 +++++----- 14 files changed, 93 insertions(+), 84 deletions(-) diff --git a/src/abis/load.py b/src/abis/load.py index a9ec698a..f52adb86 100644 --- a/src/abis/load.py +++ b/src/abis/load.py @@ -13,13 +13,10 @@ # TODO - following this issue: https://github.com/ethereum/web3.py/issues/3017 from web3.contract import Contract # type: ignore -from src.constants import PROJECT_ROOT +from src.config import config from src.logger import set_log -ABI_PATH = PROJECT_ROOT / Path("src/abis") - -WETH9_ADDRESS = Web3().to_checksum_address("0xc02aaa39b223fe8d0a0e5c4f27ead9083c756cc2") - +ABI_PATH = config.io_config.project_root_dir / Path("src/abis") log = set_log(__name__) @@ -65,7 +62,7 @@ def get_contract( def weth9(web3: Optional[Web3] = None) -> Contract | Type[Contract]: """Returns an instance of WETH9 Contract""" - return IndexedContract.WETH9.get_contract(web3, WETH9_ADDRESS) + return IndexedContract.WETH9.get_contract(web3, config.payment_config.weth_address) def erc20( diff --git a/src/fetch/payouts.py b/src/fetch/payouts.py index b1b39fbf..bcf30b34 100644 --- a/src/fetch/payouts.py +++ b/src/fetch/payouts.py @@ -13,7 +13,7 @@ from dune_client.types import Address from pandas import DataFrame, Series -from src.constants import COW_TOKEN_ADDRESS, COW_BONDING_POOL +from src.config import config from src.fetch.dune import DuneFetcher from src.fetch.prices import eth_in_token, TokenId, token_in_eth from src.models.accounting_period import AccountingPeriod @@ -23,11 +23,6 @@ from src.pg_client import MultiInstanceDBFetcher from src.utils.print_store import Category -QUOTE_REWARD_COW = 6 * 10**18 -QUOTE_REWARD_CAP_ETH = 6 * 10**14 -SERVICE_FEE_FACTOR = Fraction(15, 100) - -PROTOCOL_FEE_SAFE = Address("0xB64963f95215FDe6510657e719bd832BB8bb941B") PAYMENT_COLUMNS = { "solver", @@ -135,7 +130,7 @@ def total_eth_reward(self) -> int: def reward_scaling(self) -> Fraction: """Scaling factor for service fee The reward is multiplied by this factor""" - return 1 - SERVICE_FEE_FACTOR * self.service_fee + return 1 - config.reward_config.service_fee_factor * self.service_fee def is_overdraft(self) -> bool: """ @@ -153,7 +148,7 @@ def as_payouts(self) -> list[Transfer]: if quote_reward_cow > 0: result.append( Transfer( - token=Token(COW_TOKEN_ADDRESS), + token=Token(config.reward_config.reward_token_address), recipient=self.reward_target, amount_wei=quote_reward_cow, ) @@ -188,7 +183,8 @@ def as_payouts(self) -> list[Transfer]: token=None, recipient=( self.reward_target - if self.bonding_pool == COW_BONDING_POOL + if self.bonding_pool + == config.reward_config.cow_bonding_pool else self.solver ), amount_wei=reimbursement_eth + total_eth_reward, @@ -209,7 +205,7 @@ def as_payouts(self) -> list[Transfer]: try: result.append( Transfer( - token=Token(COW_TOKEN_ADDRESS), + token=Token(config.reward_config.reward_token_address), recipient=self.reward_target, amount_wei=reimbursement_cow + total_cow_reward, ) @@ -228,7 +224,7 @@ def as_payouts(self) -> list[Transfer]: token=None, recipient=( self.reward_target - if self.bonding_pool == COW_BONDING_POOL + if self.bonding_pool == config.reward_config.cow_bonding_pool else self.solver ), amount_wei=reimbursement_eth, @@ -241,7 +237,7 @@ def as_payouts(self) -> list[Transfer]: try: result.append( Transfer( - token=Token(COW_TOKEN_ADDRESS), + token=Token(config.reward_config.reward_token_address), recipient=self.reward_target, amount_wei=total_cow_reward, ) @@ -276,7 +272,12 @@ def extend_payment_df(pdf: DataFrame, converter: TokenConversion) -> DataFrame: # Pandas has poor support for large integers, must cast the constant to float here, # otherwise the dtype would be inferred as int64 (which overflows). pdf["quote_reward_cow"] = ( - float(min(QUOTE_REWARD_COW, converter.eth_to_token(QUOTE_REWARD_CAP_ETH))) + float( + min( + config.reward_config.quote_reward_cow, + converter.eth_to_token(config.reward_config.quote_reward_cap_native), + ) + ) * pdf["num_quotes"] ) @@ -319,7 +320,7 @@ def prepare_transfers( transfers.append( Transfer( token=None, - recipient=PROTOCOL_FEE_SAFE, + recipient=config.protocol_fee_config.protocol_fee_safe, amount_wei=final_protocol_fee_wei, ) ) @@ -327,7 +328,7 @@ def prepare_transfers( transfers.append( Transfer( token=None, - recipient=PROTOCOL_FEE_SAFE, + recipient=config.protocol_fee_config.protocol_fee_safe, amount_wei=partner_fee_tax_wei, ) ) @@ -443,12 +444,14 @@ def construct_partner_fee_payments( total_partner_fee_wei_taxed = 0 for address, value in partner_fees_wei.items(): total_partner_fee_wei_untaxed += value - if address == "0x63695Eee2c3141BDE314C5a6f89B98E62808d716": - partner_fees_wei[address] = int(0.90 * value) - total_partner_fee_wei_taxed += int(0.90 * value) + if address == config.protocol_fee_config.reduced_cut_address: + reduction_factor = 1 - config.protocol_fee_config.partner_fee_reduced_cut + partner_fees_wei[address] = int(reduction_factor * value) + total_partner_fee_wei_taxed += int(reduction_factor * value) else: - partner_fees_wei[address] = int(0.85 * value) - total_partner_fee_wei_taxed += int(0.85 * value) + reduction_factor = 1 - config.protocol_fee_config.partner_fee_cut + partner_fees_wei[address] = int(reduction_factor * value) + total_partner_fee_wei_taxed += int(reduction_factor * value) return partner_fees_wei, total_partner_fee_wei_untaxed diff --git a/src/fetch/prices.py b/src/fetch/prices.py index 97442d87..d31ef311 100644 --- a/src/fetch/prices.py +++ b/src/fetch/prices.py @@ -10,11 +10,11 @@ from coinpaprika import client as cp -from src.constants import LOG_CONFIG_FILE +from src.config import config log = logging.getLogger(__name__) logging.config.fileConfig( - fname=LOG_CONFIG_FILE.absolute(), disable_existing_loggers=False + fname=config.io_config.log_config_file.absolute(), disable_existing_loggers=False ) client = cp.Client() diff --git a/src/fetch/token_list.py b/src/fetch/token_list.py index 101c3cde..d5b32014 100644 --- a/src/fetch/token_list.py +++ b/src/fetch/token_list.py @@ -10,11 +10,11 @@ import requests -from src.constants import LOG_CONFIG_FILE +from src.config import config log = logging.getLogger(__name__) logging.config.fileConfig( - fname=LOG_CONFIG_FILE.absolute(), disable_existing_loggers=False + fname=config.io_config.log_config_file.absolute(), disable_existing_loggers=False ) ALLOWED_TOKEN_LIST_URL = "https://files.cow.fi/token_list.json" diff --git a/src/fetch/transfer_file.py b/src/fetch/transfer_file.py index ac37fe0f..5ca16fc1 100644 --- a/src/fetch/transfer_file.py +++ b/src/fetch/transfer_file.py @@ -4,7 +4,6 @@ from __future__ import annotations -import os import ssl from dataclasses import asdict @@ -14,15 +13,7 @@ from gnosis.eth.ethereum_client import EthereumClient from slack.web.client import WebClient -from src.constants import ( - SAFE_ADDRESS, - NETWORK, - NODE_URL, - AIRDROP_URL, - SAFE_URL, - FILE_OUT_DIR, - DOCS_URL, -) +from src.config import config from src.fetch.payouts import construct_payouts from src.models.accounting_period import AccountingPeriod from src.models.transfer import Transfer, CSVTransfer @@ -43,13 +34,15 @@ def manual_propose(transfers: list[Transfer], period: AccountingPeriod) -> None: f"{period.unusual_slippage_url()}" ) csv_transfers = [asdict(CSVTransfer.from_transfer(t)) for t in transfers] - FileIO(FILE_OUT_DIR).write_csv(csv_transfers, f"transfers-{period}.csv") + FileIO(config.io_config.csv_output_dir).write_csv( + csv_transfers, f"transfers-{period}.csv" + ) print(Transfer.summarize(transfers)) print( f"Please cross check these results with the dashboard linked above.\n " f"For solver payouts, paste the transfer file CSV Airdrop at:\n" - f"{AIRDROP_URL}" + f"{config.payment_config.csv_airdrop_url}" ) @@ -66,35 +59,42 @@ def auto_propose( """ # Check for required env vars early # so not to wait for query execution to realize it's not available. - signing_key = os.environ["PROPOSER_PK"] - client = EthereumClient(URI(NODE_URL)) + signing_key = config.payment_config.signing_key + assert signing_key is not None + client = EthereumClient(URI(config.node_config.node_url)) log_saver.print(Transfer.summarize(transfers), category=Category.TOTALS) transactions = prepend_unwrap_if_necessary( - client, SAFE_ADDRESS, transactions=[t.as_multisend_tx() for t in transfers] + client, + config.payment_config.payment_safe_address, + transactions=[t.as_multisend_tx() for t in transfers], ) if len(transactions) > len(transfers): log_saver.print("Prepended WETH unwrap", Category.GENERAL) log_saver.print( - f"Instructions for verifying the payout transaction can be found at\n{DOCS_URL}", + "Instructions for verifying the payout transaction can be found at\n" + f"{config.payment_config.verification_docs_url}", category=Category.GENERAL, ) if not dry_run: + slack_channel = config.io_config.slack_channel + assert slack_channel is not None + nonce = post_multisend( - safe_address=SAFE_ADDRESS, + safe_address=config.payment_config.payment_safe_address, transactions=transactions, - network=NETWORK, + network=config.payment_config.network, signing_key=signing_key, client=client, ) post_to_slack( slack_client, - channel=os.environ["SLACK_CHANNEL"], + channel=slack_channel, message=( f"Solver Rewards transaction with nonce {nonce} pending signatures.\n" - f"To sign and execute, visit:\n{SAFE_URL}\n" + f"To sign and execute, visit:\n{config.payment_config.safe_queue_url}\n" f"More details in thread" ), sub_messages=log_saver.get_values(), @@ -112,7 +112,7 @@ def auto_propose( payout_transfers_temp = construct_payouts( args.dune, orderbook=MultiInstanceDBFetcher( - [os.environ["PROD_DB_URL"], os.environ["BARN_DB_URL"]] + [config.orderbook_config.prod_db_url, config.orderbook_config.barn_db_url] ), ) payout_transfers = [] @@ -134,7 +134,7 @@ def auto_propose( transfers=payout_transfers, log_saver=dune.log_saver, slack_client=WebClient( - token=os.environ["SLACK_TOKEN"], + token=config.io_config.slack_token, # https://stackoverflow.com/questions/59808346/python-3-slack-client-ssl-sslcertverificationerror ssl=ssl_context, ), diff --git a/src/logger.py b/src/logger.py index 89bee054..d93a5d07 100644 --- a/src/logger.py +++ b/src/logger.py @@ -3,7 +3,7 @@ import logging.config from logging import Logger -from src.constants import LOG_CONFIG_FILE +from src.config import config # TODO - use this in every file that logs (and prints). @@ -13,6 +13,7 @@ def set_log(name: str) -> Logger: log = logging.getLogger(name) logging.config.fileConfig( - fname=LOG_CONFIG_FILE.absolute(), disable_existing_loggers=False + fname=config.io_config.log_config_file.absolute(), + disable_existing_loggers=False, ) return log diff --git a/src/models/token.py b/src/models/token.py index 6afc2b96..9a73d41c 100644 --- a/src/models/token.py +++ b/src/models/token.py @@ -9,7 +9,7 @@ from dune_client.types import Address -from src.constants import COW_TOKEN_ADDRESS, web3 +from src.config import config, web3 from src.utils.token_details import get_token_decimals @@ -48,7 +48,7 @@ def __init__(self, address: str | Address, decimals: Optional[int] = None): address = Address(address) self.address = address - if address == COW_TOKEN_ADDRESS: + if address == config.payment_config.cow_token_address: # Avoid Web3 Calls for main branch of program. decimals = 18 diff --git a/src/multisend.py b/src/multisend.py index 776fdc43..49ae23d7 100644 --- a/src/multisend.py +++ b/src/multisend.py @@ -12,13 +12,12 @@ from gnosis.safe.multi_send import MultiSend, MultiSendOperation, MultiSendTx from gnosis.safe.safe import Safe - -from src.constants import LOG_CONFIG_FILE, web3 +from src.config import config, web3 from src.abis.load import weth9 log = logging.getLogger(__name__) logging.config.fileConfig( - fname=LOG_CONFIG_FILE.absolute(), disable_existing_loggers=False + fname=config.io_config.log_config_file.absolute(), disable_existing_loggers=False ) # This contract address can be removed once this issue is resolved: diff --git a/src/pg_client.py b/src/pg_client.py index b7921dd5..4e753f34 100644 --- a/src/pg_client.py +++ b/src/pg_client.py @@ -8,6 +8,7 @@ from sqlalchemy import create_engine from sqlalchemy.engine import Engine +from src.config import config from src.logger import set_log from src.utils.query_file import open_query @@ -38,15 +39,23 @@ def get_solver_rewards(self, start_block: str, end_block: str) -> DataFrame: open_query("orderbook/prod_batch_rewards.sql") .replace("{{start_block}}", start_block) .replace("{{end_block}}", end_block) - .replace("{{EPSILON_LOWER}}", "10000000000000000") - .replace("{{EPSILON_UPPER}}", "12000000000000000") + .replace( + "{{EPSILON_LOWER}}", str(config.reward_config.batch_reward_cap_lower) + ) + .replace( + "{{EPSILON_UPPER}}", str(config.reward_config.batch_reward_cap_upper) + ) ) batch_reward_query_barn = ( open_query("orderbook/barn_batch_rewards.sql") .replace("{{start_block}}", start_block) .replace("{{end_block}}", end_block) - .replace("{{EPSILON_LOWER}}", "10000000000000000") - .replace("{{EPSILON_UPPER}}", "12000000000000000") + .replace( + "{{EPSILON_LOWER}}", str(config.reward_config.batch_reward_cap_lower) + ) + .replace( + "{{EPSILON_UPPER}}", str(config.reward_config.batch_reward_cap_upper) + ) ) results = [] diff --git a/src/utils/query_file.py b/src/utils/query_file.py index ec170de1..17ea5df7 100644 --- a/src/utils/query_file.py +++ b/src/utils/query_file.py @@ -6,7 +6,7 @@ import os -from src.constants import QUERY_PATH, DASHBOARD_PATH +from src.config import config def open_query(filename: str) -> str: @@ -23,9 +23,9 @@ def open_dashboard_query(filename: str) -> str: def query_file(filename: str) -> str: """Returns proper path for filename in QUERY_PATH""" - return os.path.join(QUERY_PATH, filename) + return os.path.join(config.io_config.query_dir, filename) def dashboard_file(filename: str) -> str: """Returns proper path for filename in DASHBOARD_PATH""" - return os.path.join(DASHBOARD_PATH, filename) + return os.path.join(config.io_config.dashboard_dir, filename) diff --git a/src/utils/token_details.py b/src/utils/token_details.py index 13893685..720fb542 100644 --- a/src/utils/token_details.py +++ b/src/utils/token_details.py @@ -9,12 +9,12 @@ from web3 import Web3 from src.abis.load import erc20 -from src.constants import LOG_CONFIG_FILE +from src.config import config log = logging.getLogger(__name__) logging.config.fileConfig( - fname=LOG_CONFIG_FILE.absolute(), disable_existing_loggers=False + fname=config.io_config.log_config_file.absolute(), disable_existing_loggers=False ) diff --git a/tests/unit/test_models.py b/tests/unit/test_models.py index 79c286cb..7d426dc2 100644 --- a/tests/unit/test_models.py +++ b/tests/unit/test_models.py @@ -7,7 +7,7 @@ from web3 import Web3 from src.abis.load import erc20 -from src.constants import COW_TOKEN_ADDRESS +from src.config import config from src.fetch.transfer_file import Transfer from src.models.accounting_period import AccountingPeriod from src.models.token import Token @@ -356,13 +356,13 @@ def test_from_dict(self): self.assertEqual( Transfer.from_dict( { - "token_address": COW_TOKEN_ADDRESS.address, + "token_address": config.payment_config.cow_token_address.address, "receiver": Address.from_int(1).address, "amount": "1234000000000000000", } ), Transfer( - token=Token(COW_TOKEN_ADDRESS), + token=Token(config.payment_config.cow_token_address), recipient=Address.from_int(1), amount_wei=1234 * 10**15, ), @@ -370,7 +370,7 @@ def test_from_dict(self): def test_from_dataframe(self): receiver = Address.from_int(1) - token_address = COW_TOKEN_ADDRESS.address + token_address = config.payment_config.cow_token_address.address transfer_df = pd.DataFrame( { "token_address": [None, token_address], @@ -386,7 +386,7 @@ def test_from_dataframe(self): amount_wei=12345, ), Transfer( - token=Token(COW_TOKEN_ADDRESS), + token=Token(config.payment_config.cow_token_address), recipient=receiver, amount_wei=678910, ), @@ -411,7 +411,7 @@ def test_basic_as_multisend_tx(self): ), ) erc20_transfer = Transfer( - token=Token(COW_TOKEN_ADDRESS), + token=Token(config.payment_config.cow_token_address), recipient=Address(receiver), amount_wei=15, ) @@ -419,7 +419,7 @@ def test_basic_as_multisend_tx(self): erc20_transfer.as_multisend_tx(), MultiSendTx( operation=MultiSendOperation.CALL, - to=COW_TOKEN_ADDRESS.address, + to=config.payment_config.cow_token_address.address, value=0, data=erc20().encodeABI(fn_name="transfer", args=[receiver, 15]), ), @@ -466,7 +466,7 @@ def test_summarize(self): [ Transfer(token=None, recipient=receiver, amount_wei=eth_amount), Transfer( - token=Token(COW_TOKEN_ADDRESS), + token=Token(config.payment_config.cow_token_address), recipient=receiver, amount_wei=cow_amount, ), diff --git a/tests/unit/test_multisend.py b/tests/unit/test_multisend.py index 2e84ed43..3ae0731f 100644 --- a/tests/unit/test_multisend.py +++ b/tests/unit/test_multisend.py @@ -6,7 +6,7 @@ from web3 import Web3 from src.abis.load import weth9 -from src.constants import COW_TOKEN_ADDRESS +from src.config import config from src.fetch.transfer_file import Transfer from src.models.token import Token from src.multisend import build_encoded_multisend, prepend_unwrap_if_necessary @@ -63,7 +63,7 @@ def test_prepend_unwrap(self): def test_multisend_encoding(self): receiver = Address("0xde786877a10dbb7eba25a4da65aecf47654f08ab") - cow_token = Token(COW_TOKEN_ADDRESS) + cow_token = Token(config.payment_config.cow_token_address) self.assertEqual( build_encoded_multisend([], client=self.client), "0x8d80ff0a" # MethodID diff --git a/tests/unit/test_payouts.py b/tests/unit/test_payouts.py index e52f1c17..e7e36152 100644 --- a/tests/unit/test_payouts.py +++ b/tests/unit/test_payouts.py @@ -4,7 +4,7 @@ from dune_client.types import Address from pandas import DataFrame -from src.constants import COW_TOKEN_ADDRESS +from src.config import config from src.fetch.payouts import ( extend_payment_df, normalize_address_field, @@ -326,22 +326,22 @@ def test_prepare_transfers(self): amount_wei=1, ), Transfer( - token=Token(COW_TOKEN_ADDRESS), + token=Token(config.payment_config.cow_token_address), recipient=Address(self.reward_targets[0]), amount_wei=600000000000000000, ), Transfer( - token=Token(COW_TOKEN_ADDRESS), + token=Token(config.payment_config.cow_token_address), recipient=Address(self.reward_targets[1]), amount_wei=12000000000000000000, ), Transfer( - token=Token(COW_TOKEN_ADDRESS), + token=Token(config.payment_config.cow_token_address), recipient=Address(self.reward_targets[2]), amount_wei=90000000000000000000, ), Transfer( - token=Token(COW_TOKEN_ADDRESS), + token=Token(config.payment_config.cow_token_address), recipient=Address(self.reward_targets[3]), amount_wei=int(180000000000000000000 * (1 - SERVICE_FEE_FACTOR)), ), @@ -373,7 +373,7 @@ def setUp(self) -> None: self.solver_name = "Solver1" self.reward_target = Address.from_int(2) self.bonding_pool = Address.from_int(3) - self.cow_token = Token(COW_TOKEN_ADDRESS) + self.cow_token = Token(config.payment_config.cow_token_address) self.conversion_rate = 1000 def sample_record( From 6a973d043db5a12626e5f38cff21261d21be7097 Mon Sep 17 00:00:00 2001 From: Felix Henneke Date: Mon, 4 Nov 2024 12:32:08 +0100 Subject: [PATCH 03/19] delete constants.py --- src/constants.py | 56 ------------------------------------------------ 1 file changed, 56 deletions(-) delete mode 100644 src/constants.py diff --git a/src/constants.py b/src/constants.py deleted file mode 100644 index d1d9d1e4..00000000 --- a/src/constants.py +++ /dev/null @@ -1,56 +0,0 @@ -"""Project Global Constants. """ - -import os -from pathlib import Path - -from dotenv import load_dotenv -from dune_client.types import Address -from gnosis.eth.ethereum_network import EthereumNetwork -from web3 import Web3 - - -COW_TOKEN_ADDRESS = Address("0xDEf1CA1fb7FBcDC777520aa7f396b4E015F497aB") -COW_BONDING_POOL = Address("0x5d4020b9261f01b6f8a45db929704b0ad6f5e9e6") - -PROJECT_ROOT = Path(__file__).parent.parent -FILE_OUT_DIR = PROJECT_ROOT / Path("out") -LOG_CONFIG_FILE = PROJECT_ROOT / Path("logging.conf") -QUERY_PATH = PROJECT_ROOT / Path("queries") -DASHBOARD_PATH = PROJECT_ROOT / Path("dashboards/solver-rewards-accounting") - -DOCS_URL = ( - "https://www.notion.so/cownation/Solver-Payouts-3dfee64eb3d449ed8157a652cc817a8c" -) - -# Things requiring network -load_dotenv() -ENV = os.environ -SAFE_ADDRESS = Web3.to_checksum_address( - ENV.get("SAFE_ADDRESS", "0xA03be496e67Ec29bC62F01a428683D7F9c204930") -) -# Found this exposed infura key on https://rpc.info/ -INFURA_KEY = ENV.get("INFURA_KEY", "9aa3d95b3bc440fa88ea12eaa4456161") -NETWORK_STRING = ENV.get("NETWORK", "mainnet") -NODE_URL = f"https://{NETWORK_STRING}.infura.io/v3/{INFURA_KEY}" -NETWORK = { - "mainnet": EthereumNetwork.MAINNET, - "gnosis": EthereumNetwork.GNOSIS, - "goerli": EthereumNetwork.GOERLI, -}[NETWORK_STRING] -SHORT_NAME = { - "mainnet": "eth", - "rinkeby": "rin", - "gnosis": "gno", - "goerli": "gor", -}[NETWORK_STRING] - -CSV_APP_HASH = "Qme49gESuwpSvwANmEqo34yfCkzyQehooJ5yL7aHmKJnpZ" -SAFE_URL = "https://app.safe.global" -AIRDROP_URL = ( - f"{SAFE_URL}/{SHORT_NAME}:{SAFE_ADDRESS}" - f"/apps?appUrl=https://cloudflare-ipfs.com/ipfs/{CSV_APP_HASH}/" -) -SAFE_URL = f"{SAFE_URL}/{SHORT_NAME}:{SAFE_ADDRESS}/transactions/queue" - -# Real Web3 Instance -web3 = Web3(Web3.HTTPProvider(NODE_URL)) From ebabcf1bcde1fb0d30471849342d6bde5c8e2369 Mon Sep 17 00:00:00 2001 From: Felix Henneke Date: Mon, 4 Nov 2024 14:02:30 +0100 Subject: [PATCH 04/19] fix unit test --- tests/unit/test_payouts.py | 20 ++++++++++++-------- 1 file changed, 12 insertions(+), 8 deletions(-) diff --git a/tests/unit/test_payouts.py b/tests/unit/test_payouts.py index e7e36152..c7d73323 100644 --- a/tests/unit/test_payouts.py +++ b/tests/unit/test_payouts.py @@ -13,9 +13,6 @@ TokenConversion, prepare_transfers, RewardAndPenaltyDatum, - QUOTE_REWARD_COW, - PROTOCOL_FEE_SAFE, - SERVICE_FEE_FACTOR, ) from src.models.accounting_period import AccountingPeriod from src.models.overdraft import Overdraft @@ -343,11 +340,14 @@ def test_prepare_transfers(self): Transfer( token=Token(config.payment_config.cow_token_address), recipient=Address(self.reward_targets[3]), - amount_wei=int(180000000000000000000 * (1 - SERVICE_FEE_FACTOR)), + amount_wei=int( + 180000000000000000000 + * (1 - config.reward_config.service_fee_factor) + ), ), Transfer( token=None, - recipient=PROTOCOL_FEE_SAFE, + recipient=config.protocol_fee_config.protocol_fee_safe, amount_wei=3000000000000000, ), ], @@ -392,7 +392,7 @@ def sample_record( primary_reward_eth=primary_reward, primary_reward_cow=primary_reward * self.conversion_rate, slippage_eth=slippage, - quote_reward_cow=QUOTE_REWARD_COW * num_quotes, + quote_reward_cow=config.reward_config.quote_reward_cow * num_quotes, service_fee=service_fee, ) @@ -546,7 +546,9 @@ def test_performance_reward_service_fee(self): Transfer( token=self.cow_token, recipient=self.reward_target, - amount_wei=int(primary_reward * (1 - SERVICE_FEE_FACTOR)) + amount_wei=int( + primary_reward * (1 - config.reward_config.service_fee_factor) + ) * self.conversion_rate, ), ], @@ -569,7 +571,9 @@ def test_quote_reward_service_fee(self): token=self.cow_token, recipient=self.reward_target, amount_wei=int( - 6000000000000000000 * num_quotes * (1 - SERVICE_FEE_FACTOR) + 6000000000000000000 + * num_quotes + * (1 - config.reward_config.service_fee_factor) ), ), ], From 6f6ce8ba135650d1ed4eba43d5a05e0f0502bcc2 Mon Sep 17 00:00:00 2001 From: Felix Henneke Date: Mon, 4 Nov 2024 15:32:33 +0100 Subject: [PATCH 05/19] add old default for network since tests need this variable --- src/config.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/config.py b/src/config.py index 5ca58ccd..65c14ad8 100644 --- a/src/config.py +++ b/src/config.py @@ -202,6 +202,6 @@ def get_accounting_config(network: str) -> AccountingConfig: return result -config = get_accounting_config(os.environ["NETWORK"]) +config = get_accounting_config(os.environ.get("NETWORK", "mainnet")) web3 = Web3(Web3.HTTPProvider(config.node_config.node_url)) From ff48c7bfd6eebed59d4211329471cfec682ed69b Mon Sep 17 00:00:00 2001 From: Felix Henneke Date: Mon, 4 Nov 2024 17:09:42 +0100 Subject: [PATCH 06/19] make dune key optional for now this should be removed later. we should - add special handling of secrets, or - check this variable at the start of the script --- src/config.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/config.py b/src/config.py index 65c14ad8..b44bac13 100644 --- a/src/config.py +++ b/src/config.py @@ -147,7 +147,7 @@ def get_accounting_config(network: str) -> AccountingConfig: ) safe_queue_url = f"{safe_url}/transactions/queue" - dune_api_key = os.environ["DUNE_API_KEY"] + dune_api_key = os.environ.get("DUNE_API_KEY", "") signing_key = os.getenv("PROPOSER_PK") if signing_key == "": signing_key = None From 57585348f0e7095bede66b7a0afd0d4980556b52 Mon Sep 17 00:00:00 2001 From: Felix Henneke Date: Mon, 4 Nov 2024 17:33:57 +0100 Subject: [PATCH 07/19] make all secrets optional for now --- src/config.py | 34 ++++++++++++++++++++++------------ 1 file changed, 22 insertions(+), 12 deletions(-) diff --git a/src/config.py b/src/config.py index b44bac13..869ea21c 100644 --- a/src/config.py +++ b/src/config.py @@ -125,21 +125,35 @@ def get_accounting_config(network: str) -> AccountingConfig: docs_url = "https://www.notion.so/cownation/Solver-Payouts-3dfee64eb3d449ed8157a652cc817a8c" - payment_safe_address = Web3.to_checksum_address( - os.environ.get("SAFE_ADDRESS", "0xA03be496e67Ec29bC62F01a428683D7F9c204930") - ) + # Secrets + # Found this exposed infura key on https://rpc.info/ infura_key = os.environ.get("INFURA_KEY", "9aa3d95b3bc440fa88ea12eaa4456161") + dune_api_key = os.environ.get("DUNE_API_KEY", "") + + prod_db_url = os.environ.get("PROD_DB_URL", "") + barn_db_url = os.environ.get("BARN_DB_URL", "") + + signing_key = os.getenv("PROPOSER_PK") + if signing_key == "": + signing_key = None + + slack_channel = os.getenv("SLACK_CHANNEL", None) + slack_token = os.getenv("SLACK_TOKEN", None) + node_url = f"https://{network}.infura.io/v3/{infura_key}" + # + payment_safe_address = Web3.to_checksum_address( + os.environ.get("SAFE_ADDRESS", "0xA03be496e67Ec29bC62F01a428683D7F9c204930") + ) short_name = { "mainnet": "eth", "rinkeby": "rin", "gnosis": "gno", "goerli": "gor", }[network] - csv_app_hash = "Qme49gESuwpSvwANmEqo34yfCkzyQehooJ5yL7aHmKJnpZ" safe_url = f"https://app.safe.global/{short_name}:{payment_safe_address}" airdrop_url = ( @@ -147,10 +161,6 @@ def get_accounting_config(network: str) -> AccountingConfig: ) safe_queue_url = f"{safe_url}/transactions/queue" - dune_api_key = os.environ.get("DUNE_API_KEY", "") - signing_key = os.getenv("PROPOSER_PK") - if signing_key == "": - signing_key = None result = AccountingConfig( payment_config=PaymentConfig( network=EthereumNetwork.MAINNET, @@ -167,8 +177,8 @@ def get_accounting_config(network: str) -> AccountingConfig: ), ), orderbook_config=OrderbookConfig( - prod_db_url=os.environ["PROD_DB_URL"], - barn_db_url=os.environ["BARN_DB_URL"], + prod_db_url=prod_db_url, + barn_db_url=barn_db_url, ), dune_config=DuneConfig(dune_api_key=dune_api_key, dune_blockchain="ethereum"), node_config=NodeConfig(node_url=node_url), @@ -194,8 +204,8 @@ def get_accounting_config(network: str) -> AccountingConfig: query_dir=query_dir, csv_output_dir=file_out_dir, dashboard_dir=dashboard_dir, - slack_channel=os.getenv("SLACK_CHANNEL", None), - slack_token=os.getenv("SLACK_TOKEN", None), + slack_channel=slack_channel, + slack_token=slack_token, ), ) From 146fa6508bc7d8e47bdb464abee2299bc7e63c9c Mon Sep 17 00:00:00 2001 From: Felix Henneke Date: Tue, 5 Nov 2024 10:54:50 +0100 Subject: [PATCH 08/19] fix merge commit remove token_list again --- src/fetch/token_list.py | 42 ----------------------------------------- 1 file changed, 42 deletions(-) delete mode 100644 src/fetch/token_list.py diff --git a/src/fetch/token_list.py b/src/fetch/token_list.py deleted file mode 100644 index d5b32014..00000000 --- a/src/fetch/token_list.py +++ /dev/null @@ -1,42 +0,0 @@ -""" -Standalone script for fetching the trusted token -list and pushing the data to a dune user generated view. -""" - -from __future__ import annotations - -import json -import logging.config - -import requests - -from src.config import config - -log = logging.getLogger(__name__) -logging.config.fileConfig( - fname=config.io_config.log_config_file.absolute(), disable_existing_loggers=False -) - -ALLOWED_TOKEN_LIST_URL = "https://files.cow.fi/token_list.json" - - -def parse_token_list(token_list_json: str) -> list[str]: - """ - Parses JSON-str token list as list of token dune-compatible VALUES list of EVM addresses - See PoC Query for example formatting: https://dune.com/queries/1547103 - """ - try: - token_list = json.loads(token_list_json) - except json.JSONDecodeError: - # TODO - raise properly here! - print("Could not parse JSON data!") - raise - return [ - f"('{t['address'].lower()}')" for t in token_list["tokens"] if t["chainId"] == 1 - ] - - -def get_trusted_tokens() -> list[str]: - """Returns the list of trusted buffer tradable tokens""" - response = requests.get(ALLOWED_TOKEN_LIST_URL, timeout=10) - return parse_token_list(response.text) From f97df99d59d88e2f50610665d0621fe0c003b4dd Mon Sep 17 00:00:00 2001 From: Felix Henneke Date: Tue, 5 Nov 2024 11:18:41 +0100 Subject: [PATCH 09/19] incorporate config on sippage into payment script --- src/fetch/payouts.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/fetch/payouts.py b/src/fetch/payouts.py index 130e73c8..6771f443 100644 --- a/src/fetch/payouts.py +++ b/src/fetch/payouts.py @@ -492,7 +492,7 @@ def construct_payouts( ] reward_target_df = pandas.DataFrame(dune.get_vouches()) # construct slippage df - if ignore_slippage_flag: + if ignore_slippage_flag or (not config.buffer_accounting_config.include_slippage): slippage_df_temp = pandas.merge( merged_df[["solver"]], reward_target_df[["solver", "solver_name"]], From 5b7a8cc99f55f162d04b18abfed4fbcd8b011406 Mon Sep 17 00:00:00 2001 From: Felix Henneke Date: Thu, 7 Nov 2024 11:19:38 +0100 Subject: [PATCH 10/19] use enum and static class methods for config this addresses review comments - uses an enum instead of a string for better checking of network values - move initialization of configs into individual classes: use from_network and from_env to initialize classes --- src/config.py | 262 +++++++++++++++++++++++++++++++------------------- 1 file changed, 163 insertions(+), 99 deletions(-) diff --git a/src/config.py b/src/config.py index 869ea21c..7667203f 100644 --- a/src/config.py +++ b/src/config.py @@ -1,7 +1,10 @@ """Config for solver accounting.""" +from __future__ import annotations + import os from dataclasses import dataclass +from enum import Enum from fractions import Fraction from pathlib import Path @@ -14,6 +17,13 @@ load_dotenv() +class Network(Enum): + MAINNET = "mainnet" + GNOSIS = "gnosis" + ARBITRUM_ONE = "arbitrum" + BASE = "base" + + @dataclass(frozen=True) class RewardConfig: """Configuration for reward mechanism.""" @@ -26,6 +36,26 @@ class RewardConfig: quote_reward_cap_native: int service_fee_factor: Fraction + @staticmethod + def from_network(network: Network) -> RewardConfig: + match network: + case Network.MAINNET: + return RewardConfig( + reward_token_address=Address( + "0xDEf1CA1fb7FBcDC777520aa7f396b4E015F497aB" + ), + batch_reward_cap_upper=12 * 10**15, + batch_reward_cap_lower=10 * 10**15, + quote_reward_cow=6 * 10**18, + quote_reward_cap_native=6 * 10**14, + service_fee_factor=Fraction(15, 100), + cow_bonding_pool=Address( + "0x5d4020b9261f01b6f8a45db929704b0ad6f5e9e6" + ), + ) + case _: + raise ValueError(f"No reward config set up for network {network}.") + @dataclass(frozen=True) class ProtocolFeeConfig: @@ -36,6 +66,23 @@ class ProtocolFeeConfig: partner_fee_reduced_cut: float reduced_cut_address: str + @staticmethod + def from_network(network: Network) -> ProtocolFeeConfig: + match network: + case Network.MAINNET: + return ProtocolFeeConfig( + protocol_fee_safe=Address( + "0xB64963f95215FDe6510657e719bd832BB8bb941B" + ), + partner_fee_cut=0.15, + partner_fee_reduced_cut=0.10, + reduced_cut_address="0x63695Eee2c3141BDE314C5a6f89B98E62808d716", + ) + case _: + raise ValueError( + f"No protocol fee config set up for network {network}." + ) + @dataclass(frozen=True) class BufferAccountingConfig: @@ -43,6 +90,16 @@ class BufferAccountingConfig: include_slippage: bool + @staticmethod + def from_network(network: Network) -> BufferAccountingConfig: + match network: + case Network.MAINNET: + return BufferAccountingConfig(include_slippage=True) + case _: + raise ValueError( + f"No buffer accounting config set up for network {network}." + ) + @dataclass(frozen=True) class OrderbookConfig: @@ -51,6 +108,13 @@ class OrderbookConfig: prod_db_url: str barn_db_url: str + @staticmethod + def from_env() -> OrderbookConfig: + prod_db_url = os.environ.get("PROD_DB_URL", "") + barn_db_url = os.environ.get("BARN_DB_URL", "") + + return OrderbookConfig(prod_db_url=prod_db_url, barn_db_url=barn_db_url) + @dataclass(frozen=True) class DuneConfig: @@ -59,6 +123,15 @@ class DuneConfig: dune_api_key: str dune_blockchain: str + @staticmethod + def from_network(network: Network) -> DuneConfig: + dune_api_key = os.environ.get("DUNE_API_KEY", "") + match network: + case Network.MAINNET: + return DuneConfig(dune_api_key=dune_api_key, dune_blockchain="ethereum") + case _: + raise ValueError(f"No dune config set up for network {network}.") + @dataclass(frozen=True) class NodeConfig: @@ -66,6 +139,14 @@ class NodeConfig: node_url: str + @staticmethod + def from_network(network: Network) -> NodeConfig: + # Found this exposed infura key on https://rpc.info/ + infura_key = os.environ.get("INFURA_KEY", "9aa3d95b3bc440fa88ea12eaa4456161") + node_url = f"https://{network}.infura.io/v3/{infura_key}" + + return NodeConfig(node_url=node_url) + @dataclass(frozen=True) class PaymentConfig: @@ -82,6 +163,53 @@ class PaymentConfig: verification_docs_url: str weth_address: ChecksumAddress + @staticmethod + def from_network(network: Network) -> PaymentConfig: + signing_key = os.getenv("PROPOSER_PK") + if signing_key == "": + signing_key = None + + docs_url = "https://www.notion.so/cownation/Solver-Payouts-3dfee64eb3d449ed8157a652cc817a8c" + + network_short_name = { + Network.MAINNET: "eth", + Network.GNOSIS: "gno", + } + + match network: + case Network.MAINNET: + payment_safe_address = Web3.to_checksum_address( + os.environ.get( + "SAFE_ADDRESS", "0xA03be496e67Ec29bC62F01a428683D7F9c204930" + ) + ) + short_name = network_short_name[network] + csv_app_hash = "Qme49gESuwpSvwANmEqo34yfCkzyQehooJ5yL7aHmKJnpZ" + safe_url = ( + f"https://app.safe.global/{short_name}:{payment_safe_address}" + ) + airdrop_url = f"{safe_url}/apps?appUrl=https://cloudflare-ipfs.com/ipfs/{csv_app_hash}/" + safe_queue_url = f"{safe_url}/transactions/queue" + + return PaymentConfig( + network=EthereumNetwork.MAINNET, + cow_token_address=Address( + "0xDEf1CA1fb7FBcDC777520aa7f396b4E015F497aB" + ), + payment_safe_address=Web3.to_checksum_address( + "0xA03be496e67Ec29bC62F01a428683D7F9c204930" + ), + signing_key=signing_key, + safe_queue_url=safe_queue_url, + csv_airdrop_url=airdrop_url, + verification_docs_url=docs_url, + weth_address=Web3.to_checksum_address( + "0xc02aaa39b223fe8d0a0e5c4f27ead9083c756cc2" + ), + ) + case _: + raise ValueError(f"No payment config set up for network {network}.") + @dataclass(frozen=True) class IOConfig: @@ -95,6 +223,27 @@ class IOConfig: slack_channel: str | None slack_token: str | None + @staticmethod + def from_env(): + slack_channel = os.getenv("SLACK_CHANNEL", None) + slack_token = os.getenv("SLACK_TOKEN", None) + + project_root_dir = Path(__file__).parent.parent + file_out_dir = project_root_dir / Path("out") + log_config_file = project_root_dir / Path("logging.conf") + query_dir = project_root_dir / Path("queries") + dashboard_dir = project_root_dir / Path("dashboards/solver-rewards-accounting") + + return IOConfig( + project_root_dir=project_root_dir, + log_config_file=log_config_file, + query_dir=query_dir, + csv_output_dir=file_out_dir, + dashboard_dir=dashboard_dir, + slack_channel=slack_channel, + slack_token=slack_token, + ) + @dataclass(frozen=True) class AccountingConfig: @@ -111,107 +260,22 @@ class AccountingConfig: buffer_accounting_config: BufferAccountingConfig io_config: IOConfig + @staticmethod + def from_network(network: Network) -> AccountingConfig: + """Get config for specified network.""" -def get_accounting_config(network: str) -> AccountingConfig: - """Get config for specified network.""" - - # pylint: disable=too-many-locals - - project_root_dir = Path(__file__).parent.parent - file_out_dir = project_root_dir / Path("out") - log_config_file = project_root_dir / Path("logging.conf") - query_dir = project_root_dir / Path("queries") - dashboard_dir = project_root_dir / Path("dashboards/solver-rewards-accounting") - - docs_url = "https://www.notion.so/cownation/Solver-Payouts-3dfee64eb3d449ed8157a652cc817a8c" - - # Secrets - - # Found this exposed infura key on https://rpc.info/ - infura_key = os.environ.get("INFURA_KEY", "9aa3d95b3bc440fa88ea12eaa4456161") - - dune_api_key = os.environ.get("DUNE_API_KEY", "") - - prod_db_url = os.environ.get("PROD_DB_URL", "") - barn_db_url = os.environ.get("BARN_DB_URL", "") - - signing_key = os.getenv("PROPOSER_PK") - if signing_key == "": - signing_key = None - - slack_channel = os.getenv("SLACK_CHANNEL", None) - slack_token = os.getenv("SLACK_TOKEN", None) - - node_url = f"https://{network}.infura.io/v3/{infura_key}" - - # - payment_safe_address = Web3.to_checksum_address( - os.environ.get("SAFE_ADDRESS", "0xA03be496e67Ec29bC62F01a428683D7F9c204930") - ) - short_name = { - "mainnet": "eth", - "rinkeby": "rin", - "gnosis": "gno", - "goerli": "gor", - }[network] - csv_app_hash = "Qme49gESuwpSvwANmEqo34yfCkzyQehooJ5yL7aHmKJnpZ" - safe_url = f"https://app.safe.global/{short_name}:{payment_safe_address}" - airdrop_url = ( - f"{safe_url}/apps?appUrl=https://cloudflare-ipfs.com/ipfs/{csv_app_hash}/" - ) - safe_queue_url = f"{safe_url}/transactions/queue" - - result = AccountingConfig( - payment_config=PaymentConfig( - network=EthereumNetwork.MAINNET, - cow_token_address=Address("0xDEf1CA1fb7FBcDC777520aa7f396b4E015F497aB"), - payment_safe_address=Web3.to_checksum_address( - "0xA03be496e67Ec29bC62F01a428683D7F9c204930" - ), - signing_key=signing_key, - safe_queue_url=safe_queue_url, - csv_airdrop_url=airdrop_url, - verification_docs_url=docs_url, - weth_address=Web3.to_checksum_address( - "0xc02aaa39b223fe8d0a0e5c4f27ead9083c756cc2" - ), - ), - orderbook_config=OrderbookConfig( - prod_db_url=prod_db_url, - barn_db_url=barn_db_url, - ), - dune_config=DuneConfig(dune_api_key=dune_api_key, dune_blockchain="ethereum"), - node_config=NodeConfig(node_url=node_url), - reward_config=RewardConfig( - reward_token_address=Address("0xDEf1CA1fb7FBcDC777520aa7f396b4E015F497aB"), - batch_reward_cap_upper=12 * 10**15, - batch_reward_cap_lower=10 * 10**15, - quote_reward_cow=6 * 10**18, - quote_reward_cap_native=6 * 10**14, - service_fee_factor=Fraction(15, 100), - cow_bonding_pool=Address("0x5d4020b9261f01b6f8a45db929704b0ad6f5e9e6"), - ), - protocol_fee_config=ProtocolFeeConfig( - protocol_fee_safe=Address("0xB64963f95215FDe6510657e719bd832BB8bb941B"), - partner_fee_cut=0.15, - partner_fee_reduced_cut=0.10, - reduced_cut_address="0x63695Eee2c3141BDE314C5a6f89B98E62808d716", - ), - buffer_accounting_config=BufferAccountingConfig(include_slippage=True), - io_config=IOConfig( - project_root_dir=project_root_dir, - log_config_file=log_config_file, - query_dir=query_dir, - csv_output_dir=file_out_dir, - dashboard_dir=dashboard_dir, - slack_channel=slack_channel, - slack_token=slack_token, - ), - ) - - return result + return AccountingConfig( + payment_config=PaymentConfig.from_network(network), + orderbook_config=OrderbookConfig.from_env(), + dune_config=DuneConfig.from_network(network), + node_config=NodeConfig.from_network(network), + reward_config=RewardConfig.from_network(network), + protocol_fee_config=ProtocolFeeConfig.from_network(network), + buffer_accounting_config=BufferAccountingConfig.from_network(network), + io_config=IOConfig.from_env(), + ) -config = get_accounting_config(os.environ.get("NETWORK", "mainnet")) +config = AccountingConfig.from_network(Network(os.environ.get("NETWORK", "mainnet"))) web3 = Web3(Web3.HTTPProvider(config.node_config.node_url)) From 434bc612269fa4c3a022bdeaf7e6e88c9f79c98f Mon Sep 17 00:00:00 2001 From: Felix Henneke Date: Thu, 7 Nov 2024 11:31:02 +0100 Subject: [PATCH 11/19] fix pylint and mypy mostly by adding comments --- src/config.py | 19 ++++++++++++++++--- 1 file changed, 16 insertions(+), 3 deletions(-) diff --git a/src/config.py b/src/config.py index 7667203f..a1b8bd4c 100644 --- a/src/config.py +++ b/src/config.py @@ -18,6 +18,8 @@ class Network(Enum): + """Network class for networks supported by the accounting.""" + MAINNET = "mainnet" GNOSIS = "gnosis" ARBITRUM_ONE = "arbitrum" @@ -38,6 +40,7 @@ class RewardConfig: @staticmethod def from_network(network: Network) -> RewardConfig: + """Initialize reward config for a given network.""" match network: case Network.MAINNET: return RewardConfig( @@ -68,6 +71,7 @@ class ProtocolFeeConfig: @staticmethod def from_network(network: Network) -> ProtocolFeeConfig: + """Initialize protocol fee config for a given network.""" match network: case Network.MAINNET: return ProtocolFeeConfig( @@ -92,6 +96,7 @@ class BufferAccountingConfig: @staticmethod def from_network(network: Network) -> BufferAccountingConfig: + """Initialize buffer accounting config for a given network.""" match network: case Network.MAINNET: return BufferAccountingConfig(include_slippage=True) @@ -110,6 +115,7 @@ class OrderbookConfig: @staticmethod def from_env() -> OrderbookConfig: + """Initialize orderbook config from environment variables.""" prod_db_url = os.environ.get("PROD_DB_URL", "") barn_db_url = os.environ.get("BARN_DB_URL", "") @@ -125,6 +131,7 @@ class DuneConfig: @staticmethod def from_network(network: Network) -> DuneConfig: + """Initialize dune config for a given network.""" dune_api_key = os.environ.get("DUNE_API_KEY", "") match network: case Network.MAINNET: @@ -141,6 +148,7 @@ class NodeConfig: @staticmethod def from_network(network: Network) -> NodeConfig: + """Initialize node config for a given network.""" # Found this exposed infura key on https://rpc.info/ infura_key = os.environ.get("INFURA_KEY", "9aa3d95b3bc440fa88ea12eaa4456161") node_url = f"https://{network}.infura.io/v3/{infura_key}" @@ -165,6 +173,7 @@ class PaymentConfig: @staticmethod def from_network(network: Network) -> PaymentConfig: + """Initialize payment config for a given network.""" signing_key = os.getenv("PROPOSER_PK") if signing_key == "": signing_key = None @@ -188,7 +197,10 @@ def from_network(network: Network) -> PaymentConfig: safe_url = ( f"https://app.safe.global/{short_name}:{payment_safe_address}" ) - airdrop_url = f"{safe_url}/apps?appUrl=https://cloudflare-ipfs.com/ipfs/{csv_app_hash}/" + airdrop_url = ( + f"{safe_url}/apps?appUrl=https://cloudflare-ipfs.com/ipfs/" + f"{csv_app_hash}/" + ) safe_queue_url = f"{safe_url}/transactions/queue" return PaymentConfig( @@ -224,7 +236,8 @@ class IOConfig: slack_token: str | None @staticmethod - def from_env(): + def from_env() -> IOConfig: + """Initialize io config from environment variables.""" slack_channel = os.getenv("SLACK_CHANNEL", None) slack_token = os.getenv("SLACK_TOKEN", None) @@ -262,7 +275,7 @@ class AccountingConfig: @staticmethod def from_network(network: Network) -> AccountingConfig: - """Get config for specified network.""" + """Initialize accounting config for a given network.""" return AccountingConfig( payment_config=PaymentConfig.from_network(network), From 6be9178a72e0f171e29ed4bbe8f8bc2ef60356d8 Mon Sep 17 00:00:00 2001 From: Felix Henneke Date: Tue, 12 Nov 2024 14:45:30 +0100 Subject: [PATCH 12/19] fix error in test --- tests/unit/test_payouts.py | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/tests/unit/test_payouts.py b/tests/unit/test_payouts.py index 2b017be6..95279f0d 100644 --- a/tests/unit/test_payouts.py +++ b/tests/unit/test_payouts.py @@ -604,9 +604,7 @@ def test_performance_reward_service_fee(self): Transfer( token=self.cow_token, recipient=self.reward_target, - amount_wei=int( - primary_reward * (1 - service_fee) - ) + amount_wei=int(primary_reward * (1 - service_fee)) * self.conversion_rate, ), ], @@ -629,9 +627,7 @@ def test_quote_reward_service_fee(self): token=Token(self.reward_token_address), recipient=self.reward_target, amount_wei=int( - 6000000000000000000 - * num_quotes - * (1 - service_fee)) + 6000000000000000000 * num_quotes * (1 - service_fee) ), ), ], From 9c46d503cc5e7cab00854ee93c60bb917cc2dbd3 Mon Sep 17 00:00:00 2001 From: Felix Henneke Date: Tue, 12 Nov 2024 15:00:43 +0100 Subject: [PATCH 13/19] fix tests a bit more --- tests/unit/test_payouts.py | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/tests/unit/test_payouts.py b/tests/unit/test_payouts.py index 95279f0d..f4528bd4 100644 --- a/tests/unit/test_payouts.py +++ b/tests/unit/test_payouts.py @@ -424,7 +424,8 @@ def setUp(self) -> None: self.solver_name = "Solver1" self.reward_target = Address.from_int(2) self.buffer_accounting_target = Address.from_int(3) - self.cow_token = Token(config.payment_config.cow_token_address) + self.cow_token_address = config.payment_config.cow_token_address + self.cow_token = Token(self.cow_token_address) self.conversion_rate = 1000 def sample_record( @@ -445,7 +446,7 @@ def sample_record( slippage_eth=slippage, quote_reward_cow=config.reward_config.quote_reward_cow * num_quotes, service_fee=service_fee, - reward_token_address=self.reward_token_address, + reward_token_address=self.cow_token_address, ) def test_invalid_input(self): @@ -472,7 +473,7 @@ def test_reward_datum_pm1_0_0(self): test_datum.as_payouts(), [ Transfer( - token=Token(self.reward_token_address), + token=self.cow_token, recipient=self.reward_target, amount_wei=primary_reward * self.conversion_rate, ) @@ -518,7 +519,7 @@ def test_reward_datum_0_0_1(self): test_datum.as_payouts(), [ Transfer( - token=Token(self.reward_token_address), + token=self.cow_token, recipient=self.reward_target, amount_wei=6000000000000000000 * num_quotes, ) @@ -539,7 +540,7 @@ def test_reward_datum_4_1_0(self): amount_wei=slippage, ), Transfer( - token=Token(self.reward_token_address), + token=self.cow_token, recipient=self.reward_target, amount_wei=(primary_reward) * self.conversion_rate, ), @@ -555,7 +556,7 @@ def test_reward_datum_slippage_reduces_reward(self): test_datum.as_payouts(), [ Transfer( - token=Token(self.reward_token_address), + token=self.cow_token, recipient=self.reward_target, amount_wei=(primary_reward + slippage) * self.conversion_rate, ), @@ -624,7 +625,7 @@ def test_quote_reward_service_fee(self): test_datum.as_payouts(), [ Transfer( - token=Token(self.reward_token_address), + token=self.cow_token, recipient=self.reward_target, amount_wei=int( 6000000000000000000 * num_quotes * (1 - service_fee) From 766a91a2402d017f47c5f199ec98a452ce7f9ee6 Mon Sep 17 00:00:00 2001 From: Felix Henneke Date: Thu, 14 Nov 2024 11:15:40 +0100 Subject: [PATCH 14/19] fix config import in test_prices --- tests/e2e/test_prices.py | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/tests/e2e/test_prices.py b/tests/e2e/test_prices.py index 13cf7cc5..748dff3a 100644 --- a/tests/e2e/test_prices.py +++ b/tests/e2e/test_prices.py @@ -1,12 +1,10 @@ import unittest -from datetime import datetime, timedelta +from datetime import datetime from dune_client.types import Address -from src.abis.load import WETH9_ADDRESS -from src.constants import COW_TOKEN_ADDRESS +from src.config import config from src.fetch.prices import ( - TOKEN_ADDRESS_TO_ID, TokenId, exchange_rate_atoms, usd_price, @@ -22,8 +20,8 @@ def setUp(self) -> None: self.cow_price = usd_price(TokenId.COW, self.some_date) self.eth_price = usd_price(TokenId.ETH, self.some_date) self.usdc_price = usd_price(TokenId.USDC, self.some_date) - self.cow_address = COW_TOKEN_ADDRESS - self.weth_address = Address(WETH9_ADDRESS) + self.cow_address = config.reward_config.reward_token_address + self.weth_address = Address(config.payment_config.weth_address) self.usdc_address = Address("0xa0b86991c6218b36c1d19d4a2e9eb0ce3606eb48") def test_usd_price(self): From d290f091c774d43fb5eb2437a43b2399c67b9557 Mon Sep 17 00:00:00 2001 From: Felix Henneke Date: Thu, 14 Nov 2024 12:24:45 +0100 Subject: [PATCH 15/19] add node url for testing --- .github/workflows/pull-request.yaml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/pull-request.yaml b/.github/workflows/pull-request.yaml index 43265093..6044eae9 100644 --- a/.github/workflows/pull-request.yaml +++ b/.github/workflows/pull-request.yaml @@ -26,3 +26,5 @@ jobs: - name: Unit Tests run: python -m pytest tests/unit + env: + NODE_URL: "" From 5a81073e940077397541d1b98a620d6c5cb2d552 Mon Sep 17 00:00:00 2001 From: Felix Henneke Date: Thu, 14 Nov 2024 12:54:10 +0100 Subject: [PATCH 16/19] add empty default node url --- .github/workflows/pull-request.yaml | 2 -- src/config.py | 2 +- 2 files changed, 1 insertion(+), 3 deletions(-) diff --git a/.github/workflows/pull-request.yaml b/.github/workflows/pull-request.yaml index 6044eae9..43265093 100644 --- a/.github/workflows/pull-request.yaml +++ b/.github/workflows/pull-request.yaml @@ -26,5 +26,3 @@ jobs: - name: Unit Tests run: python -m pytest tests/unit - env: - NODE_URL: "" diff --git a/src/config.py b/src/config.py index 65fcaeb2..47a3f1f5 100644 --- a/src/config.py +++ b/src/config.py @@ -151,7 +151,7 @@ def from_network(network: Network) -> NodeConfig: """Initialize node config for a given network.""" match network: case Network.MAINNET: - node_url = os.environ["NODE_URL"] + node_url = os.environ.get("NODE_URL", "") case _: raise ValueError(f"No node config set up for network {network}.") From 82a8a5b73bbca63e43c72dffc1eb76c1d0cc7c9f Mon Sep 17 00:00:00 2001 From: Haris Angelidakis <64154020+harisang@users.noreply.github.com> Date: Fri, 15 Nov 2024 03:47:14 +0200 Subject: [PATCH 17/19] Update src/config.py --- src/config.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/config.py b/src/config.py index 47a3f1f5..7ce5289d 100644 --- a/src/config.py +++ b/src/config.py @@ -62,7 +62,7 @@ def from_network(network: Network) -> RewardConfig: @dataclass(frozen=True) class ProtocolFeeConfig: - """Configuration for protocol and parner fees.""" + """Configuration for protocol and partner fees.""" protocol_fee_safe: Address partner_fee_cut: float From c3ffd419dcc90bd377777c0a2b22b89f40b6b79f Mon Sep 17 00:00:00 2001 From: Felix Henneke Date: Fri, 15 Nov 2024 10:16:41 +0100 Subject: [PATCH 18/19] add comments for protocol fee config # Conflicts: # src/config.py --- src/config.py | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/src/config.py b/src/config.py index 7ce5289d..6b4451d5 100644 --- a/src/config.py +++ b/src/config.py @@ -62,7 +62,14 @@ def from_network(network: Network) -> RewardConfig: @dataclass(frozen=True) class ProtocolFeeConfig: - """Configuration for protocol and partner fees.""" + """Configuration for protocol and partner fees. + + Attributes: + protocol_fee_safe -- address to forward protocol fees to + partner_fee_cut -- fraction of partner fees withheld from integration partners + partner_fee_reduced_cut -- reduced amount withheld from partner specified as reduced_cut_address + reduced_cut_address -- partner fee recipient who pays the reduced cut partner_fee_reduced_cut + """ protocol_fee_safe: Address partner_fee_cut: float From ac96e90c0869c7852972c457595e749636fb34f9 Mon Sep 17 00:00:00 2001 From: Felix Henneke Date: Fri, 15 Nov 2024 11:50:48 +0100 Subject: [PATCH 19/19] remove airdrop url from config --- src/config.py | 7 ------- src/fetch/transfer_file.py | 6 +----- 2 files changed, 1 insertion(+), 12 deletions(-) diff --git a/src/config.py b/src/config.py index 6b4451d5..a4cddd05 100644 --- a/src/config.py +++ b/src/config.py @@ -176,7 +176,6 @@ class PaymentConfig: payment_safe_address: ChecksumAddress signing_key: str | None safe_queue_url: str - csv_airdrop_url: str verification_docs_url: str weth_address: ChecksumAddress @@ -202,14 +201,9 @@ def from_network(network: Network) -> PaymentConfig: ) ) short_name = network_short_name[network] - csv_app_hash = "Qme49gESuwpSvwANmEqo34yfCkzyQehooJ5yL7aHmKJnpZ" safe_url = ( f"https://app.safe.global/{short_name}:{payment_safe_address}" ) - airdrop_url = ( - f"{safe_url}/apps?appUrl=https://cloudflare-ipfs.com/ipfs/" - f"{csv_app_hash}/" - ) safe_queue_url = f"{safe_url}/transactions/queue" return PaymentConfig( @@ -222,7 +216,6 @@ def from_network(network: Network) -> PaymentConfig: ), signing_key=signing_key, safe_queue_url=safe_queue_url, - csv_airdrop_url=airdrop_url, verification_docs_url=docs_url, weth_address=Web3.to_checksum_address( "0xc02aaa39b223fe8d0a0e5c4f27ead9083c756cc2" diff --git a/src/fetch/transfer_file.py b/src/fetch/transfer_file.py index 3556b423..7383ddb8 100644 --- a/src/fetch/transfer_file.py +++ b/src/fetch/transfer_file.py @@ -41,11 +41,7 @@ def manual_propose(transfers: list[Transfer], period: AccountingPeriod) -> None: ) print(Transfer.summarize(transfers)) - print( - f"Please cross check these results with the dashboard linked above.\n " - f"For solver payouts, paste the transfer file CSV Airdrop at:\n" - f"{config.payment_config.csv_airdrop_url}" - ) + print("Please cross check these results with the dashboard linked above.\n") def auto_propose(