From 2d4ae553f4dbd4acb2345dd1601a18c68f0789ba Mon Sep 17 00:00:00 2001 From: harisang Date: Mon, 2 Dec 2024 02:01:55 +0200 Subject: [PATCH 1/4] add raw batch data wrapper query --- cowprotocol/raw_data/.sqlfluff | 4 + cowprotocol/raw_data/batch_data_4351957.sql | 111 ++++++++++++++++++++ 2 files changed, 115 insertions(+) create mode 100644 cowprotocol/raw_data/.sqlfluff create mode 100644 cowprotocol/raw_data/batch_data_4351957.sql diff --git a/cowprotocol/raw_data/.sqlfluff b/cowprotocol/raw_data/.sqlfluff new file mode 100644 index 00000000..f0706857 --- /dev/null +++ b/cowprotocol/raw_data/.sqlfluff @@ -0,0 +1,4 @@ +[sqlfluff:templater:jinja:context] +start_time='2024-08-01 12:00' +end_time='2024-08-02 12:00' +blockchain='ethereum' diff --git a/cowprotocol/raw_data/batch_data_4351957.sql b/cowprotocol/raw_data/batch_data_4351957.sql new file mode 100644 index 00000000..608e3e1d --- /dev/null +++ b/cowprotocol/raw_data/batch_data_4351957.sql @@ -0,0 +1,111 @@ +with +past_batch_data_ethereum as ( + select + s.environment, + -1 as auction_id, + d.block_number, + d.block_deadline, + d.tx_hash, + d.solver, + cast(d.data.execution_cost as decimal(32, 0)) as execution_cost, --noqa: RF01 + cast(d.data.surplus as decimal(32, 0)) as surplus, --noqa: RF01 + cast(d.data.protocol_fee as decimal(32, 0)) as protocol_fee, --noqa: RF01 + cast(d.data.fee as decimal(32, 0)) as network_fee, --noqa: RF01 + cast(d.data.uncapped_payment_eth as decimal(32, 0)) as uncapped_payment_native_token, --noqa: RF01 + cast(d.data.capped_payment as decimal(32, 0)) as capped_payment, --noqa: RF01 + cast(d.data.winning_score as decimal(32, 0)) as winning_score, --noqa: RF01 + cast(d.data.reference_score as decimal(32, 0)) as reference_score --noqa: RF01 + from cowswap.raw_batch_rewards as d inner join cow_protocol_ethereum.solvers as s on d.solver = cast(s.address as varchar) where d.block_deadline < 20866925 +), + +past_batch_data_gnosis as ( --noqa: ST03 + select + 'a' as environment, + 0 as auction_id, + 0 as block_number, + 0 as block_deadline, + '0x' as tx_hash, + '0x' as solver, + 0 as execution_cost, + 0 as surplus, + 0 as protocol_fee, + 0 as network_fee, + 0 as uncapped_payment_native_token, + 0 as capped_payment, + 0 as winning_score, + 0 as reference_score + where false +), + +past_batch_data_arbitrum as ( --noqa: ST03 + select + 'a' as environment, + 0 as auction_id, + 0 as block_number, + 0 as block_deadline, + '0x' as tx_hash, + '0x' as solver, + 0 as execution_cost, + 0 as surplus, + 0 as protocol_fee, + 0 as network_fee, + 0 as uncapped_payment_native_token, + 0 as capped_payment, + 0 as winning_score, + 0 as reference_score + where 1 = 0 +) + +select + environment, + auction_id, + block_number, + block_deadline, + cast(tx_hash as varchar) as tx_hash, + cast(solver as varchar) as solver, + cast(execution_cost as decimal(32, 0)) as execution_cost, + cast(surplus as decimal(32, 0)) as surplus, + cast(protocol_fee as decimal(32, 0)) as protocol_fee, + cast(network_fee as decimal(32, 0)) as network_fee, + cast(uncapped_payment_eth as decimal(32, 0)) as uncapped_payment_native_token, + cast(capped_payment as decimal(32, 0)) as capped_payment, + cast(winning_score as decimal(32, 0)) as winning_score, + cast(reference_score as decimal(32, 0)) as reference_score +from dune.cowprotocol.dataset_batch_data_{{blockchain}}_2024_10 +union all +select + environment, + auction_id, + block_number, + block_deadline, + cast(tx_hash as varchar) as tx_hash, + cast(solver as varchar) as solver, + cast(execution_cost as decimal(32, 0)) as execution_cost, + cast(surplus as decimal(32, 0)) as surplus, + cast(protocol_fee as decimal(32, 0)) as protocol_fee, + cast(network_fee as decimal(32, 0)) as network_fee, + cast(uncapped_payment_eth as decimal(32, 0)) as uncapped_payment_native_token, + cast(capped_payment as decimal(32, 0)) as capped_payment, + cast(winning_score as decimal(32, 0)) as winning_score, + cast(reference_score as decimal(32, 0)) as reference_score +from dune.cowprotocol.dataset_batch_data_{{blockchain}}_2024_11 +union all +select + environment, + auction_id, + block_number, + block_deadline, + cast(tx_hash as varchar) as tx_hash, + cast(solver as varchar) as solver, + cast(execution_cost as decimal(32, 0)) as execution_cost, + cast(surplus as decimal(32, 0)) as surplus, + cast(protocol_fee as decimal(32, 0)) as protocol_fee, + cast(network_fee as decimal(32, 0)) as network_fee, + cast(uncapped_payment_eth as decimal(32, 0)) as uncapped_payment_native_token, + cast(capped_payment as decimal(32, 0)) as capped_payment, + cast(winning_score as decimal(32, 0)) as winning_score, + cast(reference_score as decimal(32, 0)) as reference_score +from dune.cowprotocol.dataset_batch_data_{{blockchain}}_2024_12 +union all +select * +from past_batch_data_{{blockchain}} From a00595bc87180fc5d85cd7050334fce3f2091c0a Mon Sep 17 00:00:00 2001 From: harisang Date: Mon, 2 Dec 2024 02:30:37 +0200 Subject: [PATCH 2/4] add brief description --- cowprotocol/raw_data/batch_data_4351957.sql | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/cowprotocol/raw_data/batch_data_4351957.sql b/cowprotocol/raw_data/batch_data_4351957.sql index 608e3e1d..afd8c844 100644 --- a/cowprotocol/raw_data/batch_data_4351957.sql +++ b/cowprotocol/raw_data/batch_data_4351957.sql @@ -1,3 +1,7 @@ +-- This query provides data related to rewards/payouts on a per batch auction level +-- for all auctions that had at least one winner. +-- Parameters: +-- {{blockchain}}: the chain for which we want to retrieve batch data with past_batch_data_ethereum as ( select @@ -53,7 +57,7 @@ past_batch_data_arbitrum as ( --noqa: ST03 0 as capped_payment, 0 as winning_score, 0 as reference_score - where 1 = 0 + where false ) select From 73f2cddb33fa8da3d814f9e5ce68198238eb402b Mon Sep 17 00:00:00 2001 From: harisang Date: Tue, 3 Dec 2024 03:10:09 +0200 Subject: [PATCH 3/4] increase accuracy --- cowprotocol/raw_data/batch_data_4351957.sql | 64 ++++++++++----------- 1 file changed, 32 insertions(+), 32 deletions(-) diff --git a/cowprotocol/raw_data/batch_data_4351957.sql b/cowprotocol/raw_data/batch_data_4351957.sql index afd8c844..f2533e5f 100644 --- a/cowprotocol/raw_data/batch_data_4351957.sql +++ b/cowprotocol/raw_data/batch_data_4351957.sql @@ -11,14 +11,14 @@ past_batch_data_ethereum as ( d.block_deadline, d.tx_hash, d.solver, - cast(d.data.execution_cost as decimal(32, 0)) as execution_cost, --noqa: RF01 - cast(d.data.surplus as decimal(32, 0)) as surplus, --noqa: RF01 - cast(d.data.protocol_fee as decimal(32, 0)) as protocol_fee, --noqa: RF01 - cast(d.data.fee as decimal(32, 0)) as network_fee, --noqa: RF01 - cast(d.data.uncapped_payment_eth as decimal(32, 0)) as uncapped_payment_native_token, --noqa: RF01 - cast(d.data.capped_payment as decimal(32, 0)) as capped_payment, --noqa: RF01 - cast(d.data.winning_score as decimal(32, 0)) as winning_score, --noqa: RF01 - cast(d.data.reference_score as decimal(32, 0)) as reference_score --noqa: RF01 + cast(d.data.execution_cost as decimal(38, 0)) as execution_cost, --noqa: RF01 + cast(d.data.surplus as decimal(38, 0)) as surplus, --noqa: RF01 + cast(d.data.protocol_fee as decimal(38, 0)) as protocol_fee, --noqa: RF01 + cast(d.data.fee as decimal(38, 0)) as network_fee, --noqa: RF01 + cast(d.data.uncapped_payment_eth as decimal(38, 0)) as uncapped_payment_native_token, --noqa: RF01 + cast(d.data.capped_payment as decimal(38, 0)) as capped_payment, --noqa: RF01 + cast(d.data.winning_score as decimal(38, 0)) as winning_score, --noqa: RF01 + cast(d.data.reference_score as decimal(38, 0)) as reference_score --noqa: RF01 from cowswap.raw_batch_rewards as d inner join cow_protocol_ethereum.solvers as s on d.solver = cast(s.address as varchar) where d.block_deadline < 20866925 ), @@ -67,14 +67,14 @@ select block_deadline, cast(tx_hash as varchar) as tx_hash, cast(solver as varchar) as solver, - cast(execution_cost as decimal(32, 0)) as execution_cost, - cast(surplus as decimal(32, 0)) as surplus, - cast(protocol_fee as decimal(32, 0)) as protocol_fee, - cast(network_fee as decimal(32, 0)) as network_fee, - cast(uncapped_payment_eth as decimal(32, 0)) as uncapped_payment_native_token, - cast(capped_payment as decimal(32, 0)) as capped_payment, - cast(winning_score as decimal(32, 0)) as winning_score, - cast(reference_score as decimal(32, 0)) as reference_score + cast(execution_cost as decimal(38, 0)) as execution_cost, + cast(surplus as decimal(38, 0)) as surplus, + cast(protocol_fee as decimal(38, 0)) as protocol_fee, + cast(network_fee as decimal(38, 0)) as network_fee, + cast(uncapped_payment_eth as decimal(38, 0)) as uncapped_payment_native_token, + cast(capped_payment as decimal(38, 0)) as capped_payment, + cast(winning_score as decimal(38, 0)) as winning_score, + cast(reference_score as decimal(38, 0)) as reference_score from dune.cowprotocol.dataset_batch_data_{{blockchain}}_2024_10 union all select @@ -84,14 +84,14 @@ select block_deadline, cast(tx_hash as varchar) as tx_hash, cast(solver as varchar) as solver, - cast(execution_cost as decimal(32, 0)) as execution_cost, - cast(surplus as decimal(32, 0)) as surplus, - cast(protocol_fee as decimal(32, 0)) as protocol_fee, - cast(network_fee as decimal(32, 0)) as network_fee, - cast(uncapped_payment_eth as decimal(32, 0)) as uncapped_payment_native_token, - cast(capped_payment as decimal(32, 0)) as capped_payment, - cast(winning_score as decimal(32, 0)) as winning_score, - cast(reference_score as decimal(32, 0)) as reference_score + cast(execution_cost as decimal(38, 0)) as execution_cost, + cast(surplus as decimal(38, 0)) as surplus, + cast(protocol_fee as decimal(38, 0)) as protocol_fee, + cast(network_fee as decimal(38, 0)) as network_fee, + cast(uncapped_payment_eth as decimal(38, 0)) as uncapped_payment_native_token, + cast(capped_payment as decimal(38, 0)) as capped_payment, + cast(winning_score as decimal(38, 0)) as winning_score, + cast(reference_score as decimal(38, 0)) as reference_score from dune.cowprotocol.dataset_batch_data_{{blockchain}}_2024_11 union all select @@ -101,14 +101,14 @@ select block_deadline, cast(tx_hash as varchar) as tx_hash, cast(solver as varchar) as solver, - cast(execution_cost as decimal(32, 0)) as execution_cost, - cast(surplus as decimal(32, 0)) as surplus, - cast(protocol_fee as decimal(32, 0)) as protocol_fee, - cast(network_fee as decimal(32, 0)) as network_fee, - cast(uncapped_payment_eth as decimal(32, 0)) as uncapped_payment_native_token, - cast(capped_payment as decimal(32, 0)) as capped_payment, - cast(winning_score as decimal(32, 0)) as winning_score, - cast(reference_score as decimal(32, 0)) as reference_score + cast(execution_cost as decimal(38, 0)) as execution_cost, + cast(surplus as decimal(38, 0)) as surplus, + cast(protocol_fee as decimal(38, 0)) as protocol_fee, + cast(network_fee as decimal(38, 0)) as network_fee, + cast(uncapped_payment_eth as decimal(38, 0)) as uncapped_payment_native_token, + cast(capped_payment as decimal(38, 0)) as capped_payment, + cast(winning_score as decimal(38, 0)) as winning_score, + cast(reference_score as decimal(38, 0)) as reference_score from dune.cowprotocol.dataset_batch_data_{{blockchain}}_2024_12 union all select * From 72c99a532553bc10cd7695e7e044f2d15583fd40 Mon Sep 17 00:00:00 2001 From: harisang Date: Thu, 5 Dec 2024 17:43:48 +0200 Subject: [PATCH 4/4] minor changes --- cowprotocol/raw_data/batch_data_4351957.sql | 49 ++++++++++++++------- 1 file changed, 33 insertions(+), 16 deletions(-) diff --git a/cowprotocol/raw_data/batch_data_4351957.sql b/cowprotocol/raw_data/batch_data_4351957.sql index f2533e5f..96d87e67 100644 --- a/cowprotocol/raw_data/batch_data_4351957.sql +++ b/cowprotocol/raw_data/batch_data_4351957.sql @@ -2,15 +2,32 @@ -- for all auctions that had at least one winner. -- Parameters: -- {{blockchain}}: the chain for which we want to retrieve batch data + +-- The output has the following columns: +-- environment: varchar +-- auction_id: integer +-- block_number: integer +-- block_deadline: integer +-- tx_hash: varbinary +-- solver: varbinary +-- execution_cost: decimal(38, 0) +-- surplus: decimal(38, 0) +-- protocol_fee: decimal(38, 0) +-- network_fee: decimal(38, 0) +-- uncapped_payment_native_token: decimal(38, 0) +-- capped_payment: decimal(38, 0) +-- winning_score: decimal(38, 0) +-- reference_score: decimal(38, 0) + with past_batch_data_ethereum as ( select s.environment, - -1 as auction_id, + null as auction_id, d.block_number, d.block_deadline, - d.tx_hash, - d.solver, + from_hex(d.tx_hash) as tx_hash, + from_hex(d.solver) as solver, cast(d.data.execution_cost as decimal(38, 0)) as execution_cost, --noqa: RF01 cast(d.data.surplus as decimal(38, 0)) as surplus, --noqa: RF01 cast(d.data.protocol_fee as decimal(38, 0)) as protocol_fee, --noqa: RF01 @@ -28,8 +45,8 @@ past_batch_data_gnosis as ( --noqa: ST03 0 as auction_id, 0 as block_number, 0 as block_deadline, - '0x' as tx_hash, - '0x' as solver, + 0x as tx_hash, + 0x as solver, 0 as execution_cost, 0 as surplus, 0 as protocol_fee, @@ -47,8 +64,8 @@ past_batch_data_arbitrum as ( --noqa: ST03 0 as auction_id, 0 as block_number, 0 as block_deadline, - '0x' as tx_hash, - '0x' as solver, + 0x as tx_hash, + 0x as solver, 0 as execution_cost, 0 as surplus, 0 as protocol_fee, @@ -60,13 +77,16 @@ past_batch_data_arbitrum as ( --noqa: ST03 where false ) +select * +from past_batch_data_{{blockchain}} +union all select environment, auction_id, block_number, block_deadline, - cast(tx_hash as varchar) as tx_hash, - cast(solver as varchar) as solver, + tx_hash, + solver, cast(execution_cost as decimal(38, 0)) as execution_cost, cast(surplus as decimal(38, 0)) as surplus, cast(protocol_fee as decimal(38, 0)) as protocol_fee, @@ -82,8 +102,8 @@ select auction_id, block_number, block_deadline, - cast(tx_hash as varchar) as tx_hash, - cast(solver as varchar) as solver, + tx_hash, + solver, cast(execution_cost as decimal(38, 0)) as execution_cost, cast(surplus as decimal(38, 0)) as surplus, cast(protocol_fee as decimal(38, 0)) as protocol_fee, @@ -99,8 +119,8 @@ select auction_id, block_number, block_deadline, - cast(tx_hash as varchar) as tx_hash, - cast(solver as varchar) as solver, + tx_hash, + solver, cast(execution_cost as decimal(38, 0)) as execution_cost, cast(surplus as decimal(38, 0)) as surplus, cast(protocol_fee as decimal(38, 0)) as protocol_fee, @@ -110,6 +130,3 @@ select cast(winning_score as decimal(38, 0)) as winning_score, cast(reference_score as decimal(38, 0)) as reference_score from dune.cowprotocol.dataset_batch_data_{{blockchain}}_2024_12 -union all -select * -from past_batch_data_{{blockchain}}