From 3882fdf70a2c02c15c1e2e3954e56f86094d31d3 Mon Sep 17 00:00:00 2001 From: harisang Date: Sun, 7 Jan 2024 03:22:06 +0200 Subject: [PATCH 1/5] make block range consistent --- queries/dune_v2/period_block_interval.sql | 27 ++++++++++--- queries/dune_v2/period_slippage.sql | 49 +++++++++++++++-------- queries/orderbook/batch_rewards.sql | 4 +- queries/orderbook/quote_rewards.sql | 2 +- src/queries.py | 4 +- src/scripts/gap_detector.py | 8 ++-- 6 files changed, 64 insertions(+), 30 deletions(-) diff --git a/queries/dune_v2/period_block_interval.sql b/queries/dune_v2/period_block_interval.sql index c35d34b5..8487d354 100644 --- a/queries/dune_v2/period_block_interval.sql +++ b/queries/dune_v2/period_block_interval.sql @@ -1,6 +1,23 @@ --- V3: https://dune.com/queries/1541504 +-- V3: https://dune.com/queries/3333356 +with + block_range_start as ( + select + max("number") + 1 as start_block + from + ethereum.blocks + where + time <= cast('{{StartTime}}' as timestamp) + ), + block_range_stop as ( + select + max("number") as end_block + from + ethereum.blocks + where + time <= cast('{{EndTime}}' as timestamp) + ) select - min("number") as start_block, - max("number") as end_block -from ethereum.blocks -where date_trunc('minute', time) between cast('{{StartTime}}' as timestamp) and cast('{{EndTime}}' as timestamp) + * +from + block_range_start + cross join block_range_stop \ No newline at end of file diff --git a/queries/dune_v2/period_slippage.sql b/queries/dune_v2/period_slippage.sql index 0a0b3c91..e1029c4e 100644 --- a/queries/dune_v2/period_slippage.sql +++ b/queries/dune_v2/period_slippage.sql @@ -1,7 +1,30 @@ -- https://github.com/cowprotocol/solver-rewards/pull/327 --- Query Here: https://dune.com/queries/3093726 +-- Query Here: https://dune.com/queries/3333368 with -batch_meta as ( +block_range_start as ( + select + max("number") + 1 as start_block + from + ethereum.blocks +where + time <= cast('{{StartTime}}' as timestamp) +) +,block_range_stop as ( + select + max("number") as end_block + from + ethereum.blocks + where + time <= cast('{{EndTime}}' as timestamp) + ) +,block_range as ( + select + * + from + block_range_start + cross join block_range_stop + ) +,batch_meta as ( select b.block_time, b.block_number, b.tx_hash, @@ -14,7 +37,7 @@ batch_meta as ( num_trades, b.solver_address from cow_protocol_ethereum.batches b - where b.block_time between cast('{{StartTime}}' as timestamp) and cast('{{EndTime}}' as timestamp) + where b.block_number >= (select start_block from block_range) and b.block_number<= (select end_block from block_range) and (b.solver_address = from_hex('{{SolverAddress}}') or '{{SolverAddress}}' = '0x') and (b.tx_hash = from_hex('{{TxHash}}') or '{{TxHash}}' = '0x') ) @@ -38,8 +61,8 @@ batch_meta as ( left outer join cow_protocol_ethereum.order_rewards f on f.tx_hash = t.tx_hash and f.order_uid = t.order_uid - where b.block_time between cast('{{StartTime}}' as timestamp) and cast('{{EndTime}}' as timestamp) - and t.block_time between cast('{{StartTime}}' as timestamp) and cast('{{EndTime}}' as timestamp) + where b.block_number >= (select start_block from block_range) and b.block_number <= (select end_block from block_range) + and t.block_number >= (select start_block from block_range) and t.block_number <= (select end_block from block_range) and (b.solver_address = from_hex('{{SolverAddress}}') or '{{SolverAddress}}' = '0x') and (t.tx_hash = from_hex('{{TxHash}}') or '{{TxHash}}' = '0x') ) @@ -89,7 +112,7 @@ batch_meta as ( and evt_tx_hash = b.tx_hash inner join batchwise_traders bt on evt_tx_hash = bt.tx_hash - where b.block_time between cast('{{StartTime}}' as timestamp) and cast('{{EndTime}}' as timestamp) + where b.block_number >= (select start_block from block_range) and b.block_number<= (select end_block from block_range) and 0x9008d19f58aabd9ed0d60971565aa8510560ab41 in (to, "from") and not contains(traders_in, "from") and not contains(traders_out, to) @@ -225,12 +248,6 @@ incoming_and_outgoing as ( SELECT from_hex(address_str) as address FROM ( VALUES {{TokenList}} ) as _ (address_str) ) -,block_range as ( - select min(number) as start_block, - max(number) as end_block - from ethereum.blocks - where time between cast('{{StartTime}}' as timestamp) and cast('{{EndTime}}' as timestamp) -) ,internalized_imbalances as ( select b.block_time, b.tx_hash, @@ -246,7 +263,7 @@ incoming_and_outgoing as ( join tokens.erc20 t on contract_address = from_hex(token) and blockchain = 'ethereum' - where i.block_number between (select start_block from block_range) and (select end_block from block_range) + where i.block_number >= (select start_block from block_range) and i.block_number <= (select end_block from block_range) and ('{{SolverAddress}}' = '0x' or b.solver_address = from_hex('{{SolverAddress}}')) and ('{{TxHash}}' = '0x' or b.tx_hash = from_hex('{{TxHash}}')) ) @@ -316,7 +333,7 @@ incoming_and_outgoing as ( date_trunc('hour', block_time) as hour, usd_value / units_bought as price FROM cow_protocol_ethereum.trades - WHERE block_time between cast('{{StartTime}}' as timestamp) and cast('{{EndTime}}' as timestamp) + WHERE block_number >= (select start_block from block_range) and block_number <= (select end_block from block_range) AND units_bought > 0 UNION select @@ -325,7 +342,7 @@ incoming_and_outgoing as ( date_trunc('hour', block_time) as hour, usd_value / units_sold as price FROM cow_protocol_ethereum.trades - WHERE block_time between cast('{{StartTime}}' as timestamp) and cast('{{EndTime}}' as timestamp) + WHERE block_number >= (select start_block from block_range) and block_number <= (select end_block from block_range) AND units_sold > 0 ) as combined GROUP BY hour, contract_address, decimals @@ -407,4 +424,4 @@ incoming_and_outgoing as ( solver_address, concat(environment, '-', name) ) -select * from {{CTE_NAME}} +select * from {{CTE_NAME}} \ No newline at end of file diff --git a/queries/orderbook/batch_rewards.sql b/queries/orderbook/batch_rewards.sql index f6cf575b..31e6dc25 100644 --- a/queries/orderbook/batch_rewards.sql +++ b/queries/orderbook/batch_rewards.sql @@ -18,14 +18,14 @@ WITH observed_settlements AS (SELECT AND s.tx_nonce = at.tx_nonce JOIN settlement_scores ss ON at.auction_id = ss.auction_id - WHERE ss.block_deadline > {{start_block}} + WHERE ss.block_deadline >= {{start_block}} AND ss.block_deadline <= {{end_block}}), auction_participation as (SELECT ss.auction_id, array_agg(participant) as participating_solvers FROM auction_participants JOIN settlement_scores ss ON auction_participants.auction_id = ss.auction_id - WHERE block_deadline > {{start_block}} + WHERE block_deadline >= {{start_block}} AND block_deadline <= {{end_block}} GROUP BY ss.auction_id), reward_data AS (SELECT diff --git a/queries/orderbook/quote_rewards.sql b/queries/orderbook/quote_rewards.sql index d243e674..573a011d 100644 --- a/queries/orderbook/quote_rewards.sql +++ b/queries/orderbook/quote_rewards.sql @@ -4,7 +4,7 @@ with winning_quotes as (SELECT concat('0x', encode(oq.solver, 'hex')) as solver, INNER JOIN orders o ON order_uid = uid JOIN order_quotes oq ON t.order_uid = oq.order_uid WHERE o.class = 'market' - AND block_number BETWEEN {{start_block}} AND {{end_block}} + AND block_number >= {{start_block}} AND block_number <= {{end_block}} AND oq.solver != '\x0000000000000000000000000000000000000000') SELECT solver, count(*) AS num_quotes diff --git a/src/queries.py b/src/queries.py index 1f6c4613..78b1e293 100644 --- a/src/queries.py +++ b/src/queries.py @@ -33,7 +33,7 @@ def with_params(self, params: list[QueryParameter]) -> Query: "PERIOD_BLOCK_INTERVAL": QueryData( name="Block Interval for Accounting Period", filepath="period_block_interval.sql", - q_id=1541504, + q_id=3333356, ), "VOUCH_REGISTRY": QueryData( name="Vouch Registry", @@ -43,7 +43,7 @@ def with_params(self, params: list[QueryParameter]) -> Query: "PERIOD_SLIPPAGE": QueryData( name="Solver Slippage for Period", filepath="period_slippage.sql", - q_id=3093726, + q_id=3333368, ), "DASHBOARD_SLIPPAGE": QueryData( name="Period Solver Rewards", diff --git a/src/scripts/gap_detector.py b/src/scripts/gap_detector.py index 282b0eea..6f572483 100644 --- a/src/scripts/gap_detector.py +++ b/src/scripts/gap_detector.py @@ -18,15 +18,15 @@ DB_COUNT_QUERY = ( "select count(*) as batches from settlements " - "where block_number between {{start}} and {{end}};" + "where block_number >= {{start}} and block_number <= {{end}};" ) DB_HASH_QUERY = ( "select concat('0x', encode(tx_hash, 'hex')) as tx_hash from settlements " - "where block_number between {{start}} and {{end}};" + "where block_number >= {{start}} and block_number <= {{end}};" ) -DUNE_COUNT_QUERY_ID = 2481826 -DUNE_HASH_QUERY_ID = 2532671 +DUNE_COUNT_QUERY_ID = 3333411 +DUNE_HASH_QUERY_ID = 3333413 JANUARY_2023 = 16308190 MAX_QUERYABLE_HASH_SET = 500 From f1fba54a7f40f42e75fee39ace154445f9f16bce Mon Sep 17 00:00:00 2001 From: harisang Date: Sun, 7 Jan 2024 03:27:41 +0200 Subject: [PATCH 2/5] fix some spacing issues --- queries/dune_v2/period_slippage.sql | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/queries/dune_v2/period_slippage.sql b/queries/dune_v2/period_slippage.sql index e1029c4e..f3b882fe 100644 --- a/queries/dune_v2/period_slippage.sql +++ b/queries/dune_v2/period_slippage.sql @@ -37,7 +37,7 @@ where num_trades, b.solver_address from cow_protocol_ethereum.batches b - where b.block_number >= (select start_block from block_range) and b.block_number<= (select end_block from block_range) + where b.block_number >= (select start_block from block_range) and b.block_number <= (select end_block from block_range) and (b.solver_address = from_hex('{{SolverAddress}}') or '{{SolverAddress}}' = '0x') and (b.tx_hash = from_hex('{{TxHash}}') or '{{TxHash}}' = '0x') ) @@ -112,7 +112,7 @@ where and evt_tx_hash = b.tx_hash inner join batchwise_traders bt on evt_tx_hash = bt.tx_hash - where b.block_number >= (select start_block from block_range) and b.block_number<= (select end_block from block_range) + where b.block_number >= (select start_block from block_range) and b.block_number <= (select end_block from block_range) and 0x9008d19f58aabd9ed0d60971565aa8510560ab41 in (to, "from") and not contains(traders_in, "from") and not contains(traders_out, to) From 0a1ef4dd26ed35449f06eb594e3cd33d2db90586 Mon Sep 17 00:00:00 2001 From: Haris Angelidakis <64154020+harisang@users.noreply.github.com> Date: Tue, 9 Jan 2024 11:20:48 +0200 Subject: [PATCH 3/5] Update queries/dune_v2/period_block_interval.sql Co-authored-by: Felix Henneke --- queries/dune_v2/period_block_interval.sql | 1 + 1 file changed, 1 insertion(+) diff --git a/queries/dune_v2/period_block_interval.sql b/queries/dune_v2/period_block_interval.sql index 8487d354..e6188d20 100644 --- a/queries/dune_v2/period_block_interval.sql +++ b/queries/dune_v2/period_block_interval.sql @@ -1,3 +1,4 @@ +-- https://github.com/cowprotocol/solver-rewards/pull/330 -- V3: https://dune.com/queries/3333356 with block_range_start as ( From 545067485426dbb9fd8b6fecd63be660002e6385 Mon Sep 17 00:00:00 2001 From: Haris Angelidakis <64154020+harisang@users.noreply.github.com> Date: Tue, 9 Jan 2024 11:21:36 +0200 Subject: [PATCH 4/5] Update queries/dune_v2/period_slippage.sql Co-authored-by: Felix Henneke --- queries/dune_v2/period_slippage.sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/queries/dune_v2/period_slippage.sql b/queries/dune_v2/period_slippage.sql index f3b882fe..561ffff9 100644 --- a/queries/dune_v2/period_slippage.sql +++ b/queries/dune_v2/period_slippage.sql @@ -1,4 +1,4 @@ --- https://github.com/cowprotocol/solver-rewards/pull/327 +-- https://github.com/cowprotocol/solver-rewards/pull/330 -- Query Here: https://dune.com/queries/3333368 with block_range_start as ( From 06403263492137309e513e215b2a03a65d733e98 Mon Sep 17 00:00:00 2001 From: harisang Date: Tue, 9 Jan 2024 11:37:19 +0200 Subject: [PATCH 5/5] incorporated reviewer suggestions --- queries/dune_v2/period_block_interval.sql | 25 ++++------------------ queries/dune_v2/period_slippage.sql | 26 +++++------------------ 2 files changed, 9 insertions(+), 42 deletions(-) diff --git a/queries/dune_v2/period_block_interval.sql b/queries/dune_v2/period_block_interval.sql index e6188d20..c2a10893 100644 --- a/queries/dune_v2/period_block_interval.sql +++ b/queries/dune_v2/period_block_interval.sql @@ -1,24 +1,7 @@ -- https://github.com/cowprotocol/solver-rewards/pull/330 -- V3: https://dune.com/queries/3333356 -with - block_range_start as ( - select - max("number") + 1 as start_block - from - ethereum.blocks - where - time <= cast('{{StartTime}}' as timestamp) - ), - block_range_stop as ( - select - max("number") as end_block - from - ethereum.blocks - where - time <= cast('{{EndTime}}' as timestamp) - ) select - * -from - block_range_start - cross join block_range_stop \ No newline at end of file + min("number") as start_block, + max("number") as end_block +from ethereum.blocks +where time >= cast('{{StartTime}}' as timestamp) and time < cast('{{EndTime}}' as timestamp) \ No newline at end of file diff --git a/queries/dune_v2/period_slippage.sql b/queries/dune_v2/period_slippage.sql index 561ffff9..6aedce19 100644 --- a/queries/dune_v2/period_slippage.sql +++ b/queries/dune_v2/period_slippage.sql @@ -1,28 +1,12 @@ -- https://github.com/cowprotocol/solver-rewards/pull/330 -- Query Here: https://dune.com/queries/3333368 with -block_range_start as ( - select - max("number") + 1 as start_block - from - ethereum.blocks -where - time <= cast('{{StartTime}}' as timestamp) -) -,block_range_stop as ( +block_range as ( select - max("number") as end_block - from - ethereum.blocks - where - time <= cast('{{EndTime}}' as timestamp) - ) -,block_range as ( - select - * - from - block_range_start - cross join block_range_stop + min("number") as start_block, + max("number") as end_block + from ethereum.blocks + where time >= cast('{{StartTime}}' as timestamp) and time < cast('{{EndTime}}' as timestamp) ) ,batch_meta as ( select b.block_time,