From a02ff4d416b9bc9a5e37510cfd6603fc3da983d9 Mon Sep 17 00:00:00 2001 From: viniabussafi Date: Wed, 18 Dec 2024 10:01:56 +0000 Subject: [PATCH 1/9] enrich balancer_v3 trades --- .../models/enrich_balancer_v3_dex_trades.sql | 123 ++++++++++++++++++ .../dex/models/trades/dex_trades.sql | 12 +- 2 files changed, 134 insertions(+), 1 deletion(-) create mode 100644 dbt_subprojects/dex/macros/models/enrich_balancer_v3_dex_trades.sql diff --git a/dbt_subprojects/dex/macros/models/enrich_balancer_v3_dex_trades.sql b/dbt_subprojects/dex/macros/models/enrich_balancer_v3_dex_trades.sql new file mode 100644 index 00000000000..9fc66059e09 --- /dev/null +++ b/dbt_subprojects/dex/macros/models/enrich_balancer_v3_dex_trades.sql @@ -0,0 +1,123 @@ +{% macro enrich_balancer_v3_dex_trades( + base_trades = null + , filter = null + , tokens_erc20_model = null + ) +%} + +WITH base_trades as ( + SELECT + * + FROM + {{ base_trades }} + WHERE + {{ filter }} + {% if is_incremental() %} + AND + {{ incremental_predicate('block_time') }} + {% endif %} +) +, enrichments AS ( + SELECT + base_trades.blockchain + , base_trades.project + , base_trades.version + , base_trades.block_month + , base_trades.block_date + , base_trades.block_time + , base_trades.block_number + , erc20_bought.symbol AS token_bought_symbol + , erc20_sold.symbol AS token_sold_symbol + , CASE + WHEN lower(erc20_bought.symbol) > lower(erc20_sold.symbol) THEN concat(erc20_sold.symbol, '-', erc20_bought.symbol) + ELSE concat(erc20_bought.symbol, '-', erc20_sold.symbol) + END AS token_pair + , base_trades.token_bought_amount_raw / power(10, erc20_bought.decimals) AS token_bought_amount + , base_trades.token_sold_amount_raw / power(10, erc20_sold.decimals) AS token_sold_amount + , base_trades.token_bought_amount_raw + , base_trades.token_sold_amount_raw + , base_trades.token_bought_address + , base_trades.token_sold_address + , coalesce(base_trades.taker, base_trades.tx_from) AS taker + , base_trades.maker + , base_trades.project_contract_address + , base_trades.tx_hash + , base_trades.tx_from + , base_trades.tx_to + , base_trades.evt_index + FROM + base_trades + LEFT JOIN + {{ tokens_erc20_model }} as erc20_bought + ON erc20_bought.contract_address = base_trades.token_bought_address + AND erc20_bought.blockchain = base_trades.blockchain + LEFT JOIN + {{ tokens_erc20_model }} as erc20_sold + ON erc20_sold.contract_address = base_trades.token_sold_address + AND erc20_sold.blockchain = base_trades.blockchain +) + +, enrichments_with_prices AS ( + {{ + add_amount_usd( + trades_cte = 'enrichments' + ) + }} +) + +, erc4626_prices AS( + SELECT + minute, + blockchain, + wrapped_token, + decimals, + APPROX_PERCENTILE(median_price, 0.5) AS price, + LEAD(minute, 1, NOW()) OVER (PARTITION BY wrapped_token ORDER BY minute) AS time_of_next_change + FROM {{ source('balancer_v3', 'erc4626_token_prices') }} + WHERE blockchain = 'ethereum' + GROUP BY 1, 2, 3 +) + +SELECT + blockchain + , project + , version + , block_month + , block_date + , block_time + , block_number + , token_bought_symbol + , token_sold_symbol + , token_pair + , token_bought_amount + , token_sold_amount + , token_bought_amount_raw + , token_sold_amount_raw + , COALESCE( + dexs.amount_usd, + dexs.token_bought_amount * erc4626a.price, + dexs.token_sold_amount * erc4626a.price + ) AS amount_usd + , token_bought_address + , token_sold_address + , taker + , maker + , project_contract_address + , tx_hash + , tx_from + , tx_to + , evt_index +FROM + enrichments_with_prices dexs +LEFT JOIN erc4626_prices erc4626a + ON erc4626a.wrapped_token = dexs.token_bought_address + AND erc4626a.minute <= dexs.block_time + AND dexs.block_time < erc4626a.time_of_next_change + AND dexs.blockchain = erc4626a.blockchain +LEFT JOIN erc4626_prices erc4626b + ON erc4626b.wrapped_token = dexs.token_sold_address + AND erc4626b.minute <= dexs.block_time + AND dexs.block_time < erc4626b.time_of_next_change + AND dexs.blockchain = erc4626b.blockchain + +{% endmacro %} \ No newline at end of file diff --git a/dbt_subprojects/dex/models/trades/dex_trades.sql b/dbt_subprojects/dex/models/trades/dex_trades.sql index 654e2eb5e4a..a91a64b71c9 100644 --- a/dbt_subprojects/dex/models/trades/dex_trades.sql +++ b/dbt_subprojects/dex/models/trades/dex_trades.sql @@ -53,11 +53,21 @@ WITH curve AS ( ) }} ) +,balancer_v3 AS ( + -- due to Balancer V3 having trades between ERC4626 tokens, which won't be priced on prices.usd, enrich separately + {{ + enrich_balancer_v3_dex_trades( + base_trades = ref('dex_base_trades') + , filter = "project = 'balancer' AND version = '3'" + , tokens_erc20_model = source('tokens', 'erc20') + ) + }} +) , dexs AS ( {{ enrich_dex_trades( base_trades = ref('dex_base_trades') - , filter = "project != 'curve'" + , filter = "project != 'curve' AND (project != 'balancer AND version != '3'" , tokens_erc20_model = source('tokens', 'erc20') ) }} From 4f118f3115954cfad4bb93abd93dcf5908d89387 Mon Sep 17 00:00:00 2001 From: viniabussafi Date: Wed, 18 Dec 2024 10:10:20 +0000 Subject: [PATCH 2/9] add missing ) --- dbt_subprojects/dex/models/trades/dex_trades.sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dbt_subprojects/dex/models/trades/dex_trades.sql b/dbt_subprojects/dex/models/trades/dex_trades.sql index a91a64b71c9..6348cd9d27f 100644 --- a/dbt_subprojects/dex/models/trades/dex_trades.sql +++ b/dbt_subprojects/dex/models/trades/dex_trades.sql @@ -67,7 +67,7 @@ WITH curve AS ( {{ enrich_dex_trades( base_trades = ref('dex_base_trades') - , filter = "project != 'curve' AND (project != 'balancer AND version != '3'" + , filter = "project != 'curve' AND (project != 'balancer AND version != '3')" , tokens_erc20_model = source('tokens', 'erc20') ) }} From f37ab11b7f3aece122075900531ed2ead83bc06c Mon Sep 17 00:00:00 2001 From: viniabussafi Date: Wed, 18 Dec 2024 10:18:00 +0000 Subject: [PATCH 3/9] add missing quote --- dbt_subprojects/dex/models/trades/dex_trades.sql | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/dbt_subprojects/dex/models/trades/dex_trades.sql b/dbt_subprojects/dex/models/trades/dex_trades.sql index 6348cd9d27f..754bc0cf006 100644 --- a/dbt_subprojects/dex/models/trades/dex_trades.sql +++ b/dbt_subprojects/dex/models/trades/dex_trades.sql @@ -53,7 +53,7 @@ WITH curve AS ( ) }} ) -,balancer_v3 AS ( +, balancer_v3 AS ( -- due to Balancer V3 having trades between ERC4626 tokens, which won't be priced on prices.usd, enrich separately {{ enrich_balancer_v3_dex_trades( @@ -67,7 +67,7 @@ WITH curve AS ( {{ enrich_dex_trades( base_trades = ref('dex_base_trades') - , filter = "project != 'curve' AND (project != 'balancer AND version != '3')" + , filter = "project != 'curve' AND (project != 'balancer' AND version != '3')" , tokens_erc20_model = source('tokens', 'erc20') ) }} From 10e6a74bb81920639e53770fc8ae90b31cd45e07 Mon Sep 17 00:00:00 2001 From: viniabussafi Date: Wed, 18 Dec 2024 10:21:37 +0000 Subject: [PATCH 4/9] fix group by --- .../dex/macros/models/enrich_balancer_v3_dex_trades.sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dbt_subprojects/dex/macros/models/enrich_balancer_v3_dex_trades.sql b/dbt_subprojects/dex/macros/models/enrich_balancer_v3_dex_trades.sql index 9fc66059e09..aeaf91f13a2 100644 --- a/dbt_subprojects/dex/macros/models/enrich_balancer_v3_dex_trades.sql +++ b/dbt_subprojects/dex/macros/models/enrich_balancer_v3_dex_trades.sql @@ -75,7 +75,7 @@ WITH base_trades as ( LEAD(minute, 1, NOW()) OVER (PARTITION BY wrapped_token ORDER BY minute) AS time_of_next_change FROM {{ source('balancer_v3', 'erc4626_token_prices') }} WHERE blockchain = 'ethereum' - GROUP BY 1, 2, 3 + GROUP BY 1, 2, 3, 4 ) SELECT From 5b0d787b453551cf893f482118c6041e57908775 Mon Sep 17 00:00:00 2001 From: viniabussafi Date: Wed, 18 Dec 2024 10:22:03 +0000 Subject: [PATCH 5/9] again --- .../dex/macros/models/enrich_balancer_v3_dex_trades.sql | 1 - 1 file changed, 1 deletion(-) diff --git a/dbt_subprojects/dex/macros/models/enrich_balancer_v3_dex_trades.sql b/dbt_subprojects/dex/macros/models/enrich_balancer_v3_dex_trades.sql index aeaf91f13a2..120a031c6ea 100644 --- a/dbt_subprojects/dex/macros/models/enrich_balancer_v3_dex_trades.sql +++ b/dbt_subprojects/dex/macros/models/enrich_balancer_v3_dex_trades.sql @@ -74,7 +74,6 @@ WITH base_trades as ( APPROX_PERCENTILE(median_price, 0.5) AS price, LEAD(minute, 1, NOW()) OVER (PARTITION BY wrapped_token ORDER BY minute) AS time_of_next_change FROM {{ source('balancer_v3', 'erc4626_token_prices') }} - WHERE blockchain = 'ethereum' GROUP BY 1, 2, 3, 4 ) From 745130d25b581112ca0ee371bca676c698e2ae70 Mon Sep 17 00:00:00 2001 From: viniabussafi Date: Wed, 18 Dec 2024 10:26:37 +0000 Subject: [PATCH 6/9] alias blockchain --- .../dex/macros/models/enrich_balancer_v3_dex_trades.sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dbt_subprojects/dex/macros/models/enrich_balancer_v3_dex_trades.sql b/dbt_subprojects/dex/macros/models/enrich_balancer_v3_dex_trades.sql index 120a031c6ea..094f4360d1a 100644 --- a/dbt_subprojects/dex/macros/models/enrich_balancer_v3_dex_trades.sql +++ b/dbt_subprojects/dex/macros/models/enrich_balancer_v3_dex_trades.sql @@ -78,7 +78,7 @@ WITH base_trades as ( ) SELECT - blockchain + dexs.blockchain , project , version , block_month From 155de369fb7634a51cecf6d677d806cdfad75b67 Mon Sep 17 00:00:00 2001 From: viniabussafi Date: Wed, 18 Dec 2024 10:46:07 +0000 Subject: [PATCH 7/9] add date filter to decrease run time --- dbt_subprojects/dex/models/trades/dex_trades.sql | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/dbt_subprojects/dex/models/trades/dex_trades.sql b/dbt_subprojects/dex/models/trades/dex_trades.sql index 754bc0cf006..b4bc6f60971 100644 --- a/dbt_subprojects/dex/models/trades/dex_trades.sql +++ b/dbt_subprojects/dex/models/trades/dex_trades.sql @@ -46,7 +46,7 @@ WITH curve AS ( {{ enrich_curve_dex_trades( base_trades = ref('dex_base_trades') - , filter = "project = 'curve'" + , filter = "project = 'curve' AND block_date = TIMESTAMP '2024-12-17'" , curve_ethereum = ref('curve_ethereum_base_trades') , curve_optimism = ref('curve_optimism_base_trades') , tokens_erc20_model = source('tokens', 'erc20') @@ -58,7 +58,7 @@ WITH curve AS ( {{ enrich_balancer_v3_dex_trades( base_trades = ref('dex_base_trades') - , filter = "project = 'balancer' AND version = '3'" + , filter = "project = 'balancer' AND version = '3' AND block_date = TIMESTAMP '2024-12-17'" , tokens_erc20_model = source('tokens', 'erc20') ) }} @@ -67,7 +67,7 @@ WITH curve AS ( {{ enrich_dex_trades( base_trades = ref('dex_base_trades') - , filter = "project != 'curve' AND (project != 'balancer' AND version != '3')" + , filter = "project != 'curve' AND (project != 'balancer' AND version != '3') AND block_date = TIMESTAMP '2024-12-17'" , tokens_erc20_model = source('tokens', 'erc20') ) }} From 3e0307779cff758f6dc81dfad4b65c12c494f180 Mon Sep 17 00:00:00 2001 From: viniabussafi Date: Wed, 18 Dec 2024 10:59:49 +0000 Subject: [PATCH 8/9] add balancer trades to models --- dbt_subprojects/dex/models/trades/dex_trades.sql | 1 + 1 file changed, 1 insertion(+) diff --git a/dbt_subprojects/dex/models/trades/dex_trades.sql b/dbt_subprojects/dex/models/trades/dex_trades.sql index b4bc6f60971..717d4d36fdc 100644 --- a/dbt_subprojects/dex/models/trades/dex_trades.sql +++ b/dbt_subprojects/dex/models/trades/dex_trades.sql @@ -116,6 +116,7 @@ WITH curve AS ( 'curve' , 'as_is_dexs' , 'dexs' + , 'balancer_v3' ] %} From cdc1d68aac56cbd81122596c638acc6b43d6fa93 Mon Sep 17 00:00:00 2001 From: viniabussafi Date: Wed, 18 Dec 2024 11:18:17 +0000 Subject: [PATCH 9/9] include bv2 --- dbt_subprojects/dex/models/trades/_schema.yml | 2 +- dbt_subprojects/dex/models/trades/dex_trades.sql | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/dbt_subprojects/dex/models/trades/_schema.yml b/dbt_subprojects/dex/models/trades/_schema.yml index c913109f5c5..cb10cfb6b1d 100644 --- a/dbt_subprojects/dex/models/trades/_schema.yml +++ b/dbt_subprojects/dex/models/trades/_schema.yml @@ -7,7 +7,7 @@ models: blockchain: arbitrum, avalanche_c, base, bnb, celo, ethereum, fantom, gnosis, kaia, optimism, polygon, scroll, zksync, linea, blast, sei, ronin sector: dex short_description: The `dex.trades` table captures detailed data on trades executed via decentralized exchanges (DEXs). This table contains a detailed breakdown of trade execution containing one or many trades per transaction. - contributors: 0xRob, hosuke, jeff-dude, tomfutago + contributors: 0xRob, hosuke, jeff-dude, tomfutago, viniabussafi config: tags: [ 'dex', 'trades'] description: '{{ doc("dex_trades_doc") }}' diff --git a/dbt_subprojects/dex/models/trades/dex_trades.sql b/dbt_subprojects/dex/models/trades/dex_trades.sql index 717d4d36fdc..ca7c8289878 100644 --- a/dbt_subprojects/dex/models/trades/dex_trades.sql +++ b/dbt_subprojects/dex/models/trades/dex_trades.sql @@ -32,7 +32,7 @@ ]\', "sector", "dex", - \'["hosuke", "0xrob", "jeff-dude", "tomfutago"]\') }}') + \'["hosuke", "0xrob", "jeff-dude", "tomfutago", "viniabussafi"]\') }}') }} -- keep existing dbt lineages for the following projects, as the team built themselves and use the spells throughout the entire lineage @@ -58,7 +58,7 @@ WITH curve AS ( {{ enrich_balancer_v3_dex_trades( base_trades = ref('dex_base_trades') - , filter = "project = 'balancer' AND version = '3' AND block_date = TIMESTAMP '2024-12-17'" + , filter = "(project = 'balancer' AND version = '3') AND block_date = TIMESTAMP '2024-12-17'" , tokens_erc20_model = source('tokens', 'erc20') ) }} @@ -67,7 +67,7 @@ WITH curve AS ( {{ enrich_dex_trades( base_trades = ref('dex_base_trades') - , filter = "project != 'curve' AND (project != 'balancer' AND version != '3') AND block_date = TIMESTAMP '2024-12-17'" + , filter = "project != 'curve' AND NOT (project = 'balancer' AND version = '3') AND block_date = TIMESTAMP '2024-12-17'" , tokens_erc20_model = source('tokens', 'erc20') ) }}