From 5926f1cde6d9b296d39da1cbfbc835d84e4bb744 Mon Sep 17 00:00:00 2001 From: henrystats <65398020+henrystats@users.noreply.github.com> Date: Tue, 22 Aug 2023 15:36:08 +0100 Subject: [PATCH] Dex Aggregator Feat: Lifi Optimism - Ready for review (#4082) * Dex Aggregator Migration - #1 - Lifi * push dex aggregator seed * revert changes to legacy file * test fix * test * test * smol fix * Dex Aggregator Feat: Lifi Optimism * CLEAN * adjust seed --- dbt_project.yml | 2 + models/lifi/lifi_trades.sql | 3 +- models/lifi/lifi_trades_schema.yml | 4 +- models/lifi/optimism/lifi_optimism_schema.yml | 136 ++++++++++++++++++ .../lifi/optimism/lifi_optimism_sources.yml | 13 ++ models/lifi/optimism/lifi_optimism_trades.sql | 46 ++++++ .../optimism/lifi_optimism_trades_legacy.sql | 10 ++ .../lifi/optimism/lifi_v2_optimism_trades.sql | 123 ++++++++++++++++ .../lifi_v2_optimism_trades_legacy.sql | 10 ++ seeds/dex/aggregator/dex_aggregator_seed.csv | 4 +- 10 files changed, 347 insertions(+), 4 deletions(-) create mode 100644 models/lifi/optimism/lifi_optimism_schema.yml create mode 100644 models/lifi/optimism/lifi_optimism_sources.yml create mode 100644 models/lifi/optimism/lifi_optimism_trades.sql create mode 100644 models/lifi/optimism/lifi_optimism_trades_legacy.sql create mode 100644 models/lifi/optimism/lifi_v2_optimism_trades.sql create mode 100644 models/lifi/optimism/lifi_v2_optimism_trades_legacy.sql diff --git a/dbt_project.yml b/dbt_project.yml index 80f9b932836..b5ddd523155 100644 --- a/dbt_project.yml +++ b/dbt_project.yml @@ -655,6 +655,8 @@ models: +schema: lifi fantom: +schema: lifi_fantom + optimism: + +schema: lifi_optimism astaria: +schema: astaria diff --git a/models/lifi/lifi_trades.sql b/models/lifi/lifi_trades.sql index 81c81db0ff6..df7f6d2e55a 100644 --- a/models/lifi/lifi_trades.sql +++ b/models/lifi/lifi_trades.sql @@ -1,7 +1,7 @@ {{ config( tags=['dunesql'], alias = alias('trades'), - post_hook='{{ expose_spells(\'["fantom"]\', + post_hook='{{ expose_spells(\'["fantom", "optimism"]\', "project", "lifi", \'["Henrystats"]\') }}' @@ -10,6 +10,7 @@ {% set lifi_models = [ ref('lifi_fantom_trades') +,ref('lifi_optimism_trades') ] %} diff --git a/models/lifi/lifi_trades_schema.yml b/models/lifi/lifi_trades_schema.yml index 14b8b44f1e3..c4c22570073 100644 --- a/models/lifi/lifi_trades_schema.yml +++ b/models/lifi/lifi_trades_schema.yml @@ -3,12 +3,12 @@ version: 2 models: - name: lifi_trades meta: - blockchain: fantom + blockchain: fantom, optimism sector: dex project: lifi contributors: Henrystats config: - tags: ['fantom', 'lifi','trades', 'dex', 'aggregator', 'Henrystats','cross-chain'] + tags: ['fantom', 'optimism', 'lifi','trades', 'dex', 'aggregator', 'Henrystats','cross-chain'] description: > lifi aggregator trades on all chains across all contracts and versions. This table will load dex_aggregator trades downstream. columns: diff --git a/models/lifi/optimism/lifi_optimism_schema.yml b/models/lifi/optimism/lifi_optimism_schema.yml new file mode 100644 index 00000000000..0fb8ef06ae3 --- /dev/null +++ b/models/lifi/optimism/lifi_optimism_schema.yml @@ -0,0 +1,136 @@ +version: 2 + +models: + - name: lifi_v2_optimism_trades + meta: + blockchain: optimism + sector: dex + project: lifi_v2 + contributors: Henrystats + config: + tags: ['optimism','lifi_v2','trades', 'lifi','dex'] + description: > + lifi v2 contract aggregator trades on optimism + tests: + - dbt_utils.unique_combination_of_columns: + combination_of_columns: + - block_date + - blockchain + - project + - version + - tx_hash + - evt_index + - trace_address + - check_dex_aggregator_seed: + blockchain: optimism + project: lifi + version: 2 + columns: + - &blockchain + name: blockchain + description: "Blockchain which the DEX is deployed" + - &project + name: project + description: "Project name of the DEX" + - &version + name: version + description: "Version of the contract built and deployed by the DEX project" + - &block_date + name: block_date + description: "UTC event block date of each DEX trade" + - &block_time + name: block_time + description: "UTC event block time of each DEX trade" + - &token_bought_symbol + name: token_bought_symbol + description: "Token symbol for token bought in the transaction" + - &token_sold_symbol + name: token_sold_symbol + description: "Token symbol for token sold in the transaction" + - &token_pair + name: token_pair + description: "Token symbol pair for each token involved in the transaction" + - &token_bought_amount + name: token_bought_amount + description: "Value of the token bought at time of execution in the original currency" + - &token_sold_amount + name: token_sold_amount + description: "Value of the token sold at time of execution in the original currency" + - &token_bought_amount_raw + name: token_bought_amount_raw + description: "Raw value of the token bought at time of execution in the original currency" + - &token_sold_amount_raw + name: token_sold_amount_raw + description: "Raw value of the token sold at time of execution in the original currency" + - &amount_usd + name: amount_usd + description: "USD value of the trade at time of execution" + - &token_bought_address + name: token_bought_address + description: "Contract address of the token bought" + - &token_sold_address + name: token_sold_address + description: "Contract address of the token sold" + - &taker + name: taker + description: "Address of trader who purchased a token" + - &maker + name: maker + description: "Address of trader who sold a token" + - &project_contract_address + name: project_contract_address + description: "Project contract address which executed the trade on the blockchain" + - &tx_hash + name: tx_hash + description: "Unique transaction hash value tied to each transaction on the DEX" + - &tx_from + name: tx_from + description: "Address which initiated the transaction" + - &tx_to + name: tx_to + description: "Address which received the transaction" + - &trace_address + name: trace_address + description: "" + - &evt_index + name: evt_index + description: "" + - &block_month + name: block_month + description: "UTC event block month of each DEX trade" + + - name: lifi_optimism_trades + meta: + blockchain: optimism + sector: dex + project: lifi + contributors: Henrystats + config: + tags: ['optimism','dex','trades', 'lifi'] + description: > + lifi aggregator trades on optimism across all contracts and versions. This table will load dex trades downstream. + columns: + - *blockchain + - *project + - *version + - *block_date + - *block_time + - *token_bought_symbol + - *token_sold_symbol + - *token_pair + - *token_bought_amount + - *token_sold_amount + - *token_bought_amount_raw + - *token_sold_amount_raw + - *amount_usd + - *token_bought_address + - *token_sold_address + - *taker + - *maker + - *project_contract_address + - *tx_hash + - *tx_from + - *tx_to + - *trace_address + - *evt_index + - *block_month diff --git a/models/lifi/optimism/lifi_optimism_sources.yml b/models/lifi/optimism/lifi_optimism_sources.yml new file mode 100644 index 00000000000..e86c3fba4dd --- /dev/null +++ b/models/lifi/optimism/lifi_optimism_sources.yml @@ -0,0 +1,13 @@ +version: 2 + +sources: + - name: lifi_optimism + freshness: + warn_after: { count: 12, period: hour } + description: > + Decoded event table for swaps on lifi + tables: + - name: LiFiDiamond_v2_evt_AssetSwapped + loaded_at_field: evt_block_time + - name: LiFiDiamond_v2_evt_LiFiSwappedGeneric + loaded_at_field: evt_block_time \ No newline at end of file diff --git a/models/lifi/optimism/lifi_optimism_trades.sql b/models/lifi/optimism/lifi_optimism_trades.sql new file mode 100644 index 00000000000..0f94e115a22 --- /dev/null +++ b/models/lifi/optimism/lifi_optimism_trades.sql @@ -0,0 +1,46 @@ + +{{ config( + tags=['dunesql'], + alias = alias('trades') + ) +}} + +{% set lifi_models = [ +ref('lifi_v2_optimism_trades') +] %} + + +SELECT * +FROM ( + {% for dex_model in lifi_models %} + SELECT + blockchain, + project, + version, + block_date, + block_month, + block_time, + token_bought_symbol, + token_sold_symbol, + token_pair, + token_bought_amount, + token_sold_amount, + token_bought_amount_raw, + token_sold_amount_raw, + amount_usd, + token_bought_address, + token_sold_address, + taker, + maker, + project_contract_address, + tx_hash, + tx_from, + tx_to, + trace_address, --ensure field is explicitly cast as array in base models + evt_index + FROM {{ dex_model }} + {% if not loop.last %} + UNION ALL + {% endif %} + {% endfor %} +) diff --git a/models/lifi/optimism/lifi_optimism_trades_legacy.sql b/models/lifi/optimism/lifi_optimism_trades_legacy.sql new file mode 100644 index 00000000000..94ca7c63e9d --- /dev/null +++ b/models/lifi/optimism/lifi_optimism_trades_legacy.sql @@ -0,0 +1,10 @@ + +{{ config( + tags=['legacy'], + + alias = alias('trades', legacy_model=True) + ) +}} + +SELECT + 1 as dummy \ No newline at end of file diff --git a/models/lifi/optimism/lifi_v2_optimism_trades.sql b/models/lifi/optimism/lifi_v2_optimism_trades.sql new file mode 100644 index 00000000000..0d634a71aa0 --- /dev/null +++ b/models/lifi/optimism/lifi_v2_optimism_trades.sql @@ -0,0 +1,123 @@ +{{ config( + tags=['dunesql'], + schema = 'lifi_v2_optimism', + alias = alias('trades'), + partition_by = ['block_month'], + materialized = 'incremental', + file_format = 'delta', + incremental_strategy = 'merge', + unique_key = ['block_date', 'blockchain', 'project', 'version', 'tx_hash', 'evt_index', 'trace_address'], + post_hook='{{ expose_spells(\'["optimism"]\', + "project", + "lifi_v2", + \'["Henrystats"]\') }}' + ) +}} + +{% set project_start_date = '2022-10-27' %} -- min(evet_block_time) in swapped & swapped generic events + +WITH + +{% set trade_event_tables = [ + source('lifi_optimism', 'LiFiDiamond_v2_evt_AssetSwapped') + ,source('lifi_optimism', 'LiFiDiamond_v2_evt_LiFiSwappedGeneric') +] %} + +dexs as ( + {% for trade_tables in trade_event_tables %} + SELECT + evt_block_time as block_time, + CAST(NULL as VARBINARY) as maker, + toAmount as token_bought_amount_raw, + fromAmount as token_sold_amount_raw, + CAST(NULL as double) as amount_usd, + CASE + WHEN toAssetId IN (0x, 0x0000000000000000000000000000000000000000) + THEN 0x4200000000000000000000000000000000000006 -- weth + ELSE toAssetId + END as token_bought_address, + CASE + WHEN fromAssetId IN (0x, 0x0000000000000000000000000000000000000000) + THEN 0x4200000000000000000000000000000000000006 -- weth + ELSE fromAssetId + END as token_sold_address, + contract_address as project_contract_address, + evt_tx_hash as tx_hash, + ARRAY[-1] AS trace_address, + evt_index + FROM {{ trade_tables }} p + {% if is_incremental() %} + WHERE p.evt_block_time >= date_trunc('day', now() - interval '7' Day) + {% endif %} + {% if not loop.last %} + UNION ALL + {% endif %} + {% endfor %} +) +SELECT + 'optimism' as blockchain, + 'lifi' as project, + '2' as version, + cast(date_trunc('DAY', dexs.block_time) as date) as block_date, + cast(date_trunc('MONTH', dexs.block_time) as date) as block_month, + dexs.block_time, + erc20a.symbol as token_bought_symbol, + erc20b.symbol as token_sold_symbol, + case + when lower(erc20a.symbol) > lower(erc20b.symbol) then concat(erc20b.symbol, '-', erc20a.symbol) + else concat(erc20a.symbol, '-', erc20b.symbol) + end as token_pair, + dexs.token_bought_amount_raw / power(10, erc20a.decimals) AS token_bought_amount, + dexs.token_sold_amount_raw / power(10, erc20b.decimals) AS token_sold_amount, + dexs.token_bought_amount_raw, + dexs.token_sold_amount_raw, + coalesce( + dexs.amount_usd + ,(dexs.token_bought_amount_raw / power(10, p_bought.decimals)) * p_bought.price + ,(dexs.token_sold_amount_raw / power(10, p_sold.decimals)) * p_sold.price + ) AS amount_usd, + dexs.token_bought_address, + dexs.token_sold_address, + tx."from" AS taker, -- no taker in swap event + dexs.maker, + dexs.project_contract_address, + dexs.tx_hash, + tx."from" AS tx_from, + tx.to AS tx_to, + dexs.trace_address, + dexs.evt_index +from dexs +inner join {{ source('optimism', 'transactions') }} tx + on dexs.tx_hash = tx.hash + {% if not is_incremental() %} + and tx.block_time >= TIMESTAMP '{{project_start_date}}' + {% endif %} + {% if is_incremental() %} + and tx.block_time >= date_trunc('day', now() - interval '7' Day) + {% endif %} +left join {{ ref('tokens_erc20') }} erc20a + on erc20a.contract_address = dexs.token_bought_address + and erc20a.blockchain = 'optimism' +left join {{ ref('tokens_erc20') }} erc20b + on erc20b.contract_address = dexs.token_sold_address + and erc20b.blockchain = 'optimism' +left join {{ source('prices', 'usd') }} p_bought + on p_bought.minute = date_trunc('minute', dexs.block_time) + and p_bought.contract_address = dexs.token_bought_address + and p_bought.blockchain = 'optimism' + {% if not is_incremental() %} + and p_bought.minute >= TIMESTAMP '{{project_start_date}}' + {% endif %} + {% if is_incremental() %} + and p_bought.minute >= date_trunc('day', now() - interval '7' Day) + {% endif %} +left join {{ source('prices', 'usd') }} p_sold + on p_sold.minute = date_trunc('minute', dexs.block_time) + and p_sold.contract_address = dexs.token_sold_address + and p_sold.blockchain = 'optimism' + {% if not is_incremental() %} + and p_sold.minute >= TIMESTAMP '{{project_start_date}}' + {% endif %} + {% if is_incremental() %} + and p_sold.minute >= date_trunc('day', now() - interval '7' Day) + {% endif %} \ No newline at end of file diff --git a/models/lifi/optimism/lifi_v2_optimism_trades_legacy.sql b/models/lifi/optimism/lifi_v2_optimism_trades_legacy.sql new file mode 100644 index 00000000000..ff928ffb8c4 --- /dev/null +++ b/models/lifi/optimism/lifi_v2_optimism_trades_legacy.sql @@ -0,0 +1,10 @@ +{{ config( + tags=['legacy'], + + schema = 'lifi_v2_optimism', + alias = alias('trades', legacy_model=True) + ) +}} + +SELECT +1 as dummy \ No newline at end of file diff --git a/seeds/dex/aggregator/dex_aggregator_seed.csv b/seeds/dex/aggregator/dex_aggregator_seed.csv index 278f57ec0c0..f3e28145042 100644 --- a/seeds/dex/aggregator/dex_aggregator_seed.csv +++ b/seeds/dex/aggregator/dex_aggregator_seed.csv @@ -105,4 +105,6 @@ optimism,DODO,0,2022-06-02,0x4326310124907a6c4c53fc8778d8bdcd30c3f06fa46edd72125 ethereum,tokenlon,5,2021-06-25,0xede75bd74812fc682e62f03190abe22d2fdb11e1c3cf6cc3e29b1baea515f43b,304,,0xdac17f958d2ee523a2206206994597c13d831ec7,27.967009,0xe4815ae53b124e7263f08dcdbbb757d41ed658c6,48 ethereum,tokenlon,5,2022-12-20,0xb68614d5fdcefcb9ea869dfe0a1381f115d3f8077bf82ab6517ebbb0f4772588,389,,0xdac17f958d2ee523a2206206994597c13d831ec7,47.083995,0xc02aaa39b223fe8d0a0e5c4f27ead9083c756cc2,0.039 ethereum,tokenlon,5,2021-05-24,0xbe1eda9d0666fd82196e45134909e55a504c6001a43ebc5eadfd55f9dbf8fc1f,101,,0xdac17f958d2ee523a2206206994597c13d831ec7,12905.054001,0xc02aaa39b223fe8d0a0e5c4f27ead9083c756cc2,5 -ethereum,tokenlon,5,2022-10-18,0xc168eb7ad27401aeea77e7b1c7f4dc77f8ee2453de1baec2cd3dea32c9e09c3c,384,,0xdac17f958d2ee523a2206206994597c13d831ec7,9806.7654,0xc02aaa39b223fe8d0a0e5c4f27ead9083c756cc2,7.47 \ No newline at end of file +ethereum,tokenlon,5,2022-10-18,0xc168eb7ad27401aeea77e7b1c7f4dc77f8ee2453de1baec2cd3dea32c9e09c3c,384,,0xdac17f958d2ee523a2206206994597c13d831ec7,9806.7654,0xc02aaa39b223fe8d0a0e5c4f27ead9083c756cc2,7.47 +optimism,lifi,2,2022-11-18,0xf97908921fddb1fdbaecc62e25a405ecca6f9acf39c7d2e0daf35b2e799c4797,19,-1,0x7f5c764cbc14f9669b88837ca1490cca17c31607,7.301198,0x4200000000000000000000000000000000000006,0.006 +optimism,lifi,2,2023-08-15,0x23fb004c92fad2fd7003093e8f6c6a0b3eb3830b55e736eaa4cbe044f9fea8d5,64,-1,0x94b008aa00579c1307b0ef2c499ad98a8ce58e58,931.396156,0x4200000000000000000000000000000000000006,0.505