From dce2fa17b006db8e32cdc8be36f0a8091b94db7b Mon Sep 17 00:00:00 2001 From: jeff-dude <102681548+jeff-dude@users.noreply.github.com> Date: Mon, 25 Nov 2024 12:45:35 -0500 Subject: [PATCH 01/18] revert 0x enhancements: v1 fills and deduped (#7186) * Revert "zeroex - macro for v1 fills (#7161)" This reverts commit 91241e616f5d77c7952d4723f3bebc583ffbfdea. * Revert "zeroex - macro for deduped fills (#7156)" This reverts commit 30b95c51798cd7c8da243cfad67e0f703449ae33. --- .../_project/zeroex/zeroex_v1_api_fills.sql | 577 ------------------ .../zeroex/base/zeroex_base_api_fills.sql | 251 ++++++-- .../base/zeroex_base_api_fills_deduped.sql | 102 ++++ .../zeroex/base/zeroex_base_schema.yml | 2 +- .../base/zeroex_v1_base_deduped_trades.sql | 30 - .../ethereum/zeroex_ethereum_api_fills.sql | 570 +++++++++++++---- .../zeroex_ethereum_api_fills_deduped.sql} | 30 +- .../ethereum/zeroex_ethereum_schema.yml | 6 +- .../zeroex_v1_ethereum_deduped_trades.sql | 30 - .../zeroex/zeroex_api_fills_deduped.sql | 4 +- 10 files changed, 800 insertions(+), 802 deletions(-) delete mode 100644 dbt_subprojects/dex/macros/models/_project/zeroex/zeroex_v1_api_fills.sql create mode 100644 dbt_subprojects/dex/models/_projects/zeroex/base/zeroex_base_api_fills_deduped.sql delete mode 100644 dbt_subprojects/dex/models/_projects/zeroex/base/zeroex_v1_base_deduped_trades.sql rename dbt_subprojects/dex/{macros/models/_project/zeroex/zeroex_v1_deduped_trades.sql => models/_projects/zeroex/ethereum/zeroex_ethereum_api_fills_deduped.sql} (80%) delete mode 100644 dbt_subprojects/dex/models/_projects/zeroex/ethereum/zeroex_v1_ethereum_deduped_trades.sql diff --git a/dbt_subprojects/dex/macros/models/_project/zeroex/zeroex_v1_api_fills.sql b/dbt_subprojects/dex/macros/models/_project/zeroex/zeroex_v1_api_fills.sql deleted file mode 100644 index 662d3bbd4cb..00000000000 --- a/dbt_subprojects/dex/macros/models/_project/zeroex/zeroex_v1_api_fills.sql +++ /dev/null @@ -1,577 +0,0 @@ -{% macro zeroex_evt_fills_txs(blockchain,zeroex_v3_start_date) %} -{%- set table_prefix = 'zeroex_v3_' + blockchain -%} - -SELECT - v3.evt_tx_hash AS tx_hash, - CASE - WHEN takerAddress = 0x63305728359c088a52b0b0eeec235db4d31a67fc THEN takerAddress - ELSE NULL - END AS affiliate_address, - NULL AS is_gasless, - evt_block_time AS block_time -FROM {{ source(table_prefix, 'Exchange_evt_Fill') }} v3 -WHERE ( -- nuo - v3.takerAddress = 0x63305728359c088a52b0b0eeec235db4d31a67fc - OR -- contains a bridge order - ( - v3.feeRecipientAddress = 0x1000000000000000000000000000000000000011 - AND bytearray_substring(v3.makerAssetData, 1, 4) = 0xdc1600f3 - ) -) -{% if is_incremental() %} -AND {{ incremental_predicate('evt_block_time') }} -{% else %} -AND evt_block_time >= cast('{{zeroex_v3_start_date}}' as date) -{% endif %} -{% endmacro %} - - -{% macro zeroex_v1_txs(blockchain,zeroex_v3_start_date) %} -SELECT - tr.tx_hash, - CASE - WHEN bytearray_position(INPUT, 0x869584cd) <> 0 THEN - SUBSTRING(INPUT FROM (bytearray_position(INPUT, 0x869584cd) + 16) FOR 20) - WHEN bytearray_position(INPUT, 0xfbc019a7) <> 0 THEN - SUBSTRING(INPUT FROM (bytearray_position(INPUT, 0xfbc019a7) + 16) FOR 20) - END AS affiliate_address, - CASE - WHEN (varbinary_position(input, 0x3d8d4082) <> 0 OR varbinary_position(input, 0x4f948110) <> 0) - THEN 1 - END AS is_gasless, - block_time -FROM {{ source(blockchain, 'traces') }} tr -WHERE tr.to IN ( - 0x61935cbdd02287b511119ddb11aeb42f1593b7ef, -- exchange contract - 0x6958f5e95332d93d21af0d7b9ca85b8212fee0a5, -- forwarder address - 0x4aa817c6f383c8e8ae77301d18ce48efb16fd2be, -- forwarder address - 0x4ef40d1bf0983899892946830abf99eca2dbc5ce, -- forwarder address - 0xdef1c0ded9bec7f1a1670819833240f027b25eff -- exchange proxy -) -AND ( - bytearray_position(INPUT, 0x869584cd) <> 0 - OR bytearray_position(INPUT, 0xfbc019a7) <> 0 -) -{% if is_incremental() %} -AND {{ incremental_predicate('block_time') }} -{% else %} -AND block_time >= cast('{{zeroex_v3_start_date}}' as date) -{% endif %} -{% endmacro %} - - -{% macro v3_fills_no_bridge(blockchain,zeroex_v3_start_date) %} -{%- set table_prefix = 'zeroex_v3_' + blockchain -%} - -SELECT - fills.evt_tx_hash AS tx_hash, - fills.evt_index, - fills.contract_address, - evt_block_time AS block_time, - fills.makerAddress AS maker, - fills.takerAddress AS taker, - bytearray_substring(fills.takerAssetData, 17, 20) AS taker_token, - bytearray_substring(fills.makerAssetData, 17, 20) AS maker_token, - CAST(fills.takerAssetFilledAmount AS int256) AS taker_token_amount_raw, - CAST(fills.makerAssetFilledAmount AS int256) AS maker_token_amount_raw, - 'Fill' AS type, - COALESCE(zeroex_tx.affiliate_address, fills.feeRecipientAddress) AS affiliate_address, - (zeroex_tx.tx_hash IS NOT NULL) AS swap_flag, - (fills.feeRecipientAddress = 0x86003b044f70dac0abc80ac8957305b6370893ed) AS matcha_limit_order_flag, - is_gasless -FROM {{ source(table_prefix, 'Exchange_evt_Fill') }} fills -INNER JOIN zeroex_tx ON zeroex_tx.tx_hash = fills.evt_tx_hash AND fills.evt_block_time = zeroex_tx.block_time -WHERE - -- Exclude bridge orders - (bytearray_substring(makerAssetData, 1, 4) <> 0xdc1600f3) - AND - ( - -- Include transactions with a matching tx_hash in zeroex_tx or specific feeRecipientAddress - zeroex_tx.tx_hash IS NOT NULL - OR fills.feeRecipientAddress = 0x86003b044f70dac0abc80ac8957305b6370893ed - ) - - {% if is_incremental() %} - AND {{ incremental_predicate('evt_block_time') }} - {% endif %} - {% if not is_incremental() %} - AND evt_block_time >= CAST('{{zeroex_v3_start_date}}' AS DATE) - {% endif %} -{% endmacro %} - - -{% macro v4_rfq_fills_no_bridge(blockchain, zeroex_v4_start_date) %} -{%- set table_prefix = 'zeroex_' + blockchain -%} -SELECT - fills.evt_tx_hash AS tx_hash, - fills.evt_index, - fills.contract_address, - fills.evt_block_time AS block_time, - fills.maker, - fills.taker, - fills.takerToken AS taker_token, - fills.makerToken AS maker_token, - CAST(fills.takerTokenFilledAmount AS int256) AS taker_token_amount_raw, - CAST(fills.makerTokenFilledAmount AS int256) AS maker_token_amount_raw, - 'RfqOrderFilled' AS type, - zeroex_tx.affiliate_address, - (zeroex_tx.tx_hash IS NOT NULL) AS swap_flag, - FALSE AS matcha_limit_order_flag, - is_gasless -FROM {{ source(table_prefix, 'ExchangeProxy_evt_RfqOrderFilled') }} fills -LEFT JOIN zeroex_tx - ON zeroex_tx.tx_hash = fills.evt_tx_hash - AND fills.evt_block_time = zeroex_tx.block_time - -{% if is_incremental() %} -WHERE {{ incremental_predicate('evt_block_time') }} -{% else %} -WHERE evt_block_time >= CAST('{{zeroex_v4_start_date}}' AS DATE) -{% endif %} -{% endmacro %} - - -{% macro v4_limit_fills_no_bridge(blockchain, zeroex_v4_start_date) %} -{%- set table_prefix = 'zeroex_' + blockchain -%} -SELECT - fills.evt_tx_hash AS tx_hash, - fills.evt_index, - fills.contract_address, - fills.evt_block_time AS block_time, - fills.maker, - fills.taker, - fills.takerToken AS taker_token, - fills.makerToken AS maker_token, - CAST(fills.takerTokenFilledAmount AS int256) AS taker_token_amount_raw, - CAST(fills.makerTokenFilledAmount AS int256) AS maker_token_amount_raw, - 'LimitOrderFilled' AS type, - COALESCE(zeroex_tx.affiliate_address, fills.feeRecipient) AS affiliate_address, - (zeroex_tx.tx_hash IS NOT NULL) AS swap_flag, - (fills.feeRecipient IN ( - 0x9b858be6e3047d88820f439b240deac2418a2551, - 0x86003b044f70dac0abc80ac8957305b6370893ed, - 0x5bc2419a087666148bfbe1361ae6c06d240c6131 - )) AS matcha_limit_order_flag, - is_gasless -FROM {{ source(table_prefix, 'ExchangeProxy_evt_LimitOrderFilled') }} fills -LEFT JOIN zeroex_tx - ON zeroex_tx.tx_hash = fills.evt_tx_hash - AND fills.evt_block_time = zeroex_tx.block_time - -{% if is_incremental() %} -WHERE {{ incremental_predicate('evt_block_time') }} -{% else %} -WHERE evt_block_time >= CAST('{{zeroex_v4_start_date}}' AS DATE) -{% endif %} -{% endmacro %} - -{% macro otc_fills(blockchain, zeroex_v4_start_date) %} -{%- set table_prefix = 'zeroex_' + blockchain -%} -SELECT - fills.evt_tx_hash AS tx_hash, - fills.evt_index, - fills.contract_address, - fills.evt_block_time AS block_time, - fills.maker, - 0xdef1c0ded9bec7f1a1670819833240f027b25eff AS taker, - fills.takerToken AS taker_token, - fills.makerToken AS maker_token, - CAST(fills.takerTokenFilledAmount AS int256) AS taker_token_amount_raw, - CAST(fills.makerTokenFilledAmount AS int256) AS maker_token_amount_raw, - 'OtcOrderFilled' AS type, - zeroex_tx.affiliate_address, - (zeroex_tx.tx_hash IS NOT NULL) AS swap_flag, - FALSE AS matcha_limit_order_flag, - is_gasless -FROM {{ source(table_prefix, 'ExchangeProxy_evt_OtcOrderFilled') }} fills -LEFT JOIN zeroex_tx - ON zeroex_tx.tx_hash = fills.evt_tx_hash - AND fills.evt_block_time = zeroex_tx.block_time -{% if is_incremental() %} -WHERE {{ incremental_predicate('evt_block_time') }} -{% else %} -WHERE evt_block_time >= CAST('{{zeroex_v4_start_date}}' AS DATE) -{% endif %} -{% endmacro %} - - -{% macro ERC20BridgeTransfer(blockchain, zeroex_v3_start_date) %} -SELECT - logs.tx_hash, - logs.index AS evt_index, - logs.contract_address, - logs.block_time, - bytearray_substring(data, 141, 20) AS maker, - bytearray_substring(data, 173, 20) AS taker, - bytearray_substring(data, 13, 20) AS taker_token, - bytearray_substring(data, 45, 20) AS maker_token, - bytearray_to_int256(bytearray_substring(data, 77, 20)) AS taker_token_amount_raw, - bytearray_to_int256(bytearray_substring(data, 109, 20)) AS maker_token_amount_raw, - 'ERC20BridgeTransfer' AS type, - zeroex_tx.affiliate_address, - TRUE AS swap_flag, - FALSE AS matcha_limit_order_flag, - is_gasless -FROM {{ source(blockchain, 'logs') }} logs -JOIN zeroex_tx - ON zeroex_tx.tx_hash = logs.tx_hash - AND logs.block_time = zeroex_tx.block_time -WHERE topic0 = 0x349fc08071558d8e3aa92dec9396e4e9f2dfecd6bb9065759d1932e7da43b8a9 - -{% if is_incremental() %} -AND {{ incremental_predicate('logs.block_time') }} -{% else %} -AND logs.block_time >= CAST('{{zeroex_v3_start_date}}' AS DATE) -{% endif %} -{% endmacro %} - - -{% macro BridgeFill(blockchain, zeroex_v4_start_date) %} -SELECT - logs.tx_hash, - logs.index AS evt_index, - logs.contract_address, - logs.block_time, - bytearray_substring(data, 13, 20) AS maker, - 0xdef1c0ded9bec7f1a1670819833240f027b25eff AS taker, - bytearray_substring(data, 45, 20) AS taker_token, - bytearray_substring(data, 77, 20) AS maker_token, - bytearray_to_int256(bytearray_substring(data, 109, 20)) AS taker_token_amount_raw, - bytearray_to_int256(bytearray_substring(data, 141, 20)) AS maker_token_amount_raw, - 'BridgeFill' AS type, - zeroex_tx.affiliate_address, - TRUE AS swap_flag, - FALSE AS matcha_limit_order_flag, - is_gasless -FROM {{ source(blockchain, 'logs') }} logs -JOIN zeroex_tx - ON zeroex_tx.tx_hash = logs.tx_hash - AND logs.block_time = zeroex_tx.block_time -WHERE topic0 = 0xff3bc5e46464411f331d1b093e1587d2d1aa667f5618f98a95afc4132709d3a9 - AND contract_address = 0x22f9dcf4647084d6c31b2765f6910cd85c178c18 - -{% if is_incremental() %} -AND {{ incremental_predicate('logs.block_time') }} -{% else %} -AND logs.block_time >= CAST('{{zeroex_v4_start_date}}' AS DATE) -{% endif %} -{% endmacro %} - -{% macro NewBridgeFill(blockchain, zeroex_v4_start_date) %} -SELECT - logs.tx_hash, - logs.index AS evt_index, - logs.contract_address, - logs.block_time, - bytearray_substring(data, 13, 20) AS maker, - 0xdef1c0ded9bec7f1a1670819833240f027b25eff AS taker, - bytearray_substring(data, 45, 20) AS taker_token, - bytearray_substring(data, 77, 20) AS maker_token, - bytearray_to_int256(bytearray_substring(data, 109, 20)) AS taker_token_amount_raw, - bytearray_to_int256(bytearray_substring(data, 141, 20)) AS maker_token_amount_raw, - 'NewBridgeFill' AS type, - zeroex_tx.affiliate_address, - TRUE AS swap_flag, - FALSE AS matcha_limit_order_flag, - is_gasless -FROM {{ source(blockchain, 'logs') }} logs -JOIN zeroex_tx - ON zeroex_tx.tx_hash = logs.tx_hash - AND logs.block_time = zeroex_tx.block_time -WHERE topic0 = 0xe59e71a14fe90157eedc866c4f8c767d3943d6b6b2e8cd64dddcc92ab4c55af8 - AND contract_address = 0x22f9dcf4647084d6c31b2765f6910cd85c178c18 - -{% if is_incremental() %} -AND {{ incremental_predicate('logs.block_time') }} -{% else %} -AND logs.block_time >= CAST('{{zeroex_v4_start_date}}' AS DATE) -{% endif %} -{% endmacro %} - -{% macro direct_PLP(blockchain, zeroex_v3_start_date) %} -{%- set table_prefix = 'zeroex_' + blockchain -%} -SELECT - plp.evt_tx_hash, - plp.evt_index AS evt_index, - plp.contract_address, - plp.evt_block_time AS block_time, - provider AS maker, - recipient AS taker, - inputToken AS taker_token, - outputToken AS maker_token, - CAST(inputTokenAmount AS int256) AS taker_token_amount_raw, - CAST(outputTokenAmount AS int256) AS maker_token_amount_raw, - 'LiquidityProviderSwap' AS type, - zeroex_tx.affiliate_address, - TRUE AS swap_flag, - FALSE AS matcha_limit_order_flag, - is_gasless -FROM {{ source(table_prefix, 'ExchangeProxy_evt_LiquidityProviderSwap') }} plp -JOIN zeroex_tx - ON zeroex_tx.tx_hash = plp.evt_tx_hash - AND plp.evt_block_time = zeroex_tx.block_time - -{% if is_incremental() %} -WHERE {{ incremental_predicate('evt_block_time') }} -{% else %} -WHERE evt_block_time >= CAST('{{zeroex_v3_start_date}}' AS DATE) -{% endif %} -{% endmacro %} - -{% macro direct_uniswapv2(blockchain, zeroex_v3_start_date) %} -{%- set table_prefix = 'uniswap_v2_' + blockchain -%} -SELECT - swap.evt_tx_hash AS tx_hash, - swap.evt_index, - swap.contract_address, - swap.evt_block_time AS block_time, - swap.contract_address AS maker, - 0xdef1c0ded9bec7f1a1670819833240f027b25eff AS taker, - CASE - WHEN swap.amount0In > swap.amount0Out THEN pair.token0 - ELSE pair.token1 - END AS taker_token, - CASE - WHEN swap.amount0In > swap.amount0Out THEN pair.token1 - ELSE pair.token0 - END AS maker_token, - CASE - WHEN swap.amount0In > swap.amount0Out THEN - CASE - WHEN swap.amount0In >= swap.amount0Out THEN CAST(swap.amount0In - swap.amount0Out AS int256) - ELSE CAST(0 AS int256) - END - ELSE - CASE - WHEN swap.amount1In >= swap.amount1Out THEN CAST(swap.amount1In - swap.amount1Out AS int256) - ELSE CAST(0 AS int256) - END - END AS taker_token_amount_raw, - CASE - WHEN swap.amount0In > swap.amount0Out THEN - CASE - WHEN swap.amount1Out >= swap.amount1In THEN CAST(swap.amount1Out - swap.amount1In AS int256) - ELSE CAST(0 AS int256) - END - ELSE - CASE - WHEN swap.amount0Out >= swap.amount0In THEN CAST(swap.amount0Out - swap.amount0In AS int256) - ELSE CAST(0 AS int256) - END - END AS maker_token_amount_raw, - 'Uniswap V2 Direct' AS type, - zeroex_tx.affiliate_address, - TRUE AS swap_flag, - FALSE AS matcha_limit_order_flag, - is_gasless -FROM {{ source(table_prefix, 'Pair_evt_Swap') }} swap -LEFT JOIN {{ source(table_prefix, 'Factory_evt_PairCreated') }} pair - ON pair.pair = swap.contract_address -JOIN zeroex_tx - ON zeroex_tx.tx_hash = swap.evt_tx_hash - AND swap.evt_block_time = zeroex_tx.block_time -WHERE sender = 0xdef1c0ded9bec7f1a1670819833240f027b25eff - {% if is_incremental() %} - AND {{ incremental_predicate('swap.evt_block_time') }} - {% else %} - AND swap.evt_block_time >= CAST('{{zeroex_v3_start_date}}' AS DATE) - {% endif %} -{% endmacro %} - -{% macro direct_sushiswap(blockchain, zeroex_v3_start_date) %} -{%- set table_prefix = 'sushi_' + blockchain -%} -SELECT - swap.evt_tx_hash AS tx_hash, - swap.evt_index, - swap.contract_address, - swap.evt_block_time AS block_time, - swap.contract_address AS maker, - LAST_VALUE(swap.to) OVER (PARTITION BY swap.evt_tx_hash ORDER BY swap.evt_index) AS taker, - CASE - WHEN swap.amount0In > swap.amount0Out THEN pair.token0 - ELSE pair.token1 - END AS taker_token, - CASE - WHEN swap.amount0In > swap.amount0Out THEN pair.token1 - ELSE pair.token0 - END AS maker_token, - CASE - WHEN swap.amount0In > swap.amount0Out THEN - CASE - WHEN swap.amount0In >= swap.amount0Out THEN CAST(swap.amount0In - swap.amount0Out AS int256) - ELSE CAST(0 AS int256) - END - ELSE - CASE - WHEN swap.amount1In >= swap.amount1Out THEN CAST(swap.amount1In - swap.amount1Out AS int256) - ELSE CAST(0 AS int256) - END - END AS taker_token_amount_raw, - CASE - WHEN swap.amount0In > swap.amount0Out THEN - CASE - WHEN swap.amount1Out >= swap.amount1In THEN CAST(swap.amount1Out - swap.amount1In AS int256) - ELSE CAST(0 AS int256) - END - ELSE - CASE - WHEN swap.amount0Out >= swap.amount0In THEN CAST(swap.amount0Out - swap.amount0In AS int256) - ELSE CAST(0 AS int256) - END - END AS maker_token_amount_raw, - 'Sushiswap Direct' AS type, - zeroex_tx.affiliate_address, - TRUE AS swap_flag, - FALSE AS matcha_limit_order_flag, - is_gasless -FROM {{ source(table_prefix, 'Pair_evt_Swap') }} swap -LEFT JOIN {{ source(table_prefix, 'Factory_evt_PairCreated') }} pair - ON pair.pair = swap.contract_address -JOIN zeroex_tx - ON zeroex_tx.tx_hash = swap.evt_tx_hash - AND swap.evt_block_time = zeroex_tx.block_time -WHERE sender = 0xdef1c0ded9bec7f1a1670819833240f027b25eff - {% if is_incremental() %} - AND {{ incremental_predicate('swap.evt_block_time') }} - {% else %} - AND swap.evt_block_time >= CAST('{{zeroex_v3_start_date}}' AS DATE) - {% endif %} -{% endmacro %} - - -{% macro direct_uniswapv3(blockchain,zeroex_v4_start_date) %} -{%- set table_prefix = 'uniswap_v3_' + blockchain -%} - SELECT - swap.evt_tx_hash AS tx_hash, - swap.evt_index, - swap.contract_address, - swap.evt_block_time AS block_time, - swap.contract_address AS maker, - 0xdef1c0ded9bec7f1a1670819833240f027b25eff AS taker, - CASE - WHEN amount0 < CAST(0 AS int256) THEN pair.token1 - ELSE pair.token0 - END AS taker_token, - CASE - WHEN amount0 < CAST(0 AS int256) THEN pair.token0 - ELSE pair.token1 - END AS maker_token, - CASE - WHEN amount0 < CAST(0 AS int256) THEN ABS(swap.amount1) - ELSE ABS(swap.amount0) - END AS taker_token_amount_raw, - CASE - WHEN amount0 < CAST(0 AS int256) THEN ABS(swap.amount0) - ELSE ABS(swap.amount1) - END AS maker_token_amount_raw, - 'Uniswap V3 Direct' AS type, - zeroex_tx.affiliate_address AS affiliate_address, - TRUE AS swap_flag, - FALSE AS matcha_limit_order_flag, - is_gasless - FROM {{ source(table_prefix, 'Pair_evt_Swap') }} swap - LEFT JOIN {{ source(table_prefix, 'Factory_evt_PoolCreated') }} pair - ON pair.pool = swap.contract_address - JOIN zeroex_tx - ON zeroex_tx.tx_hash = swap.evt_tx_hash - AND swap.evt_block_time = zeroex_tx.block_time - WHERE sender = 0xdef1c0ded9bec7f1a1670819833240f027b25eff - {% if is_incremental() %} - AND {{ incremental_predicate('swap.evt_block_time') }} - {% endif %} - {% if not is_incremental() %} - AND swap.evt_block_time >= CAST('{{zeroex_v4_start_date}}' AS DATE) - {% endif %} -{% endmacro %} - -{% macro trade_details(blockchain, zeroex_v3_start_date) %} -WITH results AS ( - SELECT - all_tx.tx_hash, - tx.block_number, - all_tx.evt_index, - all_tx.contract_address, - all_tx.block_time, - CAST(DATE_TRUNC('day', all_tx.block_time) AS DATE) AS block_date, - CAST(DATE_TRUNC('month', all_tx.block_time) AS DATE) AS block_month, - maker, - CASE - WHEN is_gasless = 1 AND VARBINARY_POSITION(data, 0x3a46c4e1) <> 0 THEN VARBINARY_SUBSTRING(data, 81, 20) - WHEN is_gasless = 1 AND VARBINARY_POSITION(data, 0xa98fcbf1) <> 0 THEN VARBINARY_SUBSTRING(data, 81, 20) - WHEN is_gasless = 1 AND VARBINARY_POSITION(data, 0x3d8d4082) <> 0 THEN VARBINARY_SUBSTRING(data, 177, 20) - WHEN taker = 0xdef1c0ded9bec7f1a1670819833240f027b25eff THEN tx."from" - ELSE taker - END AS taker, - taker_token, - taker_token AS token_sold_address, - ts.symbol AS taker_symbol, - maker_token, - maker_token AS token_bought_address, - ms.symbol AS maker_symbol, - CASE - WHEN LOWER(ts.symbol) > LOWER(ms.symbol) THEN CONCAT(ms.symbol, '-', ts.symbol) - ELSE CONCAT(ts.symbol, '-', ms.symbol) - END AS token_pair, - taker_token_amount_raw / POW(10, tp.decimals) AS taker_token_amount, - taker_token_amount_raw / POW(10, tp.decimals) AS token_sold_amount, - CAST(taker_token_amount_raw AS UINT256) AS taker_token_amount_raw, - maker_token_amount_raw / POW(10, mp.decimals) AS maker_token_amount, - maker_token_amount_raw / POW(10, mp.decimals) AS token_bought_amount, - CAST(maker_token_amount_raw AS UINT256) AS maker_token_amount_raw, - all_tx.type, - MAX(affiliate_address) OVER (PARTITION BY all_tx.tx_hash) AS affiliate_address, - swap_flag, - matcha_limit_order_flag, - tx."from" AS tx_from, - tx.to AS tx_to, - '{{ blockchain }}' AS blockchain - FROM all_tx - INNER JOIN {{ source(blockchain, 'transactions') }} tx - ON all_tx.tx_hash = tx.hash AND all_tx.block_time = tx.block_time - {% if is_incremental() %} - AND {{ incremental_predicate('tx.block_time') }} - {% endif %} - {% if not is_incremental() %} - AND tx.block_time >= CAST('{{ zeroex_v3_start_date }}' AS DATE) - {% endif %} - LEFT JOIN {{ source('prices', 'usd') }} tp - ON DATE_TRUNC('minute', all_tx.block_time) = tp.minute - AND CASE - WHEN all_tx.taker_token = 0xeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeee THEN 0xc02aaa39b223fe8d0a0e5c4f27ead9083c756cc2 - ELSE all_tx.taker_token - END = tp.contract_address - AND tp.blockchain = '{{ blockchain }}' - {% if is_incremental() %} - AND {{ incremental_predicate('tp.minute') }} - {% endif %} - {% if not is_incremental() %} - AND tp.minute >= CAST('{{ zeroex_v3_start_date }}' AS DATE) - {% endif %} - LEFT JOIN {{ source('prices', 'usd') }} mp - ON DATE_TRUNC('minute', all_tx.block_time) = mp.minute - AND CASE - WHEN all_tx.maker_token = 0xeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeee THEN 0xc02aaa39b223fe8d0a0e5c4f27ead9083c756cc2 - ELSE all_tx.maker_token - END = mp.contract_address - AND mp.blockchain = '{{ blockchain }}' - {% if is_incremental() %} - AND {{ incremental_predicate('mp.minute') }} - {% endif %} - {% if not is_incremental() %} - AND mp.minute >= CAST('{{ zeroex_v3_start_date }}' AS DATE) - {% endif %} - LEFT JOIN {{ source('tokens', 'erc20') }} ts - ON ts.contract_address = taker_token AND ts.blockchain = '{{ blockchain }}' - LEFT JOIN {{ source('tokens', 'erc20') }} ms - ON ms.contract_address = maker_token AND ms.blockchain = '{{ blockchain }}' -), - -results_usd AS ( - {{ - add_amount_usd( - trades_cte = 'results' - ) - }} -) -select * from results_usd -{% endmacro %} diff --git a/dbt_subprojects/dex/models/_projects/zeroex/base/zeroex_base_api_fills.sql b/dbt_subprojects/dex/models/_projects/zeroex/base/zeroex_base_api_fills.sql index c5c5d327ec1..fc3f6009fc9 100644 --- a/dbt_subprojects/dex/models/_projects/zeroex/base/zeroex_base_api_fills.sql +++ b/dbt_subprojects/dex/models/_projects/zeroex/base/zeroex_base_api_fills.sql @@ -19,65 +19,228 @@ {% set zeroex_v3_start_date = '2019-12-01' %} {% set zeroex_v4_start_date = '2021-01-06' %} -{% set blockchain = 'base' %} + +-- Test Query here: https://dune.com/queries/2834419 WITH zeroex_tx AS ( - SELECT tx_hash, - block_time as block_time, + SELECT tx_hash, max(affiliate_address) as affiliate_address, - max(is_gasless) as is_gasless + is_gasless FROM ( - {{ - zeroex_v1_txs( - blockchain = blockchain, - zeroex_v3_start_date = zeroex_v3_start_date, - ) - }} + SELECT tr.tx_hash, + CASE + WHEN bytearray_position(INPUT, 0x869584cd ) <> 0 THEN SUBSTRING(INPUT + FROM (bytearray_position(INPUT, 0x869584cd) + 16) + FOR 20) + WHEN bytearray_position(INPUT, 0xfbc019a7) <> 0 THEN SUBSTRING(INPUT + FROM (bytearray_position(INPUT, 0xfbc019a7 ) + 16) + FOR 20) + END AS affiliate_address, + case when (varbinary_position(input,0x3d8d4082) <> 0 or varbinary_position(input,0x4f948110) <> 0 ) then 1 else 0 end as is_gasless + FROM {{ source('base', 'traces') }} tr + WHERE tr.to IN ( + -- exchange contract + 0xdef1c0ded9bec7f1a1670819833240f027b25eff, + -- forwarder addresses + 0x6958f5e95332d93d21af0d7b9ca85b8212fee0a5, + 0x4aa817c6f383c8e8ae77301d18ce48efb16fd2be, + 0x4ef40d1bf0983899892946830abf99eca2dbc5ce, + -- exchange proxy + 0xdef189deaef76e379df891899eb5a00a94cbc250 + ) + AND ( + bytearray_position(INPUT, 0x869584cd ) <> 0 + OR bytearray_position(INPUT, 0xfbc019a7 ) <> 0 + ) + {% if is_incremental() %} + AND {{ incremental_predicate('block_time') }} + {% endif %} + {% if not is_incremental() %} + AND block_time >= cast('{{zeroex_v3_start_date}}' as date) + {% endif %} ) temp - group by tx_hash, block_time + group by tx_hash , is_gasless + ), -ERC20BridgeTransfer as ( - {{ - ERC20BridgeTransfer( - blockchain = blockchain, - zeroex_v3_start_date = zeroex_v3_start_date - ) - }} + +ERC20BridgeTransfer AS ( + SELECT + logs.tx_hash, + logs.block_number AS block_number, + INDEX AS evt_index, + logs.contract_address, + block_time AS block_time, + bytearray_substring(DATA, 142, 20) AS maker, + bytearray_substring(DATA, 172, 20) AS taker, + bytearray_substring(DATA, 14, 20) AS taker_token, + bytearray_substring(DATA, 45, 20) AS maker_token, + bytearray_to_uint256(bytearray_substring(DATA, 77, 20)) AS taker_token_amount_raw, + bytearray_to_uint256(bytearray_substring(DATA, 110, 20)) AS maker_token_amount_raw, + 'ERC20BridgeTransfer' AS type, + zeroex_tx.affiliate_address AS affiliate_address, + TRUE AS swap_flag, + FALSE AS matcha_limit_order_flag, + is_gasless + FROM {{ source('base', 'logs') }} logs + INNER JOIN zeroex_tx ON zeroex_tx.tx_hash = logs.tx_hash + WHERE topic0 = 0x349fc08071558d8e3aa92dec9396e4e9f2dfecd6bb9065759d1932e7da43b8a9 + + {% if is_incremental() %} + AND {{ incremental_predicate('block_time') }} + {% endif %} + {% if not is_incremental() %} + AND block_time >= cast('{{zeroex_v3_start_date}}' as date) + {% endif %} + ), -BridgeFill as ( - {{ - BridgeFill( - blockchain = blockchain, - zeroex_v4_start_date = zeroex_v4_start_date - ) - }} +BridgeFill AS ( + SELECT + logs.tx_hash, + logs.block_number AS block_number, + INDEX AS evt_index, + logs.contract_address, + block_time AS block_time, + bytearray_substring(DATA, 13, 20) AS maker, + 0xdef1c0ded9bec7f1a1670819833240f027b25eff AS taker, + bytearray_substring(DATA, 45, 20) AS taker_token, + bytearray_substring(DATA, 77, 20) AS maker_token, + bytearray_to_uint256(bytearray_substring(DATA, 109, 20)) AS taker_token_amount_raw, + bytearray_to_uint256(bytearray_substring(DATA, 141, 20)) AS maker_token_amount_raw, + 'BridgeFill' AS type, + zeroex_tx.affiliate_address AS affiliate_address, + TRUE AS swap_flag, + FALSE AS matcha_limit_order_flag, + is_gasless + FROM {{ source('base', 'logs') }} logs + INNER JOIN zeroex_tx ON zeroex_tx.tx_hash = logs.tx_hash + WHERE topic0 = 0xff3bc5e46464411f331d1b093e1587d2d1aa667f5618f98a95afc4132709d3a9 + AND contract_address = 0xdb6f1920a889355780af7570773609bd8cb1f498 + + {% if is_incremental() %} + AND {{ incremental_predicate('block_time') }} + {% endif %} + {% if not is_incremental() %} + AND block_time >= cast('{{zeroex_v4_start_date}}' as date) + {% endif %} ), -NewBridgeFill as ( - {{ - NewBridgeFill( - blockchain = blockchain, - zeroex_v4_start_date = zeroex_v4_start_date - ) - }} +NewBridgeFill AS ( + SELECT + logs.tx_hash as tx_hash, + logs.block_number AS block_number, + INDEX AS evt_index, + logs.contract_address, + block_time AS block_time, + bytearray_substring(DATA, 13, 20) AS maker, + 0xdef1c0ded9bec7f1a1670819833240f027b25eff AS taker, + bytearray_substring(DATA, 45, 20) AS taker_token, + bytearray_substring(DATA, 77, 20) AS maker_token, + bytearray_to_uint256(bytearray_substring(DATA, 109, 20)) AS taker_token_amount_raw, + bytearray_to_uint256(bytearray_substring(DATA, 141, 20)) AS maker_token_amount_raw, + 'BridgeFill' AS type, + zeroex_tx.affiliate_address AS affiliate_address, + TRUE AS swap_flag, + FALSE AS matcha_limit_order_flag, + is_gasless + FROM {{ source('base' ,'logs') }} logs + INNER JOIN zeroex_tx ON zeroex_tx.tx_hash = logs.tx_hash + WHERE topic0 = 0xe59e71a14fe90157eedc866c4f8c767d3943d6b6b2e8cd64dddcc92ab4c55af8 + AND contract_address = 0xdb6f1920a889355780af7570773609bd8cb1f498 + + {% if is_incremental() %} + AND {{ incremental_predicate('block_time') }} + {% endif %} + {% if not is_incremental() %} + AND block_time >= cast('{{zeroex_v4_start_date}}' as date) + {% endif %} ), + all_tx AS ( + SELECT * FROM ERC20BridgeTransfer - UNION ALL - SELECT * + UNION ALL SELECT * FROM BridgeFill UNION ALL SELECT * FROM NewBridgeFill -), -tbl_trade_details AS ( - {{ - trade_details( - blockchain = blockchain, - zeroex_v3_start_date = zeroex_v3_start_date - ) - }} + + ) -select * from tbl_trade_details -order by block_time desc \ No newline at end of file + +SELECT + all_tx.tx_hash, + all_tx.block_number, + all_tx.evt_index, + all_tx.contract_address, + all_tx.block_time, + cast(date_trunc('day', all_tx.block_time) AS date) AS block_date, + cast(date_trunc('month', all_tx.block_time) AS date) AS block_month, + maker, + CASE + WHEN is_gasless = 1 then case when (varbinary_substring(data,177,19) ) = 0x00000000000000000000000000000000000000 then varbinary_substring(data,81,20) else (varbinary_substring(data,177,20) ) end + WHEN taker = 0xdef1c0ded9bec7f1a1670819833240f027b25eff THEN tx."from" + ELSE taker + END AS taker, -- fix the user masked by ProxyContract issue + taker_token, + ts.symbol AS taker_symbol, + maker_token, + ms.symbol AS maker_symbol, + CASE WHEN lower(ts.symbol) > lower(ms.symbol) THEN concat(ms.symbol, '-', ts.symbol) ELSE concat(ts.symbol, '-', ms.symbol) END AS token_pair, + taker_token_amount_raw / pow(10, tp.decimals) AS taker_token_amount, + taker_token_amount_raw, + maker_token_amount_raw / pow(10, mp.decimals) AS maker_token_amount, + maker_token_amount_raw, + all_tx.type, + max(affiliate_address) over (partition by all_tx.tx_hash) as affiliate_address, + swap_flag, + matcha_limit_order_flag, + CASE WHEN maker_token IN (0xd9aaec86b65d86f6a7b5b1b0c42ffa531710b6ca,0x2ae3f1ec7f1f5012cfeab0185bfc7aa3cf0dec22,0xeb466342c4d449bc9f53a865d5cb90586f405215 ) + THEN (all_tx.maker_token_amount_raw / pow(10, mp.decimals)) * mp.price + WHEN taker_token IN (0xd9aaec86b65d86f6a7b5b1b0c42ffa531710b6ca,0x2ae3f1ec7f1f5012cfeab0185bfc7aa3cf0dec22,0xeb466342c4d449bc9f53a865d5cb90586f405215,0xd9aaec86b65d86f6a7b5b1b0c42ffa531710b6ca ) + THEN (all_tx.taker_token_amount_raw / pow(10, tp.decimals)) * tp.price + ELSE COALESCE((all_tx.maker_token_amount_raw / pow(10, mp.decimals)) * mp.price, (all_tx.taker_token_amount_raw / pow(10, tp.decimals)) * tp.price) + END AS volume_usd, + tx."from" AS tx_from, + tx.to AS tx_to, + 'base' AS blockchain +FROM all_tx +INNER JOIN {{ source('base', 'transactions')}} tx ON all_tx.tx_hash = tx.hash + +{% if is_incremental() %} +AND {{ incremental_predicate('tx.block_time') }} +{% endif %} +{% if not is_incremental() %} +AND tx.block_time >= cast('{{zeroex_v3_start_date}}' as date) +{% endif %} + +LEFT JOIN {{ source('prices', 'usd') }} tp ON date_trunc('minute', all_tx.block_time) = tp.minute +AND CASE + WHEN all_tx.taker_token = 0xeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeee THEN 0xc02aaa39b223fe8d0a0e5c4f27ead9083c756cc2 + ELSE all_tx.taker_token + END = tp.contract_address +AND tp.blockchain = 'base' + +{% if is_incremental() %} +AND {{ incremental_predicate('tp.minute') }} +{% endif %} +{% if not is_incremental() %} +AND tp.minute >= cast('{{zeroex_v3_start_date}}' as date) +{% endif %} + +LEFT JOIN {{ source('prices', 'usd') }} mp ON DATE_TRUNC('minute', all_tx.block_time) = mp.minute +AND CASE + WHEN all_tx.maker_token = 0xeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeee THEN 0xc02aaa39b223fe8d0a0e5c4f27ead9083c756cc2 + ELSE all_tx.maker_token + END = mp.contract_address +AND mp.blockchain = 'base' + +{% if is_incremental() %} +AND {{ incremental_predicate('mp.minute') }} +{% endif %} +{% if not is_incremental() %} +AND mp.minute >= cast('{{zeroex_v3_start_date}}' as date) +{% endif %} + +LEFT OUTER JOIN {{ source('tokens', 'erc20') }} ts ON ts.contract_address = taker_token and ts.blockchain = 'base' +LEFT OUTER JOIN {{ source('tokens', 'erc20') }} ms ON ms.contract_address = maker_token and ms.blockchain = 'base' \ No newline at end of file diff --git a/dbt_subprojects/dex/models/_projects/zeroex/base/zeroex_base_api_fills_deduped.sql b/dbt_subprojects/dex/models/_projects/zeroex/base/zeroex_base_api_fills_deduped.sql new file mode 100644 index 00000000000..e7f7dd0ddbd --- /dev/null +++ b/dbt_subprojects/dex/models/_projects/zeroex/base/zeroex_base_api_fills_deduped.sql @@ -0,0 +1,102 @@ +{{ config( + + schema = 'zeroex_base', + alias = 'api_fills_deduped', + materialized='incremental', + partition_by = ['block_month'], + unique_key = ['block_date', 'tx_hash', 'evt_index'], + on_schema_change='sync_all_columns', + file_format ='delta', + incremental_strategy='merge', + incremental_predicates = [incremental_predicate('DBT_INTERNAL_DEST.block_time')] + ) +}} + +{% set zeroex_v3_start_date = '2019-12-01' %} + +WITH fills_with_tx_fill_number +AS +( + SELECT row_number() OVER ( partition BY tx_hash ORDER BY evt_index ASC ) AS tx_fill_number + , * + FROM {{ ref('zeroex_base_api_fills') }} + WHERE 1=1 + AND swap_flag = true + {% if is_incremental() %} + AND {{ incremental_predicate('block_time') }} + {% endif %} + {% if not is_incremental() %} + AND block_time >= cast('{{zeroex_v3_start_date}}' as date) + {% endif %} +) +, fills_first_last +AS +( + SELECT CASE + WHEN a.taker_token = c.maker_token AND a.taker_token_amount_raw = c.maker_token_amount_raw THEN 0 + ELSE 1 END AS taker_consider_flag--from + , CASE + WHEN a.maker_token = b.taker_token AND a.maker_token_amount_raw = b.taker_token_amount_raw THEN 0 + ELSE 1 END AS maker_consider_flag + , SUM(CASE + WHEN a.maker_token = b.taker_token AND a.maker_token_amount_raw = b.taker_token_amount_raw THEN 0 + ELSE 1 END) OVER (PARTITION BY a.tx_hash ORDER BY a.evt_index DESC) hop_count + , a.* + FROM fills_with_tx_fill_number a + LEFT JOIN fills_with_tx_fill_number b ON (a.tx_hash = b.tx_hash AND a.tx_fill_number = b.tx_fill_number - 1) + LEFT JOIN fills_with_tx_fill_number c ON (a.tx_hash = c.tx_hash AND a.tx_fill_number = c.tx_fill_number + 1) +) +, deduped_bridge_fills +AS +( + SELECT tx_hash + , MAX(evt_index) AS evt_index + , MAX(affiliate_address) AS affiliate_address + , MAX(CASE WHEN taker_consider_flag = 0 THEN NULL ELSE taker_token END ) AS taker_token + , MAX(CASE WHEN maker_consider_flag = 0 THEN NULL ELSE maker_token END ) AS maker_token + , MAX(CASE WHEN taker_consider_flag = 0 THEN NULL ELSE taker_symbol END ) AS taker_symbol + , MAX(CASE WHEN maker_consider_flag = 0 THEN NULL ELSE maker_symbol END ) AS maker_symbol + , MAX(CASE WHEN taker_consider_flag = 0 THEN NULL ELSE taker_token_amount END) AS taker_token_amount + , MAX(CASE WHEN maker_consider_flag = 0 THEN NULL ELSE maker_token_amount END) AS maker_token_amount + , MAX(CASE WHEN taker_consider_flag = 0 THEN NULL ELSE taker_token_amount_raw END) AS taker_token_amount_raw + , MAX(CASE WHEN maker_consider_flag = 0 THEN NULL ELSE maker_token_amount_raw END) AS maker_token_amount_raw + , COUNT(*) AS fills_within + FROM fills_first_last a + GROUP BY tx_hash,hop_count +) +SELECT a.blockchain + , '0x API' as project + , cast('1' as varchar(10)) as version + + , a.block_date + , a.block_month + , a.block_time + , b.taker_symbol AS taker_symbol + , b.maker_symbol AS maker_symbol + , CASE + WHEN LOWER(b.taker_symbol) > LOWER(b.maker_symbol) + THEN CONCAT(COALESCE(b.maker_symbol, ''), '-', COALESCE(b.taker_symbol, '')) + ELSE CONCAT(COALESCE(b.taker_symbol, ''), '-', COALESCE(b.maker_symbol, '')) + END AS token_pair + , b.taker_token_amount + , b.maker_token_amount + , b.taker_token_amount_raw AS taker_token_amount_raw + , b.maker_token_amount_raw AS maker_token_amount_raw + , a.volume_usd + , b.taker_token + , b.maker_token + , a.taker + , a.maker + , a.affiliate_address + , a.tx_hash + , a.tx_from + , a.tx_to + , b.evt_index + , ARRAY[-1] AS trace_address + , a.type + , a.swap_flag + , b.fills_within + , a.contract_address +FROM fills_with_tx_fill_number a +INNER JOIN deduped_bridge_fills b + ON a.tx_hash = b.tx_hash AND a.evt_index = b.evt_index \ No newline at end of file diff --git a/dbt_subprojects/dex/models/_projects/zeroex/base/zeroex_base_schema.yml b/dbt_subprojects/dex/models/_projects/zeroex/base/zeroex_base_schema.yml index a0212b1a160..a6c1b0f74b8 100644 --- a/dbt_subprojects/dex/models/_projects/zeroex/base/zeroex_base_schema.yml +++ b/dbt_subprojects/dex/models/_projects/zeroex/base/zeroex_base_schema.yml @@ -82,7 +82,7 @@ models: name: blockchain description: "Blockchain which the aggregator project is deployed" - - name: zeroex_v1_base_deduped_trades + - name: zeroex_base_api_fills_deduped meta: blockchain: base project: zeroex diff --git a/dbt_subprojects/dex/models/_projects/zeroex/base/zeroex_v1_base_deduped_trades.sql b/dbt_subprojects/dex/models/_projects/zeroex/base/zeroex_v1_base_deduped_trades.sql deleted file mode 100644 index 5713bc0b849..00000000000 --- a/dbt_subprojects/dex/models/_projects/zeroex/base/zeroex_v1_base_deduped_trades.sql +++ /dev/null @@ -1,30 +0,0 @@ -{{ config( - schema = 'zeroex_v1_base', - alias = 'deduped_trades', - materialized='incremental', - partition_by = ['block_month'], - unique_key = ['block_date', 'tx_hash', 'evt_index'], - on_schema_change='sync_all_columns', - file_format ='delta', - incremental_strategy='merge', - incremental_predicates = [incremental_predicate('DBT_INTERNAL_DEST.block_time')] - ) -}} - -{% set zeroex_v3_start_date = '2019-12-01' %} -{% set blockchain = 'base' %} - -WITH -deduped_trades as ( - {{ - zeroex_v1_deduped_trades( - blockchain = blockchain, - start_date = zeroex_v3_start_date - - ) - }} - -) -select - * -from deduped_trades diff --git a/dbt_subprojects/dex/models/_projects/zeroex/ethereum/zeroex_ethereum_api_fills.sql b/dbt_subprojects/dex/models/_projects/zeroex/ethereum/zeroex_ethereum_api_fills.sql index 459792679f8..814f58dca9c 100644 --- a/dbt_subprojects/dex/models/_projects/zeroex/ethereum/zeroex_ethereum_api_fills.sql +++ b/dbt_subprojects/dex/models/_projects/zeroex/ethereum/zeroex_ethereum_api_fills.sql @@ -13,121 +13,405 @@ {% set zeroex_v3_start_date = '2019-12-01' %} {% set zeroex_v4_start_date = '2021-01-06' %} -{% set blockchain = 'ethereum' %} +-- Test Query here: https://dune.com/queries/1330551 WITH zeroex_tx AS ( - SELECT tx_hash, - block_time as block_time, + SELECT tx_hash, + block_time as block_time, max(affiliate_address) as affiliate_address, - max(is_gasless) as is_gasless - + max(is_gasless) as is_gasless FROM ( - {{ - zeroex_evt_fills_txs( - blockchain = blockchain, - zeroex_v3_start_date = zeroex_v3_start_date, - ) - }} - - union - - {{ - zeroex_v1_txs( - blockchain = blockchain, - zeroex_v3_start_date = zeroex_v3_start_date, - ) - }} + SELECT v3.evt_tx_hash AS tx_hash, + CASE + WHEN takerAddress = 0x63305728359c088a52b0b0eeec235db4d31a67fc THEN takerAddress + ELSE NULL + END AS affiliate_address, + null as is_gasless, + evt_block_time as block_time + + FROM {{ source('zeroex_v3_ethereum', 'Exchange_evt_Fill') }} v3 + WHERE ( -- nuo + v3.takerAddress = 0x63305728359c088a52b0b0eeec235db4d31a67fc + OR -- contains a bridge order + ( + v3.feeRecipientAddress = 0x1000000000000000000000000000000000000011 + AND bytearray_substring(v3.makerAssetData, 1, 4) = 0xdc1600f3 + ) + ) + + {% if is_incremental() %} + AND {{ incremental_predicate('evt_block_time') }} + {% endif %} + {% if not is_incremental() %} + AND evt_block_time >= cast('{{zeroex_v3_start_date}}' as date) + {% endif %} + + UNION ALL + SELECT tr.tx_hash, + CASE + WHEN bytearray_position(INPUT, 0x869584cd ) <> 0 THEN SUBSTRING(INPUT + FROM (bytearray_position(INPUT, 0x869584cd) + 16) + FOR 20) + WHEN bytearray_position(INPUT, 0xfbc019a7) <> 0 THEN SUBSTRING(INPUT + FROM (bytearray_position(INPUT, 0xfbc019a7 ) + 16) + FOR 20) + END AS affiliate_address, + case when (varbinary_position(input,0x3d8d4082) <> 0 or varbinary_position(input,0x4f948110) <> 0 ) then 1 end as is_gasless, + block_time + FROM {{ source('ethereum', 'traces') }} tr + WHERE tr.to IN ( + -- exchange contract + 0x61935cbdd02287b511119ddb11aeb42f1593b7ef, + -- forwarder addresses + 0x6958f5e95332d93d21af0d7b9ca85b8212fee0a5, + 0x4aa817c6f383c8e8ae77301d18ce48efb16fd2be, + 0x4ef40d1bf0983899892946830abf99eca2dbc5ce, + -- exchange proxy + 0xdef1c0ded9bec7f1a1670819833240f027b25eff + ) + AND ( + bytearray_position(INPUT, 0x869584cd ) <> 0 + OR bytearray_position(INPUT, 0xfbc019a7 ) <> 0 + ) + + {% if is_incremental() %} + AND {{ incremental_predicate('block_time') }} + {% endif %} + {% if not is_incremental() %} + AND block_time >= cast('{{zeroex_v3_start_date}}' as date) + {% endif %} ) temp group by tx_hash, block_time + ), -v3_fills_no_bridge as ( - {{ - v3_fills_no_bridge( - blockchain = blockchain, - zeroex_v3_start_date = zeroex_v3_start_date - ) - }} +v3_fills_no_bridge AS ( + SELECT + fills.evt_tx_hash AS tx_hash, + fills.evt_index, + fills.contract_address, + evt_block_time AS block_time, + fills.makerAddress AS maker, + fills.takerAddress AS taker, + bytearray_substring(fills.takerAssetData, 17, 20) AS taker_token, + bytearray_substring(fills.makerAssetData, 17, 20) AS maker_token, + cast(fills.takerAssetFilledAmount as int256) AS taker_token_amount_raw, + cast(fills.makerAssetFilledAmount as int256)AS maker_token_amount_raw, + 'Fill' AS type, + COALESCE(zeroex_tx.affiliate_address, fills.feeRecipientAddress) AS affiliate_address, + (zeroex_tx.tx_hash IS NOT NULL) AS swap_flag, + (fills.feeRecipientAddress = 0x86003b044f70dac0abc80ac8957305b6370893ed) AS matcha_limit_order_flag, + is_gasless + FROM {{ source('zeroex_v3_ethereum', 'Exchange_evt_Fill') }} fills + INNER JOIN zeroex_tx ON zeroex_tx.tx_hash = fills.evt_tx_hash and fills.evt_block_time = zeroex_tx.block_time + WHERE (bytearray_substring(makerAssetData, 1, 4) <> 0xdc1600f3) + AND (zeroex_tx.tx_hash IS NOT NULL + OR fills.feeRecipientAddress = 0x86003b044f70dac0abc80ac8957305b6370893ed) + + {% if is_incremental() %} + AND {{ incremental_predicate('evt_block_time') }} + {% endif %} + {% if not is_incremental() %} + AND evt_block_time >= cast('{{zeroex_v3_start_date}}' as date) + {% endif %} + ), -v4_rfq_fills_no_bridge as ( - {{ - v4_rfq_fills_no_bridge( - blockchain = blockchain, - zeroex_v4_start_date = zeroex_v4_start_date - ) - }} +v4_rfq_fills_no_bridge AS ( + SELECT + fills.evt_tx_hash AS tx_hash, + fills.evt_index, + fills.contract_address, + fills.evt_block_time AS block_time, + fills.maker AS maker, + fills.taker AS taker, + fills.takerToken AS taker_token, + fills.makerToken AS maker_token, + cast(fills.takerTokenFilledAmount as int256) AS taker_token_amount_raw, + cast(fills.makerTokenFilledAmount as int256) AS maker_token_amount_raw, + 'RfqOrderFilled' AS type, + zeroex_tx.affiliate_address AS affiliate_address, + (zeroex_tx.tx_hash IS NOT NULL) AS swap_flag, + FALSE AS matcha_limit_order_flag, + is_gasless + FROM {{ source('zeroex_ethereum', 'ExchangeProxy_evt_RfqOrderFilled') }} fills + LEFT JOIN zeroex_tx ON zeroex_tx.tx_hash = fills.evt_tx_hash and fills.evt_block_time = zeroex_tx.block_time + + {% if is_incremental() %} + WHERE {{ incremental_predicate('evt_block_time') }} + {% endif %} + {% if not is_incremental() %} + WHERE evt_block_time >= cast('{{zeroex_v4_start_date}}' as date) + {% endif %} ), -v4_limit_fills_no_bridge as ( - {{ - v4_limit_fills_no_bridge( - blockchain = blockchain, - zeroex_v4_start_date = zeroex_v4_start_date - ) - }} +v4_limit_fills_no_bridge AS ( + SELECT + fills.evt_tx_hash AS tx_hash, + fills.evt_index, + fills.contract_address, + fills.evt_block_time AS block_time, + fills.maker AS maker, + fills.taker AS taker, + fills.takerToken AS taker_token, + fills.makerToken AS maker_token, + cast(fills.takerTokenFilledAmount as int256) AS taker_token_amount_raw, + cast(fills.makerTokenFilledAmount as int256) AS maker_token_amount_raw, + 'LimitOrderFilled' AS type, + COALESCE(zeroex_tx.affiliate_address, fills.feeRecipient) AS affiliate_address, + (zeroex_tx.tx_hash IS NOT NULL) AS swap_flag, + (fills.feeRecipient in + (0x9b858be6e3047d88820f439b240deac2418a2551,0x86003b044f70dac0abc80ac8957305b6370893ed,0x5bc2419a087666148bfbe1361ae6c06d240c6131)) + AS matcha_limit_order_flag, + is_gasless + FROM {{ source('zeroex_ethereum', 'ExchangeProxy_evt_LimitOrderFilled') }} fills + LEFT JOIN zeroex_tx ON zeroex_tx.tx_hash = fills.evt_tx_hash and fills.evt_block_time = zeroex_tx.block_time + + {% if is_incremental() %} + WHERE {{ incremental_predicate('evt_block_time') }} + {% endif %} + {% if not is_incremental() %} + WHERE evt_block_time >= cast('{{zeroex_v4_start_date}}' as date) + {% endif %} ), -otc_fills as ( - {{ - otc_fills( - blockchain = blockchain, - zeroex_v4_start_date = zeroex_v4_start_date - ) - }} +otc_fills AS ( + SELECT + fills.evt_tx_hash AS tx_hash, + fills.evt_index, + fills.contract_address, + fills.evt_block_time AS block_time, + fills.maker AS maker, + 0xdef1c0ded9bec7f1a1670819833240f027b25eff AS taker, + fills.takerToken AS taker_token, + fills.makerToken AS maker_token, + cast(fills.takerTokenFilledAmount as int256) AS taker_token_amount_raw, + cast(fills.makerTokenFilledAmount as int256) AS maker_token_amount_raw, + 'OtcOrderFilled' AS type, + zeroex_tx.affiliate_address AS affiliate_address, + (zeroex_tx.tx_hash IS NOT NULL) AS swap_flag, + FALSE AS matcha_limit_order_flag, + is_gasless + FROM {{ source('zeroex_ethereum', 'ExchangeProxy_evt_OtcOrderFilled') }} fills + LEFT JOIN zeroex_tx ON zeroex_tx.tx_hash = fills.evt_tx_hash and fills.evt_block_time = zeroex_tx.block_time + + {% if is_incremental() %} + WHERE {{ incremental_predicate('evt_block_time') }} + {% endif %} + {% if not is_incremental() %} + WHERE evt_block_time >= cast('{{zeroex_v4_start_date}}' as date) + {% endif %} + ), -ERC20BridgeTransfer as ( - {{ - ERC20BridgeTransfer( - blockchain = blockchain, - zeroex_v3_start_date = zeroex_v3_start_date - ) - }} +ERC20BridgeTransfer AS ( + SELECT + logs.tx_hash, + INDEX AS evt_index, + logs.contract_address, + logs.block_time AS block_time, + bytearray_substring(DATA, 141, 20) AS maker, + bytearray_substring(DATA, 173, 20) AS taker, + bytearray_substring(DATA, 13, 20) AS taker_token, + bytearray_substring(DATA, 45, 20) AS maker_token, + bytearray_to_int256(bytearray_substring(DATA, 77, 20)) AS taker_token_amount_raw, + bytearray_to_int256(bytearray_substring(DATA, 109, 20)) AS maker_token_amount_raw, + 'ERC20BridgeTransfer' AS type, + zeroex_tx.affiliate_address AS affiliate_address, + TRUE AS swap_flag, + FALSE AS matcha_limit_order_flag, + is_gasless + FROM {{ source('ethereum', 'logs') }} logs + JOIN zeroex_tx ON zeroex_tx.tx_hash = logs.tx_hash and logs.block_time = zeroex_tx.block_time + WHERE topic0 = 0x349fc08071558d8e3aa92dec9396e4e9f2dfecd6bb9065759d1932e7da43b8a9 + + {% if is_incremental() %} + AND {{ incremental_predicate('logs.block_time') }} + {% endif %} + {% if not is_incremental() %} + AND logs.block_time >= cast('{{zeroex_v3_start_date}}' as date) + {% endif %} + ), -BridgeFill as ( - {{ - BridgeFill( - blockchain = blockchain, - zeroex_v4_start_date = zeroex_v4_start_date - ) - }} +BridgeFill AS ( + SELECT + logs.tx_hash, + INDEX AS evt_index, + logs.contract_address, + logs.block_time AS block_time, + bytearray_substring(DATA, 13, 20) AS maker, + 0xdef1c0ded9bec7f1a1670819833240f027b25eff AS taker, + bytearray_substring(DATA, 45, 20) AS taker_token, + bytearray_substring(DATA, 77, 20) AS maker_token, + bytearray_to_int256(bytearray_substring(DATA, 109, 20)) AS taker_token_amount_raw, + bytearray_to_int256(bytearray_substring(DATA, 141, 20)) AS maker_token_amount_raw, + 'BridgeFill' AS type, + zeroex_tx.affiliate_address AS affiliate_address, + TRUE AS swap_flag, + FALSE AS matcha_limit_order_flag, + is_gasless + FROM {{ source('ethereum', 'logs') }} logs + JOIN zeroex_tx ON zeroex_tx.tx_hash = logs.tx_hash and logs.block_time = zeroex_tx.block_time + WHERE topic0 = 0xff3bc5e46464411f331d1b093e1587d2d1aa667f5618f98a95afc4132709d3a9 + AND contract_address = 0x22f9dcf4647084d6c31b2765f6910cd85c178c18 + + {% if is_incremental() %} + AND {{ incremental_predicate('logs.block_time') }} + {% endif %} + {% if not is_incremental() %} + AND logs.block_time >= cast('{{zeroex_v4_start_date}}' as date) + {% endif %} ), -NewBridgeFill as ( - {{ - NewBridgeFill( - blockchain = blockchain, - zeroex_v4_start_date = zeroex_v4_start_date - ) - }} +NewBridgeFill AS ( + SELECT + logs.tx_hash, + INDEX AS evt_index, + logs.contract_address, + logs.block_time AS block_time, + bytearray_substring(DATA, 13, 20) AS maker, + 0xdef1c0ded9bec7f1a1670819833240f027b25eff AS taker, + bytearray_substring(DATA, 45, 20) AS taker_token, + bytearray_substring(DATA, 77, 20) AS maker_token, + bytearray_to_int256(bytearray_substring(DATA, 109, 20)) AS taker_token_amount_raw, + bytearray_to_int256(bytearray_substring(DATA, 141, 20)) AS maker_token_amount_raw, + 'NewBridgeFill' AS type, + zeroex_tx.affiliate_address AS affiliate_address, + TRUE AS swap_flag, + FALSE AS matcha_limit_order_flag, + is_gasless + FROM {{ source('ethereum' ,'logs') }} logs + JOIN zeroex_tx ON zeroex_tx.tx_hash = logs.tx_hash and logs.block_time = zeroex_tx.block_time + WHERE topic0 = 0xe59e71a14fe90157eedc866c4f8c767d3943d6b6b2e8cd64dddcc92ab4c55af8 + AND contract_address = 0x22f9dcf4647084d6c31b2765f6910cd85c178c18 + + {% if is_incremental() %} + AND {{ incremental_predicate('logs.block_time') }} + {% endif %} + {% if not is_incremental() %} + AND logs.block_time >= cast('{{zeroex_v4_start_date}}' as date) + {% endif %} ), -direct_PLP as ( - {{ - direct_PLP( - blockchain = blockchain, - zeroex_v3_start_date = zeroex_v3_start_date - ) - }} +direct_PLP AS ( + SELECT + plp.evt_tx_hash, + plp.evt_index AS evt_index, + plp.contract_address, + plp.evt_block_time AS block_time, + provider AS maker, + recipient AS taker, + inputToken AS taker_token, + outputToken AS maker_token, + cast(inputTokenAmount as int256) AS taker_token_amount_raw, + cast(outputTokenAmount as int256)AS maker_token_amount_raw, + 'LiquidityProviderSwap' AS type, + zeroex_tx.affiliate_address AS affiliate_address, + TRUE AS swap_flag, + FALSE AS matcha_limit_order_flag, + is_gasless + FROM {{ source('zeroex_ethereum', 'ExchangeProxy_evt_LiquidityProviderSwap') }} plp + JOIN zeroex_tx ON zeroex_tx.tx_hash = plp.evt_tx_hash and plp.evt_block_time = zeroex_tx.block_time + + {% if is_incremental() %} + WHERE {{ incremental_predicate('evt_block_time') }} + {% endif %} + {% if not is_incremental() %} + WHERE evt_block_time >= cast('{{zeroex_v3_start_date}}' as date) + {% endif %} + ), -direct_uniswapv2 as ( - {{ - direct_uniswapv2( - blockchain = blockchain, - zeroex_v3_start_date = zeroex_v3_start_date - ) - }} +direct_uniswapv2 AS ( + SELECT + swap.evt_tx_hash AS tx_hash, + swap.evt_index, + swap.contract_address, + swap.evt_block_time AS block_time, + swap.contract_address AS maker, + 0xdef1c0ded9bec7f1a1670819833240f027b25eff as taker, + CASE WHEN swap.amount0In > swap.amount0Out THEN pair.token0 ELSE pair.token1 END AS taker_token, + CASE WHEN swap.amount0In > swap.amount0Out THEN pair.token1 ELSE pair.token0 END AS maker_token, + CASE WHEN swap.amount0In > swap.amount0Out THEN + CASE WHEN swap.amount0In >= swap.amount0Out THEN cast(swap.amount0In - swap.amount0Out as int256) ELSE cast(0 as int256) END ELSE + CASE WHEN swap.amount1In >= swap.amount1Out THEN cast(swap.amount1In - swap.amount1Out as int256) ELSE cast(0 as int256) END END AS taker_token_amount_raw, + CASE WHEN swap.amount0In > swap.amount0Out THEN + CASE WHEN swap.amount1Out >= swap.amount1In THEN cast(swap.amount1Out - swap.amount1In as int256) ELSE cast(0 as int256) END ELSE + CASE WHEN swap.amount0Out >= swap.amount0In THEN cast(swap.amount0Out - swap.amount0In as int256) ELSE cast(0 as int256) END END AS maker_token_amount_raw, + 'Uniswap V2 Direct' AS type, + zeroex_tx.affiliate_address AS affiliate_address, + TRUE AS swap_flag, + FALSE AS matcha_limit_order_flag, + is_gasless + FROM {{ source('uniswap_v2_ethereum', 'Pair_evt_Swap') }} swap + LEFT JOIN {{ source('uniswap_v2_ethereum', 'Factory_evt_PairCreated') }} pair ON pair.pair = swap.contract_address + JOIN zeroex_tx ON zeroex_tx.tx_hash = swap.evt_tx_hash and swap.evt_block_time = zeroex_tx.block_time + WHERE sender = 0xdef1c0ded9bec7f1a1670819833240f027b25eff + + {% if is_incremental() %} + AND {{ incremental_predicate('swap.evt_block_time') }} + {% endif %} + {% if not is_incremental() %} + AND swap.evt_block_time >= cast('{{zeroex_v3_start_date}}' as date) + {% endif %} + ), -direct_sushiswap as ( - {{ - direct_sushiswap( - blockchain = blockchain, - zeroex_v3_start_date = zeroex_v3_start_date - ) - }} +direct_sushiswap AS ( + SELECT + swap.evt_tx_hash AS tx_hash, + swap.evt_index, + swap.contract_address, + swap.evt_block_time AS block_time, + swap.contract_address AS maker, + LAST_VALUE(swap.to) OVER (PARTITION BY swap.evt_tx_hash ORDER BY swap.evt_index) AS taker, + CASE WHEN swap.amount0In > swap.amount0Out THEN pair.token0 ELSE pair.token1 END AS taker_token, + CASE WHEN swap.amount0In > swap.amount0Out THEN pair.token1 ELSE pair.token0 END AS maker_token, + CASE WHEN swap.amount0In > swap.amount0Out THEN + CASE WHEN swap.amount0In >= swap.amount0Out THEN cast(swap.amount0In - swap.amount0Out as int256) ELSE cast(0 as int256) END ELSE + CASE WHEN swap.amount1In >= swap.amount1Out THEN cast(swap.amount1In - swap.amount1Out as int256) ELSE cast(0 as int256) END END AS taker_token_amount_raw, + CASE WHEN swap.amount0In > swap.amount0Out THEN + CASE WHEN swap.amount1Out >= swap.amount1In THEN cast(swap.amount1Out - swap.amount1In as int256) ELSE cast(0 as int256) END ELSE + CASE WHEN swap.amount0Out >= swap.amount0In THEN cast(swap.amount0Out - swap.amount0In as int256) ELSE cast(0 as int256) END END AS maker_token_amount_raw, + + 'Sushiswap Direct' AS type, + zeroex_tx.affiliate_address AS affiliate_address, + TRUE AS swap_flag, + FALSE AS matcha_limit_order_flag, + is_gasless + FROM {{ source('sushi_ethereum', 'Pair_evt_Swap') }} swap + LEFT JOIN {{ source('sushi_ethereum', 'Factory_evt_PairCreated') }} pair ON pair.pair = swap.contract_address + JOIN zeroex_tx ON zeroex_tx.tx_hash = swap.evt_tx_hash and swap.evt_block_time = zeroex_tx.block_time + WHERE sender = 0xdef1c0ded9bec7f1a1670819833240f027b25eff + + {% if is_incremental() %} + AND {{ incremental_predicate('swap.evt_block_time') }} + {% endif %} + {% if not is_incremental() %} + AND swap.evt_block_time >= cast('{{zeroex_v3_start_date}}' as date) + {% endif %} ), -direct_uniswapv3 as ( - {{ - direct_uniswapv3( - blockchain = blockchain, - zeroex_v4_start_date = zeroex_v4_start_date - ) - }} +direct_uniswapv3 AS ( + SELECT + swap.evt_tx_hash AS tx_hash, + swap.evt_index, + swap.contract_address, + swap.evt_block_time AS block_time, + swap.contract_address AS maker, + 0xdef1c0ded9bec7f1a1670819833240f027b25eff as taker, + CASE WHEN amount0 < cast(0 as int256) THEN pair.token1 ELSE pair.token0 END AS taker_token, + CASE WHEN amount0 < cast(0 as int256) THEN pair.token0 ELSE pair.token1 END AS maker_token, + CASE WHEN amount0 < cast(0 as int256) THEN ABS(swap.amount1) ELSE ABS(swap.amount0) END AS taker_token_amount_raw, + CASE WHEN amount0 < cast(0 as int256) THEN ABS(swap.amount0) ELSE ABS(swap.amount1) END AS maker_token_amount_raw, + 'Uniswap V3 Direct' AS type, + zeroex_tx.affiliate_address AS affiliate_address, + TRUE AS swap_flag, + FALSE AS matcha_limit_order_flag, + is_gasless + FROM {{ source('uniswap_v3_ethereum', 'Pair_evt_Swap') }} swap + LEFT JOIN {{ source('uniswap_v3_ethereum', 'Factory_evt_PoolCreated') }} pair ON pair.pool = swap.contract_address + JOIN zeroex_tx ON zeroex_tx.tx_hash = swap.evt_tx_hash and swap.evt_block_time = zeroex_tx.block_time + WHERE sender = 0xdef1c0ded9bec7f1a1670819833240f027b25eff + + {% if is_incremental() %} + AND {{ incremental_predicate('swap.evt_block_time') }} + {% endif %} + {% if not is_incremental() %} + AND swap.evt_block_time >= cast('{{zeroex_v4_start_date}}' as date) + {% endif %} + ), all_tx AS ( SELECT * @@ -152,14 +436,86 @@ all_tx AS ( FROM v4_limit_fills_no_bridge UNION ALL SELECT * FROM otc_fills -), -tbl_trade_details AS ( - {{ - trade_details( - blockchain = blockchain, - zeroex_v3_start_date = zeroex_v3_start_date - ) - }} ) -select * from tbl_trade_details -order by block_time desc \ No newline at end of file + +SELECT + all_tx.tx_hash, + tx.block_number, + all_tx.evt_index, + all_tx.contract_address, + all_tx.block_time, + cast(date_trunc('day', all_tx.block_time) AS date) AS block_date, + cast(date_trunc('month', all_tx.block_time) AS date) AS block_month, + maker, + CASE + when is_gasless = 1 and varbinary_position (data, 0x3a46c4e1) <> 0 then varbinary_substring(data,81,20) + when is_gasless = 1 and varbinary_position (data, 0xa98fcbf1) <> 0 then varbinary_substring(data,81,20) + when is_gasless = 1 and varbinary_position (data, 0x3d8d4082) <> 0 then varbinary_substring(data,177,20) + + WHEN taker = 0xdef1c0ded9bec7f1a1670819833240f027b25eff THEN tx."from" + ELSE taker + END AS taker, -- fix the user masked by ProxyContract issue + taker_token, + ts.symbol AS taker_symbol, + maker_token, + ms.symbol AS maker_symbol, + CASE WHEN lower(ts.symbol) > lower(ms.symbol) THEN concat(ms.symbol, '-', ts.symbol) ELSE concat(ts.symbol, '-', ms.symbol) END AS token_pair, + taker_token_amount_raw / pow(10, tp.decimals) AS taker_token_amount, + cast(taker_token_amount_raw as uint256) as taker_token_amount_raw, + maker_token_amount_raw / pow(10, mp.decimals) AS maker_token_amount, + cast(maker_token_amount_raw as uint256) as maker_token_amount_raw, + all_tx.type, + max(affiliate_address) over (partition by all_tx.tx_hash) as affiliate_address, + swap_flag, + matcha_limit_order_flag, + CASE WHEN maker_token IN (0xc02aaa39b223fe8d0a0e5c4f27ead9083c756cc2,0xa0b86991c6218b36c1d19d4a2e9eb0ce3606eb48,0xdac17f958d2ee523a2206206994597c13d831ec7, + 0x4fabb145d64652a948d72533023f6e7a623c7c53,0x6b175474e89094c44da98b954eedeac495271d0f,0xae7ab96520de3a18e5e111b5eaab095312d7fe84) AND mp.price IS NOT NULL + THEN (all_tx.maker_token_amount_raw / pow(10, mp.decimals)) * mp.price + WHEN taker_token IN (0xc02aaa39b223fe8d0a0e5c4f27ead9083c756cc2,0xa0b86991c6218b36c1d19d4a2e9eb0ce3606eb48,0xdac17f958d2ee523a2206206994597c13d831ec7, + 0x4fabb145d64652a948d72533023f6e7a623c7c53,0x6b175474e89094c44da98b954eedeac495271d0f,0xae7ab96520de3a18e5e111b5eaab095312d7fe84) AND tp.price IS NOT NULL + THEN (all_tx.taker_token_amount_raw / pow(10, tp.decimals)) * tp.price + ELSE COALESCE((all_tx.maker_token_amount_raw / pow(10, mp.decimals)) * mp.price, (all_tx.taker_token_amount_raw / pow(10, tp.decimals)) * tp.price) + END AS volume_usd, + tx."from" AS tx_from, + tx.to AS tx_to, + 'ethereum' AS blockchain +FROM all_tx +INNER JOIN {{ source('ethereum', 'transactions')}} tx ON all_tx.tx_hash = tx.hash and all_tx.block_time = tx.block_time + +{% if is_incremental() %} +AND {{ incremental_predicate('tx.block_time') }} +{% endif %} +{% if not is_incremental() %} +AND tx.block_time >= cast('{{zeroex_v3_start_date}}' as date) +{% endif %} + +LEFT JOIN {{ source('prices', 'usd') }} tp ON date_trunc('minute', all_tx.block_time) = tp.minute +AND CASE + WHEN all_tx.taker_token = 0xeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeee THEN 0xc02aaa39b223fe8d0a0e5c4f27ead9083c756cc2 + ELSE all_tx.taker_token + END = tp.contract_address +AND tp.blockchain = 'ethereum' + +{% if is_incremental() %} +AND {{ incremental_predicate('tp.minute') }} +{% endif %} +{% if not is_incremental() %} +AND tp.minute >= cast('{{zeroex_v3_start_date}}' as date) +{% endif %} + +LEFT JOIN {{ source('prices', 'usd') }} mp ON DATE_TRUNC('minute', all_tx.block_time) = mp.minute +AND CASE + WHEN all_tx.maker_token = 0xeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeee THEN 0xc02aaa39b223fe8d0a0e5c4f27ead9083c756cc2 + ELSE all_tx.maker_token + END = mp.contract_address +AND mp.blockchain = 'ethereum' + +{% if is_incremental() %} +AND {{ incremental_predicate('mp.minute') }} +{% endif %} +{% if not is_incremental() %} +AND mp.minute >= cast('{{zeroex_v3_start_date}}' as date) +{% endif %} + +LEFT OUTER JOIN {{ source('tokens', 'erc20') }} ts ON ts.contract_address = taker_token and ts.blockchain = 'ethereum' +LEFT OUTER JOIN {{ source('tokens', 'erc20') }} ms ON ms.contract_address = maker_token and ms.blockchain = 'ethereum' diff --git a/dbt_subprojects/dex/macros/models/_project/zeroex/zeroex_v1_deduped_trades.sql b/dbt_subprojects/dex/models/_projects/zeroex/ethereum/zeroex_ethereum_api_fills_deduped.sql similarity index 80% rename from dbt_subprojects/dex/macros/models/_project/zeroex/zeroex_v1_deduped_trades.sql rename to dbt_subprojects/dex/models/_projects/zeroex/ethereum/zeroex_ethereum_api_fills_deduped.sql index 76f95860aaa..9a23ab938c9 100644 --- a/dbt_subprojects/dex/macros/models/_project/zeroex/zeroex_v1_deduped_trades.sql +++ b/dbt_subprojects/dex/models/_projects/zeroex/ethereum/zeroex_ethereum_api_fills_deduped.sql @@ -1,20 +1,36 @@ -{% macro zeroex_v1_deduped_trades(blockchain, start_date) %} +{{ config( -{% set table_name = 'zeroex_' ~ blockchain ~ '_api_fills' %} + schema = 'zeroex_ethereum', + alias = 'api_fills_deduped', + materialized='incremental', + partition_by = ['block_month'], + unique_key = ['block_date', 'tx_hash', 'evt_index'], + on_schema_change='sync_all_columns', + file_format ='delta', + incremental_strategy='merge', + incremental_predicates = [incremental_predicate('DBT_INTERNAL_DEST.block_time')] + ) +}} + +{% set zeroex_v3_start_date = '2019-12-01' %} +--the code used to create the data for the insertion into the dex.trades table +--this code is also the deduped version for the fills tables. +--only the data for 0x API fills. +--dependent on:zeroex_ethereum.api_fills. WITH fills_with_tx_fill_number AS ( SELECT row_number() OVER ( partition BY tx_hash ORDER BY evt_index ASC ) AS tx_fill_number , * - FROM {{ ref(table_name) }} + FROM {{ ref('zeroex_ethereum_api_fills') }} WHERE 1=1 AND swap_flag = true {% if is_incremental() %} AND {{ incremental_predicate('block_time') }} {% endif %} {% if not is_incremental() %} - AND block_time >= DATE '{{start_date}}' + AND block_time >= cast('{{zeroex_v3_start_date}}' as date) {% endif %} ) , fills_first_last @@ -50,7 +66,7 @@ AS , MAX(CASE WHEN maker_consider_flag = 0 THEN NULL ELSE maker_token_amount_raw END) AS maker_token_amount_raw , COUNT(*) AS fills_within FROM fills_first_last a - GROUP BY tx_hash + GROUP BY tx_hash,hop_count ) SELECT a.blockchain , '0x API' as project @@ -82,6 +98,4 @@ SELECT a.blockchain , a.contract_address FROM fills_with_tx_fill_number a INNER JOIN deduped_bridge_fills b - ON a.tx_hash = b.tx_hash AND a.evt_index = b.evt_index -{% endmacro %} - + ON a.tx_hash = b.tx_hash AND a.evt_index = b.evt_index \ No newline at end of file diff --git a/dbt_subprojects/dex/models/_projects/zeroex/ethereum/zeroex_ethereum_schema.yml b/dbt_subprojects/dex/models/_projects/zeroex/ethereum/zeroex_ethereum_schema.yml index 134e8bc863f..1337fcc8435 100644 --- a/dbt_subprojects/dex/models/_projects/zeroex/ethereum/zeroex_ethereum_schema.yml +++ b/dbt_subprojects/dex/models/_projects/zeroex/ethereum/zeroex_ethereum_schema.yml @@ -5,7 +5,7 @@ models: meta: blockchain: ethereum project: zeroex - contributors: danning.sui, bakabhai993, hosuke, rantum + contributors: danning.sui, bakabhai993, hosuke config: tags: ['ethereum','0x','dex_aggregator','dex','aggregator'] description: > @@ -106,11 +106,11 @@ models: name: blockchain - - name: zeroex_v1_ethereum_deduped_trades + - name: zeroex_ethereum_api_fills_deduped meta: blockchain: ethereum project: zeroex - contributors: bakabhai993, rantum + contributors: bakabhai993 config: tags: ['ethereum','0x','dex_aggregator','dex','aggregator'] description: > diff --git a/dbt_subprojects/dex/models/_projects/zeroex/ethereum/zeroex_v1_ethereum_deduped_trades.sql b/dbt_subprojects/dex/models/_projects/zeroex/ethereum/zeroex_v1_ethereum_deduped_trades.sql deleted file mode 100644 index 5b0debbdab0..00000000000 --- a/dbt_subprojects/dex/models/_projects/zeroex/ethereum/zeroex_v1_ethereum_deduped_trades.sql +++ /dev/null @@ -1,30 +0,0 @@ -{{ config( - schema = 'zeroex_v1_ethereum', - alias = 'deduped_trades', - materialized='incremental', - partition_by = ['block_month'], - unique_key = ['block_date', 'tx_hash', 'evt_index'], - on_schema_change='sync_all_columns', - file_format ='delta', - incremental_strategy='merge', - incremental_predicates = [incremental_predicate('DBT_INTERNAL_DEST.block_time')] - ) -}} - -{% set zeroex_v3_start_date = '2019-12-01' %} -{% set blockchain = 'ethereum' %} - -WITH -deduped_trades as ( - {{ - zeroex_v1_deduped_trades( - blockchain = blockchain, - start_date = zeroex_v3_start_date - - ) - }} - -) -select - * -from deduped_trades diff --git a/dbt_subprojects/dex/models/_projects/zeroex/zeroex_api_fills_deduped.sql b/dbt_subprojects/dex/models/_projects/zeroex/zeroex_api_fills_deduped.sql index 2b00a9a3f86..07f52a7d2f9 100644 --- a/dbt_subprojects/dex/models/_projects/zeroex/zeroex_api_fills_deduped.sql +++ b/dbt_subprojects/dex/models/_projects/zeroex/zeroex_api_fills_deduped.sql @@ -12,9 +12,9 @@ {% set zeroex_models = [ ref('zeroex_arbitrum_api_fills_deduped') ,ref('zeroex_avalanche_c_api_fills_deduped') - ,ref('zeroex_v1_base_deduped_trades') + ,ref('zeroex_base_api_fills_deduped') ,ref('zeroex_celo_api_fills_deduped') - ,ref('zeroex_v1_ethereum_deduped_trades') + ,ref('zeroex_ethereum_api_fills_deduped') ,ref('zeroex_fantom_api_fills_deduped') ,ref('zeroex_optimism_api_fills_deduped') ,ref('zeroex_polygon_api_fills_deduped') From c475f441eb2eb49fcb7770633fb29399bc38c870 Mon Sep 17 00:00:00 2001 From: jeff-dude <102681548+jeff-dude@users.noreply.github.com> Date: Mon, 25 Nov 2024 16:22:48 -0500 Subject: [PATCH 02/18] Revert 0x changes: downstream is broken (#7189) * Revert "zid selection update (#7178)" This reverts commit 11e45bbc1610577673823eb3df4b19a463ee683a. * Revert "zeroex - v2 macro model replacements (#7166)" This reverts commit 51d47e46dde928ddab907220749908365214da21. * Revert "add zeroex_v2_mode trades (#7167)" This reverts commit 241e6a344c87b35d756878bd1f541a2d57041dc0. --- .../models/_project/zeroex/zeroex_v2.sql | 73 +++-- .../arbitrum/zeroex_arbitrum_schema.yml | 4 +- .../zeroex_arbitrum_settler_trades.sql | 310 ++++++++++++++++++ .../arbitrum/zeroex_v2_arbitrum_trades.sql | 71 ---- .../avalanche_c/zeroex_avalanche_c_schema.yml | 4 +- .../zeroex_avalanche_c_settler_trades.sql | 310 ++++++++++++++++++ .../zeroex_v2_avalanche_c_trades.sql | 70 ---- .../zeroex/base/zeroex_v2_base_trades.sql | 13 +- .../zeroex/blast/zeroex_blast_schema.yml | 4 +- .../blast/zeroex_blast_settler_trades.sql | 301 +++++++++++++++++ .../zeroex/blast/zeroex_v2_blast_trades.sql | 71 ---- .../ethereum/zeroex_v2_ethereum_trades.sql | 13 +- .../zeroex/mantle/zeroex_mantle_schema.yml | 4 +- .../mantle/zeroex_mantle_settler_trades.sql | 296 +++++++++++++++++ .../zeroex/mantle/zeroex_v2_mantle_trades.sql | 71 ---- .../zeroex/mode/zeroex_mode_schema.yml | 111 ------- .../zeroex/mode/zeroex_v2_mode_trades.sql | 61 ---- .../polygon/zeroex_v2_polygon_trades.sql | 13 +- .../zeroex/zeroex_api_fills_deduped.sql | 11 +- .../models/_projects/zeroex/zeroex_schema.yml | 2 +- .../seeds/_project/zeroex/mode/_schema.yml | 11 - .../mode/zeroex_v2_mode_trades_sample.csv | 4 - 22 files changed, 1275 insertions(+), 553 deletions(-) create mode 100644 dbt_subprojects/dex/models/_projects/zeroex/arbitrum/zeroex_arbitrum_settler_trades.sql delete mode 100644 dbt_subprojects/dex/models/_projects/zeroex/arbitrum/zeroex_v2_arbitrum_trades.sql create mode 100644 dbt_subprojects/dex/models/_projects/zeroex/avalanche_c/zeroex_avalanche_c_settler_trades.sql delete mode 100644 dbt_subprojects/dex/models/_projects/zeroex/avalanche_c/zeroex_v2_avalanche_c_trades.sql create mode 100644 dbt_subprojects/dex/models/_projects/zeroex/blast/zeroex_blast_settler_trades.sql delete mode 100644 dbt_subprojects/dex/models/_projects/zeroex/blast/zeroex_v2_blast_trades.sql create mode 100644 dbt_subprojects/dex/models/_projects/zeroex/mantle/zeroex_mantle_settler_trades.sql delete mode 100644 dbt_subprojects/dex/models/_projects/zeroex/mantle/zeroex_v2_mantle_trades.sql delete mode 100644 dbt_subprojects/dex/models/_projects/zeroex/mode/zeroex_mode_schema.yml delete mode 100644 dbt_subprojects/dex/models/_projects/zeroex/mode/zeroex_v2_mode_trades.sql delete mode 100644 dbt_subprojects/dex/seeds/_project/zeroex/mode/_schema.yml delete mode 100644 dbt_subprojects/dex/seeds/_project/zeroex/mode/zeroex_v2_mode_trades_sample.csv diff --git a/dbt_subprojects/dex/macros/models/_project/zeroex/zeroex_v2.sql b/dbt_subprojects/dex/macros/models/_project/zeroex/zeroex_v2.sql index 601061e71ba..4589ba128b7 100644 --- a/dbt_subprojects/dex/macros/models/_project/zeroex/zeroex_v2.sql +++ b/dbt_subprojects/dex/macros/models/_project/zeroex/zeroex_v2.sql @@ -1,15 +1,15 @@ {% macro zeroex_settler_txs_cte(blockchain, start_date) %} WITH tbl_addresses AS ( SELECT - varbinary_to_int256 (topic1) as token_id, - bytearray_substring(logs.topic3,13,20) as settler_address, + token_id, + "to" AS settler_address, block_time AS begin_block_time, block_number AS begin_block_number FROM - {{ source( blockchain, 'logs') }} + {{ source('nft', 'transfers') }} WHERE contract_address = 0x00000000000004533fe15556b1e086bb1a72ceae - and topic0 = 0xaa94c583a45742b26ac5274d230aea34ab334ed5722264aa5673010e612bc0b2 + AND blockchain = '{{ blockchain }}' ), tbl_end_times AS ( @@ -38,8 +38,7 @@ settler_trace_data AS ( "to" AS contract_address, varbinary_substring(input,1,4) AS method_id, varbinary_substring(input,varbinary_position(input,0xfd3ad6d4)+132,32) tracker, - a.settler_address, - row_number() OVER (PARTITION BY tr.tx_hash ORDER BY trace_address) AS rn + a.settler_address FROM {{ source(blockchain, 'traces') }} AS tr JOIN @@ -62,14 +61,15 @@ settler_txs AS ( method_id, contract_address, settler_address, - varbinary_substring(tracker,2,12) AS zid, + MAX(varbinary_substring(tracker,2,12)) AS zid, CASE - WHEN method_id = 0x1fff991f THEN varbinary_substring(tracker,14,3) - WHEN method_id = 0xfd3ad6d4 THEN varbinary_substring(tracker,13,3) + WHEN method_id = 0x1fff991f THEN MAX(varbinary_substring(tracker,14,3)) + WHEN method_id = 0xfd3ad6d4 THEN MAX(varbinary_substring(tracker,13,3)) END AS tag FROM settler_trace_data - WHERE rn = 1 + GROUP BY + 1,2,3,4,5,6 ) SELECT * FROM settler_txs @@ -171,6 +171,30 @@ prices AS ( {% endif %} ), +fills AS ( + WITH signatures AS ( + SELECT DISTINCT signature + FROM {{ source(blockchain, 'logs_decoded') }} l + JOIN tbl_trades tt ON tt.tx_hash = l.tx_hash AND l.block_time = tt.block_time AND l.block_number = tt.block_number + WHERE event_name IN ('TokenExchange', 'OtcOrderFilled', 'SellBaseToken', 'Swap', 'BuyGem', 'DODOSwap', 'SellGem', 'Submitted') + {% if is_incremental() %} + AND {{ incremental_predicate('l.block_time') }} + {% else %} + AND l.block_time >= DATE '{{start_date}}' + {% endif %} + ) + SELECT tt.tx_hash, tt.block_number, tt.block_time, COUNT(*) AS fills_within + FROM {{ source(blockchain, 'logs') }} l + JOIN signatures ON signature = topic0 + JOIN tbl_trades tt ON tt.tx_hash = l.tx_hash AND l.block_time = tt.block_time AND l.block_number = tt.block_number + {% if is_incremental() %} + WHERE {{ incremental_predicate('l.block_time') }} + {% else %} + WHERE l.block_time >= DATE '{{start_date}}' + {% endif %} + GROUP BY 1,2,3 +), + results AS ( SELECT '{{blockchain}}' AS blockchain, @@ -200,7 +224,8 @@ results AS ( maker_amount / POW(10,COALESCE(tm.decimals,pm.decimals)) AS maker_token_amount, maker_amount / POW(10,COALESCE(tm.decimals,pm.decimals)) AS token_bought_amount, maker_amount / POW(10,COALESCE(tm.decimals,pm.decimals)) * pm.price AS maker_amount, - tag + tag, + fills_within FROM tbl_trades trades JOIN @@ -210,6 +235,8 @@ results AS ( {% else %} AND tr.block_time >= DATE '{{start_date}}' {% endif %} + LEFT JOIN + fills f ON f.tx_hash = trades.tx_hash AND f.block_time = trades.block_time AND f.block_number = trades.block_number LEFT JOIN tokens tt ON tt.blockchain = '{{blockchain}}' AND tt.contract_address = taker_token LEFT JOIN @@ -269,28 +296,4 @@ order by block_time desc {% macro zeroex_v2_trades_indirect(blockchain, start_date) %} {{ zeroex_v2_trades(blockchain, start_date, false) }} -{% endmacro %} - -{% macro zeroex_v2_trades_fills_count(blockchain, start_date) %} - WITH signatures AS ( - SELECT DISTINCT signature - FROM {{ source(blockchain, 'logs_decoded') }} l - JOIN tbl_trades tt ON tt.tx_hash = l.tx_hash AND l.block_time = tt.block_time AND l.block_number = tt.block_number - WHERE event_name IN ('TokenExchange', 'OtcOrderFilled', 'SellBaseToken', 'Swap', 'BuyGem', 'DODOSwap', 'SellGem', 'Submitted') - {% if is_incremental() %} - AND {{ incremental_predicate('l.block_time') }} - {% else %} - AND l.block_time >= DATE '{{start_date}}' - {% endif %} - ) - SELECT tt.tx_hash, tt.block_number, tt.block_time, COUNT(*) AS fills_within - FROM {{ source(blockchain, 'logs') }} l - JOIN signatures ON signature = topic0 - JOIN tbl_trades tt ON tt.tx_hash = l.tx_hash AND l.block_time = tt.block_time AND l.block_number = tt.block_number - {% if is_incremental() %} - WHERE {{ incremental_predicate('l.block_time') }} - {% else %} - WHERE l.block_time >= DATE '{{start_date}}' - {% endif %} - GROUP BY 1,2,3 {% endmacro %} \ No newline at end of file diff --git a/dbt_subprojects/dex/models/_projects/zeroex/arbitrum/zeroex_arbitrum_schema.yml b/dbt_subprojects/dex/models/_projects/zeroex/arbitrum/zeroex_arbitrum_schema.yml index 3ca18250dcb..3498e9df2ef 100644 --- a/dbt_subprojects/dex/models/_projects/zeroex/arbitrum/zeroex_arbitrum_schema.yml +++ b/dbt_subprojects/dex/models/_projects/zeroex/arbitrum/zeroex_arbitrum_schema.yml @@ -206,7 +206,7 @@ models: name: protocol_fee_paid_eth description: "The protocol fee paid in ETH" - - name: zeroex_v2_arbitrum_trades + - name: zeroex_arbitrum_settler_trades meta: blockchain: arbitrum project: zeroex @@ -214,7 +214,7 @@ models: config: tags: ['arbitrum','0x','dex_aggregator','dex','aggregator'] description: > - 0x API erc20 trades through 0x v2 contracts. No fills, only deduped transactions. + 0x API erc20 trades through 0x Settler contracts. No fills, only deduped transactions. data_tests: - dbt_utils.unique_combination_of_columns: combination_of_columns: diff --git a/dbt_subprojects/dex/models/_projects/zeroex/arbitrum/zeroex_arbitrum_settler_trades.sql b/dbt_subprojects/dex/models/_projects/zeroex/arbitrum/zeroex_arbitrum_settler_trades.sql new file mode 100644 index 00000000000..815021d5009 --- /dev/null +++ b/dbt_subprojects/dex/models/_projects/zeroex/arbitrum/zeroex_arbitrum_settler_trades.sql @@ -0,0 +1,310 @@ +{{ config( + schema = 'zeroex_arbitrum', + alias = 'settler_trades', + materialized='incremental', + partition_by = ['block_month'], + unique_key = ['block_month', 'block_date', 'tx_hash', 'evt_index'], + on_schema_change='sync_all_columns', + file_format ='delta', + incremental_strategy='merge', + incremental_predicates = [incremental_predicate('DBT_INTERNAL_DEST.block_time')] +)}} + +{% set zeroex_settler_start_date = '2024-07-15' %} + +WITH tbl_addresses AS ( + SELECT + blockchain, + token_id, + to AS settler_address, + block_time AS begin_block_time, + block_number AS begin_block_number + FROM + {{ source('nft', 'transfers') }} + WHERE + contract_address = 0x00000000000004533fe15556b1e086bb1a72ceae + AND blockchain = 'arbitrum' + and block_time > TIMESTAMP '2024-05-23' +), + +tbl_end_times AS ( + SELECT + *, + LEAD(begin_block_time) OVER (PARTITION BY blockchain, token_id ORDER BY begin_block_time) AS end_block_time, + LEAD(begin_block_number) OVER (PARTITION BY blockchain, token_id ORDER BY begin_block_number) AS end_block_number + FROM + tbl_addresses +), + +result_0x_settler_addresses AS ( + SELECT + * + FROM + tbl_end_times + WHERE + settler_address != 0x0000000000000000000000000000000000000000 +), + +settler_txs AS ( + SELECT + tx_hash, + block_time, + block_number, + method_id, + contract_address, + settler_address, + MAX(varbinary_substring(tracker,2,12)) AS zid, + CASE + WHEN method_id = 0x1fff991f THEN MAX(varbinary_substring(tracker,14,3)) + WHEN method_id = 0xfd3ad6d4 THEN MAX(varbinary_substring(tracker,13,3)) + END AS tag + FROM ( + SELECT + tr.tx_hash, + block_number, + block_time, + "to" AS contract_address, + varbinary_substring(input,1,4) AS method_id, + varbinary_substring(input,varbinary_position(input,0xfd3ad6d4)+132,32) tracker, + a.settler_address + FROM + {{ source('arbitrum', 'traces') }} AS tr + LEFT JOIN + result_0x_settler_addresses a ON a.settler_address = tr.to AND a.blockchain = 'arbitrum' AND tr.block_time > a.begin_block_time + WHERE + (a.settler_address is not null or tr.to in (0x0000000000001fF3684f28c67538d4D072C22734)) + AND varbinary_substring(input,1,4) IN (0x1fff991f, 0xfd3ad6d4) + {% if is_incremental() %} + AND {{ incremental_predicate('block_time') }} + {% else %} + AND block_time >= DATE '{{zeroex_settler_start_date}}' + {% endif %} + ) + GROUP BY + 1,2,3,4,5,6 +), +tbl_trades as ( + +with tbl_all_logs AS ( + SELECT + logs.tx_hash, + logs.block_time, + logs.block_number, + index, + case when ( (varbinary_substring(logs.topic2, 13, 20) = logs.tx_from) OR + (varbinary_substring(logs.topic2, 13, 20) = first_value(bytearray_substring(logs.topic1,13,20)) over (partition by logs.tx_hash order by index) ) or + topic0 = 0x7fcf532c15f0a6db0bd6d0e038bea71d30d808c7d98cb3bf7268a95bf5081b65) + then 1 end as valid, + coalesce(bytearray_substring(logs.topic2,13,20), first_value(bytearray_substring(logs.topic1,13,20)) over (partition by logs.tx_hash order by index) ) as taker, + logs.contract_address as maker_token, + first_value(logs.contract_address) over (partition by logs.tx_hash order by index) as taker_token, + first_value(try_cast(bytearray_to_uint256(bytearray_substring(DATA, 22,11)) as int256) ) over (partition by logs.tx_hash order by index) as taker_amount, + try_cast(bytearray_to_uint256(bytearray_substring(DATA, 22,11)) as int256) as maker_amount, + method_id, + tag, + st.settler_address, + zid, + st.settler_address as contract_address + FROM + {{ source('arbitrum', 'logs') }} AS logs + JOIN + settler_txs st ON st.tx_hash = logs.tx_hash + AND logs.block_time = st.block_time + AND st.block_number = logs.block_number + AND ( (st.settler_address = bytearray_substring(logs.topic1,13,20)) + or (st.settler_address= bytearray_substring(logs.topic2,13,20)) + or logs.tx_from = bytearray_substring(logs.topic1,13,20) + or logs.tx_from = bytearray_substring(logs.topic2,13,20) + or logs.tx_to = varbinary_substring(logs.topic1,13,20) + ) + WHERE + topic0 IN ( 0x7fcf532c15f0a6db0bd6d0e038bea71d30d808c7d98cb3bf7268a95bf5081b65, + 0xddf252ad1be2c89b69c2b068fc378daa952ba7f163c4a11628f55a4df523b3ef, + 0xe1fffcc4923d04b559f4d29a8bfc6cda04eb5b0d3c460751c2402c5c5cc9109c ) + and topic1 != 0x0000000000000000000000000000000000000000000000000000000000000000 + and zid != 0xa00000000000000000000000 + {% if is_incremental() %} + AND {{ incremental_predicate('logs.block_time') }} + {% else %} + AND logs.block_time >= DATE '{{zeroex_settler_start_date}}' + {% endif %} + ), + tbl_valid_logs as ( + select * + , row_number() over (partition by tx_hash order by index desc) rn + from tbl_all_logs + where taker_token != maker_token + ) + select * from tbl_valid_logs where rn = 1 +), + + +tokens AS ( + with token_list as ( + select + distinct maker_token as token + from + tbl_trades + + union distinct + + select + distinct taker_token as token + from tbl_trades + ) + + select * + from + token_list tl + join + {{ source('tokens', 'erc20') }} AS te ON te.contract_address = tl.token + WHERE + te.blockchain = 'arbitrum' +), + +prices AS ( + SELECT DISTINCT + pu.* + FROM + {{ source('prices', 'usd') }} AS pu + JOIN + tbl_trades ON (pu.contract_address = taker_token OR pu.contract_address = maker_token) AND date_trunc('minute',block_time) = minute + WHERE + pu.blockchain = 'arbitrum' + {% if is_incremental() %} + AND {{ incremental_predicate('minute') }} + {% else %} + AND minute >= DATE '{{zeroex_settler_start_date}}' + {% endif %} +), + +fills as ( + with signatures as ( + select distinct signature + from {{ source('arbitrum', 'logs_decoded') }} l + join tbl_trades tt on tt.tx_hash = l.tx_hash and l.block_time = tt.block_time and l.block_number = tt.block_number + and event_name in ('TokenExchange', 'OtcOrderFilled', 'SellarbitrumToken', 'Swap', 'BuyGem', 'DODOSwap', 'SellGem', 'Submitted') + WHERE 1=1 + {% if is_incremental() %} + AND {{ incremental_predicate('l.block_time') }} + {% else %} + AND l.block_time >= DATE '{{zeroex_settler_start_date}}' + {% endif %} + ) + + select tt.tx_hash, tt.block_number, tt.block_time, count(*) fills_within + from {{ source('arbitrum', 'logs') }} l + join signatures on signature = topic0 + join tbl_trades tt on tt.tx_hash = l.tx_hash and l.block_time = tt.block_time and l.block_number = tt.block_number + WHERE 1=1 + {% if is_incremental() %} + AND {{ incremental_predicate('l.block_time') }} + {% else %} + AND l.block_time >= DATE '{{zeroex_settler_start_date}}' + {% endif %} + group by 1,2,3 + ), + +results AS ( + SELECT + trades.block_time, + trades.block_number, + zid, + trades.contract_address, + method_id, + trades.tx_hash, + "from" AS tx_from, + "to" AS tx_to, + trades.index AS tx_index, + case when varbinary_substring(tr.data,1,4) = 0x500c22bc then "from" else taker end as taker, + CAST(NULL AS varbinary) AS maker, + taker_token, + pt.price, + COALESCE(tt.symbol, pt.symbol) AS taker_symbol, + taker_amount AS taker_token_amount_raw, + taker_amount / POW(10,COALESCE(tt.decimals,pt.decimals)) AS taker_token_amount, + taker_amount / POW(10,COALESCE(tt.decimals,pt.decimals)) * pt.price AS taker_amount, + maker_token, + COALESCE(tm.symbol, pm.symbol) AS maker_symbol, + maker_amount AS maker_token_amount_raw, + maker_amount / POW(10,COALESCE(tm.decimals,pm.decimals)) AS maker_token_amount, + maker_amount / POW(10,COALESCE(tm.decimals,pm.decimals)) * pm.price AS maker_amount, + tag, + fills_within + FROM + tbl_trades trades + JOIN + {{ source('arbitrum', 'transactions') }} tr ON tr.hash = trades.tx_hash AND tr.block_time = trades.block_time AND tr.block_number = trades.block_number + LEFT JOIN + fills f ON f.tx_hash = trades.tx_hash AND f.block_time = trades.block_time AND f.block_number = trades.block_number + LEFT JOIN + tokens tt ON tt.blockchain = 'arbitrum' AND tt.contract_address = taker_token + LEFT JOIN + tokens tm ON tm.blockchain = 'arbitrum' AND tm.contract_address = maker_token + LEFT JOIN + prices pt ON pt.blockchain = 'arbitrum' AND pt.contract_address = taker_token AND pt.minute = DATE_TRUNC('minute', trades.block_time) + LEFT JOIN + prices pm ON pm.blockchain = 'arbitrum' AND pm.contract_address = maker_token AND pm.minute = DATE_TRUNC('minute', trades.block_time) + WHERE + 1=1 + {% if is_incremental() %} + AND {{ incremental_predicate('tr.block_time') }} + {% else %} + AND tr.block_time >= DATE '{{zeroex_settler_start_date}}' + {% endif %} +), + +results_usd AS ( + SELECT + 'arbitrum' AS blockchain, + '0x-API' AS project, + 'settler' AS version, + DATE_TRUNC('day', block_time) block_date, + DATE_TRUNC('month', block_time) AS block_month, + block_time, + taker_symbol, + maker_symbol, + CASE WHEN LOWER(taker_symbol) > LOWER(maker_symbol) THEN CONCAT(maker_symbol, '-', taker_symbol) ELSE CONCAT(taker_symbol, '-', maker_symbol) END AS token_pair, + taker_token_amount, + maker_token_amount, + taker_token_amount_raw, + maker_token_amount_raw, + CASE WHEN maker_token IN (0x82af49447d8a07e3bd95bd0d56f35241523fbab1, + 0xaf88d065e77c8cc2239327c5edb3a432268e5831, + 0xff970a61a04b1ca14834a43f5de4533ebddb5cc8, + 0xfd086bc7cd5c481dcc9c85ebe478a1c0b69fcbb9, + 0x912ce59144191c1204e64559fe8253a0e49e6548) AND maker_amount IS NOT NULL + THEN maker_amount + WHEN taker_token IN (0x82af49447d8a07e3bd95bd0d56f35241523fbab1, + 0xaf88d065e77c8cc2239327c5edb3a432268e5831, + 0xff970a61a04b1ca14834a43f5de4533ebddb5cc8, + 0xfd086bc7cd5c481dcc9c85ebe478a1c0b69fcbb9, + 0x912ce59144191c1204e64559fe8253a0e49e6548) AND taker_amount IS NOT NULL + THEN taker_amount + ELSE COALESCE(maker_amount, taker_amount) + END AS volume_usd, + taker_token, + maker_token, + taker, + maker, + tag, + zid, + tx_hash, + tx_from, + tx_to, + tx_index AS evt_index, + (ARRAY[-1]) AS trace_address, + 'settler' AS type, + TRUE AS swap_flag, + fills_within, + contract_address + FROM + results +) + +SELECT DISTINCT + * +FROM + results_usd +ORDER BY + block_time DESC \ No newline at end of file diff --git a/dbt_subprojects/dex/models/_projects/zeroex/arbitrum/zeroex_v2_arbitrum_trades.sql b/dbt_subprojects/dex/models/_projects/zeroex/arbitrum/zeroex_v2_arbitrum_trades.sql deleted file mode 100644 index 95cd5eec698..00000000000 --- a/dbt_subprojects/dex/models/_projects/zeroex/arbitrum/zeroex_v2_arbitrum_trades.sql +++ /dev/null @@ -1,71 +0,0 @@ -{{ config( - schema = 'zeroex_v2_arbitrum', - alias = 'trades', - materialized='incremental', - partition_by = ['block_month'], - unique_key = ['block_month', 'block_date', 'tx_hash', 'evt_index'], - on_schema_change='sync_all_columns', - file_format ='delta', - incremental_strategy='merge', - incremental_predicates = [incremental_predicate('DBT_INTERNAL_DEST.block_time')] -)}} - -{% set zeroex_settler_start_date = '2024-07-15' %} -{% set blockchain = 'arbitrum' %} - -WITH zeroex_tx AS ( - {{ - zeroex_settler_txs_cte( - blockchain = blockchain, - start_date = zeroex_settler_start_date - ) - }} -), -zeroex_v2_trades_direct AS ( - {{ - zeroex_v2_trades_direct( - blockchain = blockchain, - start_date = zeroex_settler_start_date - - ) - }} -), -zeroex_v2_trades_indirect AS ( - {{ - zeroex_v2_trades_indirect( - blockchain = blockchain, - start_date = zeroex_settler_start_date - - ) - }} -), -tbl_trades AS ( - SELECT * - FROM zeroex_v2_trades_direct - UNION ALL - SELECT * - FROM zeroex_v2_trades_indirect -), -fills_count as( - {{ - zeroex_v2_trades_fills_count( - blockchain = blockchain, - start_date = zeroex_settler_start_date - ) - }} -), -trade_details as ( - {{ - zeroex_v2_trades_detail( - blockchain = blockchain, - start_date = zeroex_settler_start_date - - ) - }} - -) -select - t.*, fills_within - from trade_details t - left join fills_count f on t.tx_hash = f.tx_hash and t.block_time = f.block_time - \ No newline at end of file diff --git a/dbt_subprojects/dex/models/_projects/zeroex/avalanche_c/zeroex_avalanche_c_schema.yml b/dbt_subprojects/dex/models/_projects/zeroex/avalanche_c/zeroex_avalanche_c_schema.yml index 98b01576297..48deac58eea 100644 --- a/dbt_subprojects/dex/models/_projects/zeroex/avalanche_c/zeroex_avalanche_c_schema.yml +++ b/dbt_subprojects/dex/models/_projects/zeroex/avalanche_c/zeroex_avalanche_c_schema.yml @@ -150,7 +150,7 @@ models: name: fills_within description: "fills in then multihop, if present" - - name: zeroex_v2_avalanche_c_trades + - name: zeroex_avalanche_c_settler_trades meta: blockchain: avalanche_c project: zeroex @@ -158,7 +158,7 @@ models: config: tags: ['avalanche_c','0x','dex_aggregator','dex','aggregator'] description: > - 0x API erc20 trades through 0x v2 contracts. No fills, only deduped transactions. + 0x API erc20 trades through 0x Settler contracts. No fills, only deduped transactions. data_tests: - dbt_utils.unique_combination_of_columns: combination_of_columns: diff --git a/dbt_subprojects/dex/models/_projects/zeroex/avalanche_c/zeroex_avalanche_c_settler_trades.sql b/dbt_subprojects/dex/models/_projects/zeroex/avalanche_c/zeroex_avalanche_c_settler_trades.sql new file mode 100644 index 00000000000..4434aa4ad0c --- /dev/null +++ b/dbt_subprojects/dex/models/_projects/zeroex/avalanche_c/zeroex_avalanche_c_settler_trades.sql @@ -0,0 +1,310 @@ +{{ config( + schema = 'zeroex_avalanche_c', + alias = 'settler_trades', + materialized='incremental', + partition_by = ['block_month'], + unique_key = ['block_month', 'block_date', 'tx_hash', 'evt_index'], + on_schema_change='sync_all_columns', + file_format ='delta', + incremental_strategy='merge', + incremental_predicates = [incremental_predicate('DBT_INTERNAL_DEST.block_time')] +)}} + +{% set zeroex_settler_start_date = '2024-07-15' %} + +WITH tbl_addresses AS ( + SELECT + blockchain, + token_id, + to AS settler_address, + block_time AS begin_block_time, + block_number AS begin_block_number + FROM + {{ source('nft', 'transfers') }} + WHERE + contract_address = 0x00000000000004533fe15556b1e086bb1a72ceae + AND blockchain = 'avalanche_c' + and block_time > TIMESTAMP '2024-05-23' +), + +tbl_end_times AS ( + SELECT + *, + LEAD(begin_block_time) OVER (PARTITION BY blockchain, token_id ORDER BY begin_block_time) AS end_block_time, + LEAD(begin_block_number) OVER (PARTITION BY blockchain, token_id ORDER BY begin_block_number) AS end_block_number + FROM + tbl_addresses +), + +result_0x_settler_addresses AS ( + SELECT + * + FROM + tbl_end_times + WHERE + settler_address != 0x0000000000000000000000000000000000000000 +), + +settler_txs AS ( + SELECT + tx_hash, + block_time, + block_number, + method_id, + contract_address, + settler_address, + MAX(varbinary_substring(tracker,2,12)) AS zid, + CASE + WHEN method_id = 0x1fff991f THEN MAX(varbinary_substring(tracker,14,3)) + WHEN method_id = 0xfd3ad6d4 THEN MAX(varbinary_substring(tracker,13,3)) + END AS tag + FROM ( + SELECT + tr.tx_hash, + block_number, + block_time, + "to" AS contract_address, + varbinary_substring(input,1,4) AS method_id, + varbinary_substring(input,varbinary_position(input,0xfd3ad6d4)+132,32) tracker, + a.settler_address + FROM + {{ source('avalanche_c', 'traces') }} AS tr + LEFT JOIN + result_0x_settler_addresses a ON a.settler_address = tr.to AND a.blockchain = 'avalanche_c' AND tr.block_time > a.begin_block_time + WHERE + (a.settler_address IS NOT NULL OR tr.to = 0x0000000000005E88410CcDFaDe4a5EfaE4b49562 ) + AND varbinary_substring(input,1,4) IN (0x1fff991f, 0xfd3ad6d4) + {% if is_incremental() %} + AND {{ incremental_predicate('block_time') }} + {% else %} + AND block_time >= DATE '{{zeroex_settler_start_date}}' + {% endif %} + ) + GROUP BY + 1,2,3,4,5,6 +), +tbl_trades as ( + +with tbl_all_logs AS ( + SELECT + logs.tx_hash, + logs.block_time, + logs.block_number, + index, + case when ( (varbinary_substring(logs.topic2, 13, 20) = logs.tx_from) OR + (varbinary_substring(logs.topic2, 13, 20) = first_value(bytearray_substring(logs.topic1,13,20)) over (partition by logs.tx_hash order by index) ) or + topic0 = 0x7fcf532c15f0a6db0bd6d0e038bea71d30d808c7d98cb3bf7268a95bf5081b65) + then 1 end as valid, + coalesce(bytearray_substring(logs.topic2,13,20), first_value(bytearray_substring(logs.topic1,13,20)) over (partition by logs.tx_hash order by index) ) as taker, + logs.contract_address as maker_token, + first_value(logs.contract_address) over (partition by logs.tx_hash order by index) as taker_token, + first_value(try_cast(bytearray_to_uint256(bytearray_substring(DATA, 22,11)) as int256) ) over (partition by logs.tx_hash order by index) as taker_amount, + try_cast(bytearray_to_uint256(bytearray_substring(DATA, 22,11)) as int256) as maker_amount, + method_id, + tag, + st.settler_address, + zid, + st.settler_address as contract_address + FROM + {{ source('avalanche_c', 'logs') }} AS logs + JOIN + settler_txs st ON st.tx_hash = logs.tx_hash + AND logs.block_time = st.block_time + AND st.block_number = logs.block_number + AND ( (st.settler_address = bytearray_substring(logs.topic1,13,20)) + or (st.settler_address= bytearray_substring(logs.topic2,13,20)) + or logs.tx_from = bytearray_substring(logs.topic1,13,20) + or logs.tx_from = bytearray_substring(logs.topic2,13,20) + or logs.tx_to = varbinary_substring(logs.topic1,13,20) + ) + WHERE + topic0 IN ( 0x7fcf532c15f0a6db0bd6d0e038bea71d30d808c7d98cb3bf7268a95bf5081b65, + 0xddf252ad1be2c89b69c2b068fc378daa952ba7f163c4a11628f55a4df523b3ef, + 0xe1fffcc4923d04b559f4d29a8bfc6cda04eb5b0d3c460751c2402c5c5cc9109c ) + and topic1 != 0x0000000000000000000000000000000000000000000000000000000000000000 + and zid != 0xa00000000000000000000000 + {% if is_incremental() %} + AND {{ incremental_predicate('logs.block_time') }} + {% else %} + AND logs.block_time >= DATE '{{zeroex_settler_start_date}}' + {% endif %} + ), + tbl_valid_logs as ( + select * + , row_number() over (partition by tx_hash order by valid, index desc) rn + from tbl_all_logs + where taker_token != maker_token + ) + select * from tbl_valid_logs where rn = 1 +), + + +tokens AS ( + with token_list as ( + select + distinct maker_token as token + from + tbl_trades + + union distinct + + select + distinct taker_token as token + from tbl_trades + ) + + select * + from + token_list tl + join + {{ source('tokens', 'erc20') }} AS te ON te.contract_address = tl.token + WHERE + te.blockchain = 'avalanche_c' +), + +prices AS ( + SELECT DISTINCT + pu.* + FROM + {{ source('prices', 'usd') }} AS pu + JOIN + tbl_trades ON (pu.contract_address = taker_token OR pu.contract_address = maker_token) AND date_trunc('minute',block_time) = minute + WHERE + pu.blockchain = 'avalanche_c' + {% if is_incremental() %} + AND {{ incremental_predicate('minute') }} + {% else %} + AND minute >= DATE '{{zeroex_settler_start_date}}' + {% endif %} +), + +fills as ( + with signatures as ( + select distinct signature + from {{ source('avalanche_c', 'logs_decoded') }} l + join tbl_trades tt on tt.tx_hash = l.tx_hash and l.block_time = tt.block_time and l.block_number = tt.block_number + and event_name in ('TokenExchange', 'OtcOrderFilled', 'Sellavalanche_cToken', 'Swap', 'BuyGem', 'DODOSwap', 'SellGem', 'Submitted') + WHERE 1=1 + {% if is_incremental() %} + AND {{ incremental_predicate('l.block_time') }} + {% else %} + AND l.block_time >= DATE '{{zeroex_settler_start_date}}' + {% endif %} + ) + + select tt.tx_hash, tt.block_number, tt.block_time, count(*) fills_within + from {{ source('avalanche_c', 'logs') }} l + join signatures on signature = topic0 + join tbl_trades tt on tt.tx_hash = l.tx_hash and l.block_time = tt.block_time and l.block_number = tt.block_number + WHERE 1=1 + {% if is_incremental() %} + AND {{ incremental_predicate('l.block_time') }} + {% else %} + AND l.block_time >= DATE '{{zeroex_settler_start_date}}' + {% endif %} + group by 1,2,3 + ), + +results AS ( + SELECT + trades.block_time, + trades.block_number, + zid, + trades.contract_address, + method_id, + trades.tx_hash, + "from" AS tx_from, + "to" AS tx_to, + trades.index AS tx_index, + case when varbinary_substring(tr.data,1,4) = 0x500c22bc then "from" else taker end as taker, + CAST(NULL AS varbinary) AS maker, + taker_token, + pt.price, + COALESCE(tt.symbol, pt.symbol) AS taker_symbol, + taker_amount AS taker_token_amount_raw, + taker_amount / POW(10,COALESCE(tt.decimals,pt.decimals)) AS taker_token_amount, + taker_amount / POW(10,COALESCE(tt.decimals,pt.decimals)) * pt.price AS taker_amount, + maker_token, + COALESCE(tm.symbol, pm.symbol) AS maker_symbol, + maker_amount AS maker_token_amount_raw, + maker_amount / POW(10,COALESCE(tm.decimals,pm.decimals)) AS maker_token_amount, + maker_amount / POW(10,COALESCE(tm.decimals,pm.decimals)) * pm.price AS maker_amount, + tag, + fills_within + FROM + tbl_trades trades + JOIN + {{ source('avalanche_c', 'transactions') }} tr ON tr.hash = trades.tx_hash AND tr.block_time = trades.block_time AND tr.block_number = trades.block_number + LEFT JOIN + fills f ON f.tx_hash = trades.tx_hash AND f.block_time = trades.block_time AND f.block_number = trades.block_number + LEFT JOIN + tokens tt ON tt.blockchain = 'avalanche_c' AND tt.contract_address = taker_token + LEFT JOIN + tokens tm ON tm.blockchain = 'avalanche_c' AND tm.contract_address = maker_token + LEFT JOIN + prices pt ON pt.blockchain = 'avalanche_c' AND pt.contract_address = taker_token AND pt.minute = DATE_TRUNC('minute', trades.block_time) + LEFT JOIN + prices pm ON pm.blockchain = 'avalanche_c' AND pm.contract_address = maker_token AND pm.minute = DATE_TRUNC('minute', trades.block_time) + WHERE + 1=1 + {% if is_incremental() %} + AND {{ incremental_predicate('tr.block_time') }} + {% else %} + AND tr.block_time >= DATE '{{zeroex_settler_start_date}}' + {% endif %} +), + +results_usd AS ( + SELECT + 'avalanche_c' AS blockchain, + '0x-API' AS project, + 'settler' AS version, + DATE_TRUNC('day', block_time) block_date, + DATE_TRUNC('month', block_time) AS block_month, + block_time, + taker_symbol, + maker_symbol, + CASE WHEN LOWER(taker_symbol) > LOWER(maker_symbol) THEN CONCAT(maker_symbol, '-', taker_symbol) ELSE CONCAT(taker_symbol, '-', maker_symbol) END AS token_pair, + taker_token_amount, + maker_token_amount, + taker_token_amount_raw, + maker_token_amount_raw, + CASE WHEN maker_token IN (0xb31f66aa3c1e785363f0875a1b74e27b85fd66c7, + 0xb97ef9ef8734c71904d8002f8b6bc66dd9c48a6e, + 0x152b9d0fdc40c096757f570a51e494bd4b943e50, + 0x49d5c2bdffac6ce2bfdb6640f4f80f226bc10bab, + 0x9702230a8ea53601f5cd2dc00fdbc13d4df4a8c7) AND maker_amount IS NOT NULL + THEN maker_amount + WHEN taker_token IN (0xb31f66aa3c1e785363f0875a1b74e27b85fd66c7, + 0xb97ef9ef8734c71904d8002f8b6bc66dd9c48a6e, + 0x152b9d0fdc40c096757f570a51e494bd4b943e50, + 0x49d5c2bdffac6ce2bfdb6640f4f80f226bc10bab, + 0x9702230a8ea53601f5cd2dc00fdbc13d4df4a8c7) AND taker_amount IS NOT NULL + THEN taker_amount + ELSE COALESCE(maker_amount, taker_amount) + END AS volume_usd, + taker_token, + maker_token, + taker, + maker, + tag, + zid, + tx_hash, + tx_from, + tx_to, + tx_index AS evt_index, + (ARRAY[-1]) AS trace_address, + 'settler' AS type, + TRUE AS swap_flag, + fills_within, + contract_address + FROM + results +) + +SELECT DISTINCT + * +FROM + results_usd +ORDER BY + block_time DESC \ No newline at end of file diff --git a/dbt_subprojects/dex/models/_projects/zeroex/avalanche_c/zeroex_v2_avalanche_c_trades.sql b/dbt_subprojects/dex/models/_projects/zeroex/avalanche_c/zeroex_v2_avalanche_c_trades.sql deleted file mode 100644 index a24523639a1..00000000000 --- a/dbt_subprojects/dex/models/_projects/zeroex/avalanche_c/zeroex_v2_avalanche_c_trades.sql +++ /dev/null @@ -1,70 +0,0 @@ -{{ config( - schema = 'zeroex_v2_avalanche_c', - alias = 'trades', - materialized='incremental', - partition_by = ['block_month'], - unique_key = ['block_month', 'block_date', 'tx_hash', 'evt_index'], - on_schema_change='sync_all_columns', - file_format ='delta', - incremental_strategy='merge', - incremental_predicates = [incremental_predicate('DBT_INTERNAL_DEST.block_time')] -)}} - -{% set zeroex_settler_start_date = '2024-07-15' %} -{% set blockchain = 'avalanche_c' %} - -WITH zeroex_tx AS ( - {{ - zeroex_settler_txs_cte( - blockchain = blockchain, - start_date = zeroex_settler_start_date - ) - }} -), -zeroex_v2_trades_direct AS ( - {{ - zeroex_v2_trades_direct( - blockchain = blockchain, - start_date = zeroex_settler_start_date - - ) - }} -), -zeroex_v2_trades_indirect AS ( - {{ - zeroex_v2_trades_indirect( - blockchain = blockchain, - start_date = zeroex_settler_start_date - - ) - }} -), -tbl_trades AS ( - SELECT * - FROM zeroex_v2_trades_direct - UNION ALL - SELECT * - FROM zeroex_v2_trades_indirect -), -fills_count as( - {{ - zeroex_v2_trades_fills_count( - blockchain = blockchain, - start_date = zeroex_settler_start_date - ) - }} -), -trade_details as ( - {{ - zeroex_v2_trades_detail( - blockchain = blockchain, - start_date = zeroex_settler_start_date - - ) - }} - -) -select - t.*, fills_within - from trade_details t - left join fills_count f on t.tx_hash = f.tx_hash and t.block_time = f.block_time \ No newline at end of file diff --git a/dbt_subprojects/dex/models/_projects/zeroex/base/zeroex_v2_base_trades.sql b/dbt_subprojects/dex/models/_projects/zeroex/base/zeroex_v2_base_trades.sql index 2dc75f30a85..d69c3174508 100644 --- a/dbt_subprojects/dex/models/_projects/zeroex/base/zeroex_v2_base_trades.sql +++ b/dbt_subprojects/dex/models/_projects/zeroex/base/zeroex_v2_base_trades.sql @@ -46,14 +46,6 @@ tbl_trades AS ( SELECT * FROM zeroex_v2_trades_indirect ), -fills_count as( - {{ - zeroex_v2_trades_fills_count( - blockchain = blockchain, - start_date = zeroex_settler_start_date - ) - }} -), trade_details as ( {{ zeroex_v2_trades_detail( @@ -65,6 +57,5 @@ trade_details as ( ) select - t.*, fills_within - from trade_details t - left join fills_count f on t.tx_hash = f.tx_hash and t.block_time = f.block_time + * + from trade_details diff --git a/dbt_subprojects/dex/models/_projects/zeroex/blast/zeroex_blast_schema.yml b/dbt_subprojects/dex/models/_projects/zeroex/blast/zeroex_blast_schema.yml index 4361c49a24b..598e21561d9 100644 --- a/dbt_subprojects/dex/models/_projects/zeroex/blast/zeroex_blast_schema.yml +++ b/dbt_subprojects/dex/models/_projects/zeroex/blast/zeroex_blast_schema.yml @@ -1,7 +1,7 @@ version: 2 models: - - name: zeroex_v2_blast_trades + - name: zeroex_blast_settler_trades meta: blockchain: blast project: zeroex @@ -9,7 +9,7 @@ models: config: tags: ['blast','0x','dex_aggregator','dex','aggregator'] description: > - 0x API erc20 trades through 0x v2 contracts. No fills, only deduped transactions. + 0x API erc20 trades through 0x Settler contracts. No fills, only deduped transactions. data_tests: - dbt_utils.unique_combination_of_columns: combination_of_columns: diff --git a/dbt_subprojects/dex/models/_projects/zeroex/blast/zeroex_blast_settler_trades.sql b/dbt_subprojects/dex/models/_projects/zeroex/blast/zeroex_blast_settler_trades.sql new file mode 100644 index 00000000000..9fe304e77ba --- /dev/null +++ b/dbt_subprojects/dex/models/_projects/zeroex/blast/zeroex_blast_settler_trades.sql @@ -0,0 +1,301 @@ +{{ config( + schema = 'zeroex_blast', + alias = 'settler_trades', + materialized='incremental', + partition_by = ['block_month'], + unique_key = ['block_month', 'block_date', 'tx_hash', 'evt_index'], + on_schema_change='sync_all_columns', + file_format ='delta', + incremental_strategy='merge', + incremental_predicates = [incremental_predicate('DBT_INTERNAL_DEST.block_time')] +)}} + +{% set zeroex_settler_start_date = '2024-07-15' %} + +WITH tbl_addresses AS ( + SELECT + blockchain, + token_id, + to AS settler_address, + block_time AS begin_block_time, + block_number AS begin_block_number + FROM + {{ source('nft', 'transfers') }} + WHERE + contract_address = 0x00000000000004533fe15556b1e086bb1a72ceae + AND blockchain = 'blast' + and block_time > TIMESTAMP '2024-05-23' +), + +tbl_end_times AS ( + SELECT + *, + LEAD(begin_block_time) OVER (PARTITION BY blockchain, token_id ORDER BY begin_block_time) AS end_block_time, + LEAD(begin_block_number) OVER (PARTITION BY blockchain, token_id ORDER BY begin_block_number) AS end_block_number + FROM + tbl_addresses +), + +result_0x_settler_addresses AS ( + SELECT + * + FROM + tbl_end_times + WHERE + settler_address != 0x0000000000000000000000000000000000000000 +), + +settler_txs AS ( + SELECT + tx_hash, + block_time, + block_number, + method_id, + contract_address, + settler_address, + MAX(varbinary_substring(tracker,2,12)) AS zid, + CASE + WHEN method_id = 0x1fff991f THEN MAX(varbinary_substring(tracker,14,3)) + WHEN method_id = 0xfd3ad6d4 THEN MAX(varbinary_substring(tracker,13,3)) + END AS tag + FROM ( + SELECT + tr.tx_hash, + block_number, + block_time, + "to" AS contract_address, + varbinary_substring(input,1,4) AS method_id, + varbinary_substring(input,varbinary_position(input,0xfd3ad6d4)+132,32) tracker, + a.settler_address + FROM + {{ source('blast', 'traces') }} AS tr + LEFT JOIN + result_0x_settler_addresses a ON a.settler_address = tr.to AND a.blockchain = 'blast' AND tr.block_time > a.begin_block_time + WHERE + (a.settler_address IS NOT NULL OR tr.to = 0x0000000000001fF3684f28c67538d4D072C22734) + AND varbinary_substring(input,1,4) IN (0x1fff991f, 0xfd3ad6d4) + {% if is_incremental() %} + AND {{ incremental_predicate('block_time') }} + {% else %} + AND block_time >= DATE '{{zeroex_settler_start_date}}' + {% endif %} + ) + GROUP BY + 1,2,3,4,5,6 +), +tbl_trades as ( + +with tbl_all_logs AS ( + SELECT + logs.tx_hash, + logs.block_time, + logs.block_number, + index, + coalesce(bytearray_substring(logs.topic2,13,20), first_value(bytearray_substring(logs.topic1,13,20)) over (partition by logs.tx_hash order by index) ) as taker, + logs.contract_address as maker_token, + first_value(logs.contract_address) over (partition by logs.tx_hash order by index) as taker_token, + first_value(try_cast(bytearray_to_uint256(bytearray_substring(DATA, 22,11)) as int256) ) over (partition by logs.tx_hash order by index) as taker_amount, + try_cast(bytearray_to_uint256(bytearray_substring(DATA, 22,11)) as int256) as maker_amount, + method_id, + tag, + st.settler_address, + zid, + st.settler_address as contract_address + FROM + {{ source('blast', 'logs') }} AS logs + JOIN + settler_txs st ON st.tx_hash = logs.tx_hash AND logs.block_time = st.block_time AND st.block_number = logs.block_number + AND ( (st.settler_address = bytearray_substring(logs.topic1,13,20)) + or (st.settler_address= bytearray_substring(logs.topic2,13,20)) + or logs.tx_from = varbinary_substring(logs.topic1,13,20) + or logs.tx_from = varbinary_substring(logs.topic2,13,20) + or logs.tx_to = varbinary_substring(logs.topic1,13,20) + ) + WHERE + topic0 IN ( 0x7fcf532c15f0a6db0bd6d0e038bea71d30d808c7d98cb3bf7268a95bf5081b65, + 0xddf252ad1be2c89b69c2b068fc378daa952ba7f163c4a11628f55a4df523b3ef, + 0xe1fffcc4923d04b559f4d29a8bfc6cda04eb5b0d3c460751c2402c5c5cc9109c ) + and topic1 != 0x0000000000000000000000000000000000000000000000000000000000000000 + and zid != 0xa00000000000000000000000 + {% if is_incremental() %} + AND {{ incremental_predicate('logs.block_time') }} + {% else %} + AND logs.block_time >= DATE '{{zeroex_settler_start_date}}' + {% endif %} + ), + tbl_valid_logs as ( + select * + , row_number() over (partition by tx_hash order by index desc) rn + from tbl_all_logs + where maker_token != taker_token + ) + select * from tbl_valid_logs + where rn = 1 +), + + +tokens AS ( + with token_list as ( + select + distinct maker_token as token + from + tbl_trades + + union distinct + + select + distinct taker_token as token + from tbl_trades + ) + + select * + from + token_list tl + join + {{ source('tokens', 'erc20') }} AS te ON te.contract_address = tl.token + WHERE + te.blockchain = 'blast' +), + +prices AS ( + SELECT DISTINCT + pu.* + FROM + {{ source('prices', 'usd') }} AS pu + JOIN + tbl_trades ON (pu.contract_address = taker_token OR pu.contract_address = maker_token) AND date_trunc('minute',block_time) = minute + WHERE + pu.blockchain = 'blast' + {% if is_incremental() %} + AND {{ incremental_predicate('minute') }} + {% else %} + AND minute >= DATE '{{zeroex_settler_start_date}}' + {% endif %} +), + +fills as ( + with signatures as ( + select distinct signature + from {{ source('blast', 'logs_decoded') }} l + join tbl_trades tt on tt.tx_hash = l.tx_hash and l.block_time = tt.block_time and l.block_number = tt.block_number + and event_name in ('TokenExchange', 'OtcOrderFilled', 'SellblastToken', 'Swap', 'BuyGem', 'DODOSwap', 'SellGem', 'Submitted') + WHERE 1=1 + {% if is_incremental() %} + AND {{ incremental_predicate('l.block_time') }} + {% else %} + AND l.block_time >= DATE '{{zeroex_settler_start_date}}' + {% endif %} + ) + + select tt.tx_hash, tt.block_number, tt.block_time, count(*) fills_within + from {{ source('blast', 'logs') }} l + join signatures on signature = topic0 + join tbl_trades tt on tt.tx_hash = l.tx_hash and l.block_time = tt.block_time and l.block_number = tt.block_number + WHERE 1=1 + {% if is_incremental() %} + AND {{ incremental_predicate('l.block_time') }} + {% else %} + AND l.block_time >= DATE '{{zeroex_settler_start_date}}' + {% endif %} + group by 1,2,3 + ), + +results AS ( + SELECT + trades.block_time, + trades.block_number, + zid, + trades.contract_address, + method_id, + trades.tx_hash, + "from" AS tx_from, + "to" AS tx_to, + trades.index AS tx_index, + case when varbinary_substring(tr.data,1,4) = 0x500c22bc then "from" else taker end as taker, + CAST(NULL AS varbinary) AS maker, + taker_token, + pt.price, + COALESCE(tt.symbol, pt.symbol) AS taker_symbol, + taker_amount AS taker_token_amount_raw, + taker_amount / POW(10,COALESCE(tt.decimals,pt.decimals)) AS taker_token_amount, + taker_amount / POW(10,COALESCE(tt.decimals,pt.decimals)) * pt.price AS taker_amount, + maker_token, + COALESCE(tm.symbol, pm.symbol) AS maker_symbol, + maker_amount AS maker_token_amount_raw, + maker_amount / POW(10,COALESCE(tm.decimals,pm.decimals)) AS maker_token_amount, + maker_amount / POW(10,COALESCE(tm.decimals,pm.decimals)) * pm.price AS maker_amount, + tag, + fills_within + FROM + tbl_trades trades + JOIN + {{ source('blast', 'transactions') }} tr ON tr.hash = trades.tx_hash AND tr.block_time = trades.block_time AND tr.block_number = trades.block_number + LEFT JOIN + fills f ON f.tx_hash = trades.tx_hash AND f.block_time = trades.block_time AND f.block_number = trades.block_number + LEFT JOIN + tokens tt ON tt.blockchain = 'blast' AND tt.contract_address = taker_token + LEFT JOIN + tokens tm ON tm.blockchain = 'blast' AND tm.contract_address = maker_token + LEFT JOIN + prices pt ON pt.blockchain = 'blast' AND pt.contract_address = taker_token AND pt.minute = DATE_TRUNC('minute', trades.block_time) + LEFT JOIN + prices pm ON pm.blockchain = 'blast' AND pm.contract_address = maker_token AND pm.minute = DATE_TRUNC('minute', trades.block_time) + WHERE + 1=1 + {% if is_incremental() %} + AND {{ incremental_predicate('tr.block_time') }} + {% else %} + AND tr.block_time >= DATE '{{zeroex_settler_start_date}}' + {% endif %} +), + +results_usd AS ( + SELECT + 'blast' AS blockchain, + '0x-API' AS project, + 'settler' AS version, + DATE_TRUNC('day', block_time) block_date, + DATE_TRUNC('month', block_time) AS block_month, + block_time, + taker_symbol, + maker_symbol, + CASE WHEN LOWER(taker_symbol) > LOWER(maker_symbol) THEN CONCAT(maker_symbol, '-', taker_symbol) ELSE CONCAT(taker_symbol, '-', maker_symbol) END AS token_pair, + taker_token_amount, + maker_token_amount, + taker_token_amount_raw, + maker_token_amount_raw, + CASE WHEN maker_token IN (0x4300000000000000000000000000000000000004, + 0x4300000000000000000000000000000000000003, + 0xb1a5700fa2358173fe465e6ea4ff52e36e88e2ad ) AND maker_amount IS NOT NULL + THEN maker_amount + WHEN taker_token IN (0x4300000000000000000000000000000000000004, + 0x4300000000000000000000000000000000000003, + 0xb1a5700fa2358173fe465e6ea4ff52e36e88e2ad ) AND taker_amount IS NOT NULL + THEN taker_amount + ELSE COALESCE(maker_amount, taker_amount) + END AS volume_usd, + taker_token, + maker_token, + taker, + maker, + tag, + zid, + tx_hash, + tx_from, + tx_to, + tx_index AS evt_index, + (ARRAY[-1]) AS trace_address, + 'settler' AS type, + TRUE AS swap_flag, + fills_within, + contract_address + FROM + results +) + +SELECT DISTINCT + * +FROM + results_usd +ORDER BY + block_time DESC \ No newline at end of file diff --git a/dbt_subprojects/dex/models/_projects/zeroex/blast/zeroex_v2_blast_trades.sql b/dbt_subprojects/dex/models/_projects/zeroex/blast/zeroex_v2_blast_trades.sql deleted file mode 100644 index c382761f69d..00000000000 --- a/dbt_subprojects/dex/models/_projects/zeroex/blast/zeroex_v2_blast_trades.sql +++ /dev/null @@ -1,71 +0,0 @@ -{{ config( - schema = 'zeroex_v2_blast', - alias = 'trades', - materialized='incremental', - partition_by = ['block_month'], - unique_key = ['block_month', 'block_date', 'tx_hash', 'evt_index'], - on_schema_change='sync_all_columns', - file_format ='delta', - incremental_strategy='merge', - incremental_predicates = [incremental_predicate('DBT_INTERNAL_DEST.block_time')] -)}} - -{% set zeroex_settler_start_date = '2024-07-15' %} -{% set blockchain = 'blast' %} - -WITH zeroex_tx AS ( - {{ - zeroex_settler_txs_cte( - blockchain = blockchain, - start_date = zeroex_settler_start_date - ) - }} -), -zeroex_v2_trades_direct AS ( - {{ - zeroex_v2_trades_direct( - blockchain = blockchain, - start_date = zeroex_settler_start_date - - ) - }} -), -zeroex_v2_trades_indirect AS ( - {{ - zeroex_v2_trades_indirect( - blockchain = blockchain, - start_date = zeroex_settler_start_date - - ) - }} -), -tbl_trades AS ( - SELECT * - FROM zeroex_v2_trades_direct - UNION ALL - SELECT * - FROM zeroex_v2_trades_indirect -), -fills_count as( - {{ - zeroex_v2_trades_fills_count( - blockchain = blockchain, - start_date = zeroex_settler_start_date - ) - }} -), -trade_details as ( - {{ - zeroex_v2_trades_detail( - blockchain = blockchain, - start_date = zeroex_settler_start_date - - ) - }} - -) -select - t.*, fills_within - from trade_details t - left join fills_count f on t.tx_hash = f.tx_hash and t.block_time = f.block_time - \ No newline at end of file diff --git a/dbt_subprojects/dex/models/_projects/zeroex/ethereum/zeroex_v2_ethereum_trades.sql b/dbt_subprojects/dex/models/_projects/zeroex/ethereum/zeroex_v2_ethereum_trades.sql index 256d8476c7d..ceea2b3d384 100644 --- a/dbt_subprojects/dex/models/_projects/zeroex/ethereum/zeroex_v2_ethereum_trades.sql +++ b/dbt_subprojects/dex/models/_projects/zeroex/ethereum/zeroex_v2_ethereum_trades.sql @@ -46,14 +46,6 @@ tbl_trades AS ( SELECT * FROM zeroex_v2_trades_indirect ), -fills_count as( - {{ - zeroex_v2_trades_fills_count( - blockchain = blockchain, - start_date = zeroex_settler_start_date - ) - }} -), trade_details as ( {{ zeroex_v2_trades_detail( @@ -65,6 +57,5 @@ trade_details as ( ) select - t.*, fills_within - from trade_details t - left join fills_count f on t.tx_hash = f.tx_hash and t.block_time = f.block_time + * + from trade_details diff --git a/dbt_subprojects/dex/models/_projects/zeroex/mantle/zeroex_mantle_schema.yml b/dbt_subprojects/dex/models/_projects/zeroex/mantle/zeroex_mantle_schema.yml index 418999bf150..bbbabcb2e22 100644 --- a/dbt_subprojects/dex/models/_projects/zeroex/mantle/zeroex_mantle_schema.yml +++ b/dbt_subprojects/dex/models/_projects/zeroex/mantle/zeroex_mantle_schema.yml @@ -1,7 +1,7 @@ version: 2 models: - - name: zeroex_v2_mantle_trades + - name: zeroex_mantle_settler_trades meta: blockchain: mantle project: zeroex @@ -9,7 +9,7 @@ models: config: tags: ['mantle','0x','dex_aggregator','dex','aggregator'] description: > - 0x API erc20 trades through 0x v2 contracts. No fills, only deduped transactions. + 0x API erc20 trades through 0x Settler contracts. No fills, only deduped transactions. data_tests: - dbt_utils.unique_combination_of_columns: combination_of_columns: diff --git a/dbt_subprojects/dex/models/_projects/zeroex/mantle/zeroex_mantle_settler_trades.sql b/dbt_subprojects/dex/models/_projects/zeroex/mantle/zeroex_mantle_settler_trades.sql new file mode 100644 index 00000000000..35bd8dc974e --- /dev/null +++ b/dbt_subprojects/dex/models/_projects/zeroex/mantle/zeroex_mantle_settler_trades.sql @@ -0,0 +1,296 @@ +{{ config( + schema = 'zeroex_mantle', + alias = 'settler_trades', + materialized='incremental', + partition_by = ['block_month'], + unique_key = ['block_month', 'block_date', 'tx_hash', 'evt_index'], + on_schema_change='sync_all_columns', + file_format ='delta', + incremental_strategy='merge', + incremental_predicates = [incremental_predicate('DBT_INTERNAL_DEST.block_time')] +)}} + +{% set zeroex_settler_start_date = '2024-07-15' %} + +WITH tbl_addresses AS ( + SELECT + blockchain, + token_id, + to AS settler_address, + block_time AS begin_block_time, + block_number AS begin_block_number + FROM + {{ source('nft', 'transfers') }} + WHERE + contract_address = 0x00000000000004533fe15556b1e086bb1a72ceae + AND blockchain = 'mantle' + and block_time > TIMESTAMP '2024-05-23' +), + +tbl_end_times AS ( + SELECT + *, + LEAD(begin_block_time) OVER (PARTITION BY blockchain, token_id ORDER BY begin_block_time) AS end_block_time, + LEAD(begin_block_number) OVER (PARTITION BY blockchain, token_id ORDER BY begin_block_number) AS end_block_number + FROM + tbl_addresses +), + +result_0x_settler_addresses AS ( + SELECT + * + FROM + tbl_end_times + WHERE + settler_address != 0x0000000000000000000000000000000000000000 +), + +settler_txs AS ( + SELECT + tx_hash, + block_time, + block_number, + method_id, + contract_address, + settler_address, + MAX(varbinary_substring(tracker,2,12)) AS zid, + CASE + WHEN method_id = 0x1fff991f THEN MAX(varbinary_substring(tracker,14,3)) + WHEN method_id = 0xfd3ad6d4 THEN MAX(varbinary_substring(tracker,13,3)) + END AS tag + FROM ( + SELECT + tr.tx_hash, + block_number, + block_time, + "to" AS contract_address, + varbinary_substring(input,1,4) AS method_id, + varbinary_substring(input,varbinary_position(input,0xfd3ad6d4)+132,32) tracker, + a.settler_address + FROM + {{ source('mantle', 'traces') }} AS tr + JOIN + result_0x_settler_addresses a ON a.settler_address = tr.to AND a.blockchain = 'mantle' AND tr.block_time > a.begin_block_time + WHERE + (a.settler_address IS NOT NULL OR tr.to = 0xca11bde05977b3631167028862be2a173976ca11) + AND varbinary_substring(input,1,4) IN (0x1fff991f, 0xfd3ad6d4) + {% if is_incremental() %} + AND {{ incremental_predicate('block_time') }} + {% else %} + AND block_time >= DATE '{{zeroex_settler_start_date}}' + {% endif %} + ) + GROUP BY + 1,2,3,4,5,6 +), +tbl_trades as ( + +with tbl_all_logs AS ( + SELECT + logs.tx_hash, + logs.block_time, + logs.block_number, + index, + coalesce(bytearray_substring(logs.topic2,13,20), first_value(bytearray_substring(logs.topic1,13,20)) over (partition by logs.tx_hash order by index) ) as taker, + logs.contract_address as maker_token, + first_value(logs.contract_address) over (partition by logs.tx_hash order by index) as taker_token, + first_value(try_cast(bytearray_to_uint256(bytearray_substring(DATA, 22,11)) as int256) ) over (partition by logs.tx_hash order by index) as taker_amount, + try_cast(bytearray_to_uint256(bytearray_substring(DATA, 22,11)) as int256) as maker_amount, + method_id, + tag, + st.settler_address, + zid, + st.settler_address as contract_address + FROM + {{ source('mantle', 'logs') }} AS logs + JOIN + settler_txs st ON st.tx_hash = logs.tx_hash AND logs.block_time = st.block_time AND st.block_number = logs.block_number + + WHERE + topic0 IN ( 0x7fcf532c15f0a6db0bd6d0e038bea71d30d808c7d98cb3bf7268a95bf5081b65, + 0xddf252ad1be2c89b69c2b068fc378daa952ba7f163c4a11628f55a4df523b3ef, + 0xe1fffcc4923d04b559f4d29a8bfc6cda04eb5b0d3c460751c2402c5c5cc9109c ) + and topic1 != 0x0000000000000000000000000000000000000000000000000000000000000000 + and zid != 0xa00000000000000000000000 + {% if is_incremental() %} + AND {{ incremental_predicate('logs.block_time') }} + {% else %} + AND logs.block_time >= DATE '{{zeroex_settler_start_date}}' + {% endif %} + ), + tbl_valid_logs as ( + select * + , row_number() over (partition by tx_hash order by index desc) rn + from tbl_all_logs + where maker_token != taker_token + ) + select * from tbl_valid_logs + where rn = 1 +), + + +tokens AS ( + with token_list as ( + select + distinct maker_token as token + from + tbl_trades + + union distinct + + select + distinct taker_token as token + from tbl_trades + ) + + select * + from + token_list tl + join + {{ source('tokens', 'erc20') }} AS te ON te.contract_address = tl.token + WHERE + te.blockchain = 'mantle' +), + +prices AS ( + SELECT DISTINCT + pu.* + FROM + {{ source('prices', 'usd') }} AS pu + JOIN + tbl_trades ON (pu.contract_address = taker_token OR pu.contract_address = maker_token) AND date_trunc('minute',block_time) = minute + WHERE + pu.blockchain = 'mantle' + {% if is_incremental() %} + AND {{ incremental_predicate('minute') }} + {% else %} + AND minute >= DATE '{{zeroex_settler_start_date}}' + {% endif %} +), + +fills as ( + with signatures as ( + select distinct signature + from {{ source('mantle', 'logs_decoded') }} l + join tbl_trades tt on tt.tx_hash = l.tx_hash and l.block_time = tt.block_time and l.block_number = tt.block_number + and event_name in ('TokenExchange', 'OtcOrderFilled', 'SellmantleToken', 'Swap', 'BuyGem', 'DODOSwap', 'SellGem', 'Submitted') + WHERE 1=1 + {% if is_incremental() %} + AND {{ incremental_predicate('l.block_time') }} + {% else %} + AND l.block_time >= DATE '{{zeroex_settler_start_date}}' + {% endif %} + ) + + select tt.tx_hash, tt.block_number, tt.block_time, count(*) fills_within + from {{ source('mantle', 'logs') }} l + join signatures on signature = topic0 + join tbl_trades tt on tt.tx_hash = l.tx_hash and l.block_time = tt.block_time and l.block_number = tt.block_number + WHERE 1=1 + {% if is_incremental() %} + AND {{ incremental_predicate('l.block_time') }} + {% else %} + AND l.block_time >= DATE '{{zeroex_settler_start_date}}' + {% endif %} + group by 1,2,3 + ), + +results AS ( + SELECT + trades.block_time, + trades.block_number, + zid, + trades.contract_address, + method_id, + trades.tx_hash, + "from" AS tx_from, + "to" AS tx_to, + trades.index AS tx_index, + case when varbinary_substring(tr.data,1,4) = 0x500c22bc then "from" else taker end as taker, + CAST(NULL AS varbinary) AS maker, + taker_token, + pt.price, + COALESCE(tt.symbol, pt.symbol) AS taker_symbol, + taker_amount AS taker_token_amount_raw, + taker_amount / POW(10,COALESCE(tt.decimals,pt.decimals)) AS taker_token_amount, + taker_amount / POW(10,COALESCE(tt.decimals,pt.decimals)) * pt.price AS taker_amount, + maker_token, + COALESCE(tm.symbol, pm.symbol) AS maker_symbol, + maker_amount AS maker_token_amount_raw, + maker_amount / POW(10,COALESCE(tm.decimals,pm.decimals)) AS maker_token_amount, + maker_amount / POW(10,COALESCE(tm.decimals,pm.decimals)) * pm.price AS maker_amount, + tag, + fills_within + FROM + tbl_trades trades + JOIN + {{ source('mantle', 'transactions') }} tr ON tr.hash = trades.tx_hash AND tr.block_time = trades.block_time AND tr.block_number = trades.block_number + LEFT JOIN + fills f ON f.tx_hash = trades.tx_hash AND f.block_time = trades.block_time AND f.block_number = trades.block_number + LEFT JOIN + tokens tt ON tt.blockchain = 'mantle' AND tt.contract_address = taker_token + LEFT JOIN + tokens tm ON tm.blockchain = 'mantle' AND tm.contract_address = maker_token + LEFT JOIN + prices pt ON pt.blockchain = 'mantle' AND pt.contract_address = taker_token AND pt.minute = DATE_TRUNC('minute', trades.block_time) + LEFT JOIN + prices pm ON pm.blockchain = 'mantle' AND pm.contract_address = maker_token AND pm.minute = DATE_TRUNC('minute', trades.block_time) + WHERE + 1=1 + {% if is_incremental() %} + AND {{ incremental_predicate('tr.block_time') }} + {% else %} + AND tr.block_time >= DATE '{{zeroex_settler_start_date}}' + {% endif %} +), + +results_usd AS ( + SELECT + 'mantle' AS blockchain, + '0x-API' AS project, + 'settler' AS version, + DATE_TRUNC('day', block_time) block_date, + DATE_TRUNC('month', block_time) AS block_month, + block_time, + taker_symbol, + maker_symbol, + CASE WHEN LOWER(taker_symbol) > LOWER(maker_symbol) THEN CONCAT(maker_symbol, '-', taker_symbol) ELSE CONCAT(taker_symbol, '-', maker_symbol) END AS token_pair, + taker_token_amount, + maker_token_amount, + taker_token_amount_raw, + maker_token_amount_raw, + CASE WHEN maker_token IN (0x4300000000000000000000000000000000000004, + 0x4300000000000000000000000000000000000003, + 0xb1a5700fa2358173fe465e6ea4ff52e36e88e2ad ) AND maker_amount IS NOT NULL + THEN maker_amount + WHEN taker_token IN (0x4300000000000000000000000000000000000004, + 0x4300000000000000000000000000000000000003, + 0xb1a5700fa2358173fe465e6ea4ff52e36e88e2ad ) AND taker_amount IS NOT NULL + THEN taker_amount + ELSE COALESCE(maker_amount, taker_amount) + END AS volume_usd, + taker_token, + maker_token, + taker, + maker, + tag, + zid, + tx_hash, + tx_from, + tx_to, + tx_index AS evt_index, + (ARRAY[-1]) AS trace_address, + 'settler' AS type, + TRUE AS swap_flag, + fills_within, + contract_address + FROM + results +) + +SELECT DISTINCT + * +FROM + results_usd +ORDER BY + block_time DESC \ No newline at end of file diff --git a/dbt_subprojects/dex/models/_projects/zeroex/mantle/zeroex_v2_mantle_trades.sql b/dbt_subprojects/dex/models/_projects/zeroex/mantle/zeroex_v2_mantle_trades.sql deleted file mode 100644 index 88ba80757a3..00000000000 --- a/dbt_subprojects/dex/models/_projects/zeroex/mantle/zeroex_v2_mantle_trades.sql +++ /dev/null @@ -1,71 +0,0 @@ -{{ config( - schema = 'zeroex_v2_mantle', - alias = 'trades', - materialized='incremental', - partition_by = ['block_month'], - unique_key = ['block_month', 'block_date', 'tx_hash', 'evt_index'], - on_schema_change='sync_all_columns', - file_format ='delta', - incremental_strategy='merge', - incremental_predicates = [incremental_predicate('DBT_INTERNAL_DEST.block_time')] -)}} - -{% set zeroex_settler_start_date = '2024-07-15' %} -{% set blockchain = 'mantle' %} - -WITH zeroex_tx AS ( - {{ - zeroex_settler_txs_cte( - blockchain = blockchain, - start_date = zeroex_settler_start_date - ) - }} -), -zeroex_v2_trades_direct AS ( - {{ - zeroex_v2_trades_direct( - blockchain = blockchain, - start_date = zeroex_settler_start_date - - ) - }} -), -zeroex_v2_trades_indirect AS ( - {{ - zeroex_v2_trades_indirect( - blockchain = blockchain, - start_date = zeroex_settler_start_date - - ) - }} -), -tbl_trades AS ( - SELECT * - FROM zeroex_v2_trades_direct - UNION ALL - SELECT * - FROM zeroex_v2_trades_indirect -), -fills_count as( - {{ - zeroex_v2_trades_fills_count( - blockchain = blockchain, - start_date = zeroex_settler_start_date - ) - }} -), -trade_details as ( - {{ - zeroex_v2_trades_detail( - blockchain = blockchain, - start_date = zeroex_settler_start_date - - ) - }} - -) -select - t.*, fills_within - from trade_details t - left join fills_count f on t.tx_hash = f.tx_hash and t.block_time = f.block_time - \ No newline at end of file diff --git a/dbt_subprojects/dex/models/_projects/zeroex/mode/zeroex_mode_schema.yml b/dbt_subprojects/dex/models/_projects/zeroex/mode/zeroex_mode_schema.yml deleted file mode 100644 index 080d07968bf..00000000000 --- a/dbt_subprojects/dex/models/_projects/zeroex/mode/zeroex_mode_schema.yml +++ /dev/null @@ -1,111 +0,0 @@ -version: 2 - -models: - - name: zeroex_v2_mode_trades - meta: - blockchain: mode - project: zeroex - contributors: rantum - config: - tags: ['mode','0x','dex_aggregator','dex','aggregator'] - description: > - 0x API erc20 trades through 0x v2 contracts. No fills, only deduped transactions. - data_tests: - - dbt_utils.unique_combination_of_columns: - combination_of_columns: - - block_month - - block_date - - tx_hash - - evt_index - - check_seed: - seed_file: ref('zeroex_v2_mode_trades_sample') - match_columns: - - tx_hash - check_columns: - - taker - - maker_token - - taker_token - columns: - - &blockchain - name: blockchain - description: "Blockchain which the aggregator project is deployed" - - &block_date - name: block_date - description: "UTC event block date of each trade" - - &block_month - name: block_month - description: "UTC event block month of each trade" - - &block_time - name: block_time - description: "UTC event block time of each trade" - - &taker_symbol - name: taker_symbol - description: "Symbol of the token taker sells" - - &maker_symbol - name: maker_symbol - description: "Symbol of the token taker buys" - - &token_pair - name: token_pair - description: "Token pair traded" - - &taker_token_amount - name: taker_token_amount - description: "The after-decimal amount of the token taker sells" - - &taker_token_amount_raw - name: taker_token_amount_raw - description: "The raw amount of the token taker sells" - - &maker_token_amount - name: maker_token_amount - description: "The after-decimal amount of the token taker buys" - - &maker_token_amount_raw - name: maker_token_amount_raw - description: "The raw amount of the token taker buys" - - &volume_usd - name: volume_usd - description: "Trading volume measured in USD value" - - &taker_token - name: taker_token - description: "Contract address of the token taker sells" - - &maker_token - name: maker_token - description: "Contract address of the token taker buys" - - &maker - name: maker - description: "buyer of the trade" - - &taker - name: taker - description: "seller of the trade" - - &affiliate_address - name: affiliate_address - description: "The recipient address of the affiliate, or the applications that is using 0x API, for receiving affiliate fee" - - &tx_hash - name: tx_hash - description: "Transaction hash of the fill" - - &tx_from - name: tx_from - description: "Address which initiated the trade" - - &tx_to - name: tx_to - description: "Address which received the trade" - - &evt_index - name: evt_index - description: "Index of the corresponding order filled event" - - &type - name: type - description: "The liquidity route the order went thru" - - &swap_flag - name: swap_flag - description: "If the swap was filled/consumed thru 0x API" - - &contract_address - name: contract_address - desctiption: "The address of the contract which fired the fill/swap event" - - &fills_within - name: fills_within - description: "fills in then multihop, if present" - - - - - - - - \ No newline at end of file diff --git a/dbt_subprojects/dex/models/_projects/zeroex/mode/zeroex_v2_mode_trades.sql b/dbt_subprojects/dex/models/_projects/zeroex/mode/zeroex_v2_mode_trades.sql deleted file mode 100644 index 03a74e6e745..00000000000 --- a/dbt_subprojects/dex/models/_projects/zeroex/mode/zeroex_v2_mode_trades.sql +++ /dev/null @@ -1,61 +0,0 @@ -{{ config( - schema = 'zeroex_v2_mode', - alias = 'trades', - materialized='incremental', - partition_by = ['block_month'], - unique_key = ['block_month', 'block_date', 'tx_hash', 'evt_index'], - on_schema_change='sync_all_columns', - file_format ='delta', - incremental_strategy='merge', - incremental_predicates = [incremental_predicate('DBT_INTERNAL_DEST.block_time')] -)}} - -{% set zeroex_settler_start_date = '2024-07-15' %} -{% set blockchain = 'mode' %} - -WITH zeroex_tx AS ( - {{ - zeroex_settler_txs_cte( - blockchain = blockchain, - start_date = zeroex_settler_start_date - ) - }} -), -zeroex_v2_trades_direct AS ( - {{ - zeroex_v2_trades_direct( - blockchain = blockchain, - start_date = zeroex_settler_start_date - - ) - }} -), -zeroex_v2_trades_indirect AS ( - {{ - zeroex_v2_trades_indirect( - blockchain = blockchain, - start_date = zeroex_settler_start_date - - ) - }} -), -tbl_trades AS ( - SELECT * - FROM zeroex_v2_trades_direct - UNION ALL - SELECT * - FROM zeroex_v2_trades_indirect -), -trade_details as ( - {{ - zeroex_v2_trades_detail( - blockchain = blockchain, - start_date = zeroex_settler_start_date - - ) - }} - -) -select - * - from trade_details diff --git a/dbt_subprojects/dex/models/_projects/zeroex/polygon/zeroex_v2_polygon_trades.sql b/dbt_subprojects/dex/models/_projects/zeroex/polygon/zeroex_v2_polygon_trades.sql index 6c45e4fbcff..1177284b18a 100644 --- a/dbt_subprojects/dex/models/_projects/zeroex/polygon/zeroex_v2_polygon_trades.sql +++ b/dbt_subprojects/dex/models/_projects/zeroex/polygon/zeroex_v2_polygon_trades.sql @@ -46,14 +46,6 @@ tbl_trades AS ( SELECT * FROM zeroex_v2_trades_indirect ), -fills_count as( - {{ - zeroex_v2_trades_fills_count( - blockchain = blockchain, - start_date = zeroex_settler_start_date - ) - }} -), trade_details as ( {{ zeroex_v2_trades_detail( @@ -65,6 +57,5 @@ trade_details as ( ) select - t.*, fills_within - from trade_details t - left join fills_count f on t.tx_hash = f.tx_hash and t.block_time = f.block_time + * + from trade_details diff --git a/dbt_subprojects/dex/models/_projects/zeroex/zeroex_api_fills_deduped.sql b/dbt_subprojects/dex/models/_projects/zeroex/zeroex_api_fills_deduped.sql index 07f52a7d2f9..51563852cb7 100644 --- a/dbt_subprojects/dex/models/_projects/zeroex/zeroex_api_fills_deduped.sql +++ b/dbt_subprojects/dex/models/_projects/zeroex/zeroex_api_fills_deduped.sql @@ -1,7 +1,7 @@ {{ config( schema = 'zeroex' , alias = 'api_fills_deduped' - , post_hook='{{ expose_spells(\'["arbitrum", "avalanche_c", "base", "bnb", "celo", "ethereum", "fantom", "optimism", "polygon","scroll", "linea","blast","mantle","mode"]\', + , post_hook='{{ expose_spells(\'["arbitrum", "avalanche_c", "base", "bnb", "celo", "ethereum", "fantom", "optimism", "polygon","scroll", "linea","blast","mantle"]\', "project", "zeroex", \'["rantum","bakabhai993"]\') }}' @@ -27,13 +27,12 @@ ,ref('zeroex_v2_polygon_trades') ,ref('zeroex_optimism_settler_trades') ,ref('zeroex_bnb_settler_trades') - ,ref('zeroex_v2_avalanche_c_trades') - ,ref('zeroex_v2_arbitrum_trades') + ,ref('zeroex_avalanche_c_settler_trades') + ,ref('zeroex_arbitrum_settler_trades') ,ref('zeroex_scroll_settler_trades') ,ref('zeroex_linea_settler_trades') - ,ref('zeroex_v2_mode_trades') - ,ref('zeroex_v2_blast_trades') - ,ref('zeroex_v2_mantle_trades') + ,ref('zeroex_blast_settler_trades') + ,ref('zeroex_mantle_settler_trades') ] %} diff --git a/dbt_subprojects/dex/models/_projects/zeroex/zeroex_schema.yml b/dbt_subprojects/dex/models/_projects/zeroex/zeroex_schema.yml index 7cab763d18a..c18ff3eb4d2 100644 --- a/dbt_subprojects/dex/models/_projects/zeroex/zeroex_schema.yml +++ b/dbt_subprojects/dex/models/_projects/zeroex/zeroex_schema.yml @@ -105,7 +105,7 @@ models: - name: zeroex_api_fills_deduped meta: - blockchain: ethereum, optimism, polygon, arbitrum, fantom, avalanche, bnb, base, scroll, linea, blast, mantle, mode + blockchain: ethereum, optimism, polygon, arbitrum, fantom, avalanche, bnb, base, scroll, linea, blast, mantle sector: dex project: zeroex contributors: rantum, bakabhai993 diff --git a/dbt_subprojects/dex/seeds/_project/zeroex/mode/_schema.yml b/dbt_subprojects/dex/seeds/_project/zeroex/mode/_schema.yml deleted file mode 100644 index 0f47baa5698..00000000000 --- a/dbt_subprojects/dex/seeds/_project/zeroex/mode/_schema.yml +++ /dev/null @@ -1,11 +0,0 @@ -version: 2 - -seeds: - - name: zeroex_v2_mode_trades_sample - config: - column_types: - tx_hash: varbinary - taker: varbinary - taker_token: varbinary - maker_token: varbinary - taker_token_amount: double \ No newline at end of file diff --git a/dbt_subprojects/dex/seeds/_project/zeroex/mode/zeroex_v2_mode_trades_sample.csv b/dbt_subprojects/dex/seeds/_project/zeroex/mode/zeroex_v2_mode_trades_sample.csv deleted file mode 100644 index 0f4fc0c98d2..00000000000 --- a/dbt_subprojects/dex/seeds/_project/zeroex/mode/zeroex_v2_mode_trades_sample.csv +++ /dev/null @@ -1,4 +0,0 @@ -tx_hash,taker,taker_token,maker_token,taker_token_amount -0xd7ace13835ceb119b071c508e8ee262fcecb5a0f2a1bfdd1b9b1f04afa73d94d,0x44c20316a06371ca5acb975974c18abfa7a14a5a,0xf0f161fda2712db8b566946122a5af183995e2ed,0x4200000000000000000000000000000000000006, -0xd10163ac93593667d3922a05c09580856b1b4c1dc015738c67968912a42c46dc,0x8a6bfcae15e729fd1440574108437dea281a9b3e,0x6a660e56fa3b630a786cc4ae98859f8532d03de9,0x2416092f143378750bb29b79ed961ab195cceea5,238973 -0xd84a08f6d48b5c34cde908452602088cdb42beceef29074b8a8d5c7e45f2a3dc,0x8a6bfcae15e729fd1440574108437dea281a9b3e,0x4200000000000000000000000000000000000006,0x6a660e56fa3b630a786cc4ae98859f8532d03de9, From a5039ee60d50b93010b8a6c8a1192c152d4af74a Mon Sep 17 00:00:00 2001 From: Rantum Date: Tue, 26 Nov 2024 09:44:25 -0800 Subject: [PATCH 03/18] add exclude tag to nft spells (#7196) --- .../_projects/zeroex/ethereum/zeroex_ethereum_nft_fills.sql | 1 + 1 file changed, 1 insertion(+) diff --git a/dbt_subprojects/dex/models/_projects/zeroex/ethereum/zeroex_ethereum_nft_fills.sql b/dbt_subprojects/dex/models/_projects/zeroex/ethereum/zeroex_ethereum_nft_fills.sql index faf0b2ce872..862995c9162 100644 --- a/dbt_subprojects/dex/models/_projects/zeroex/ethereum/zeroex_ethereum_nft_fills.sql +++ b/dbt_subprojects/dex/models/_projects/zeroex/ethereum/zeroex_ethereum_nft_fills.sql @@ -1,4 +1,5 @@ {{ config( + tags=['prod_exclude'], schema = 'zeroex_ethereum', alias = 'nft_fills', materialized='incremental', From 7c7b706da8c68aff27e850253d60bd4edefb0812 Mon Sep 17 00:00:00 2001 From: 0xRob <83790096+0xRobin@users.noreply.github.com> Date: Thu, 28 Nov 2024 11:55:56 +0100 Subject: [PATCH 04/18] Dune index: cleanup models (#7173) * clean up index and stats models * syntax * split out gas fees metrics by chain * syntax fix * syntax fix * split out transactions * setup evm macros * column ref * split out transfers * run CI incremental (revert this before merge) * fix compile * restore stats models * restore stats schemas * Revert "run CI incremental (revert this before merge)" This reverts commit d54411ca923ca766db63920c512c813ef0f8afd2. --- .../sector/metrics/metrics_fees_evm.sql | 18 ++ .../metrics/metrics_transactions_evm.sql | 19 ++ .../sector/metrics/metrics_transfers_evm.sql} | 18 +- .../models/_metrics/dune_index/_schema.yml | 7 +- .../dune_index/metrics_dune_index_daily.sql | 74 +++++- .../dune_index/metrics_dune_index_stats.sql | 2 +- .../models/_metrics/fees/_schema.yml | 20 +- .../_metrics/fees/chains/bitcoin/_schema.yml | 15 ++ .../metrics_bitcoin_gas_fees_daily.sql | 46 ++++ .../_metrics/fees/chains/evm/_schema.yml | 236 +++++++++++++++++ .../evm/metrics_arbitrum_gas_fees_daily.sql | 15 ++ .../metrics_avalanche_c_gas_fees_daily.sql | 15 ++ .../evm/metrics_base_gas_fees_daily.sql | 15 ++ .../evm/metrics_blast_gas_fees_daily.sql | 15 ++ .../chains/evm/metrics_bnb_gas_fees_daily.sql | 15 ++ .../evm/metrics_celo_gas_fees_daily.sql | 15 ++ .../evm/metrics_ethereum_gas_fees_daily.sql | 15 ++ .../evm/metrics_fantom_gas_fees_daily.sql | 15 ++ .../evm/metrics_gnosis_gas_fees_daily.sql | 15 ++ .../evm/metrics_linea_gas_fees_daily.sql | 15 ++ .../evm/metrics_mantle_gas_fees_daily.sql | 15 ++ .../evm/metrics_optimism_gas_fees_daily.sql | 15 ++ .../evm/metrics_polygon_gas_fees_daily.sql | 15 ++ .../evm/metrics_scroll_gas_fees_daily.sql | 15 ++ .../chains/evm/metrics_sei_gas_fees_daily.sql | 15 ++ .../evm/metrics_zkevm_gas_fees_daily.sql | 15 ++ .../evm/metrics_zksync_gas_fees_daily.sql | 15 ++ .../evm/metrics_zora_gas_fees_daily.sql | 15 ++ .../_metrics/fees/chains/solana/_schema.yml | 15 ++ .../solana/metrics_solana_gas_fees_daily.sql | 60 +++++ .../_metrics/fees/chains/tron/_schema.yml | 24 ++ .../tron/metrics_tron_gas_fees_daily.sql | 39 +++ .../{ => chains}/tron/tron_fee_correction.sql | 0 .../fees/metrics_fees_index_daily.sql | 38 --- .../_metrics/fees/metrics_gas_fees_daily.sql | 140 +++------- .../_metrics/fees/metrics_gas_fees_stats.sql | 2 +- .../models/_metrics/fees/tron/_schema.yml | 12 - .../models/_metrics/transactions/_schema.yml | 22 +- .../transactions/chains/bitcoin/_schema.yml | 15 ++ .../metrics_bitcoin_transactions_daily.sql | 26 ++ .../transactions/chains/evm/_schema.yml | 249 ++++++++++++++++++ .../metrics_arbitrum_transactions_daily.sql | 15 ++ ...metrics_avalanche_c_transactions_daily.sql | 15 ++ .../evm/metrics_base_transactions_daily.sql | 15 ++ .../evm/metrics_blast_transactions_daily.sql | 15 ++ .../evm/metrics_bnb_transactions_daily.sql | 15 ++ .../evm/metrics_celo_transactions_daily.sql | 15 ++ .../metrics_ethereum_transactions_daily.sql | 15 ++ .../evm/metrics_fantom_transactions_daily.sql | 15 ++ .../evm/metrics_gnosis_transactions_daily.sql | 15 ++ .../evm/metrics_linea_transactions_daily.sql | 15 ++ .../evm/metrics_mantle_transactions_daily.sql | 15 ++ .../metrics_optimism_transactions_daily.sql | 15 ++ .../metrics_polygon_transactions_daily.sql | 15 ++ .../evm/metrics_scroll_transactions_daily.sql | 15 ++ .../evm/metrics_sei_transactions_daily.sql | 15 ++ .../evm/metrics_tron_transactions_daily.sql | 15 ++ .../evm/metrics_zkevm_transactions_daily.sql | 15 ++ .../evm/metrics_zksync_transactions_daily.sql | 15 ++ .../evm/metrics_zora_transactions_daily.sql | 15 ++ .../transactions/chains/solana/_schema.yml | 15 ++ .../metrics_solana_transactions_daily.sql | 27 ++ .../metrics_transactions_daily.sql | 109 +++----- .../metrics_transactions_index_daily.sql | 38 --- .../metrics_transactions_stats.sql | 2 +- .../models/_metrics/transfers/_schema.yml | 51 +--- .../metrics_bitcoin_transfers_daily.sql} | 6 +- .../_metrics/transfers/chains/evm/_schema.yml | 249 ++++++++++++++++++ .../evm/metrics_arbitrum_transfers_daily.sql | 15 ++ .../metrics_avalanche_c_transfers_daily.sql | 15 ++ .../evm/metrics_base_transfers_daily.sql | 15 ++ .../evm/metrics_blast_transfers_daily.sql | 15 ++ .../evm/metrics_bnb_transfers_daily.sql | 15 ++ .../evm/metrics_celo_transfers_daily.sql | 15 ++ .../evm/metrics_ethereum_transfers_daily.sql | 15 ++ .../evm/metrics_fantom_transfers_daily.sql | 15 ++ .../evm/metrics_gnosis_transfers_daily.sql | 15 ++ .../evm/metrics_linea_transfers_daily.sql | 15 ++ .../evm/metrics_mantle_transfers_daily.sql | 15 ++ .../evm/metrics_optimism_transfers_daily.sql | 15 ++ .../evm/metrics_polygon_transfers_daily.sql | 15 ++ .../evm/metrics_scroll_transfers_daily.sql | 15 ++ .../evm/metrics_sei_transfers_daily.sql | 15 ++ .../evm/metrics_tron_transfers_daily.sql | 15 ++ .../evm/metrics_zkevm_transfers_daily.sql | 15 ++ .../evm/metrics_zksync_transfers_daily.sql | 15 ++ .../evm/metrics_zora_transfers_daily.sql | 15 ++ .../metrics_solana_transfers_daily.sql} | 6 +- .../transfers/metrics_transfers_daily.sql | 69 +++-- .../metrics_transfers_index_daily.sql | 38 --- .../transfers/metrics_transfers_stats.sql | 2 +- 91 files changed, 2103 insertions(+), 446 deletions(-) create mode 100644 dbt_subprojects/daily_spellbook/macros/sector/metrics/metrics_fees_evm.sql create mode 100644 dbt_subprojects/daily_spellbook/macros/sector/metrics/metrics_transactions_evm.sql rename dbt_subprojects/daily_spellbook/{models/_metrics/transfers/metrics_net_transfers_daily.sql => macros/sector/metrics/metrics_transfers_evm.sql} (88%) create mode 100644 dbt_subprojects/daily_spellbook/models/_metrics/fees/chains/bitcoin/_schema.yml create mode 100644 dbt_subprojects/daily_spellbook/models/_metrics/fees/chains/bitcoin/metrics_bitcoin_gas_fees_daily.sql create mode 100644 dbt_subprojects/daily_spellbook/models/_metrics/fees/chains/evm/_schema.yml create mode 100644 dbt_subprojects/daily_spellbook/models/_metrics/fees/chains/evm/metrics_arbitrum_gas_fees_daily.sql create mode 100644 dbt_subprojects/daily_spellbook/models/_metrics/fees/chains/evm/metrics_avalanche_c_gas_fees_daily.sql create mode 100644 dbt_subprojects/daily_spellbook/models/_metrics/fees/chains/evm/metrics_base_gas_fees_daily.sql create mode 100644 dbt_subprojects/daily_spellbook/models/_metrics/fees/chains/evm/metrics_blast_gas_fees_daily.sql create mode 100644 dbt_subprojects/daily_spellbook/models/_metrics/fees/chains/evm/metrics_bnb_gas_fees_daily.sql create mode 100644 dbt_subprojects/daily_spellbook/models/_metrics/fees/chains/evm/metrics_celo_gas_fees_daily.sql create mode 100644 dbt_subprojects/daily_spellbook/models/_metrics/fees/chains/evm/metrics_ethereum_gas_fees_daily.sql create mode 100644 dbt_subprojects/daily_spellbook/models/_metrics/fees/chains/evm/metrics_fantom_gas_fees_daily.sql create mode 100644 dbt_subprojects/daily_spellbook/models/_metrics/fees/chains/evm/metrics_gnosis_gas_fees_daily.sql create mode 100644 dbt_subprojects/daily_spellbook/models/_metrics/fees/chains/evm/metrics_linea_gas_fees_daily.sql create mode 100644 dbt_subprojects/daily_spellbook/models/_metrics/fees/chains/evm/metrics_mantle_gas_fees_daily.sql create mode 100644 dbt_subprojects/daily_spellbook/models/_metrics/fees/chains/evm/metrics_optimism_gas_fees_daily.sql create mode 100644 dbt_subprojects/daily_spellbook/models/_metrics/fees/chains/evm/metrics_polygon_gas_fees_daily.sql create mode 100644 dbt_subprojects/daily_spellbook/models/_metrics/fees/chains/evm/metrics_scroll_gas_fees_daily.sql create mode 100644 dbt_subprojects/daily_spellbook/models/_metrics/fees/chains/evm/metrics_sei_gas_fees_daily.sql create mode 100644 dbt_subprojects/daily_spellbook/models/_metrics/fees/chains/evm/metrics_zkevm_gas_fees_daily.sql create mode 100644 dbt_subprojects/daily_spellbook/models/_metrics/fees/chains/evm/metrics_zksync_gas_fees_daily.sql create mode 100644 dbt_subprojects/daily_spellbook/models/_metrics/fees/chains/evm/metrics_zora_gas_fees_daily.sql create mode 100644 dbt_subprojects/daily_spellbook/models/_metrics/fees/chains/solana/_schema.yml create mode 100644 dbt_subprojects/daily_spellbook/models/_metrics/fees/chains/solana/metrics_solana_gas_fees_daily.sql create mode 100644 dbt_subprojects/daily_spellbook/models/_metrics/fees/chains/tron/_schema.yml create mode 100644 dbt_subprojects/daily_spellbook/models/_metrics/fees/chains/tron/metrics_tron_gas_fees_daily.sql rename dbt_subprojects/daily_spellbook/models/_metrics/fees/{ => chains}/tron/tron_fee_correction.sql (100%) delete mode 100644 dbt_subprojects/daily_spellbook/models/_metrics/fees/metrics_fees_index_daily.sql delete mode 100644 dbt_subprojects/daily_spellbook/models/_metrics/fees/tron/_schema.yml create mode 100644 dbt_subprojects/daily_spellbook/models/_metrics/transactions/chains/bitcoin/_schema.yml create mode 100644 dbt_subprojects/daily_spellbook/models/_metrics/transactions/chains/bitcoin/metrics_bitcoin_transactions_daily.sql create mode 100644 dbt_subprojects/daily_spellbook/models/_metrics/transactions/chains/evm/_schema.yml create mode 100644 dbt_subprojects/daily_spellbook/models/_metrics/transactions/chains/evm/metrics_arbitrum_transactions_daily.sql create mode 100644 dbt_subprojects/daily_spellbook/models/_metrics/transactions/chains/evm/metrics_avalanche_c_transactions_daily.sql create mode 100644 dbt_subprojects/daily_spellbook/models/_metrics/transactions/chains/evm/metrics_base_transactions_daily.sql create mode 100644 dbt_subprojects/daily_spellbook/models/_metrics/transactions/chains/evm/metrics_blast_transactions_daily.sql create mode 100644 dbt_subprojects/daily_spellbook/models/_metrics/transactions/chains/evm/metrics_bnb_transactions_daily.sql create mode 100644 dbt_subprojects/daily_spellbook/models/_metrics/transactions/chains/evm/metrics_celo_transactions_daily.sql create mode 100644 dbt_subprojects/daily_spellbook/models/_metrics/transactions/chains/evm/metrics_ethereum_transactions_daily.sql create mode 100644 dbt_subprojects/daily_spellbook/models/_metrics/transactions/chains/evm/metrics_fantom_transactions_daily.sql create mode 100644 dbt_subprojects/daily_spellbook/models/_metrics/transactions/chains/evm/metrics_gnosis_transactions_daily.sql create mode 100644 dbt_subprojects/daily_spellbook/models/_metrics/transactions/chains/evm/metrics_linea_transactions_daily.sql create mode 100644 dbt_subprojects/daily_spellbook/models/_metrics/transactions/chains/evm/metrics_mantle_transactions_daily.sql create mode 100644 dbt_subprojects/daily_spellbook/models/_metrics/transactions/chains/evm/metrics_optimism_transactions_daily.sql create mode 100644 dbt_subprojects/daily_spellbook/models/_metrics/transactions/chains/evm/metrics_polygon_transactions_daily.sql create mode 100644 dbt_subprojects/daily_spellbook/models/_metrics/transactions/chains/evm/metrics_scroll_transactions_daily.sql create mode 100644 dbt_subprojects/daily_spellbook/models/_metrics/transactions/chains/evm/metrics_sei_transactions_daily.sql create mode 100644 dbt_subprojects/daily_spellbook/models/_metrics/transactions/chains/evm/metrics_tron_transactions_daily.sql create mode 100644 dbt_subprojects/daily_spellbook/models/_metrics/transactions/chains/evm/metrics_zkevm_transactions_daily.sql create mode 100644 dbt_subprojects/daily_spellbook/models/_metrics/transactions/chains/evm/metrics_zksync_transactions_daily.sql create mode 100644 dbt_subprojects/daily_spellbook/models/_metrics/transactions/chains/evm/metrics_zora_transactions_daily.sql create mode 100644 dbt_subprojects/daily_spellbook/models/_metrics/transactions/chains/solana/_schema.yml create mode 100644 dbt_subprojects/daily_spellbook/models/_metrics/transactions/chains/solana/metrics_solana_transactions_daily.sql delete mode 100644 dbt_subprojects/daily_spellbook/models/_metrics/transactions/metrics_transactions_index_daily.sql rename dbt_subprojects/daily_spellbook/models/_metrics/transfers/{metrics_net_bitcoin_transfers_daily.sql => chains/bitcoin/metrics_bitcoin_transfers_daily.sql} (97%) create mode 100644 dbt_subprojects/daily_spellbook/models/_metrics/transfers/chains/evm/_schema.yml create mode 100644 dbt_subprojects/daily_spellbook/models/_metrics/transfers/chains/evm/metrics_arbitrum_transfers_daily.sql create mode 100644 dbt_subprojects/daily_spellbook/models/_metrics/transfers/chains/evm/metrics_avalanche_c_transfers_daily.sql create mode 100644 dbt_subprojects/daily_spellbook/models/_metrics/transfers/chains/evm/metrics_base_transfers_daily.sql create mode 100644 dbt_subprojects/daily_spellbook/models/_metrics/transfers/chains/evm/metrics_blast_transfers_daily.sql create mode 100644 dbt_subprojects/daily_spellbook/models/_metrics/transfers/chains/evm/metrics_bnb_transfers_daily.sql create mode 100644 dbt_subprojects/daily_spellbook/models/_metrics/transfers/chains/evm/metrics_celo_transfers_daily.sql create mode 100644 dbt_subprojects/daily_spellbook/models/_metrics/transfers/chains/evm/metrics_ethereum_transfers_daily.sql create mode 100644 dbt_subprojects/daily_spellbook/models/_metrics/transfers/chains/evm/metrics_fantom_transfers_daily.sql create mode 100644 dbt_subprojects/daily_spellbook/models/_metrics/transfers/chains/evm/metrics_gnosis_transfers_daily.sql create mode 100644 dbt_subprojects/daily_spellbook/models/_metrics/transfers/chains/evm/metrics_linea_transfers_daily.sql create mode 100644 dbt_subprojects/daily_spellbook/models/_metrics/transfers/chains/evm/metrics_mantle_transfers_daily.sql create mode 100644 dbt_subprojects/daily_spellbook/models/_metrics/transfers/chains/evm/metrics_optimism_transfers_daily.sql create mode 100644 dbt_subprojects/daily_spellbook/models/_metrics/transfers/chains/evm/metrics_polygon_transfers_daily.sql create mode 100644 dbt_subprojects/daily_spellbook/models/_metrics/transfers/chains/evm/metrics_scroll_transfers_daily.sql create mode 100644 dbt_subprojects/daily_spellbook/models/_metrics/transfers/chains/evm/metrics_sei_transfers_daily.sql create mode 100644 dbt_subprojects/daily_spellbook/models/_metrics/transfers/chains/evm/metrics_tron_transfers_daily.sql create mode 100644 dbt_subprojects/daily_spellbook/models/_metrics/transfers/chains/evm/metrics_zkevm_transfers_daily.sql create mode 100644 dbt_subprojects/daily_spellbook/models/_metrics/transfers/chains/evm/metrics_zksync_transfers_daily.sql create mode 100644 dbt_subprojects/daily_spellbook/models/_metrics/transfers/chains/evm/metrics_zora_transfers_daily.sql rename dbt_subprojects/daily_spellbook/models/_metrics/transfers/{metrics_net_solana_transfers_daily.sql => chains/solana/metrics_solana_transfers_daily.sql} (96%) delete mode 100644 dbt_subprojects/daily_spellbook/models/_metrics/transfers/metrics_transfers_index_daily.sql diff --git a/dbt_subprojects/daily_spellbook/macros/sector/metrics/metrics_fees_evm.sql b/dbt_subprojects/daily_spellbook/macros/sector/metrics/metrics_fees_evm.sql new file mode 100644 index 00000000000..412b4c7728d --- /dev/null +++ b/dbt_subprojects/daily_spellbook/macros/sector/metrics/metrics_fees_evm.sql @@ -0,0 +1,18 @@ +{% macro metrics_fees_evm(blockchain) %} + +select + blockchain + , block_date + , sum(tx_fee_usd) as gas_fees_usd +from + {{ source('gas', 'fees') }} +where blockchain = '{{blockchain}}' +{% if is_incremental() %} +and + {{ incremental_predicate('block_date') }} +{% endif %} +group by + blockchain + ,block_date + +{% endmacro %} diff --git a/dbt_subprojects/daily_spellbook/macros/sector/metrics/metrics_transactions_evm.sql b/dbt_subprojects/daily_spellbook/macros/sector/metrics/metrics_transactions_evm.sql new file mode 100644 index 00000000000..1ac65276fb3 --- /dev/null +++ b/dbt_subprojects/daily_spellbook/macros/sector/metrics/metrics_transactions_evm.sql @@ -0,0 +1,19 @@ +{% macro metrics_transactions_evm(blockchain) %} + +select + blockchain + , block_date + , approx_distinct(tx_hash) as tx_count --max 2% error, which is fine +from + {{ source('tokens', 'transfers') }} +where + blockchain = '{{ blockchain }}' + and amount_usd >=1 + {% if is_incremental() %} + and {{ incremental_predicate('block_date') }} + {% endif %} +group by + blockchain + , block_date + +{% endmacro %} diff --git a/dbt_subprojects/daily_spellbook/models/_metrics/transfers/metrics_net_transfers_daily.sql b/dbt_subprojects/daily_spellbook/macros/sector/metrics/metrics_transfers_evm.sql similarity index 88% rename from dbt_subprojects/daily_spellbook/models/_metrics/transfers/metrics_net_transfers_daily.sql rename to dbt_subprojects/daily_spellbook/macros/sector/metrics/metrics_transfers_evm.sql index 584a68edc7a..f96fdd425c8 100644 --- a/dbt_subprojects/daily_spellbook/models/_metrics/transfers/metrics_net_transfers_daily.sql +++ b/dbt_subprojects/daily_spellbook/macros/sector/metrics/metrics_transfers_evm.sql @@ -1,13 +1,4 @@ -{{ config( - schema = 'metrics' - , alias = 'net_transfers_daily' - , materialized = 'incremental' - , file_format = 'delta' - , incremental_strategy = 'merge' - , unique_key = ['blockchain', 'block_date'] - , incremental_predicates = [incremental_predicate('DBT_INTERNAL_DEST.block_date')] - ) -}} +{% macro metrics_transfers_evm(blockchain) %} with raw_transfers as ( select @@ -19,7 +10,7 @@ with raw_transfers as ( from {{ source('tokens', 'transfers') }} where - 1 = 1 + blockchain = '{{blockchain}}' {% if is_incremental() %} and {{ incremental_predicate('block_date') }} {% endif %} @@ -40,7 +31,7 @@ with raw_transfers as ( from {{ source('tokens', 'transfers') }} where - 1 = 1 + blockchain = '{{blockchain}}' {% if is_incremental() %} and {{ incremental_predicate('block_date') }} {% endif %} @@ -60,6 +51,7 @@ with raw_transfers as ( inner join {{ source('labels', 'owner_details') }} as od on oa.owner_key = od.owner_key + where oa.blockchain = '{{blockchain}}' ), transfers_amount as ( select t.blockchain @@ -108,3 +100,5 @@ where group by blockchain , block_date + +{% endmacro %} diff --git a/dbt_subprojects/daily_spellbook/models/_metrics/dune_index/_schema.yml b/dbt_subprojects/daily_spellbook/models/_metrics/dune_index/_schema.yml index 0c19c70acf8..eed04ce6a8c 100644 --- a/dbt_subprojects/daily_spellbook/models/_metrics/dune_index/_schema.yml +++ b/dbt_subprojects/daily_spellbook/models/_metrics/dune_index/_schema.yml @@ -4,7 +4,7 @@ models: - name: metrics_dune_index_daily meta: sector: metrics - contributors: jeff-dude + contributors: jeff-dude, 0xRob config: tags: ['metrics', 'dune', 'index', 'daily'] description: "Combine transactions, transfers and fees index values to get a daily dune index value" @@ -13,10 +13,11 @@ models: combination_of_columns: - blockchain - block_date + - name: metrics_dune_index_stats meta: sector: metrics contributors: jeff-dude config: - tags: ['metrics', 'dune', 'index', 'daily'] - description: "View containing various time aggregations of the dune index" \ No newline at end of file + tags: [ 'metrics', 'dune', 'index', 'daily' ] + description: "View containing various time aggregations of the dune index" diff --git a/dbt_subprojects/daily_spellbook/models/_metrics/dune_index/metrics_dune_index_daily.sql b/dbt_subprojects/daily_spellbook/models/_metrics/dune_index/metrics_dune_index_daily.sql index e5f68fe3345..593d5136189 100644 --- a/dbt_subprojects/daily_spellbook/models/_metrics/dune_index/metrics_dune_index_daily.sql +++ b/dbt_subprojects/daily_spellbook/models/_metrics/dune_index/metrics_dune_index_daily.sql @@ -1,21 +1,71 @@ {{ config( schema = 'metrics' , alias = 'dune_index_daily' - , materialized = 'view' + , materialized = 'incremental' + , file_format = 'delta' + , incremental_strategy = 'merge' + , unique_key = ['blockchain', 'block_date'] + , incremental_predicates = [incremental_predicate('DBT_INTERNAL_DEST.block_date')] ) }} +{% set baseline_date = '2018-01-01' %} +{% set start_date = '2015-08-21' %} + +with +fees as ( + select + blockchain + , block_date + , gas_fees_usd + , (gas_fees_usd / (select sum(gas_fees_usd) from {{ ref('metrics_gas_fees_daily') }} where block_date = date '{{ baseline_date }}')) * 10 as fees_index + from + {{ ref('metrics_gas_fees_daily') }} + where + block_date >= date '{{ start_date }}' + {% if is_incremental() %} + and {{ incremental_predicate('block_date') }} + {% endif %} +), +transactions as ( + select + blockchain + , block_date + , tx_count + , (tx_count / cast((select sum(tx_count) from {{ ref('metrics_transactions_daily') }} where block_date = date '{{ baseline_date }}') as double)) * 10 as tx_index + from + {{ ref('metrics_transactions_daily') }} + where + block_date >= date '{{ start_date }}' + {% if is_incremental() %} + and {{ incremental_predicate('block_date') }} + {% endif %} +) +,transfers as ( + select + blockchain + , block_date + , net_transfer_amount_usd + , (net_transfer_amount_usd / cast((select sum(net_transfer_amount_usd) from {{ ref('metrics_transfers_daily') }} where block_date = date '{{ baseline_date }}') as double)) * 10 as transfers_index + from + {{ ref('metrics_transfers_daily') }} + where + block_date >= date '{{ start_date }}' + {% if is_incremental() %} + and {{ incremental_predicate('block_date') }} + {% endif %} +) + select blockchain , block_date - , coalesce(f.fees_index,0) as fees_index - , coalesce(tr.transfers_index,0) as transfers_index - , coalesce(tx.tx_index,0) as tx_index - , coalesce(f.fees_index,0)*0.45 + coalesce(tr.transfers_index,0)*0.45 + coalesce(tx.tx_index,0)*0.10 as dune_index -from {{ ref('metrics_fees_index_daily') }} as f -left join - {{ ref('metrics_transfers_index_daily') }} as tr - using (blockchain, block_date) -left join - {{ ref('metrics_transactions_index_daily') }} as tx - using (blockchain, block_date) \ No newline at end of file + , 0.45 * coalesce(fees_index,0) + 0.45 * coalesce(transfers_index,0) + 0.10 * coalesce(tx_index,0) as dune_index + , coalesce(fees_index,0) as fees_index + , coalesce(transfers_index,0) as transfers_index + , coalesce(tx_index,0) as tx_index + , gas_fees_usd + , tx_count + , net_transfer_amount_usd +from fees +left join transfers using (blockchain, block_date) +left join transactions using (blockchain, block_date) diff --git a/dbt_subprojects/daily_spellbook/models/_metrics/dune_index/metrics_dune_index_stats.sql b/dbt_subprojects/daily_spellbook/models/_metrics/dune_index/metrics_dune_index_stats.sql index 7beb905d72b..c33a292b079 100644 --- a/dbt_subprojects/daily_spellbook/models/_metrics/dune_index/metrics_dune_index_stats.sql +++ b/dbt_subprojects/daily_spellbook/models/_metrics/dune_index/metrics_dune_index_stats.sql @@ -157,4 +157,4 @@ from inner join weekly_stats as w on d.blockchain = w.blockchain inner join monthly_stats as m - on d.blockchain = m.blockchain \ No newline at end of file + on d.blockchain = m.blockchain diff --git a/dbt_subprojects/daily_spellbook/models/_metrics/fees/_schema.yml b/dbt_subprojects/daily_spellbook/models/_metrics/fees/_schema.yml index 19335852960..8a20c3dc635 100644 --- a/dbt_subprojects/daily_spellbook/models/_metrics/fees/_schema.yml +++ b/dbt_subprojects/daily_spellbook/models/_metrics/fees/_schema.yml @@ -8,27 +8,11 @@ models: config: tags: ['metrics', 'fees', 'gas', 'daily'] description: "Sum of total fees spent per day across all chains available in gas.fees and gas_solana.fees tables" - data_tests: - - dbt_utils.unique_combination_of_columns: - combination_of_columns: - - blockchain - - block_date + - name: metrics_gas_fees_stats meta: sector: metrics contributors: jeff-dude config: tags: ['metrics', 'fees', 'gas'] - description: "View of gas fees per blockchain aggregated to various levels. The goal is to output one row per chain with stats availble for use in counter visuals." - - name: metrics_fees_index_daily - meta: - sector: metrics - contributors: jeff-dude - config: - tags: ['metrics', 'fees', 'gas', 'index', 'daily'] - description: "Each day, per chain, compare the adoption of activity based on total gas fees relative to the baseline expectation" - data_tests: - - dbt_utils.unique_combination_of_columns: - combination_of_columns: - - blockchain - - block_date \ No newline at end of file + description: "View of gas fees per blockchain aggregated to various levels. The goal is to output one row per chain." diff --git a/dbt_subprojects/daily_spellbook/models/_metrics/fees/chains/bitcoin/_schema.yml b/dbt_subprojects/daily_spellbook/models/_metrics/fees/chains/bitcoin/_schema.yml new file mode 100644 index 00000000000..fa94e015cd8 --- /dev/null +++ b/dbt_subprojects/daily_spellbook/models/_metrics/fees/chains/bitcoin/_schema.yml @@ -0,0 +1,15 @@ +version: 2 + +models: + - name: metrics_bitcoin_gas_fees_daily + meta: + sector: metrics + contributors: jeff-dude + config: + tags: ['metrics', 'fees', 'gas', 'daily'] + description: "Sum of total fees spent per day" + data_tests: + - dbt_utils.unique_combination_of_columns: + combination_of_columns: + - blockchain + - block_date diff --git a/dbt_subprojects/daily_spellbook/models/_metrics/fees/chains/bitcoin/metrics_bitcoin_gas_fees_daily.sql b/dbt_subprojects/daily_spellbook/models/_metrics/fees/chains/bitcoin/metrics_bitcoin_gas_fees_daily.sql new file mode 100644 index 00000000000..89cd4f55b75 --- /dev/null +++ b/dbt_subprojects/daily_spellbook/models/_metrics/fees/chains/bitcoin/metrics_bitcoin_gas_fees_daily.sql @@ -0,0 +1,46 @@ +{{ config( + schema = 'metrics_bitcoin' + , alias = 'gas_fees_daily' + , materialized = 'incremental' + , file_format = 'delta' + , incremental_strategy = 'merge' + , unique_key = ['blockchain', 'block_date'] + , incremental_predicates = [incremental_predicate('DBT_INTERNAL_DEST.block_date')] + ) +}} + +with prices as ( + select + day + , price + from + {{ source('prices', 'usd_daily') }} + where + symbol = 'BTC' + and blockchain is null + {% if is_incremental() %} + and {{ incremental_predicate('day') }} + {% endif %} +) +, bitcoin_fees as ( + select + date as block_date + , sum(total_fees) as daily_fee + from + {{ source('bitcoin', 'blocks') }} + where + date < cast(date_trunc('day', now()) as date) --exclude current day to match prices.usd_daily + {% if is_incremental() %} + and {{ incremental_predicate('date') }} + {% endif %} + group by + date +) +select + 'bitcoin' as blockchain + , block_date + , (daily_fee * price) as gas_fees_usd +from + bitcoin_fees +inner join prices + on block_date = day diff --git a/dbt_subprojects/daily_spellbook/models/_metrics/fees/chains/evm/_schema.yml b/dbt_subprojects/daily_spellbook/models/_metrics/fees/chains/evm/_schema.yml new file mode 100644 index 00000000000..58105e44cd9 --- /dev/null +++ b/dbt_subprojects/daily_spellbook/models/_metrics/fees/chains/evm/_schema.yml @@ -0,0 +1,236 @@ +version: 2 + +models: + - name: metrics_arbitrum_gas_fees_daily + meta: + sector: metrics + contributors: jeff-dude, 0xRob + config: + tags: ['metrics', 'fees', 'gas', 'daily'] + description: "Sum of total fees spent per day" + data_tests: + - dbt_utils.unique_combination_of_columns: + combination_of_columns: + - blockchain + - block_date + + - name: metrics_avalanche_c_gas_fees_daily + meta: + sector: metrics + contributors: jeff-dude, 0xRob + config: + tags: ['metrics', 'fees', 'gas', 'daily'] + description: "Sum of total fees spent per day" + data_tests: + - dbt_utils.unique_combination_of_columns: + combination_of_columns: + - blockchain + - block_date + + - name: metrics_base_gas_fees_daily + meta: + sector: metrics + contributors: jeff-dude, 0xRob + config: + tags: ['metrics', 'fees', 'gas', 'daily'] + description: "Sum of total fees spent per day" + data_tests: + - dbt_utils.unique_combination_of_columns: + combination_of_columns: + - blockchain + - block_date + + - name: metrics_blast_gas_fees_daily + meta: + sector: metrics + contributors: jeff-dude, 0xRob + config: + tags: ['metrics', 'fees', 'gas', 'daily'] + description: "Sum of total fees spent per day" + data_tests: + - dbt_utils.unique_combination_of_columns: + combination_of_columns: + - blockchain + - block_date + + - name: metrics_bnb_gas_fees_daily + meta: + sector: metrics + contributors: jeff-dude, 0xRob + config: + tags: ['metrics', 'fees', 'gas', 'daily'] + description: "Sum of total fees spent per day" + data_tests: + - dbt_utils.unique_combination_of_columns: + combination_of_columns: + - blockchain + - block_date + + - name: metrics_celo_gas_fees_daily + meta: + sector: metrics + contributors: jeff-dude, 0xRob + config: + tags: ['metrics', 'fees', 'gas', 'daily'] + description: "Sum of total fees spent per day" + data_tests: + - dbt_utils.unique_combination_of_columns: + combination_of_columns: + - blockchain + - block_date + + - name: metrics_ethereum_gas_fees_daily + meta: + sector: metrics + contributors: jeff-dude, 0xRob + config: + tags: ['metrics', 'fees', 'gas', 'daily'] + description: "Sum of total fees spent per day" + data_tests: + - dbt_utils.unique_combination_of_columns: + combination_of_columns: + - blockchain + - block_date + + - name: metrics_fantom_gas_fees_daily + meta: + sector: metrics + contributors: jeff-dude, 0xRob + config: + tags: ['metrics', 'fees', 'gas', 'daily'] + description: "Sum of total fees spent per day" + data_tests: + - dbt_utils.unique_combination_of_columns: + combination_of_columns: + - blockchain + - block_date + + - name: metrics_gnosis_gas_fees_daily + meta: + sector: metrics + contributors: jeff-dude, 0xRob + config: + tags: ['metrics', 'fees', 'gas', 'daily'] + description: "Sum of total fees spent per day" + data_tests: + - dbt_utils.unique_combination_of_columns: + combination_of_columns: + - blockchain + - block_date + + - name: metrics_linea_gas_fees_daily + meta: + sector: metrics + contributors: jeff-dude, 0xRob + config: + tags: ['metrics', 'fees', 'gas', 'daily'] + description: "Sum of total fees spent per day" + data_tests: + - dbt_utils.unique_combination_of_columns: + combination_of_columns: + - blockchain + - block_date + + - name: metrics_mantle_gas_fees_daily + meta: + sector: metrics + contributors: jeff-dude, 0xRob + config: + tags: ['metrics', 'fees', 'gas', 'daily'] + description: "Sum of total fees spent per day" + data_tests: + - dbt_utils.unique_combination_of_columns: + combination_of_columns: + - blockchain + - block_date + + - name: metrics_optimism_gas_fees_daily + meta: + sector: metrics + contributors: jeff-dude, 0xRob + config: + tags: ['metrics', 'fees', 'gas', 'daily'] + description: "Sum of total fees spent per day" + data_tests: + - dbt_utils.unique_combination_of_columns: + combination_of_columns: + - blockchain + - block_date + + - name: metrics_polygon_gas_fees_daily + meta: + sector: metrics + contributors: jeff-dude, 0xRob + config: + tags: ['metrics', 'fees', 'gas', 'daily'] + description: "Sum of total fees spent per day" + data_tests: + - dbt_utils.unique_combination_of_columns: + combination_of_columns: + - blockchain + - block_date + + - name: metrics_scroll_gas_fees_daily + meta: + sector: metrics + contributors: jeff-dude, 0xRob + config: + tags: ['metrics', 'fees', 'gas', 'daily'] + description: "Sum of total fees spent per day" + data_tests: + - dbt_utils.unique_combination_of_columns: + combination_of_columns: + - blockchain + - block_date + + - name: metrics_sei_gas_fees_daily + meta: + sector: metrics + contributors: jeff-dude, 0xRob + config: + tags: ['metrics', 'fees', 'gas', 'daily'] + description: "Sum of total fees spent per day" + data_tests: + - dbt_utils.unique_combination_of_columns: + combination_of_columns: + - blockchain + - block_date + + - name: metrics_zkevm_gas_fees_daily + meta: + sector: metrics + contributors: jeff-dude, 0xRob + config: + tags: ['metrics', 'fees', 'gas', 'daily'] + description: "Sum of total fees spent per day" + data_tests: + - dbt_utils.unique_combination_of_columns: + combination_of_columns: + - blockchain + - block_date + + - name: metrics_zksync_gas_fees_daily + meta: + sector: metrics + contributors: jeff-dude, 0xRob + config: + tags: ['metrics', 'fees', 'gas', 'daily'] + description: "Sum of total fees spent per day" + data_tests: + - dbt_utils.unique_combination_of_columns: + combination_of_columns: + - blockchain + - block_date + + - name: metrics_zora_gas_fees_daily + meta: + sector: metrics + contributors: jeff-dude, 0xRob + config: + tags: ['metrics', 'fees', 'gas', 'daily'] + description: "Sum of total fees spent per day" + data_tests: + - dbt_utils.unique_combination_of_columns: + combination_of_columns: + - blockchain + - block_date diff --git a/dbt_subprojects/daily_spellbook/models/_metrics/fees/chains/evm/metrics_arbitrum_gas_fees_daily.sql b/dbt_subprojects/daily_spellbook/models/_metrics/fees/chains/evm/metrics_arbitrum_gas_fees_daily.sql new file mode 100644 index 00000000000..2e939bf2771 --- /dev/null +++ b/dbt_subprojects/daily_spellbook/models/_metrics/fees/chains/evm/metrics_arbitrum_gas_fees_daily.sql @@ -0,0 +1,15 @@ +{% set blockchain = 'arbitrum' %} + +{{ config( + schema = 'metrics_' + blockchain + , alias = 'gas_fees_daily' + , materialized = 'incremental' + , file_format = 'delta' + , incremental_strategy = 'merge' + , unique_key = ['blockchain', 'block_date'] + , incremental_predicates = [incremental_predicate('DBT_INTERNAL_DEST.block_date')] + ) +}} + + +{{ metrics_fees_evm(blockchain) }} diff --git a/dbt_subprojects/daily_spellbook/models/_metrics/fees/chains/evm/metrics_avalanche_c_gas_fees_daily.sql b/dbt_subprojects/daily_spellbook/models/_metrics/fees/chains/evm/metrics_avalanche_c_gas_fees_daily.sql new file mode 100644 index 00000000000..f83d076533c --- /dev/null +++ b/dbt_subprojects/daily_spellbook/models/_metrics/fees/chains/evm/metrics_avalanche_c_gas_fees_daily.sql @@ -0,0 +1,15 @@ +{% set blockchain = 'avalanche_c' %} + +{{ config( + schema = 'metrics_' + blockchain + , alias = 'gas_fees_daily' + , materialized = 'incremental' + , file_format = 'delta' + , incremental_strategy = 'merge' + , unique_key = ['blockchain', 'block_date'] + , incremental_predicates = [incremental_predicate('DBT_INTERNAL_DEST.block_date')] + ) +}} + + +{{ metrics_fees_evm(blockchain) }} diff --git a/dbt_subprojects/daily_spellbook/models/_metrics/fees/chains/evm/metrics_base_gas_fees_daily.sql b/dbt_subprojects/daily_spellbook/models/_metrics/fees/chains/evm/metrics_base_gas_fees_daily.sql new file mode 100644 index 00000000000..082759b8855 --- /dev/null +++ b/dbt_subprojects/daily_spellbook/models/_metrics/fees/chains/evm/metrics_base_gas_fees_daily.sql @@ -0,0 +1,15 @@ +{% set blockchain = 'base' %} + +{{ config( + schema = 'metrics_' + blockchain + , alias = 'gas_fees_daily' + , materialized = 'incremental' + , file_format = 'delta' + , incremental_strategy = 'merge' + , unique_key = ['blockchain', 'block_date'] + , incremental_predicates = [incremental_predicate('DBT_INTERNAL_DEST.block_date')] + ) +}} + + +{{ metrics_fees_evm(blockchain) }} diff --git a/dbt_subprojects/daily_spellbook/models/_metrics/fees/chains/evm/metrics_blast_gas_fees_daily.sql b/dbt_subprojects/daily_spellbook/models/_metrics/fees/chains/evm/metrics_blast_gas_fees_daily.sql new file mode 100644 index 00000000000..632e8ebf4f4 --- /dev/null +++ b/dbt_subprojects/daily_spellbook/models/_metrics/fees/chains/evm/metrics_blast_gas_fees_daily.sql @@ -0,0 +1,15 @@ +{% set blockchain = 'blast' %} + +{{ config( + schema = 'metrics_' + blockchain + , alias = 'gas_fees_daily' + , materialized = 'incremental' + , file_format = 'delta' + , incremental_strategy = 'merge' + , unique_key = ['blockchain', 'block_date'] + , incremental_predicates = [incremental_predicate('DBT_INTERNAL_DEST.block_date')] + ) +}} + + +{{ metrics_fees_evm(blockchain) }} diff --git a/dbt_subprojects/daily_spellbook/models/_metrics/fees/chains/evm/metrics_bnb_gas_fees_daily.sql b/dbt_subprojects/daily_spellbook/models/_metrics/fees/chains/evm/metrics_bnb_gas_fees_daily.sql new file mode 100644 index 00000000000..dec556a7b3c --- /dev/null +++ b/dbt_subprojects/daily_spellbook/models/_metrics/fees/chains/evm/metrics_bnb_gas_fees_daily.sql @@ -0,0 +1,15 @@ +{% set blockchain = 'bnb' %} + +{{ config( + schema = 'metrics_' + blockchain + , alias = 'gas_fees_daily' + , materialized = 'incremental' + , file_format = 'delta' + , incremental_strategy = 'merge' + , unique_key = ['blockchain', 'block_date'] + , incremental_predicates = [incremental_predicate('DBT_INTERNAL_DEST.block_date')] + ) +}} + + +{{ metrics_fees_evm(blockchain) }} diff --git a/dbt_subprojects/daily_spellbook/models/_metrics/fees/chains/evm/metrics_celo_gas_fees_daily.sql b/dbt_subprojects/daily_spellbook/models/_metrics/fees/chains/evm/metrics_celo_gas_fees_daily.sql new file mode 100644 index 00000000000..15da711d9f7 --- /dev/null +++ b/dbt_subprojects/daily_spellbook/models/_metrics/fees/chains/evm/metrics_celo_gas_fees_daily.sql @@ -0,0 +1,15 @@ +{% set blockchain = 'celo' %} + +{{ config( + schema = 'metrics_' + blockchain + , alias = 'gas_fees_daily' + , materialized = 'incremental' + , file_format = 'delta' + , incremental_strategy = 'merge' + , unique_key = ['blockchain', 'block_date'] + , incremental_predicates = [incremental_predicate('DBT_INTERNAL_DEST.block_date')] + ) +}} + + +{{ metrics_fees_evm(blockchain) }} diff --git a/dbt_subprojects/daily_spellbook/models/_metrics/fees/chains/evm/metrics_ethereum_gas_fees_daily.sql b/dbt_subprojects/daily_spellbook/models/_metrics/fees/chains/evm/metrics_ethereum_gas_fees_daily.sql new file mode 100644 index 00000000000..b22fe1e34be --- /dev/null +++ b/dbt_subprojects/daily_spellbook/models/_metrics/fees/chains/evm/metrics_ethereum_gas_fees_daily.sql @@ -0,0 +1,15 @@ +{% set blockchain = 'ethereum' %} + +{{ config( + schema = 'metrics_' + blockchain + , alias = 'gas_fees_daily' + , materialized = 'incremental' + , file_format = 'delta' + , incremental_strategy = 'merge' + , unique_key = ['blockchain', 'block_date'] + , incremental_predicates = [incremental_predicate('DBT_INTERNAL_DEST.block_date')] + ) +}} + + +{{ metrics_fees_evm(blockchain) }} diff --git a/dbt_subprojects/daily_spellbook/models/_metrics/fees/chains/evm/metrics_fantom_gas_fees_daily.sql b/dbt_subprojects/daily_spellbook/models/_metrics/fees/chains/evm/metrics_fantom_gas_fees_daily.sql new file mode 100644 index 00000000000..92e6f7bb581 --- /dev/null +++ b/dbt_subprojects/daily_spellbook/models/_metrics/fees/chains/evm/metrics_fantom_gas_fees_daily.sql @@ -0,0 +1,15 @@ +{% set blockchain = 'fantom' %} + +{{ config( + schema = 'metrics_' + blockchain + , alias = 'gas_fees_daily' + , materialized = 'incremental' + , file_format = 'delta' + , incremental_strategy = 'merge' + , unique_key = ['blockchain', 'block_date'] + , incremental_predicates = [incremental_predicate('DBT_INTERNAL_DEST.block_date')] + ) +}} + + +{{ metrics_fees_evm(blockchain) }} diff --git a/dbt_subprojects/daily_spellbook/models/_metrics/fees/chains/evm/metrics_gnosis_gas_fees_daily.sql b/dbt_subprojects/daily_spellbook/models/_metrics/fees/chains/evm/metrics_gnosis_gas_fees_daily.sql new file mode 100644 index 00000000000..8cfd81990d9 --- /dev/null +++ b/dbt_subprojects/daily_spellbook/models/_metrics/fees/chains/evm/metrics_gnosis_gas_fees_daily.sql @@ -0,0 +1,15 @@ +{% set blockchain = 'gnosis' %} + +{{ config( + schema = 'metrics_' + blockchain + , alias = 'gas_fees_daily' + , materialized = 'incremental' + , file_format = 'delta' + , incremental_strategy = 'merge' + , unique_key = ['blockchain', 'block_date'] + , incremental_predicates = [incremental_predicate('DBT_INTERNAL_DEST.block_date')] + ) +}} + + +{{ metrics_fees_evm(blockchain) }} diff --git a/dbt_subprojects/daily_spellbook/models/_metrics/fees/chains/evm/metrics_linea_gas_fees_daily.sql b/dbt_subprojects/daily_spellbook/models/_metrics/fees/chains/evm/metrics_linea_gas_fees_daily.sql new file mode 100644 index 00000000000..a9c10dd0e13 --- /dev/null +++ b/dbt_subprojects/daily_spellbook/models/_metrics/fees/chains/evm/metrics_linea_gas_fees_daily.sql @@ -0,0 +1,15 @@ +{% set blockchain = 'linea' %} + +{{ config( + schema = 'metrics_' + blockchain + , alias = 'gas_fees_daily' + , materialized = 'incremental' + , file_format = 'delta' + , incremental_strategy = 'merge' + , unique_key = ['blockchain', 'block_date'] + , incremental_predicates = [incremental_predicate('DBT_INTERNAL_DEST.block_date')] + ) +}} + + +{{ metrics_fees_evm(blockchain) }} diff --git a/dbt_subprojects/daily_spellbook/models/_metrics/fees/chains/evm/metrics_mantle_gas_fees_daily.sql b/dbt_subprojects/daily_spellbook/models/_metrics/fees/chains/evm/metrics_mantle_gas_fees_daily.sql new file mode 100644 index 00000000000..3743c06acc8 --- /dev/null +++ b/dbt_subprojects/daily_spellbook/models/_metrics/fees/chains/evm/metrics_mantle_gas_fees_daily.sql @@ -0,0 +1,15 @@ +{% set blockchain = 'mantle' %} + +{{ config( + schema = 'metrics_' + blockchain + , alias = 'gas_fees_daily' + , materialized = 'incremental' + , file_format = 'delta' + , incremental_strategy = 'merge' + , unique_key = ['blockchain', 'block_date'] + , incremental_predicates = [incremental_predicate('DBT_INTERNAL_DEST.block_date')] + ) +}} + + +{{ metrics_fees_evm(blockchain) }} diff --git a/dbt_subprojects/daily_spellbook/models/_metrics/fees/chains/evm/metrics_optimism_gas_fees_daily.sql b/dbt_subprojects/daily_spellbook/models/_metrics/fees/chains/evm/metrics_optimism_gas_fees_daily.sql new file mode 100644 index 00000000000..b96b6f9d60b --- /dev/null +++ b/dbt_subprojects/daily_spellbook/models/_metrics/fees/chains/evm/metrics_optimism_gas_fees_daily.sql @@ -0,0 +1,15 @@ +{% set blockchain = 'optimism' %} + +{{ config( + schema = 'metrics_' + blockchain + , alias = 'gas_fees_daily' + , materialized = 'incremental' + , file_format = 'delta' + , incremental_strategy = 'merge' + , unique_key = ['blockchain', 'block_date'] + , incremental_predicates = [incremental_predicate('DBT_INTERNAL_DEST.block_date')] + ) +}} + + +{{ metrics_fees_evm(blockchain) }} diff --git a/dbt_subprojects/daily_spellbook/models/_metrics/fees/chains/evm/metrics_polygon_gas_fees_daily.sql b/dbt_subprojects/daily_spellbook/models/_metrics/fees/chains/evm/metrics_polygon_gas_fees_daily.sql new file mode 100644 index 00000000000..8bcf90f4b01 --- /dev/null +++ b/dbt_subprojects/daily_spellbook/models/_metrics/fees/chains/evm/metrics_polygon_gas_fees_daily.sql @@ -0,0 +1,15 @@ +{% set blockchain = 'polygon' %} + +{{ config( + schema = 'metrics_' + blockchain + , alias = 'gas_fees_daily' + , materialized = 'incremental' + , file_format = 'delta' + , incremental_strategy = 'merge' + , unique_key = ['blockchain', 'block_date'] + , incremental_predicates = [incremental_predicate('DBT_INTERNAL_DEST.block_date')] + ) +}} + + +{{ metrics_fees_evm(blockchain) }} diff --git a/dbt_subprojects/daily_spellbook/models/_metrics/fees/chains/evm/metrics_scroll_gas_fees_daily.sql b/dbt_subprojects/daily_spellbook/models/_metrics/fees/chains/evm/metrics_scroll_gas_fees_daily.sql new file mode 100644 index 00000000000..eb6fbbd25fe --- /dev/null +++ b/dbt_subprojects/daily_spellbook/models/_metrics/fees/chains/evm/metrics_scroll_gas_fees_daily.sql @@ -0,0 +1,15 @@ +{% set blockchain = 'scroll' %} + +{{ config( + schema = 'metrics_' + blockchain + , alias = 'gas_fees_daily' + , materialized = 'incremental' + , file_format = 'delta' + , incremental_strategy = 'merge' + , unique_key = ['blockchain', 'block_date'] + , incremental_predicates = [incremental_predicate('DBT_INTERNAL_DEST.block_date')] + ) +}} + + +{{ metrics_fees_evm(blockchain) }} diff --git a/dbt_subprojects/daily_spellbook/models/_metrics/fees/chains/evm/metrics_sei_gas_fees_daily.sql b/dbt_subprojects/daily_spellbook/models/_metrics/fees/chains/evm/metrics_sei_gas_fees_daily.sql new file mode 100644 index 00000000000..3df03b06885 --- /dev/null +++ b/dbt_subprojects/daily_spellbook/models/_metrics/fees/chains/evm/metrics_sei_gas_fees_daily.sql @@ -0,0 +1,15 @@ +{% set blockchain = 'sei' %} + +{{ config( + schema = 'metrics_' + blockchain + , alias = 'gas_fees_daily' + , materialized = 'incremental' + , file_format = 'delta' + , incremental_strategy = 'merge' + , unique_key = ['blockchain', 'block_date'] + , incremental_predicates = [incremental_predicate('DBT_INTERNAL_DEST.block_date')] + ) +}} + + +{{ metrics_fees_evm(blockchain) }} diff --git a/dbt_subprojects/daily_spellbook/models/_metrics/fees/chains/evm/metrics_zkevm_gas_fees_daily.sql b/dbt_subprojects/daily_spellbook/models/_metrics/fees/chains/evm/metrics_zkevm_gas_fees_daily.sql new file mode 100644 index 00000000000..c946fdd4c47 --- /dev/null +++ b/dbt_subprojects/daily_spellbook/models/_metrics/fees/chains/evm/metrics_zkevm_gas_fees_daily.sql @@ -0,0 +1,15 @@ +{% set blockchain = 'zkevm' %} + +{{ config( + schema = 'metrics_' + blockchain + , alias = 'gas_fees_daily' + , materialized = 'incremental' + , file_format = 'delta' + , incremental_strategy = 'merge' + , unique_key = ['blockchain', 'block_date'] + , incremental_predicates = [incremental_predicate('DBT_INTERNAL_DEST.block_date')] + ) +}} + + +{{ metrics_fees_evm(blockchain) }} diff --git a/dbt_subprojects/daily_spellbook/models/_metrics/fees/chains/evm/metrics_zksync_gas_fees_daily.sql b/dbt_subprojects/daily_spellbook/models/_metrics/fees/chains/evm/metrics_zksync_gas_fees_daily.sql new file mode 100644 index 00000000000..949e3ac28fb --- /dev/null +++ b/dbt_subprojects/daily_spellbook/models/_metrics/fees/chains/evm/metrics_zksync_gas_fees_daily.sql @@ -0,0 +1,15 @@ +{% set blockchain = 'zksync' %} + +{{ config( + schema = 'metrics_' + blockchain + , alias = 'gas_fees_daily' + , materialized = 'incremental' + , file_format = 'delta' + , incremental_strategy = 'merge' + , unique_key = ['blockchain', 'block_date'] + , incremental_predicates = [incremental_predicate('DBT_INTERNAL_DEST.block_date')] + ) +}} + + +{{ metrics_fees_evm(blockchain) }} diff --git a/dbt_subprojects/daily_spellbook/models/_metrics/fees/chains/evm/metrics_zora_gas_fees_daily.sql b/dbt_subprojects/daily_spellbook/models/_metrics/fees/chains/evm/metrics_zora_gas_fees_daily.sql new file mode 100644 index 00000000000..b22a703f410 --- /dev/null +++ b/dbt_subprojects/daily_spellbook/models/_metrics/fees/chains/evm/metrics_zora_gas_fees_daily.sql @@ -0,0 +1,15 @@ +{% set blockchain = 'zora' %} + +{{ config( + schema = 'metrics_' + blockchain + , alias = 'gas_fees_daily' + , materialized = 'incremental' + , file_format = 'delta' + , incremental_strategy = 'merge' + , unique_key = ['blockchain', 'block_date'] + , incremental_predicates = [incremental_predicate('DBT_INTERNAL_DEST.block_date')] + ) +}} + + +{{ metrics_fees_evm(blockchain) }} diff --git a/dbt_subprojects/daily_spellbook/models/_metrics/fees/chains/solana/_schema.yml b/dbt_subprojects/daily_spellbook/models/_metrics/fees/chains/solana/_schema.yml new file mode 100644 index 00000000000..44116ea0c4c --- /dev/null +++ b/dbt_subprojects/daily_spellbook/models/_metrics/fees/chains/solana/_schema.yml @@ -0,0 +1,15 @@ +version: 2 + +models: + - name: metrics_solana_gas_fees_daily + meta: + sector: metrics + contributors: jeff-dude + config: + tags: ['metrics', 'fees', 'gas', 'daily'] + description: "Sum of total fees spent per day" + data_tests: + - dbt_utils.unique_combination_of_columns: + combination_of_columns: + - blockchain + - block_date diff --git a/dbt_subprojects/daily_spellbook/models/_metrics/fees/chains/solana/metrics_solana_gas_fees_daily.sql b/dbt_subprojects/daily_spellbook/models/_metrics/fees/chains/solana/metrics_solana_gas_fees_daily.sql new file mode 100644 index 00000000000..1eb0afcb862 --- /dev/null +++ b/dbt_subprojects/daily_spellbook/models/_metrics/fees/chains/solana/metrics_solana_gas_fees_daily.sql @@ -0,0 +1,60 @@ +{{ config( + schema = 'metrics_solana' + , alias = 'gas_fees_daily' + , materialized = 'incremental' + , file_format = 'delta' + , incremental_strategy = 'merge' + , unique_key = ['blockchain', 'block_date'] + , incremental_predicates = [incremental_predicate('DBT_INTERNAL_DEST.block_date')] + ) +}} + +with fees as ( + select + blockchain + , block_date + , sum(tx_fee_usd) as gas_fees_usd + from + {{ source('gas', 'fees') }} + where blockchain = 'solana' + {% if is_incremental() %} + and + {{ incremental_predicate('block_date') }} + {% endif %} + group by + blockchain + , block_date +), solana_vote_fees as ( + -- solana vote fees are stored in a different spell due to data volume & lack of value-add for materializing the fee breakdown + select + blockchain + , block_date + , sum(tx_fee_usd) as gas_fees_usd + from + {{ source('gas_solana', 'vote_fees') }} + {% if is_incremental() %} + where + {{ incremental_predicate('block_date') }} + {% endif %} + group by + blockchain + , block_date +), combined_fees as ( + select + fees.blockchain + , fees.block_date + , fees.gas_fees_usd + coalesce(solana_vote_fees.gas_fees_usd, 0) as gas_fees_usd + from + fees + left join + solana_vote_fees + on + fees.blockchain = solana_vote_fees.blockchain + and fees.block_date = solana_vote_fees.block_date +) + +select + blockchain + ,block_date + ,gas_fees_usd +from combined_fees diff --git a/dbt_subprojects/daily_spellbook/models/_metrics/fees/chains/tron/_schema.yml b/dbt_subprojects/daily_spellbook/models/_metrics/fees/chains/tron/_schema.yml new file mode 100644 index 00000000000..30a48fd7599 --- /dev/null +++ b/dbt_subprojects/daily_spellbook/models/_metrics/fees/chains/tron/_schema.yml @@ -0,0 +1,24 @@ +version: 2 + +models: + - name: tron_fee_correction + meta: + sector: metrics + contributors: 0xRob + description: "correction needed to compensate for subsized energy fees" + data_tests: + - dbt_utils.unique_combination_of_columns: + combination_of_columns: + - day + - name: metrics_tron_gas_fees_daily + meta: + sector: metrics + contributors: jeff-dude, 0xRob + config: + tags: ['metrics', 'fees', 'gas', 'daily'] + description: "Sum of total fees spent per day" + data_tests: + - dbt_utils.unique_combination_of_columns: + combination_of_columns: + - blockchain + - block_date diff --git a/dbt_subprojects/daily_spellbook/models/_metrics/fees/chains/tron/metrics_tron_gas_fees_daily.sql b/dbt_subprojects/daily_spellbook/models/_metrics/fees/chains/tron/metrics_tron_gas_fees_daily.sql new file mode 100644 index 00000000000..1de205e2724 --- /dev/null +++ b/dbt_subprojects/daily_spellbook/models/_metrics/fees/chains/tron/metrics_tron_gas_fees_daily.sql @@ -0,0 +1,39 @@ +{{ config( + schema = 'metrics_tron' + , alias = 'gas_fees_daily' + , materialized = 'incremental' + , file_format = 'delta' + , incremental_strategy = 'merge' + , unique_key = ['blockchain', 'block_date'] + , incremental_predicates = [incremental_predicate('DBT_INTERNAL_DEST.block_date')] + ) +}} + +with fees as ( + select + blockchain + , block_date + , sum(tx_fee_usd) as gas_fees_usd + from + {{ source('gas', 'fees') }} + where blockchain = 'tron' + {% if is_incremental() %} + and + {{ incremental_predicate('block_date') }} + {% endif %} + group by + blockchain + , block_date +) + +select + blockchain + ,block_date + ,gas_fees_usd * coalesce(t.trx_fee_ratio,0.0) as gas_fees_usd -- apply correction to account for subsidized fees +from fees +left join {{ref('tron_fee_correction')}} t + on block_date = t.day + and blockchain = 'tron' + {% if is_incremental() %} + and {{ incremental_predicate('day') }} + {% endif %} diff --git a/dbt_subprojects/daily_spellbook/models/_metrics/fees/tron/tron_fee_correction.sql b/dbt_subprojects/daily_spellbook/models/_metrics/fees/chains/tron/tron_fee_correction.sql similarity index 100% rename from dbt_subprojects/daily_spellbook/models/_metrics/fees/tron/tron_fee_correction.sql rename to dbt_subprojects/daily_spellbook/models/_metrics/fees/chains/tron/tron_fee_correction.sql diff --git a/dbt_subprojects/daily_spellbook/models/_metrics/fees/metrics_fees_index_daily.sql b/dbt_subprojects/daily_spellbook/models/_metrics/fees/metrics_fees_index_daily.sql deleted file mode 100644 index dd777d49048..00000000000 --- a/dbt_subprojects/daily_spellbook/models/_metrics/fees/metrics_fees_index_daily.sql +++ /dev/null @@ -1,38 +0,0 @@ -{{ config( - schema = 'metrics' - , alias = 'fees_index_daily' - , materialized = 'view' - ) -}} - -{% set baseline_date = '2018-01-01' %} -{% set start_date = '2015-08-21' %} - -with baseline as ( - select - sum(gas_fees_usd) as baseline_gas_fees_usd -- sum is required due to blockchain being second unique key in source - from - {{ ref('metrics_gas_fees_daily') }} - where - block_date = date '{{ baseline_date }}' -), daily as ( - select - blockchain - , block_date - , gas_fees_usd - from - {{ ref('metrics_gas_fees_daily') }} - where - block_date >= date '{{ start_date }}' -) -select - d.blockchain - , d.block_date - , d.gas_fees_usd - , b.baseline_gas_fees_usd - , (d.gas_fees_usd / b.baseline_gas_fees_usd) * 10 as fees_index -from - daily as d -left join - baseline as b - on 1 = 1 diff --git a/dbt_subprojects/daily_spellbook/models/_metrics/fees/metrics_gas_fees_daily.sql b/dbt_subprojects/daily_spellbook/models/_metrics/fees/metrics_gas_fees_daily.sql index 72c3c7ac935..4d577b17c5f 100644 --- a/dbt_subprojects/daily_spellbook/models/_metrics/fees/metrics_gas_fees_daily.sql +++ b/dbt_subprojects/daily_spellbook/models/_metrics/fees/metrics_gas_fees_daily.sql @@ -1,112 +1,44 @@ {{ config( schema = 'metrics' , alias = 'gas_fees_daily' - , materialized = 'incremental' - , file_format = 'delta' - , incremental_strategy = 'merge' - , unique_key = ['blockchain', 'block_date'] - , incremental_predicates = [incremental_predicate('DBT_INTERNAL_DEST.block_date')] + , materialized = 'view' ) }} -with fees as ( - select - blockchain - , block_date - , sum(tx_fee_usd) as gas_fees_usd - from - {{ source('gas', 'fees') }} - {% if is_incremental() %} - where - {{ incremental_predicate('block_date') }} - {% endif %} - group by - blockchain - , block_date -), solana_vote_fees as ( - -- solana vote fees are stored in a different spell due to data volume & lack of value-add for materializing the fee breakdown - select - blockchain - , block_date - , sum(tx_fee_usd) as gas_fees_usd - from - {{ source('gas_solana', 'vote_fees') }} - {% if is_incremental() %} - where - {{ incremental_predicate('block_date') }} - {% endif %} - group by +{% set chains = [ + 'arbitrum' + , 'avalanche_c' + , 'base' + , 'bitcoin' + , 'blast' + , 'bnb' + , 'celo' + , 'ethereum' + , 'fantom' + , 'gnosis' + , 'linea' + , 'mantle' + , 'optimism' + , 'polygon' + , 'scroll' + , 'sei' + , 'solana' + , 'tron' + , 'zkevm' + , 'zksync' + , 'zora' +] %} + +SELECT * +FROM ( + {% for blockchain in chains %} + SELECT blockchain - , block_date -), combined_fees as ( - select - fees.blockchain - , fees.block_date - , fees.gas_fees_usd + coalesce(solana_vote_fees.gas_fees_usd, 0) as gas_fees_usd - from - fees - left join - solana_vote_fees - on - fees.blockchain = solana_vote_fees.blockchain - and fees.block_date = solana_vote_fees.block_date -), bitcoin_fees as ( - with prices as ( - select - day - , price - from - {{ source('prices', 'usd_daily') }} - where - symbol = 'BTC' - and blockchain is null - {% if is_incremental() %} - and {{ incremental_predicate('day') }} - {% endif %} - ), btc as ( - select - date as block_date - , sum(total_fees) as daily_fee - from - {{ source('bitcoin', 'blocks') }} - where - date < cast(date_trunc('day', now()) as date) --exclude current day to match prices.usd_daily - {% if is_incremental() %} - and {{ incremental_predicate('date') }} - {% endif %} - group by - date - ) - select - 'bitcoin' as blockchain - , btc.block_date - , btc.daily_fee as gas_fees_raw - , prices.price as daily_price - , (btc.daily_fee * prices.price) as gas_fees_usd - from - btc - inner join - prices - on btc.block_date = prices.day + ,block_date + ,gas_fees_usd + FROM {{ ref('metrics_' + blockchain + '_gas_fees_daily') }} + {% if not loop.last %} + UNION ALL + {% endif %} + {% endfor %} ) -select - blockchain - ,block_date - ,case when blockchain = 'tron' - then gas_fees_usd * coalesce(t.trx_fee_ratio,0.0) -- apply correction to account for subsidized fees - else gas_fees_usd end - as gas_fees_usd -from combined_fees -left join {{ref('tron_fee_correction')}} t - on block_date = t.day - and blockchain = 'tron' - {% if is_incremental() %} - and {{ incremental_predicate('day') }} - {% endif %} -union all -select - blockchain - , block_date - , gas_fees_usd -from - bitcoin_fees \ No newline at end of file diff --git a/dbt_subprojects/daily_spellbook/models/_metrics/fees/metrics_gas_fees_stats.sql b/dbt_subprojects/daily_spellbook/models/_metrics/fees/metrics_gas_fees_stats.sql index ded73862c35..c1a860c28f5 100644 --- a/dbt_subprojects/daily_spellbook/models/_metrics/fees/metrics_gas_fees_stats.sql +++ b/dbt_subprojects/daily_spellbook/models/_metrics/fees/metrics_gas_fees_stats.sql @@ -155,4 +155,4 @@ from inner join weekly_stats as w on d.blockchain = w.blockchain inner join monthly_stats as m - on d.blockchain = m.blockchain \ No newline at end of file + on d.blockchain = m.blockchain diff --git a/dbt_subprojects/daily_spellbook/models/_metrics/fees/tron/_schema.yml b/dbt_subprojects/daily_spellbook/models/_metrics/fees/tron/_schema.yml deleted file mode 100644 index cc3aa2a265b..00000000000 --- a/dbt_subprojects/daily_spellbook/models/_metrics/fees/tron/_schema.yml +++ /dev/null @@ -1,12 +0,0 @@ -version: 2 - -models: - - name: tron_fee_correction - meta: - sector: metrics - contributors: 0xRob - description: "correction needed to compensate for subsized energy fees" - data_tests: - - dbt_utils.unique_combination_of_columns: - combination_of_columns: - - day diff --git a/dbt_subprojects/daily_spellbook/models/_metrics/transactions/_schema.yml b/dbt_subprojects/daily_spellbook/models/_metrics/transactions/_schema.yml index 5f6e1f051ca..82c58a98985 100644 --- a/dbt_subprojects/daily_spellbook/models/_metrics/transactions/_schema.yml +++ b/dbt_subprojects/daily_spellbook/models/_metrics/transactions/_schema.yml @@ -8,27 +8,11 @@ models: config: tags: ['metrics', 'transactions', 'daily'] description: "Sum of total tx's per day across all chains" - data_tests: - - dbt_utils.unique_combination_of_columns: - combination_of_columns: - - blockchain - - block_date + - name: metrics_transactions_stats meta: sector: metrics contributors: jeff-dude config: - tags: ['metrics', 'transactions'] - description: "View of tx's per blockchain aggregated to various levels. The goal is to output one row per chain with stats availble for use in counter visuals." - - name: metrics_transactions_index_daily - meta: - sector: metrics - contributors: jeff-dude - config: - tags: ['metrics', 'transactions', 'index', 'daily'] - description: "Each day, per chain, compare the adoption of total transactions relative to the baseline expectation" - data_tests: - - dbt_utils.unique_combination_of_columns: - combination_of_columns: - - blockchain - - block_date \ No newline at end of file + tags: [ 'metrics', 'transactions' ] + description: "View of tx's per blockchain aggregated to various levels. The goal is to output one row per chain." diff --git a/dbt_subprojects/daily_spellbook/models/_metrics/transactions/chains/bitcoin/_schema.yml b/dbt_subprojects/daily_spellbook/models/_metrics/transactions/chains/bitcoin/_schema.yml new file mode 100644 index 00000000000..aa54dcc9b6d --- /dev/null +++ b/dbt_subprojects/daily_spellbook/models/_metrics/transactions/chains/bitcoin/_schema.yml @@ -0,0 +1,15 @@ +version: 2 + +models: + - name: metrics_bitcoin_transactions_daily + meta: + sector: metrics + contributors: jeff-dude + config: + tags: ['metrics', 'transactions', 'daily'] + description: "Sum of total tx's per day" + data_tests: + - dbt_utils.unique_combination_of_columns: + combination_of_columns: + - blockchain + - block_date diff --git a/dbt_subprojects/daily_spellbook/models/_metrics/transactions/chains/bitcoin/metrics_bitcoin_transactions_daily.sql b/dbt_subprojects/daily_spellbook/models/_metrics/transactions/chains/bitcoin/metrics_bitcoin_transactions_daily.sql new file mode 100644 index 00000000000..378c3e6b82d --- /dev/null +++ b/dbt_subprojects/daily_spellbook/models/_metrics/transactions/chains/bitcoin/metrics_bitcoin_transactions_daily.sql @@ -0,0 +1,26 @@ +{{ config( + schema = 'metrics_bitcoin' + , alias = 'transactions_daily' + , materialized = 'incremental' + , file_format = 'delta' + , incremental_strategy = 'merge' + , unique_key = ['blockchain', 'block_date'] + , incremental_predicates = [incremental_predicate('DBT_INTERNAL_DEST.block_date')] + ) +}} + +select + blockchain + , block_date + , approx_distinct(tx_id) as tx_count +from + {{ source('transfers_bitcoin', 'satoshi') }} +where + 1 = 1 + and amount_transfer_usd > 1 + {% if is_incremental() %} + and {{ incremental_predicate('block_date') }} + {% endif %} +group by + blockchain + , block_date diff --git a/dbt_subprojects/daily_spellbook/models/_metrics/transactions/chains/evm/_schema.yml b/dbt_subprojects/daily_spellbook/models/_metrics/transactions/chains/evm/_schema.yml new file mode 100644 index 00000000000..aa0e43b7912 --- /dev/null +++ b/dbt_subprojects/daily_spellbook/models/_metrics/transactions/chains/evm/_schema.yml @@ -0,0 +1,249 @@ +version: 2 + +models: + - name: metrics_arbitrum_transactions_daily + meta: + sector: metrics + contributors: jeff-dude, 0xRob + config: + tags: ['metrics', 'transactions', 'daily'] + description: "Sum of total tx's per day" + data_tests: + - dbt_utils.unique_combination_of_columns: + combination_of_columns: + - blockchain + - block_date + + - name: metrics_avalanche_c_transactions_daily + meta: + sector: metrics + contributors: jeff-dude, 0xRob + config: + tags: ['metrics', 'transactions', 'daily'] + description: "Sum of total tx's per day" + data_tests: + - dbt_utils.unique_combination_of_columns: + combination_of_columns: + - blockchain + - block_date + + - name: metrics_base_transactions_daily + meta: + sector: metrics + contributors: jeff-dude, 0xRob + config: + tags: ['metrics', 'transactions', 'daily'] + description: "Sum of total tx's per day" + data_tests: + - dbt_utils.unique_combination_of_columns: + combination_of_columns: + - blockchain + - block_date + + - name: metrics_blast_transactions_daily + meta: + sector: metrics + contributors: jeff-dude, 0xRob + config: + tags: ['metrics', 'transactions', 'daily'] + description: "Sum of total tx's per day" + data_tests: + - dbt_utils.unique_combination_of_columns: + combination_of_columns: + - blockchain + - block_date + + - name: metrics_bnb_transactions_daily + meta: + sector: metrics + contributors: jeff-dude, 0xRob + config: + tags: ['metrics', 'transactions', 'daily'] + description: "Sum of total tx's per day" + data_tests: + - dbt_utils.unique_combination_of_columns: + combination_of_columns: + - blockchain + - block_date + + - name: metrics_celo_transactions_daily + meta: + sector: metrics + contributors: jeff-dude, 0xRob + config: + tags: ['metrics', 'transactions', 'daily'] + description: "Sum of total tx's per day" + data_tests: + - dbt_utils.unique_combination_of_columns: + combination_of_columns: + - blockchain + - block_date + + - name: metrics_ethereum_transactions_daily + meta: + sector: metrics + contributors: jeff-dude, 0xRob + config: + tags: ['metrics', 'transactions', 'daily'] + description: "Sum of total tx's per day" + data_tests: + - dbt_utils.unique_combination_of_columns: + combination_of_columns: + - blockchain + - block_date + + - name: metrics_fantom_transactions_daily + meta: + sector: metrics + contributors: jeff-dude, 0xRob + config: + tags: ['metrics', 'transactions', 'daily'] + description: "Sum of total tx's per day" + data_tests: + - dbt_utils.unique_combination_of_columns: + combination_of_columns: + - blockchain + - block_date + + - name: metrics_gnosis_transactions_daily + meta: + sector: metrics + contributors: jeff-dude, 0xRob + config: + tags: ['metrics', 'transactions', 'daily'] + description: "Sum of total tx's per day" + data_tests: + - dbt_utils.unique_combination_of_columns: + combination_of_columns: + - blockchain + - block_date + + - name: metrics_linea_transactions_daily + meta: + sector: metrics + contributors: jeff-dude, 0xRob + config: + tags: ['metrics', 'transactions', 'daily'] + description: "Sum of total tx's per day" + data_tests: + - dbt_utils.unique_combination_of_columns: + combination_of_columns: + - blockchain + - block_date + + - name: metrics_mantle_transactions_daily + meta: + sector: metrics + contributors: jeff-dude, 0xRob + config: + tags: ['metrics', 'transactions', 'daily'] + description: "Sum of total tx's per day" + data_tests: + - dbt_utils.unique_combination_of_columns: + combination_of_columns: + - blockchain + - block_date + + - name: metrics_optimism_transactions_daily + meta: + sector: metrics + contributors: jeff-dude, 0xRob + config: + tags: ['metrics', 'transactions', 'daily'] + description: "Sum of total tx's per day" + data_tests: + - dbt_utils.unique_combination_of_columns: + combination_of_columns: + - blockchain + - block_date + + - name: metrics_polygon_transactions_daily + meta: + sector: metrics + contributors: jeff-dude, 0xRob + config: + tags: ['metrics', 'transactions', 'daily'] + description: "Sum of total tx's per day" + data_tests: + - dbt_utils.unique_combination_of_columns: + combination_of_columns: + - blockchain + - block_date + + - name: metrics_scroll_transactions_daily + meta: + sector: metrics + contributors: jeff-dude, 0xRob + config: + tags: ['metrics', 'transactions', 'daily'] + description: "Sum of total tx's per day" + data_tests: + - dbt_utils.unique_combination_of_columns: + combination_of_columns: + - blockchain + - block_date + + - name: metrics_sei_transactions_daily + meta: + sector: metrics + contributors: jeff-dude, 0xRob + config: + tags: ['metrics', 'transactions', 'daily'] + description: "Sum of total tx's per day" + data_tests: + - dbt_utils.unique_combination_of_columns: + combination_of_columns: + - blockchain + - block_date + + - name: metrics_tron_transactions_daily + meta: + sector: metrics + contributors: jeff-dude, 0xRob + config: + tags: ['metrics', 'transactions', 'daily'] + description: "Sum of total tx's per day" + data_tests: + - dbt_utils.unique_combination_of_columns: + combination_of_columns: + - blockchain + - block_date + + - name: metrics_zkevm_transactions_daily + meta: + sector: metrics + contributors: jeff-dude, 0xRob + config: + tags: ['metrics', 'transactions', 'daily'] + description: "Sum of total tx's per day" + data_tests: + - dbt_utils.unique_combination_of_columns: + combination_of_columns: + - blockchain + - block_date + + - name: metrics_zksync_transactions_daily + meta: + sector: metrics + contributors: jeff-dude, 0xRob + config: + tags: ['metrics', 'transactions', 'daily'] + description: "Sum of total tx's per day" + data_tests: + - dbt_utils.unique_combination_of_columns: + combination_of_columns: + - blockchain + - block_date + + - name: metrics_zora_transactions_daily + meta: + sector: metrics + contributors: jeff-dude, 0xRob + config: + tags: ['metrics', 'transactions', 'daily'] + description: "Sum of total tx's per day" + data_tests: + - dbt_utils.unique_combination_of_columns: + combination_of_columns: + - blockchain + - block_date diff --git a/dbt_subprojects/daily_spellbook/models/_metrics/transactions/chains/evm/metrics_arbitrum_transactions_daily.sql b/dbt_subprojects/daily_spellbook/models/_metrics/transactions/chains/evm/metrics_arbitrum_transactions_daily.sql new file mode 100644 index 00000000000..0293fd2b2c8 --- /dev/null +++ b/dbt_subprojects/daily_spellbook/models/_metrics/transactions/chains/evm/metrics_arbitrum_transactions_daily.sql @@ -0,0 +1,15 @@ +{% set blockchain = 'arbitrum' %} + +{{ config( + schema = 'metrics_' + blockchain + , alias = 'transactions_daily' + , materialized = 'incremental' + , file_format = 'delta' + , incremental_strategy = 'merge' + , unique_key = ['blockchain', 'block_date'] + , incremental_predicates = [incremental_predicate('DBT_INTERNAL_DEST.block_date')] + ) +}} + + +{{ metrics_transactions_evm(blockchain) }} diff --git a/dbt_subprojects/daily_spellbook/models/_metrics/transactions/chains/evm/metrics_avalanche_c_transactions_daily.sql b/dbt_subprojects/daily_spellbook/models/_metrics/transactions/chains/evm/metrics_avalanche_c_transactions_daily.sql new file mode 100644 index 00000000000..d2e90789069 --- /dev/null +++ b/dbt_subprojects/daily_spellbook/models/_metrics/transactions/chains/evm/metrics_avalanche_c_transactions_daily.sql @@ -0,0 +1,15 @@ +{% set blockchain = 'avalanche_c' %} + +{{ config( + schema = 'metrics_' + blockchain + , alias = 'transactions_daily' + , materialized = 'incremental' + , file_format = 'delta' + , incremental_strategy = 'merge' + , unique_key = ['blockchain', 'block_date'] + , incremental_predicates = [incremental_predicate('DBT_INTERNAL_DEST.block_date')] + ) +}} + + +{{ metrics_transactions_evm(blockchain) }} diff --git a/dbt_subprojects/daily_spellbook/models/_metrics/transactions/chains/evm/metrics_base_transactions_daily.sql b/dbt_subprojects/daily_spellbook/models/_metrics/transactions/chains/evm/metrics_base_transactions_daily.sql new file mode 100644 index 00000000000..9d3ee77d25b --- /dev/null +++ b/dbt_subprojects/daily_spellbook/models/_metrics/transactions/chains/evm/metrics_base_transactions_daily.sql @@ -0,0 +1,15 @@ +{% set blockchain = 'base' %} + +{{ config( + schema = 'metrics_' + blockchain + , alias = 'transactions_daily' + , materialized = 'incremental' + , file_format = 'delta' + , incremental_strategy = 'merge' + , unique_key = ['blockchain', 'block_date'] + , incremental_predicates = [incremental_predicate('DBT_INTERNAL_DEST.block_date')] + ) +}} + + +{{ metrics_transactions_evm(blockchain) }} diff --git a/dbt_subprojects/daily_spellbook/models/_metrics/transactions/chains/evm/metrics_blast_transactions_daily.sql b/dbt_subprojects/daily_spellbook/models/_metrics/transactions/chains/evm/metrics_blast_transactions_daily.sql new file mode 100644 index 00000000000..7c5046ea9c2 --- /dev/null +++ b/dbt_subprojects/daily_spellbook/models/_metrics/transactions/chains/evm/metrics_blast_transactions_daily.sql @@ -0,0 +1,15 @@ +{% set blockchain = 'blast' %} + +{{ config( + schema = 'metrics_' + blockchain + , alias = 'transactions_daily' + , materialized = 'incremental' + , file_format = 'delta' + , incremental_strategy = 'merge' + , unique_key = ['blockchain', 'block_date'] + , incremental_predicates = [incremental_predicate('DBT_INTERNAL_DEST.block_date')] + ) +}} + + +{{ metrics_transactions_evm(blockchain) }} diff --git a/dbt_subprojects/daily_spellbook/models/_metrics/transactions/chains/evm/metrics_bnb_transactions_daily.sql b/dbt_subprojects/daily_spellbook/models/_metrics/transactions/chains/evm/metrics_bnb_transactions_daily.sql new file mode 100644 index 00000000000..c3feff1e770 --- /dev/null +++ b/dbt_subprojects/daily_spellbook/models/_metrics/transactions/chains/evm/metrics_bnb_transactions_daily.sql @@ -0,0 +1,15 @@ +{% set blockchain = 'bnb' %} + +{{ config( + schema = 'metrics_' + blockchain + , alias = 'transactions_daily' + , materialized = 'incremental' + , file_format = 'delta' + , incremental_strategy = 'merge' + , unique_key = ['blockchain', 'block_date'] + , incremental_predicates = [incremental_predicate('DBT_INTERNAL_DEST.block_date')] + ) +}} + + +{{ metrics_transactions_evm(blockchain) }} diff --git a/dbt_subprojects/daily_spellbook/models/_metrics/transactions/chains/evm/metrics_celo_transactions_daily.sql b/dbt_subprojects/daily_spellbook/models/_metrics/transactions/chains/evm/metrics_celo_transactions_daily.sql new file mode 100644 index 00000000000..94822b61bb1 --- /dev/null +++ b/dbt_subprojects/daily_spellbook/models/_metrics/transactions/chains/evm/metrics_celo_transactions_daily.sql @@ -0,0 +1,15 @@ +{% set blockchain = 'celo' %} + +{{ config( + schema = 'metrics_' + blockchain + , alias = 'transactions_daily' + , materialized = 'incremental' + , file_format = 'delta' + , incremental_strategy = 'merge' + , unique_key = ['blockchain', 'block_date'] + , incremental_predicates = [incremental_predicate('DBT_INTERNAL_DEST.block_date')] + ) +}} + + +{{ metrics_transactions_evm(blockchain) }} diff --git a/dbt_subprojects/daily_spellbook/models/_metrics/transactions/chains/evm/metrics_ethereum_transactions_daily.sql b/dbt_subprojects/daily_spellbook/models/_metrics/transactions/chains/evm/metrics_ethereum_transactions_daily.sql new file mode 100644 index 00000000000..6f053536b70 --- /dev/null +++ b/dbt_subprojects/daily_spellbook/models/_metrics/transactions/chains/evm/metrics_ethereum_transactions_daily.sql @@ -0,0 +1,15 @@ +{% set blockchain = 'ethereum' %} + +{{ config( + schema = 'metrics_' + blockchain + , alias = 'transactions_daily' + , materialized = 'incremental' + , file_format = 'delta' + , incremental_strategy = 'merge' + , unique_key = ['blockchain', 'block_date'] + , incremental_predicates = [incremental_predicate('DBT_INTERNAL_DEST.block_date')] + ) +}} + + +{{ metrics_transactions_evm(blockchain) }} diff --git a/dbt_subprojects/daily_spellbook/models/_metrics/transactions/chains/evm/metrics_fantom_transactions_daily.sql b/dbt_subprojects/daily_spellbook/models/_metrics/transactions/chains/evm/metrics_fantom_transactions_daily.sql new file mode 100644 index 00000000000..9d8043425fb --- /dev/null +++ b/dbt_subprojects/daily_spellbook/models/_metrics/transactions/chains/evm/metrics_fantom_transactions_daily.sql @@ -0,0 +1,15 @@ +{% set blockchain = 'fantom' %} + +{{ config( + schema = 'metrics_' + blockchain + , alias = 'transactions_daily' + , materialized = 'incremental' + , file_format = 'delta' + , incremental_strategy = 'merge' + , unique_key = ['blockchain', 'block_date'] + , incremental_predicates = [incremental_predicate('DBT_INTERNAL_DEST.block_date')] + ) +}} + + +{{ metrics_transactions_evm(blockchain) }} diff --git a/dbt_subprojects/daily_spellbook/models/_metrics/transactions/chains/evm/metrics_gnosis_transactions_daily.sql b/dbt_subprojects/daily_spellbook/models/_metrics/transactions/chains/evm/metrics_gnosis_transactions_daily.sql new file mode 100644 index 00000000000..b9f6ba8a83a --- /dev/null +++ b/dbt_subprojects/daily_spellbook/models/_metrics/transactions/chains/evm/metrics_gnosis_transactions_daily.sql @@ -0,0 +1,15 @@ +{% set blockchain = 'gnosis' %} + +{{ config( + schema = 'metrics_' + blockchain + , alias = 'transactions_daily' + , materialized = 'incremental' + , file_format = 'delta' + , incremental_strategy = 'merge' + , unique_key = ['blockchain', 'block_date'] + , incremental_predicates = [incremental_predicate('DBT_INTERNAL_DEST.block_date')] + ) +}} + + +{{ metrics_transactions_evm(blockchain) }} diff --git a/dbt_subprojects/daily_spellbook/models/_metrics/transactions/chains/evm/metrics_linea_transactions_daily.sql b/dbt_subprojects/daily_spellbook/models/_metrics/transactions/chains/evm/metrics_linea_transactions_daily.sql new file mode 100644 index 00000000000..05c80ad240a --- /dev/null +++ b/dbt_subprojects/daily_spellbook/models/_metrics/transactions/chains/evm/metrics_linea_transactions_daily.sql @@ -0,0 +1,15 @@ +{% set blockchain = 'linea' %} + +{{ config( + schema = 'metrics_' + blockchain + , alias = 'transactions_daily' + , materialized = 'incremental' + , file_format = 'delta' + , incremental_strategy = 'merge' + , unique_key = ['blockchain', 'block_date'] + , incremental_predicates = [incremental_predicate('DBT_INTERNAL_DEST.block_date')] + ) +}} + + +{{ metrics_transactions_evm(blockchain) }} diff --git a/dbt_subprojects/daily_spellbook/models/_metrics/transactions/chains/evm/metrics_mantle_transactions_daily.sql b/dbt_subprojects/daily_spellbook/models/_metrics/transactions/chains/evm/metrics_mantle_transactions_daily.sql new file mode 100644 index 00000000000..6e98f940369 --- /dev/null +++ b/dbt_subprojects/daily_spellbook/models/_metrics/transactions/chains/evm/metrics_mantle_transactions_daily.sql @@ -0,0 +1,15 @@ +{% set blockchain = 'mantle' %} + +{{ config( + schema = 'metrics_' + blockchain + , alias = 'transactions_daily' + , materialized = 'incremental' + , file_format = 'delta' + , incremental_strategy = 'merge' + , unique_key = ['blockchain', 'block_date'] + , incremental_predicates = [incremental_predicate('DBT_INTERNAL_DEST.block_date')] + ) +}} + + +{{ metrics_transactions_evm(blockchain) }} diff --git a/dbt_subprojects/daily_spellbook/models/_metrics/transactions/chains/evm/metrics_optimism_transactions_daily.sql b/dbt_subprojects/daily_spellbook/models/_metrics/transactions/chains/evm/metrics_optimism_transactions_daily.sql new file mode 100644 index 00000000000..766ea462934 --- /dev/null +++ b/dbt_subprojects/daily_spellbook/models/_metrics/transactions/chains/evm/metrics_optimism_transactions_daily.sql @@ -0,0 +1,15 @@ +{% set blockchain = 'optimism' %} + +{{ config( + schema = 'metrics_' + blockchain + , alias = 'transactions_daily' + , materialized = 'incremental' + , file_format = 'delta' + , incremental_strategy = 'merge' + , unique_key = ['blockchain', 'block_date'] + , incremental_predicates = [incremental_predicate('DBT_INTERNAL_DEST.block_date')] + ) +}} + + +{{ metrics_transactions_evm(blockchain) }} diff --git a/dbt_subprojects/daily_spellbook/models/_metrics/transactions/chains/evm/metrics_polygon_transactions_daily.sql b/dbt_subprojects/daily_spellbook/models/_metrics/transactions/chains/evm/metrics_polygon_transactions_daily.sql new file mode 100644 index 00000000000..e3cebf43868 --- /dev/null +++ b/dbt_subprojects/daily_spellbook/models/_metrics/transactions/chains/evm/metrics_polygon_transactions_daily.sql @@ -0,0 +1,15 @@ +{% set blockchain = 'polygon' %} + +{{ config( + schema = 'metrics_' + blockchain + , alias = 'transactions_daily' + , materialized = 'incremental' + , file_format = 'delta' + , incremental_strategy = 'merge' + , unique_key = ['blockchain', 'block_date'] + , incremental_predicates = [incremental_predicate('DBT_INTERNAL_DEST.block_date')] + ) +}} + + +{{ metrics_transactions_evm(blockchain) }} diff --git a/dbt_subprojects/daily_spellbook/models/_metrics/transactions/chains/evm/metrics_scroll_transactions_daily.sql b/dbt_subprojects/daily_spellbook/models/_metrics/transactions/chains/evm/metrics_scroll_transactions_daily.sql new file mode 100644 index 00000000000..59d31f62ab9 --- /dev/null +++ b/dbt_subprojects/daily_spellbook/models/_metrics/transactions/chains/evm/metrics_scroll_transactions_daily.sql @@ -0,0 +1,15 @@ +{% set blockchain = 'scroll' %} + +{{ config( + schema = 'metrics_' + blockchain + , alias = 'transactions_daily' + , materialized = 'incremental' + , file_format = 'delta' + , incremental_strategy = 'merge' + , unique_key = ['blockchain', 'block_date'] + , incremental_predicates = [incremental_predicate('DBT_INTERNAL_DEST.block_date')] + ) +}} + + +{{ metrics_transactions_evm(blockchain) }} diff --git a/dbt_subprojects/daily_spellbook/models/_metrics/transactions/chains/evm/metrics_sei_transactions_daily.sql b/dbt_subprojects/daily_spellbook/models/_metrics/transactions/chains/evm/metrics_sei_transactions_daily.sql new file mode 100644 index 00000000000..4fde7ded5d5 --- /dev/null +++ b/dbt_subprojects/daily_spellbook/models/_metrics/transactions/chains/evm/metrics_sei_transactions_daily.sql @@ -0,0 +1,15 @@ +{% set blockchain = 'sei' %} + +{{ config( + schema = 'metrics_' + blockchain + , alias = 'transactions_daily' + , materialized = 'incremental' + , file_format = 'delta' + , incremental_strategy = 'merge' + , unique_key = ['blockchain', 'block_date'] + , incremental_predicates = [incremental_predicate('DBT_INTERNAL_DEST.block_date')] + ) +}} + + +{{ metrics_transactions_evm(blockchain) }} diff --git a/dbt_subprojects/daily_spellbook/models/_metrics/transactions/chains/evm/metrics_tron_transactions_daily.sql b/dbt_subprojects/daily_spellbook/models/_metrics/transactions/chains/evm/metrics_tron_transactions_daily.sql new file mode 100644 index 00000000000..7a56a34abfe --- /dev/null +++ b/dbt_subprojects/daily_spellbook/models/_metrics/transactions/chains/evm/metrics_tron_transactions_daily.sql @@ -0,0 +1,15 @@ +{% set blockchain = 'tron' %} + +{{ config( + schema = 'metrics_' + blockchain + , alias = 'transactions_daily' + , materialized = 'incremental' + , file_format = 'delta' + , incremental_strategy = 'merge' + , unique_key = ['blockchain', 'block_date'] + , incremental_predicates = [incremental_predicate('DBT_INTERNAL_DEST.block_date')] + ) +}} + + +{{ metrics_transactions_evm(blockchain) }} diff --git a/dbt_subprojects/daily_spellbook/models/_metrics/transactions/chains/evm/metrics_zkevm_transactions_daily.sql b/dbt_subprojects/daily_spellbook/models/_metrics/transactions/chains/evm/metrics_zkevm_transactions_daily.sql new file mode 100644 index 00000000000..bc1b4c9ba6b --- /dev/null +++ b/dbt_subprojects/daily_spellbook/models/_metrics/transactions/chains/evm/metrics_zkevm_transactions_daily.sql @@ -0,0 +1,15 @@ +{% set blockchain = 'zkevm' %} + +{{ config( + schema = 'metrics_' + blockchain + , alias = 'transactions_daily' + , materialized = 'incremental' + , file_format = 'delta' + , incremental_strategy = 'merge' + , unique_key = ['blockchain', 'block_date'] + , incremental_predicates = [incremental_predicate('DBT_INTERNAL_DEST.block_date')] + ) +}} + + +{{ metrics_transactions_evm(blockchain) }} diff --git a/dbt_subprojects/daily_spellbook/models/_metrics/transactions/chains/evm/metrics_zksync_transactions_daily.sql b/dbt_subprojects/daily_spellbook/models/_metrics/transactions/chains/evm/metrics_zksync_transactions_daily.sql new file mode 100644 index 00000000000..73bf7c01fc2 --- /dev/null +++ b/dbt_subprojects/daily_spellbook/models/_metrics/transactions/chains/evm/metrics_zksync_transactions_daily.sql @@ -0,0 +1,15 @@ +{% set blockchain = 'zksync' %} + +{{ config( + schema = 'metrics_' + blockchain + , alias = 'transactions_daily' + , materialized = 'incremental' + , file_format = 'delta' + , incremental_strategy = 'merge' + , unique_key = ['blockchain', 'block_date'] + , incremental_predicates = [incremental_predicate('DBT_INTERNAL_DEST.block_date')] + ) +}} + + +{{ metrics_transactions_evm(blockchain) }} diff --git a/dbt_subprojects/daily_spellbook/models/_metrics/transactions/chains/evm/metrics_zora_transactions_daily.sql b/dbt_subprojects/daily_spellbook/models/_metrics/transactions/chains/evm/metrics_zora_transactions_daily.sql new file mode 100644 index 00000000000..cf28eb3b1d1 --- /dev/null +++ b/dbt_subprojects/daily_spellbook/models/_metrics/transactions/chains/evm/metrics_zora_transactions_daily.sql @@ -0,0 +1,15 @@ +{% set blockchain = 'zora' %} + +{{ config( + schema = 'metrics_' + blockchain + , alias = 'transactions_daily' + , materialized = 'incremental' + , file_format = 'delta' + , incremental_strategy = 'merge' + , unique_key = ['blockchain', 'block_date'] + , incremental_predicates = [incremental_predicate('DBT_INTERNAL_DEST.block_date')] + ) +}} + + +{{ metrics_transactions_evm(blockchain) }} diff --git a/dbt_subprojects/daily_spellbook/models/_metrics/transactions/chains/solana/_schema.yml b/dbt_subprojects/daily_spellbook/models/_metrics/transactions/chains/solana/_schema.yml new file mode 100644 index 00000000000..e314dcbfb2a --- /dev/null +++ b/dbt_subprojects/daily_spellbook/models/_metrics/transactions/chains/solana/_schema.yml @@ -0,0 +1,15 @@ +version: 2 + +models: + - name: metrics_solana_transactions_daily + meta: + sector: metrics + contributors: jeff-dude + config: + tags: ['metrics', 'transactions', 'daily'] + description: "Sum of total tx's per day" + data_tests: + - dbt_utils.unique_combination_of_columns: + combination_of_columns: + - blockchain + - block_date diff --git a/dbt_subprojects/daily_spellbook/models/_metrics/transactions/chains/solana/metrics_solana_transactions_daily.sql b/dbt_subprojects/daily_spellbook/models/_metrics/transactions/chains/solana/metrics_solana_transactions_daily.sql new file mode 100644 index 00000000000..141f01f5824 --- /dev/null +++ b/dbt_subprojects/daily_spellbook/models/_metrics/transactions/chains/solana/metrics_solana_transactions_daily.sql @@ -0,0 +1,27 @@ +{{ config( + schema = 'metrics_solana' + , alias = 'transactions_daily' + , materialized = 'incremental' + , file_format = 'delta' + , incremental_strategy = 'merge' + , unique_key = ['blockchain', 'block_date'] + , incremental_predicates = [incremental_predicate('DBT_INTERNAL_DEST.block_date')] + ) +}} + +select + 'solana' as blockchain + , block_date + , approx_distinct(tx_id) as tx_count +from + {{ source('tokens_solana', 'transfers') }} +where + 1 = 1 + and action != 'wrap' + and amount_usd > 1 + {% if is_incremental() %} + and {{ incremental_predicate('block_date') }} + {% endif %} +group by + 'solana' + , block_date diff --git a/dbt_subprojects/daily_spellbook/models/_metrics/transactions/metrics_transactions_daily.sql b/dbt_subprojects/daily_spellbook/models/_metrics/transactions/metrics_transactions_daily.sql index 62b6a2777aa..915451e5703 100644 --- a/dbt_subprojects/daily_spellbook/models/_metrics/transactions/metrics_transactions_daily.sql +++ b/dbt_subprojects/daily_spellbook/models/_metrics/transactions/metrics_transactions_daily.sql @@ -1,83 +1,44 @@ {{ config( schema = 'metrics' , alias = 'transactions_daily' - , materialized = 'incremental' - , file_format = 'delta' - , incremental_strategy = 'merge' - , unique_key = ['blockchain', 'block_date'] - , incremental_predicates = [incremental_predicate('DBT_INTERNAL_DEST.block_date')] + , materialized = 'view' ) }} -with evm as ( - select - blockchain - , block_date - , approx_distinct(tx_hash) as tx_count --max 2% error, which is fine - from - {{ source('tokens', 'transfers') }} - where - 1 = 1 - and amount_usd >=1 - {% if is_incremental() %} - and {{ incremental_predicate('block_date') }} - {% endif %} - group by - blockchain - , block_date -) -, solana as ( - select - 'solana' as blockchain - , block_date - , approx_distinct(tx_id) as tx_count - from - {{ source('tokens_solana', 'transfers') }} - where - 1 = 1 - and action != 'wrap' - and amount_usd > 1 - {% if is_incremental() %} - and {{ incremental_predicate('block_date') }} - {% endif %} - group by - 'solana' - , block_date -) -, bitcoin as ( - select +{% set chains = [ + 'arbitrum' + , 'avalanche_c' + , 'base' + , 'bitcoin' + , 'blast' + , 'bnb' + , 'celo' + , 'ethereum' + , 'fantom' + , 'gnosis' + , 'linea' + , 'mantle' + , 'optimism' + , 'polygon' + , 'scroll' + , 'sei' + , 'solana' + , 'tron' + , 'zkevm' + , 'zksync' + , 'zora' +] %} + +SELECT * +FROM ( + {% for blockchain in chains %} + SELECT blockchain - , block_date - , approx_distinct(tx_id) as tx_count - from - {{ source('transfers_bitcoin', 'satoshi') }} - where - 1 = 1 - and amount_transfer_usd > 1 - {% if is_incremental() %} - and {{ incremental_predicate('block_date') }} + ,block_date + ,tx_count + FROM {{ ref('metrics_' + blockchain + '_transactions_daily') }} + {% if not loop.last %} + UNION ALL {% endif %} - group by - blockchain - , block_date + {% endfor %} ) -select - blockchain - , block_date - , tx_count -from - evm -union all -select - blockchain - , block_date - , tx_count -from - solana -union all -select - blockchain - , block_date - , tx_count -from - bitcoin \ No newline at end of file diff --git a/dbt_subprojects/daily_spellbook/models/_metrics/transactions/metrics_transactions_index_daily.sql b/dbt_subprojects/daily_spellbook/models/_metrics/transactions/metrics_transactions_index_daily.sql deleted file mode 100644 index 194eb76b218..00000000000 --- a/dbt_subprojects/daily_spellbook/models/_metrics/transactions/metrics_transactions_index_daily.sql +++ /dev/null @@ -1,38 +0,0 @@ -{{ config( - schema = 'metrics' - , alias = 'transactions_index_daily' - , materialized = 'view' - ) -}} - -{% set baseline_date = '2018-01-01' %} -{% set start_date = '2015-08-21' %} - -with baseline as ( - select - sum(tx_count) as baseline_tx_count -- sum is required due to blockchain being second unique key in source - from - {{ ref('metrics_transactions_daily') }} - where - block_date = date '{{ baseline_date }}' -), daily as ( - select - blockchain - , block_date - , tx_count - from - {{ ref('metrics_transactions_daily') }} - where - block_date >= date '{{ start_date }}' -) -select - d.blockchain - , d.block_date - , d.tx_count - , b.baseline_tx_count - , (cast(d.tx_count as double) / cast(b.baseline_tx_count as double)) * 10 as tx_index -from - daily as d -left join - baseline as b - on 1 = 1 diff --git a/dbt_subprojects/daily_spellbook/models/_metrics/transactions/metrics_transactions_stats.sql b/dbt_subprojects/daily_spellbook/models/_metrics/transactions/metrics_transactions_stats.sql index a03620af097..39063698f34 100644 --- a/dbt_subprojects/daily_spellbook/models/_metrics/transactions/metrics_transactions_stats.sql +++ b/dbt_subprojects/daily_spellbook/models/_metrics/transactions/metrics_transactions_stats.sql @@ -155,4 +155,4 @@ from inner join weekly_stats as w on d.blockchain = w.blockchain inner join monthly_stats as m - on d.blockchain = m.blockchain \ No newline at end of file + on d.blockchain = m.blockchain diff --git a/dbt_subprojects/daily_spellbook/models/_metrics/transfers/_schema.yml b/dbt_subprojects/daily_spellbook/models/_metrics/transfers/_schema.yml index e03f33be0d9..7330b573af4 100644 --- a/dbt_subprojects/daily_spellbook/models/_metrics/transfers/_schema.yml +++ b/dbt_subprojects/daily_spellbook/models/_metrics/transfers/_schema.yml @@ -1,7 +1,7 @@ version: 2 models: - - name: metrics_net_transfers_daily + - name: metrics_transfers_daily meta: sector: metrics contributors: jeff-dude @@ -18,62 +18,35 @@ models: with the positive net received (aka received) it adds up to 0 since the net sent should equal the net received. For example, if I give you $1 that's -1 sent (from me) and +1 received (by you) which adds to 0. This query counts the received to show that $1 was effectively transfered." - data_tests: - - dbt_utils.unique_combination_of_columns: - combination_of_columns: - - blockchain - - block_date - - name: metrics_net_solana_transfers_daily + + - name: metrics_transfers_stats meta: sector: metrics contributors: jeff-dude config: - tags: ['metrics', 'net_transfers', 'solana'] - description: *net_transfers_description - data_tests: - - dbt_utils.unique_combination_of_columns: - combination_of_columns: - - blockchain - - block_date - - name: metrics_net_bitcoin_transfers_daily + tags: ['metrics', 'transfers'] + description: "View of transfers per blockchain aggregated to various levels. The goal is to output one row per chain." + + - name: metrics_solana_transfers_daily meta: sector: metrics contributors: jeff-dude config: - tags: ['metrics', 'net_transfers', 'bitcoin'] + tags: ['metrics', 'net_transfers', 'solana'] description: *net_transfers_description data_tests: - dbt_utils.unique_combination_of_columns: combination_of_columns: - blockchain - block_date - - name: metrics_transfers_daily - meta: - sector: metrics - contributors: jeff-dude - config: - tags: ['metrics', 'transfers', 'daily'] - description: | - "Sum of total transfer amount per day across all chain." - data_tests: - - dbt_utils.unique_combination_of_columns: - combination_of_columns: - - blockchain - - block_date - - name: metrics_transfers_stats - meta: - sector: metrics - contributors: jeff-dude - config: - tags: ['metrics', 'transfers'] - description: "View of transfers per blockchain aggregated to various levels. The goal is to output one row per chain with stats availble for use in counter visuals." - - name: metrics_transfers_index_daily + + - name: metrics_bitcoin_transfers_daily meta: sector: metrics contributors: jeff-dude config: - tags: ['metrics', 'transfers', 'index', 'daily'] - description: "Each day, per chain, compare the adoption of total transfers relative to the baseline expectation" + tags: ['metrics', 'net_transfers', 'bitcoin'] + description: *net_transfers_description data_tests: - dbt_utils.unique_combination_of_columns: combination_of_columns: diff --git a/dbt_subprojects/daily_spellbook/models/_metrics/transfers/metrics_net_bitcoin_transfers_daily.sql b/dbt_subprojects/daily_spellbook/models/_metrics/transfers/chains/bitcoin/metrics_bitcoin_transfers_daily.sql similarity index 97% rename from dbt_subprojects/daily_spellbook/models/_metrics/transfers/metrics_net_bitcoin_transfers_daily.sql rename to dbt_subprojects/daily_spellbook/models/_metrics/transfers/chains/bitcoin/metrics_bitcoin_transfers_daily.sql index f193edc8d53..711518ccf4b 100644 --- a/dbt_subprojects/daily_spellbook/models/_metrics/transfers/metrics_net_bitcoin_transfers_daily.sql +++ b/dbt_subprojects/daily_spellbook/models/_metrics/transfers/chains/bitcoin/metrics_bitcoin_transfers_daily.sql @@ -1,6 +1,6 @@ {{ config( - schema = 'metrics' - , alias = 'net_bitcoin_transfers_daily' + schema = 'metrics_bitcoin' + , alias = 'transfers_daily' , materialized = 'incremental' , file_format = 'delta' , incremental_strategy = 'merge' @@ -110,4 +110,4 @@ where net_transfer_amount_usd > 0 group by blockchain - , block_date \ No newline at end of file + , block_date diff --git a/dbt_subprojects/daily_spellbook/models/_metrics/transfers/chains/evm/_schema.yml b/dbt_subprojects/daily_spellbook/models/_metrics/transfers/chains/evm/_schema.yml new file mode 100644 index 00000000000..c4b82af4cbd --- /dev/null +++ b/dbt_subprojects/daily_spellbook/models/_metrics/transfers/chains/evm/_schema.yml @@ -0,0 +1,249 @@ +version: 2 + +models: + - name: metrics_arbitrum_transfers_daily + meta: + sector: metrics + contributors: jeff-dude, 0xRob + config: + tags: ['metrics', 'transfers', 'daily'] + description: "Total transfer amount per day" + data_tests: + - dbt_utils.unique_combination_of_columns: + combination_of_columns: + - blockchain + - block_date + + - name: metrics_avalanche_c_transfers_daily + meta: + sector: metrics + contributors: jeff-dude, 0xRob + config: + tags: ['metrics', 'transfers', 'daily'] + description: "Total transfer amount per day" + data_tests: + - dbt_utils.unique_combination_of_columns: + combination_of_columns: + - blockchain + - block_date + + - name: metrics_base_transfers_daily + meta: + sector: metrics + contributors: jeff-dude, 0xRob + config: + tags: ['metrics', 'transfers', 'daily'] + description: "Total transfer amount per day" + data_tests: + - dbt_utils.unique_combination_of_columns: + combination_of_columns: + - blockchain + - block_date + + - name: metrics_blast_transfers_daily + meta: + sector: metrics + contributors: jeff-dude, 0xRob + config: + tags: ['metrics', 'transfers', 'daily'] + description: "Total transfer amount per day" + data_tests: + - dbt_utils.unique_combination_of_columns: + combination_of_columns: + - blockchain + - block_date + + - name: metrics_bnb_transfers_daily + meta: + sector: metrics + contributors: jeff-dude, 0xRob + config: + tags: ['metrics', 'transfers', 'daily'] + description: "Total transfer amount per day" + data_tests: + - dbt_utils.unique_combination_of_columns: + combination_of_columns: + - blockchain + - block_date + + - name: metrics_celo_transfers_daily + meta: + sector: metrics + contributors: jeff-dude, 0xRob + config: + tags: ['metrics', 'transfers', 'daily'] + description: "Total transfer amount per day" + data_tests: + - dbt_utils.unique_combination_of_columns: + combination_of_columns: + - blockchain + - block_date + + - name: metrics_ethereum_transfers_daily + meta: + sector: metrics + contributors: jeff-dude, 0xRob + config: + tags: ['metrics', 'transfers', 'daily'] + description: "Total transfer amount per day" + data_tests: + - dbt_utils.unique_combination_of_columns: + combination_of_columns: + - blockchain + - block_date + + - name: metrics_fantom_transfers_daily + meta: + sector: metrics + contributors: jeff-dude, 0xRob + config: + tags: ['metrics', 'transfers', 'daily'] + description: "Total transfer amount per day" + data_tests: + - dbt_utils.unique_combination_of_columns: + combination_of_columns: + - blockchain + - block_date + + - name: metrics_gnosis_transfers_daily + meta: + sector: metrics + contributors: jeff-dude, 0xRob + config: + tags: ['metrics', 'transfers', 'daily'] + description: "Total transfer amount per day" + data_tests: + - dbt_utils.unique_combination_of_columns: + combination_of_columns: + - blockchain + - block_date + + - name: metrics_linea_transfers_daily + meta: + sector: metrics + contributors: jeff-dude, 0xRob + config: + tags: ['metrics', 'transfers', 'daily'] + description: "Total transfer amount per day" + data_tests: + - dbt_utils.unique_combination_of_columns: + combination_of_columns: + - blockchain + - block_date + + - name: metrics_mantle_transfers_daily + meta: + sector: metrics + contributors: jeff-dude, 0xRob + config: + tags: ['metrics', 'transfers', 'daily'] + description: "Total transfer amount per day" + data_tests: + - dbt_utils.unique_combination_of_columns: + combination_of_columns: + - blockchain + - block_date + + - name: metrics_optimism_transfers_daily + meta: + sector: metrics + contributors: jeff-dude, 0xRob + config: + tags: ['metrics', 'transfers', 'daily'] + description: "Total transfer amount per day" + data_tests: + - dbt_utils.unique_combination_of_columns: + combination_of_columns: + - blockchain + - block_date + + - name: metrics_polygon_transfers_daily + meta: + sector: metrics + contributors: jeff-dude, 0xRob + config: + tags: ['metrics', 'transfers', 'daily'] + description: "Total transfer amount per day" + data_tests: + - dbt_utils.unique_combination_of_columns: + combination_of_columns: + - blockchain + - block_date + + - name: metrics_scroll_transfers_daily + meta: + sector: metrics + contributors: jeff-dude, 0xRob + config: + tags: ['metrics', 'transfers', 'daily'] + description: "Total transfer amount per day" + data_tests: + - dbt_utils.unique_combination_of_columns: + combination_of_columns: + - blockchain + - block_date + + - name: metrics_sei_transfers_daily + meta: + sector: metrics + contributors: jeff-dude, 0xRob + config: + tags: ['metrics', 'transfers', 'daily'] + description: "Total transfer amount per day" + data_tests: + - dbt_utils.unique_combination_of_columns: + combination_of_columns: + - blockchain + - block_date + + - name: metrics_tron_transfers_daily + meta: + sector: metrics + contributors: jeff-dude, 0xRob + config: + tags: ['metrics', 'transfers', 'daily'] + description: "Total transfer amount per day" + data_tests: + - dbt_utils.unique_combination_of_columns: + combination_of_columns: + - blockchain + - block_date + + - name: metrics_zkevm_transfers_daily + meta: + sector: metrics + contributors: jeff-dude, 0xRob + config: + tags: ['metrics', 'transfers', 'daily'] + description: "Total transfer amount per day" + data_tests: + - dbt_utils.unique_combination_of_columns: + combination_of_columns: + - blockchain + - block_date + + - name: metrics_zksync_transfers_daily + meta: + sector: metrics + contributors: jeff-dude, 0xRob + config: + tags: ['metrics', 'transfers', 'daily'] + description: "Total transfer amount per day" + data_tests: + - dbt_utils.unique_combination_of_columns: + combination_of_columns: + - blockchain + - block_date + + - name: metrics_zora_transfers_daily + meta: + sector: metrics + contributors: jeff-dude, 0xRob + config: + tags: ['metrics', 'transfers', 'daily'] + description: "Total transfer amount per day" + data_tests: + - dbt_utils.unique_combination_of_columns: + combination_of_columns: + - blockchain + - block_date diff --git a/dbt_subprojects/daily_spellbook/models/_metrics/transfers/chains/evm/metrics_arbitrum_transfers_daily.sql b/dbt_subprojects/daily_spellbook/models/_metrics/transfers/chains/evm/metrics_arbitrum_transfers_daily.sql new file mode 100644 index 00000000000..272c9cf2fcd --- /dev/null +++ b/dbt_subprojects/daily_spellbook/models/_metrics/transfers/chains/evm/metrics_arbitrum_transfers_daily.sql @@ -0,0 +1,15 @@ +{% set blockchain = 'arbitrum' %} + +{{ config( + schema = 'metrics_' + blockchain + , alias = 'transfers_daily' + , materialized = 'incremental' + , file_format = 'delta' + , incremental_strategy = 'merge' + , unique_key = ['blockchain', 'block_date'] + , incremental_predicates = [incremental_predicate('DBT_INTERNAL_DEST.block_date')] + ) +}} + + +{{ metrics_transfers_evm(blockchain) }} diff --git a/dbt_subprojects/daily_spellbook/models/_metrics/transfers/chains/evm/metrics_avalanche_c_transfers_daily.sql b/dbt_subprojects/daily_spellbook/models/_metrics/transfers/chains/evm/metrics_avalanche_c_transfers_daily.sql new file mode 100644 index 00000000000..5d768c36edc --- /dev/null +++ b/dbt_subprojects/daily_spellbook/models/_metrics/transfers/chains/evm/metrics_avalanche_c_transfers_daily.sql @@ -0,0 +1,15 @@ +{% set blockchain = 'avalanche_c' %} + +{{ config( + schema = 'metrics_' + blockchain + , alias = 'transfers_daily' + , materialized = 'incremental' + , file_format = 'delta' + , incremental_strategy = 'merge' + , unique_key = ['blockchain', 'block_date'] + , incremental_predicates = [incremental_predicate('DBT_INTERNAL_DEST.block_date')] + ) +}} + + +{{ metrics_transfers_evm(blockchain) }} diff --git a/dbt_subprojects/daily_spellbook/models/_metrics/transfers/chains/evm/metrics_base_transfers_daily.sql b/dbt_subprojects/daily_spellbook/models/_metrics/transfers/chains/evm/metrics_base_transfers_daily.sql new file mode 100644 index 00000000000..8b9099a6035 --- /dev/null +++ b/dbt_subprojects/daily_spellbook/models/_metrics/transfers/chains/evm/metrics_base_transfers_daily.sql @@ -0,0 +1,15 @@ +{% set blockchain = 'base' %} + +{{ config( + schema = 'metrics_' + blockchain + , alias = 'transfers_daily' + , materialized = 'incremental' + , file_format = 'delta' + , incremental_strategy = 'merge' + , unique_key = ['blockchain', 'block_date'] + , incremental_predicates = [incremental_predicate('DBT_INTERNAL_DEST.block_date')] + ) +}} + + +{{ metrics_transfers_evm(blockchain) }} diff --git a/dbt_subprojects/daily_spellbook/models/_metrics/transfers/chains/evm/metrics_blast_transfers_daily.sql b/dbt_subprojects/daily_spellbook/models/_metrics/transfers/chains/evm/metrics_blast_transfers_daily.sql new file mode 100644 index 00000000000..c35a554a42a --- /dev/null +++ b/dbt_subprojects/daily_spellbook/models/_metrics/transfers/chains/evm/metrics_blast_transfers_daily.sql @@ -0,0 +1,15 @@ +{% set blockchain = 'blast' %} + +{{ config( + schema = 'metrics_' + blockchain + , alias = 'transfers_daily' + , materialized = 'incremental' + , file_format = 'delta' + , incremental_strategy = 'merge' + , unique_key = ['blockchain', 'block_date'] + , incremental_predicates = [incremental_predicate('DBT_INTERNAL_DEST.block_date')] + ) +}} + + +{{ metrics_transfers_evm(blockchain) }} diff --git a/dbt_subprojects/daily_spellbook/models/_metrics/transfers/chains/evm/metrics_bnb_transfers_daily.sql b/dbt_subprojects/daily_spellbook/models/_metrics/transfers/chains/evm/metrics_bnb_transfers_daily.sql new file mode 100644 index 00000000000..0fd3b6a25b2 --- /dev/null +++ b/dbt_subprojects/daily_spellbook/models/_metrics/transfers/chains/evm/metrics_bnb_transfers_daily.sql @@ -0,0 +1,15 @@ +{% set blockchain = 'bnb' %} + +{{ config( + schema = 'metrics_' + blockchain + , alias = 'transfers_daily' + , materialized = 'incremental' + , file_format = 'delta' + , incremental_strategy = 'merge' + , unique_key = ['blockchain', 'block_date'] + , incremental_predicates = [incremental_predicate('DBT_INTERNAL_DEST.block_date')] + ) +}} + + +{{ metrics_transfers_evm(blockchain) }} diff --git a/dbt_subprojects/daily_spellbook/models/_metrics/transfers/chains/evm/metrics_celo_transfers_daily.sql b/dbt_subprojects/daily_spellbook/models/_metrics/transfers/chains/evm/metrics_celo_transfers_daily.sql new file mode 100644 index 00000000000..3e9b393ccce --- /dev/null +++ b/dbt_subprojects/daily_spellbook/models/_metrics/transfers/chains/evm/metrics_celo_transfers_daily.sql @@ -0,0 +1,15 @@ +{% set blockchain = 'celo' %} + +{{ config( + schema = 'metrics_' + blockchain + , alias = 'transfers_daily' + , materialized = 'incremental' + , file_format = 'delta' + , incremental_strategy = 'merge' + , unique_key = ['blockchain', 'block_date'] + , incremental_predicates = [incremental_predicate('DBT_INTERNAL_DEST.block_date')] + ) +}} + + +{{ metrics_transfers_evm(blockchain) }} diff --git a/dbt_subprojects/daily_spellbook/models/_metrics/transfers/chains/evm/metrics_ethereum_transfers_daily.sql b/dbt_subprojects/daily_spellbook/models/_metrics/transfers/chains/evm/metrics_ethereum_transfers_daily.sql new file mode 100644 index 00000000000..f1b7aa5c1d8 --- /dev/null +++ b/dbt_subprojects/daily_spellbook/models/_metrics/transfers/chains/evm/metrics_ethereum_transfers_daily.sql @@ -0,0 +1,15 @@ +{% set blockchain = 'ethereum' %} + +{{ config( + schema = 'metrics_' + blockchain + , alias = 'transfers_daily' + , materialized = 'incremental' + , file_format = 'delta' + , incremental_strategy = 'merge' + , unique_key = ['blockchain', 'block_date'] + , incremental_predicates = [incremental_predicate('DBT_INTERNAL_DEST.block_date')] + ) +}} + + +{{ metrics_transfers_evm(blockchain) }} diff --git a/dbt_subprojects/daily_spellbook/models/_metrics/transfers/chains/evm/metrics_fantom_transfers_daily.sql b/dbt_subprojects/daily_spellbook/models/_metrics/transfers/chains/evm/metrics_fantom_transfers_daily.sql new file mode 100644 index 00000000000..6acd73adc56 --- /dev/null +++ b/dbt_subprojects/daily_spellbook/models/_metrics/transfers/chains/evm/metrics_fantom_transfers_daily.sql @@ -0,0 +1,15 @@ +{% set blockchain = 'fantom' %} + +{{ config( + schema = 'metrics_' + blockchain + , alias = 'transfers_daily' + , materialized = 'incremental' + , file_format = 'delta' + , incremental_strategy = 'merge' + , unique_key = ['blockchain', 'block_date'] + , incremental_predicates = [incremental_predicate('DBT_INTERNAL_DEST.block_date')] + ) +}} + + +{{ metrics_transfers_evm(blockchain) }} diff --git a/dbt_subprojects/daily_spellbook/models/_metrics/transfers/chains/evm/metrics_gnosis_transfers_daily.sql b/dbt_subprojects/daily_spellbook/models/_metrics/transfers/chains/evm/metrics_gnosis_transfers_daily.sql new file mode 100644 index 00000000000..9b916f51eaf --- /dev/null +++ b/dbt_subprojects/daily_spellbook/models/_metrics/transfers/chains/evm/metrics_gnosis_transfers_daily.sql @@ -0,0 +1,15 @@ +{% set blockchain = 'gnosis' %} + +{{ config( + schema = 'metrics_' + blockchain + , alias = 'transfers_daily' + , materialized = 'incremental' + , file_format = 'delta' + , incremental_strategy = 'merge' + , unique_key = ['blockchain', 'block_date'] + , incremental_predicates = [incremental_predicate('DBT_INTERNAL_DEST.block_date')] + ) +}} + + +{{ metrics_transfers_evm(blockchain) }} diff --git a/dbt_subprojects/daily_spellbook/models/_metrics/transfers/chains/evm/metrics_linea_transfers_daily.sql b/dbt_subprojects/daily_spellbook/models/_metrics/transfers/chains/evm/metrics_linea_transfers_daily.sql new file mode 100644 index 00000000000..d7aa7dab9e2 --- /dev/null +++ b/dbt_subprojects/daily_spellbook/models/_metrics/transfers/chains/evm/metrics_linea_transfers_daily.sql @@ -0,0 +1,15 @@ +{% set blockchain = 'linea' %} + +{{ config( + schema = 'metrics_' + blockchain + , alias = 'transfers_daily' + , materialized = 'incremental' + , file_format = 'delta' + , incremental_strategy = 'merge' + , unique_key = ['blockchain', 'block_date'] + , incremental_predicates = [incremental_predicate('DBT_INTERNAL_DEST.block_date')] + ) +}} + + +{{ metrics_transfers_evm(blockchain) }} diff --git a/dbt_subprojects/daily_spellbook/models/_metrics/transfers/chains/evm/metrics_mantle_transfers_daily.sql b/dbt_subprojects/daily_spellbook/models/_metrics/transfers/chains/evm/metrics_mantle_transfers_daily.sql new file mode 100644 index 00000000000..035f2c1a820 --- /dev/null +++ b/dbt_subprojects/daily_spellbook/models/_metrics/transfers/chains/evm/metrics_mantle_transfers_daily.sql @@ -0,0 +1,15 @@ +{% set blockchain = 'mantle' %} + +{{ config( + schema = 'metrics_' + blockchain + , alias = 'transfers_daily' + , materialized = 'incremental' + , file_format = 'delta' + , incremental_strategy = 'merge' + , unique_key = ['blockchain', 'block_date'] + , incremental_predicates = [incremental_predicate('DBT_INTERNAL_DEST.block_date')] + ) +}} + + +{{ metrics_transfers_evm(blockchain) }} diff --git a/dbt_subprojects/daily_spellbook/models/_metrics/transfers/chains/evm/metrics_optimism_transfers_daily.sql b/dbt_subprojects/daily_spellbook/models/_metrics/transfers/chains/evm/metrics_optimism_transfers_daily.sql new file mode 100644 index 00000000000..7179d55e91a --- /dev/null +++ b/dbt_subprojects/daily_spellbook/models/_metrics/transfers/chains/evm/metrics_optimism_transfers_daily.sql @@ -0,0 +1,15 @@ +{% set blockchain = 'optimism' %} + +{{ config( + schema = 'metrics_' + blockchain + , alias = 'transfers_daily' + , materialized = 'incremental' + , file_format = 'delta' + , incremental_strategy = 'merge' + , unique_key = ['blockchain', 'block_date'] + , incremental_predicates = [incremental_predicate('DBT_INTERNAL_DEST.block_date')] + ) +}} + + +{{ metrics_transfers_evm(blockchain) }} diff --git a/dbt_subprojects/daily_spellbook/models/_metrics/transfers/chains/evm/metrics_polygon_transfers_daily.sql b/dbt_subprojects/daily_spellbook/models/_metrics/transfers/chains/evm/metrics_polygon_transfers_daily.sql new file mode 100644 index 00000000000..725a9236cee --- /dev/null +++ b/dbt_subprojects/daily_spellbook/models/_metrics/transfers/chains/evm/metrics_polygon_transfers_daily.sql @@ -0,0 +1,15 @@ +{% set blockchain = 'polygon' %} + +{{ config( + schema = 'metrics_' + blockchain + , alias = 'transfers_daily' + , materialized = 'incremental' + , file_format = 'delta' + , incremental_strategy = 'merge' + , unique_key = ['blockchain', 'block_date'] + , incremental_predicates = [incremental_predicate('DBT_INTERNAL_DEST.block_date')] + ) +}} + + +{{ metrics_transfers_evm(blockchain) }} diff --git a/dbt_subprojects/daily_spellbook/models/_metrics/transfers/chains/evm/metrics_scroll_transfers_daily.sql b/dbt_subprojects/daily_spellbook/models/_metrics/transfers/chains/evm/metrics_scroll_transfers_daily.sql new file mode 100644 index 00000000000..9c3f5418d5c --- /dev/null +++ b/dbt_subprojects/daily_spellbook/models/_metrics/transfers/chains/evm/metrics_scroll_transfers_daily.sql @@ -0,0 +1,15 @@ +{% set blockchain = 'scroll' %} + +{{ config( + schema = 'metrics_' + blockchain + , alias = 'transfers_daily' + , materialized = 'incremental' + , file_format = 'delta' + , incremental_strategy = 'merge' + , unique_key = ['blockchain', 'block_date'] + , incremental_predicates = [incremental_predicate('DBT_INTERNAL_DEST.block_date')] + ) +}} + + +{{ metrics_transfers_evm(blockchain) }} diff --git a/dbt_subprojects/daily_spellbook/models/_metrics/transfers/chains/evm/metrics_sei_transfers_daily.sql b/dbt_subprojects/daily_spellbook/models/_metrics/transfers/chains/evm/metrics_sei_transfers_daily.sql new file mode 100644 index 00000000000..17d9ec4a367 --- /dev/null +++ b/dbt_subprojects/daily_spellbook/models/_metrics/transfers/chains/evm/metrics_sei_transfers_daily.sql @@ -0,0 +1,15 @@ +{% set blockchain = 'sei' %} + +{{ config( + schema = 'metrics_' + blockchain + , alias = 'transfers_daily' + , materialized = 'incremental' + , file_format = 'delta' + , incremental_strategy = 'merge' + , unique_key = ['blockchain', 'block_date'] + , incremental_predicates = [incremental_predicate('DBT_INTERNAL_DEST.block_date')] + ) +}} + + +{{ metrics_transfers_evm(blockchain) }} diff --git a/dbt_subprojects/daily_spellbook/models/_metrics/transfers/chains/evm/metrics_tron_transfers_daily.sql b/dbt_subprojects/daily_spellbook/models/_metrics/transfers/chains/evm/metrics_tron_transfers_daily.sql new file mode 100644 index 00000000000..035d9485a6c --- /dev/null +++ b/dbt_subprojects/daily_spellbook/models/_metrics/transfers/chains/evm/metrics_tron_transfers_daily.sql @@ -0,0 +1,15 @@ +{% set blockchain = 'tron' %} + +{{ config( + schema = 'metrics_' + blockchain + , alias = 'transfers_daily' + , materialized = 'incremental' + , file_format = 'delta' + , incremental_strategy = 'merge' + , unique_key = ['blockchain', 'block_date'] + , incremental_predicates = [incremental_predicate('DBT_INTERNAL_DEST.block_date')] + ) +}} + + +{{ metrics_transfers_evm(blockchain) }} diff --git a/dbt_subprojects/daily_spellbook/models/_metrics/transfers/chains/evm/metrics_zkevm_transfers_daily.sql b/dbt_subprojects/daily_spellbook/models/_metrics/transfers/chains/evm/metrics_zkevm_transfers_daily.sql new file mode 100644 index 00000000000..a77bc732309 --- /dev/null +++ b/dbt_subprojects/daily_spellbook/models/_metrics/transfers/chains/evm/metrics_zkevm_transfers_daily.sql @@ -0,0 +1,15 @@ +{% set blockchain = 'zkevm' %} + +{{ config( + schema = 'metrics_' + blockchain + , alias = 'transfers_daily' + , materialized = 'incremental' + , file_format = 'delta' + , incremental_strategy = 'merge' + , unique_key = ['blockchain', 'block_date'] + , incremental_predicates = [incremental_predicate('DBT_INTERNAL_DEST.block_date')] + ) +}} + + +{{ metrics_transfers_evm(blockchain) }} diff --git a/dbt_subprojects/daily_spellbook/models/_metrics/transfers/chains/evm/metrics_zksync_transfers_daily.sql b/dbt_subprojects/daily_spellbook/models/_metrics/transfers/chains/evm/metrics_zksync_transfers_daily.sql new file mode 100644 index 00000000000..a941a3965e2 --- /dev/null +++ b/dbt_subprojects/daily_spellbook/models/_metrics/transfers/chains/evm/metrics_zksync_transfers_daily.sql @@ -0,0 +1,15 @@ +{% set blockchain = 'zksync' %} + +{{ config( + schema = 'metrics_' + blockchain + , alias = 'transfers_daily' + , materialized = 'incremental' + , file_format = 'delta' + , incremental_strategy = 'merge' + , unique_key = ['blockchain', 'block_date'] + , incremental_predicates = [incremental_predicate('DBT_INTERNAL_DEST.block_date')] + ) +}} + + +{{ metrics_transfers_evm(blockchain) }} diff --git a/dbt_subprojects/daily_spellbook/models/_metrics/transfers/chains/evm/metrics_zora_transfers_daily.sql b/dbt_subprojects/daily_spellbook/models/_metrics/transfers/chains/evm/metrics_zora_transfers_daily.sql new file mode 100644 index 00000000000..21bdfb3d180 --- /dev/null +++ b/dbt_subprojects/daily_spellbook/models/_metrics/transfers/chains/evm/metrics_zora_transfers_daily.sql @@ -0,0 +1,15 @@ +{% set blockchain = 'zora' %} + +{{ config( + schema = 'metrics_' + blockchain + , alias = 'transfers_daily' + , materialized = 'incremental' + , file_format = 'delta' + , incremental_strategy = 'merge' + , unique_key = ['blockchain', 'block_date'] + , incremental_predicates = [incremental_predicate('DBT_INTERNAL_DEST.block_date')] + ) +}} + + +{{ metrics_transfers_evm(blockchain) }} diff --git a/dbt_subprojects/daily_spellbook/models/_metrics/transfers/metrics_net_solana_transfers_daily.sql b/dbt_subprojects/daily_spellbook/models/_metrics/transfers/chains/solana/metrics_solana_transfers_daily.sql similarity index 96% rename from dbt_subprojects/daily_spellbook/models/_metrics/transfers/metrics_net_solana_transfers_daily.sql rename to dbt_subprojects/daily_spellbook/models/_metrics/transfers/chains/solana/metrics_solana_transfers_daily.sql index 5a2472b35bb..88b7e629e8d 100644 --- a/dbt_subprojects/daily_spellbook/models/_metrics/transfers/metrics_net_solana_transfers_daily.sql +++ b/dbt_subprojects/daily_spellbook/models/_metrics/transfers/chains/solana/metrics_solana_transfers_daily.sql @@ -1,6 +1,6 @@ {{ config( - schema = 'metrics' - , alias = 'net_solana_transfers_daily' + schema = 'metrics_solana' + , alias = 'transfers_daily' , materialized = 'incremental' , file_format = 'delta' , incremental_strategy = 'merge' @@ -92,4 +92,4 @@ where net_transfer_amount_usd > 0 group by blockchain - , block_date \ No newline at end of file + , block_date diff --git a/dbt_subprojects/daily_spellbook/models/_metrics/transfers/metrics_transfers_daily.sql b/dbt_subprojects/daily_spellbook/models/_metrics/transfers/metrics_transfers_daily.sql index 4a81fe41984..108611d314f 100644 --- a/dbt_subprojects/daily_spellbook/models/_metrics/transfers/metrics_transfers_daily.sql +++ b/dbt_subprojects/daily_spellbook/models/_metrics/transfers/metrics_transfers_daily.sql @@ -5,32 +5,43 @@ ) }} -select - blockchain - , block_date - , transfer_amount_usd_sent - , transfer_amount_usd_received - , transfer_amount_usd - , net_transfer_amount_usd -from - {{ ref('metrics_net_transfers_daily') }} -union all -select - blockchain - , block_date - , transfer_amount_usd_sent - , transfer_amount_usd_received - , transfer_amount_usd - , net_transfer_amount_usd -from - {{ ref('metrics_net_solana_transfers_daily') }} -union all -select - blockchain - , block_date - , transfer_amount_usd_sent - , transfer_amount_usd_received - , transfer_amount_usd - , net_transfer_amount_usd -from - {{ ref('metrics_net_bitcoin_transfers_daily') }} +{% set chains = [ + 'arbitrum' + , 'avalanche_c' + , 'base' + , 'bitcoin' + , 'blast' + , 'bnb' + , 'celo' + , 'ethereum' + , 'fantom' + , 'gnosis' + , 'linea' + , 'mantle' + , 'optimism' + , 'polygon' + , 'scroll' + , 'sei' + , 'solana' + , 'tron' + , 'zkevm' + , 'zksync' + , 'zora' +] %} + +SELECT * +FROM ( + {% for blockchain in chains %} + SELECT + blockchain + , block_date + , transfer_amount_usd_sent + , transfer_amount_usd_received + , transfer_amount_usd + , net_transfer_amount_usd + FROM {{ ref('metrics_' + blockchain + '_transfers_daily') }} + {% if not loop.last %} + UNION ALL + {% endif %} + {% endfor %} +) diff --git a/dbt_subprojects/daily_spellbook/models/_metrics/transfers/metrics_transfers_index_daily.sql b/dbt_subprojects/daily_spellbook/models/_metrics/transfers/metrics_transfers_index_daily.sql deleted file mode 100644 index f13e4e1648c..00000000000 --- a/dbt_subprojects/daily_spellbook/models/_metrics/transfers/metrics_transfers_index_daily.sql +++ /dev/null @@ -1,38 +0,0 @@ -{{ config( - schema = 'metrics' - , alias = 'transfers_index_daily' - , materialized = 'view' - ) -}} - -{% set baseline_date = '2018-01-01' %} -{% set start_date = '2015-08-21' %} - -with baseline as ( - select - sum(net_transfer_amount_usd) as baseline_net_transfer_amount_usd -- sum is required due to blockchain being second unique key in source - from - {{ ref('metrics_transfers_daily') }} - where - block_date = date '{{ baseline_date }}' -), daily as ( - select - blockchain - , block_date - , net_transfer_amount_usd - from - {{ ref('metrics_transfers_daily') }} - where - block_date >= date '{{ start_date }}' -) -select - d.blockchain - , d.block_date - , d.net_transfer_amount_usd - , b.baseline_net_transfer_amount_usd - , (d.net_transfer_amount_usd / b.baseline_net_transfer_amount_usd) * 10 as transfers_index -from - daily as d -left join - baseline as b - on 1 = 1 diff --git a/dbt_subprojects/daily_spellbook/models/_metrics/transfers/metrics_transfers_stats.sql b/dbt_subprojects/daily_spellbook/models/_metrics/transfers/metrics_transfers_stats.sql index dfe6afb49b5..45d7124b597 100644 --- a/dbt_subprojects/daily_spellbook/models/_metrics/transfers/metrics_transfers_stats.sql +++ b/dbt_subprojects/daily_spellbook/models/_metrics/transfers/metrics_transfers_stats.sql @@ -155,4 +155,4 @@ from inner join weekly_stats as w on d.blockchain = w.blockchain inner join monthly_stats as m - on d.blockchain = m.blockchain \ No newline at end of file + on d.blockchain = m.blockchain From 392af004b68dc7bfd5eafa51c84defd26945eb94 Mon Sep 17 00:00:00 2001 From: maybeyonas Date: Thu, 28 Nov 2024 16:26:08 +0530 Subject: [PATCH 05/18] adding more rwa tokens and schema update (#7191) * adding more rwa tokens and schema update * column fixes --- .../rwa/arbitrum/rwa_arbitrum_schema.yml | 6 ++ .../rwa/arbitrum/rwa_arbitrum_tokens.sql | 86 +++++++++++++++++-- 2 files changed, 87 insertions(+), 5 deletions(-) diff --git a/dbt_subprojects/daily_spellbook/models/_sector/rwa/arbitrum/rwa_arbitrum_schema.yml b/dbt_subprojects/daily_spellbook/models/_sector/rwa/arbitrum/rwa_arbitrum_schema.yml index 575bafe15c1..003910de111 100644 --- a/dbt_subprojects/daily_spellbook/models/_sector/rwa/arbitrum/rwa_arbitrum_schema.yml +++ b/dbt_subprojects/daily_spellbook/models/_sector/rwa/arbitrum/rwa_arbitrum_schema.yml @@ -30,6 +30,12 @@ models: - &type name: type description: "Type of Asset" + - &decimals + name: decimals + description: "Decimals of the asset" + - &category + name: category + description: "RWA category for the Asset" - name: rwa_arbitrum_dex_pools diff --git a/dbt_subprojects/daily_spellbook/models/_sector/rwa/arbitrum/rwa_arbitrum_tokens.sql b/dbt_subprojects/daily_spellbook/models/_sector/rwa/arbitrum/rwa_arbitrum_tokens.sql index 3e38fcc57aa..6bc56f2101d 100644 --- a/dbt_subprojects/daily_spellbook/models/_sector/rwa/arbitrum/rwa_arbitrum_tokens.sql +++ b/dbt_subprojects/daily_spellbook/models/_sector/rwa/arbitrum/rwa_arbitrum_tokens.sql @@ -15,14 +15,90 @@ select token_address, symbol, project, - type + type, + decimals, + category from (values - (0x59D9356E565Ab3A36dD77763Fc0d87fEaf85508C, 'USDM' , 'Mountain Protocol' , 'RWA' ), - (0xfc90518D5136585ba45e34ED5E1D108BD3950CFa, 'USD+' , 'Dinari' , 'RWA' ), - (0x9d2f299715d94d8a7e6f5eaa8e654e8c74a988a7, 'FXS' , 'Frax Finance' , 'Governance' ) + (0x59D9356E565Ab3A36dD77763Fc0d87fEaf85508C, 'USDM' , 'Mountain Protocol' , 'RWA' , 18, 'Stablecoin'), + (0xfc90518D5136585ba45e34ED5E1D108BD3950CFa, 'USD+' , 'Dinari' , 'RWA' , 6 , 'Stocks'), + (0x9d2f299715d94d8a7e6f5eaa8e654e8c74a988a7, 'FXS' , 'Frax Finance' , 'Governance' , 18, 'Stablecoin'), + (0x667fd83e24ca1d935d36717d305d54fa0cac991c, 'FRAX' , 'Frax Finance' , 'RWA' , 18, 'Stablecoin'), + (0xCe38e140fC3982a6bCEbc37b040913EF2Cd6C5a7, 'AAPL.d' , 'Dinari' , 'RWA' , 18, 'Stocks'), + (0xd8F728AdB72a46Ae2c92234AE8870D04907786C5, 'AMD.d' , 'Dinari' , 'RWA' , 18, 'Stocks'), + (0x8240aFFe697CdE618AD05c3c8963f5Bfe152650b, 'AMZN.d' , 'Dinari' , 'RWA' , 18, 'Stocks'), + (0x3619ca1e96c629f7D71C1b03dc0Ee56479356228, 'ARKB.d' , 'Dinari' , 'RWA' , 18, 'Stocks'), + (0x67BaD479F77488f0f427584e267e66086a7Da43A, 'ARM.d' , 'Dinari' , 'RWA' , 18, 'Stocks'), + (0x769fF50fD49900a6c53b2aF049eACB83dAD52Bdf, 'BITB.d' , 'Dinari' , 'RWA' , 18, 'Stocks'), + (0x2B7c643b42409F352B936BF07e0538ba20979Bff, 'BRRR.d' , 'Dinari' , 'RWA' , 18, 'Stocks'), + (0x2824eFE5CeDB3BC8730E412981997daC7C7640C2, 'BTCO.d' , 'Dinari' , 'RWA' , 18, 'Stocks'), + (0xc52915Fe75dc8db9fb6306f43AAef1344E0837AB, 'BTCW.d' , 'Dinari' , 'RWA' , 18, 'Stocks'), + (0x46b979440ac257151ee5a5bc9597b76386907fa1, 'COIN.d' , 'Dinari' , 'RWA' , 18, 'Stocks'), + (0xDD92f0723a7318e684A88532CAC2421E3cC9968e, 'DEFI.d' , 'Dinari' , 'RWA' , 18, 'Stocks'), + (0x3c9f23db4ddc5655f7be636358d319a3de1ff0c4, 'DIS.d' , 'Dinari' , 'RWA' , 18, 'Stocks'), + (0xa6f344abc6e2501b2b303fcbba99cd89f136b5fb, 'EZBC.d' , 'Dinari' , 'RWA' , 18, 'Stocks'), + (0x026fdf3024953cb2e8982bc11c67d336f37a5044, 'FBTC.d' , 'Dinari' , 'RWA' , 18, 'Stocks'), + (0xb1284f6b3e487e3f773e9ad40f337c3b3cda5c69, 'GBTC.d' , 'Dinari' , 'RWA' , 18, 'Stocks'), + (0x8e50d11a54cff859b202b7fe5225353be0646410, 'GOOGL.d' , 'Dinari' , 'RWA' , 18, 'Stocks'), + (0x0c59f6b96d3cac58240429c7659ec107f8b1efa7, 'HODL.d' , 'Dinari' , 'RWA' , 18, 'Stocks'), + (0xc1ba16afdcb3a41242944c9faaccd9fb6f2b428c, 'IBIT.d' , 'Dinari' , 'RWA' , 18, 'Stocks'), + (0x0c29891dc5060618c779e2a45fbe4808aa5ae6ad, 'MCD.d' , 'Dinari' , 'RWA' , 18, 'Stocks'), + (0x519062155b0591627c8a0c0958110a8c5639dca6, 'META.d' , 'Dinari' , 'RWA' , 18, 'Stocks'), + (0x77308f8b63a99b24b262d930e0218ed2f49f8475, 'MSFT.d' , 'Dinari' , 'RWA' , 18, 'Stocks'), + (0x3ad63b3c0ea6d7a093ff98fde040baddc389ecdc, 'NFLX.d' , 'Dinari' , 'RWA' , 18, 'Stocks'), + (0x14297be295ab922458277be046e89f73382bdf8e, 'NLY.d' , 'Dinari' , 'RWA' , 18, 'Stocks'), + (0x4dafffddea93ddf1e0e7b61e844331455053ce5c, 'NVDA.d' , 'Dinari' , 'RWA' , 18, 'Stocks'), + (0xf1f18f765f118c3598cc54dcac1d0e12066263fe, 'PFE.d' , 'Dinari' , 'RWA' , 18, 'Stocks'), + (0x118346c2bb9d24412ed58c53bf9bb6f61a20d7ec, 'PLD.d' , 'Dinari' , 'RWA' , 18, 'Stocks'), + (0x5b6424769823e82a1829b0a8bcaf501bffd90d25, 'PYPL.d' , 'Dinari' , 'RWA' , 18, 'Stocks'), + (0x0b5ac0d7dcf6964609a12af4f6c6f3c257070193, 'RIOT.d' , 'Dinari' , 'RWA' , 18, 'Stocks'), + (0xf4bd09b048248876e39fcf2e0cdf1aee1240a9d2, 'SPY.d' , 'Dinari' , 'RWA' , 18, 'Stocks'), + (0xd883bcf80b2b085fa40cc3e2416b4ab1cbca649e, 'SQ.d' , 'Dinari' , 'RWA' , 18, 'Stocks'), + (0x36d37b6cbca364cf1d843eff8c2f6824491bcf81, 'TSLA.d' , 'Dinari' , 'RWA' , 18, 'Stocks'), + (0x9c46e1b70d447b770dbfc8d450543a431af6df3a, 'USFR.d' , 'Dinari' , 'RWA' , 18, 'Stocks'), + (0xeb0d1360a14c3b162f2974daa5d218e0c1090146, 'YUM.d' , 'Dinari' , 'RWA' , 18, 'Stocks'), + (0x0a2919147b871a0fc90f04944e31fad56d9af666, 'SLX.d' , 'Dinari' , 'RWA' , 18, 'Stocks'), + (0xc20770116f2821d550574c2b9ce1b4baa7012377, 'WEAT.d' , 'Dinari' , 'RWA' , 18, 'Stocks'), + (0x13f950ee286a5be0254065d4b66420fc0e57adfc, 'WOOD.d' , 'Dinari' , 'RWA' , 18, 'Stocks'), + (0xad6a646c1b262586ef3a8b6c6304e7c9218ecac4, 'PHO.d' , 'Dinari' , 'RWA' , 18, 'Stocks'), + (0x97ec5dada8262bd922bffd54a93f5a11efe0b136, 'RDDT.d' , 'Dinari' , 'RWA' , 18, 'Stocks'), + (0xb415998a7bb6f11dc589e0eb20adf586ba32f12a, 'GME.d' , 'Dinari' , 'RWA' , 18, 'Stocks'), + (0x28bf1c9ee2eb746a2d61a0bec97a344028171d6c, 'AMC.d' , 'Dinari' , 'RWA' , 18, 'Stocks'), + (0xa0eefbba9c18925fb6bab26281806772398cca1f, 'UBER.d' , 'Dinari' , 'RWA' , 18, 'Stocks'), + (0x337842843512192b798a5592053ce8e2245651f8, 'CSCO.d' , 'Dinari' , 'RWA' , 18, 'Stocks'), + (0x289594b223ab31832726eb99871fa99adac583e5, 'ADBE.d' , 'Dinari' , 'RWA' , 18, 'Stocks'), + (0x7c5fed5e0f8d05748cc12ffe1ca400b07de0f983, 'AVGO.d' , 'Dinari' , 'RWA' , 18, 'Stocks'), + (0x315a2dca4b1b633d3a707c71d96243534c02f7c4, 'XOM.d' , 'Dinari' , 'RWA' , 18, 'Stocks'), + (0x0e929101c4fa7c91eaa64d7216161ba3eee387fe, 'CVX.d' , 'Dinari' , 'RWA' , 18, 'Stocks'), + (0x50e2b96f958133cbda56d1a62d156e812fe97828, 'JNJ.d' , 'Dinari' , 'RWA' , 18, 'Stocks'), + (0x44966bf47a494b36dfb407afb334a9226cdf90bc, 'AZN.d' , 'Dinari' , 'RWA' , 18, 'Stocks'), + (0x50f7b76a0b888e5d7196f1a472d2e567d425c761, 'TJX.d' , 'Dinari' , 'RWA' , 18, 'Stocks'), + (0xb90e1ee5f4194388e71c11dc2a4eea16da391ce4, 'PG.d' , 'Dinari' , 'RWA' , 18, 'Stocks'), + (0x79ECCE8E2D17603877Ff15BC29804CbCB590EC08, 'fBILL' , 'Fortunafi' , 'RWA' , 18, 'Treasury Bills'), + (0x45bafad5a6a531Bc18Cf6CE5B02C58eA4D20589b, 'ifBILL' , 'Fortunafi' , 'RWA' , 18, 'Treasury Bills'), + (0x20c64dee8fda5269a78f2d5bdba861ca1d83df7a, 'bHIGH' , 'BackedFi' , 'RWA' , 18, 'Bonds'), + (0xbbcb0356bB9e6B3Faa5CbF9E5F36185d53403Ac9, 'bCOIN' , 'BackedFi' , 'RWA' , 18, 'Bonds'), + (0x2f11EeeE0bf21E7661A22dbBbb9068F4ad191b86, 'bNIU' , 'BackedFi' , 'RWA' , 18, 'Bonds'), + (0xAde6057FcAfa57d6d51FFa341C64ce4814995995, 'bZPR1' , 'BackedFi' , 'RWA' , 18, 'Bonds'), + (0x2F123cF3F37CE3328CC9B5b8415f9EC5109b45e7, 'bC3M' , 'BackedFi' , 'RWA' , 18, 'Bonds'), + (0x0f76d32cdccdcbd602a55af23eaf58fd1ee17245, 'bERNA' , 'BackedFi' , 'RWA' , 18, 'Bonds'), + (0x3f95aa88ddbb7d9d484aa3d482bf0a80009c52c9, 'bERNX' , 'BackedFi' , 'RWA' , 18, 'Bonds'), + (0xF84D28A8D28292842dD73D1c5F99476A80b6666A, 'TBILL' , 'OpenEden' , 'RWA' , 6 , 'Treasury Bills'), + (0xB88a5Ac00917a02d82c7cd6CEBd73E2852d43574, 'SWEEP' , 'Sweep' , 'RWA' , 18, 'Stablecoin'), + (0x35e050d3C0eC2d29D269a8EcEa763a183bDF9A9D, 'USDY' , 'Ondo Finance' , 'RWA' , 18, 'Treasury Bonds'), + (0x9B6226dd0191a77d032F56A6d383044EE99944C3, 'flrEUR' , 'Florence Finance' , 'RWA' , 18, 'Stablecoin'), + (0x0BBF664D46becc28593368c97236FAa0fb397595, 'KNOX' , 'Reserve Protocol' , 'RWA' , 18, 'Stablecoin'), + (0x0000206329b97DB379d5E1Bf586BbDB969C63274, 'USDA' , 'Angle' , 'RWA' , 18, 'Stablecoin'), + (0xFA5Ed56A203466CbBC2430a43c66b9D8723528E7, 'agEUR' , 'Angle' , 'RWA' , 18, 'Stablecoin'), + (0x323665443cef804a3b5206103304bd4872ea4253, 'USDV' , 'Verified USD' , 'RWA' , 6 , 'Stablecoin'), + (0xB9e4765BCE2609bC1949592059B17Ea72fEe6C6A, 'BENJI' , 'FranklinTempleton' , 'RWA' , 18, 'U.S. govt Securities'), + (0x89B1e7068bF8E3232dD8f16c35cAc45bDA584f4E, 'SWEEPR' , 'Sweep' , 'Governance' , 18, 'Stablecoin'), + (0x3269a3C00AB86c753856fD135d97b87FACB0d848, 'FFM' , 'Florence Finance' , 'Governance' , 18, 'Stablecoin'), + (0xCa5Ca9083702c56b481D1eec86F1776FDbd2e594, 'RSR' , 'Reserve Protocol' , 'Governance' , 18, 'Stablecoin') ) as t( token_address, symbol, project, - type + type, + decimals, + category ) From 502a89b33ed73da08dd9d7531b406108befaecc4 Mon Sep 17 00:00:00 2001 From: tomfutago <35136350+tomfutago@users.noreply.github.com> Date: Thu, 28 Nov 2024 10:56:14 +0000 Subject: [PATCH 06/18] fix: cbBTC decimals (#7192) --- .../ethereum/covers/nexusmutual_ethereum_covers_v2.sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dbt_subprojects/daily_spellbook/models/nexusmutual/ethereum/covers/nexusmutual_ethereum_covers_v2.sql b/dbt_subprojects/daily_spellbook/models/nexusmutual/ethereum/covers/nexusmutual_ethereum_covers_v2.sql index 448047f7964..9e6f803b750 100644 --- a/dbt_subprojects/daily_spellbook/models/nexusmutual/ethereum/covers/nexusmutual_ethereum_covers_v2.sql +++ b/dbt_subprojects/daily_spellbook/models/nexusmutual/ethereum/covers/nexusmutual_ethereum_covers_v2.sql @@ -88,7 +88,7 @@ cover_premiums as ( when 7 then 'cbBTC' else 'NA' end as cover_asset, - c.sum_assured / if(c.cover_asset = 6, 1e6, 1e18) as sum_assured, + c.sum_assured / case c.cover_asset when 6 then 1e6 when 7 then 1e8 else 1e18 end as sum_assured, case c.payment_asset when 0 then 'ETH' when 1 then 'DAI' From fc71eac30e3e66c445c77cbabfa4a735ed3533f9 Mon Sep 17 00:00:00 2001 From: 0xRob <83790096+0xRobin@users.noreply.github.com> Date: Thu, 28 Nov 2024 11:56:28 +0100 Subject: [PATCH 07/18] enable excluded boost model (#7193) * enable model * remove cast to int and leave as uint256 --------- Co-authored-by: jeff-dude <102681548+jeff-dude@users.noreply.github.com> --- .../daily_spellbook/models/boost/boost_deployed.sql | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/dbt_subprojects/daily_spellbook/models/boost/boost_deployed.sql b/dbt_subprojects/daily_spellbook/models/boost/boost_deployed.sql index 2b7e0c586ea..40be3f50a3b 100644 --- a/dbt_subprojects/daily_spellbook/models/boost/boost_deployed.sql +++ b/dbt_subprojects/daily_spellbook/models/boost/boost_deployed.sql @@ -1,7 +1,6 @@ {{ config( schema='boost', - tags = ['prod_exclude'], alias='deployed', materialized='incremental', file_format='delta', @@ -33,7 +32,7 @@ select TRY(from_unixtime(end_time)) as end_time, reward_amount_raw, reward_token_address, - cast(max_participants as int) as max_participants, + max_participants, creation_time, creator as creator_address from From e2962b93d55617edf043dcbff4b9fa2bd7ece085 Mon Sep 17 00:00:00 2001 From: maybeyonas Date: Thu, 28 Nov 2024 16:53:11 +0530 Subject: [PATCH 08/18] [swell] - add base (not the chain) swell rsweth withdrawals tables (#7179) * batman * sources for withdrawal * updating descriptions * add unique test and comma fix * code fixes * attempt fix for incremental * another incremental fix attempt * fuck it we ball ? * dont think the previous incremental balled * adding seed * trying to fix seed testing * lets try this * decimal fixes --- .../ethereum/swell_rsweth_ethereum_schema.yml | 91 ++++++++++++++++++ .../swell_rsweth_ethereum_withdrawals.sql | 94 +++++++++++++++++++ .../seeds/swell/rsweth/ethereum/schema.yml | 24 +++++ ...swell_rsweth_ethereum_withdrawals_seed.csv | 3 + .../swell_rsweth_ethereum_sources.yml | 70 ++++++++++++++ 5 files changed, 282 insertions(+) create mode 100644 dbt_subprojects/daily_spellbook/models/swell/rsweth/ethereum/swell_rsweth_ethereum_schema.yml create mode 100644 dbt_subprojects/daily_spellbook/models/swell/rsweth/ethereum/swell_rsweth_ethereum_withdrawals.sql create mode 100644 dbt_subprojects/daily_spellbook/seeds/swell/rsweth/ethereum/schema.yml create mode 100644 dbt_subprojects/daily_spellbook/seeds/swell/rsweth/ethereum/swell_rsweth_ethereum_withdrawals_seed.csv create mode 100644 sources/swell/rsweth/ethereum/swell_rsweth_ethereum_sources.yml diff --git a/dbt_subprojects/daily_spellbook/models/swell/rsweth/ethereum/swell_rsweth_ethereum_schema.yml b/dbt_subprojects/daily_spellbook/models/swell/rsweth/ethereum/swell_rsweth_ethereum_schema.yml new file mode 100644 index 00000000000..2ac88a66d10 --- /dev/null +++ b/dbt_subprojects/daily_spellbook/models/swell/rsweth/ethereum/swell_rsweth_ethereum_schema.yml @@ -0,0 +1,91 @@ +version: 2 + +models: + - name: swell_rsweth_ethereum_withdrawals + meta: + blockchain: ethereum + project: swell + contributors: maybeYonas + + config: + tags: ['swell', 'restaking', 'lrt', 'withdrawals'] + description: "rswETH withdrawals" + data_tests: + - dbt_utils.unique_combination_of_columns: + combination_of_columns: + - token_id + - check_seed: + seed_file: ref('swell_rsweth_ethereum_withdrawals_seed') + match_columns: + - token_id + check_columns: + - request_tx_hash + - request_block_number + - request_index + - owner + - rswETH_amount + - rswETH_request_rate + - processed_tx_hash + - processed_block_number + - processed_index + - rswETH_processed_rate + - claim_tx_hash + - claim_block_number + - claim_index + - ETH_amount + + columns: + - &request_block_time + name: request_block_time + description: "timestamp at withdrawal request" + - &request_tx_hash + name: request_tx_hash + description: "hash of tx at withdrawal request" + - &request_block_number + name: request_block_number + description: "block at withdrawal request" + - &request_index + name: request_index + description: "event index at at withdrawal request" + - &token_id + name: token_id + description: "unique id of withdrawal" + - &owner + name: owner + description: "owner of withdrawn eth" + - &rswETH_amount + name: rswETH_amount + description: "amount of rsweth burned" + - &rswETH_request_rate + name: rswETH_request_rate + description: "rsweth to eth rate at withdrawal request" + - &processed_block_time + name: processed_block_time + description: "timestamp at withdrawal ready for completion" + - &processed_tx_hash + name: processed_tx_hash + description: "hash of tx at withdrawal ready for completion" + - &processed_block_number + name: processed_block_number + description: "block at withdrawal ready for completion" + - &processed_index + name: processed_index + description: "event index at at withdrawal ready for completion" + - &rswETH_processed_rate + name: rswETH_processed_rate + description: "rsweth to eth rate at withdrawal completion" + - &claim_block_time + name: claim_block_time + description: "timestamp at withdrawal claimed" + - &claim_tx_hash + name: claim_tx_hash + description: "hash of tx at withdrawal claimed" + - &claim_block_number + name: claim_block_number + description: "block at withdrawal claimed" + - &claim_index + name: claim_index + description: "event index at at withdrawal claimed" + - Ð_amount + name: ETH_amount + description: "eth amount claimed by user" \ No newline at end of file diff --git a/dbt_subprojects/daily_spellbook/models/swell/rsweth/ethereum/swell_rsweth_ethereum_withdrawals.sql b/dbt_subprojects/daily_spellbook/models/swell/rsweth/ethereum/swell_rsweth_ethereum_withdrawals.sql new file mode 100644 index 00000000000..1f0a6730ae6 --- /dev/null +++ b/dbt_subprojects/daily_spellbook/models/swell/rsweth/ethereum/swell_rsweth_ethereum_withdrawals.sql @@ -0,0 +1,94 @@ +{{ config( + schema = 'swell_rsweth_ethereum', + alias = 'withdrawals', + materialized = 'incremental', + file_format = 'delta', + incremental_strategy = 'merge', + unique_key = ['token_id'], + post_hook='{{ expose_spells(\'["ethereum"]\', + "project", + "swell", + \'["maybeYonas"]\') }}' + ) +}} + +{% set incremental %} + where evt_block_time >= ( + select min(request_block_time) + from {{ this }} + where claim_block_time is null + ) +{% endset %} + +with +rsweth_decoded_withdrawal_requests as ( + select + evt_block_time, evt_tx_hash, evt_block_number, evt_index, + lastTokenIdProcessed, + owner, + tokenId, + amount, + timestamp, + rateWhenCreated + from {{source ('swell_v3_ethereum', 'RswEXIT_evt_WithdrawRequestCreated')}} + -- from swell_v3_ethereum.RswEXIT_evt_WithdrawRequestCreated + {% if is_incremental() %} + {{incremental}} + {% endif %} +), +rsweth_decoded_withdrawal_claimed as ( + select + evt_block_time, evt_tx_hash, evt_block_number, evt_index, + owner, + tokenId, + exitClaimedETH + from {{source ('swell_v3_ethereum', 'RswEXIT_evt_WithdrawalClaimed')}} + -- from swell_v3_ethereum.RswEXIT_evt_WithdrawalClaimed + {% if is_incremental() %} + {{incremental}} + {% endif %} + +), +rsweth_decoded_withdrawal_processed as ( + select + evt_block_time, evt_tx_hash, evt_block_number, evt_index, + fromTokenId, + toTokenId, + processedRate, + processedExitingETH, + processedExitedETH + from {{source ('swell_v3_ethereum', 'RswEXIT_evt_WithdrawalsProcessed')}} + -- from swell_v3_ethereum.RswEXIT_evt_WithdrawalsProcessed + {% if is_incremental() %} + {{incremental}} + {% endif %} +) + +select + r.evt_block_time as request_block_time, + r.evt_tx_hash as request_tx_hash, + r.evt_block_number as request_block_number, + r.evt_index as request_index, + r.tokenId as token_id, + r.owner as owner, + r.amount/1e18 as rswETH_amount, + r.rateWhenCreated/1e18 as rswETH_request_rate, + + p.evt_block_time as processed_block_time, + p.evt_tx_hash as processed_tx_hash, + p.evt_block_number as processed_block_number, + p.evt_index as processed_index, + p.processedRate/1e18 as rswETH_processed_rate, + + c.evt_block_time as claim_block_time, + c.evt_tx_hash as claim_tx_hash, + c.evt_block_number as claim_block_number, + c.evt_index as claim_index, + c.exitClaimedETH/1e18 as ETH_amount +from rsweth_decoded_withdrawal_requests r + left join rsweth_decoded_withdrawal_processed p + on r.tokenId >= p.fromTokenId + and r.tokenId <= p.toTokenId + left join rsweth_decoded_withdrawal_claimed c + on r.tokenId = c.tokenId +order by 1 desc \ No newline at end of file diff --git a/dbt_subprojects/daily_spellbook/seeds/swell/rsweth/ethereum/schema.yml b/dbt_subprojects/daily_spellbook/seeds/swell/rsweth/ethereum/schema.yml new file mode 100644 index 00000000000..84344e87d47 --- /dev/null +++ b/dbt_subprojects/daily_spellbook/seeds/swell/rsweth/ethereum/schema.yml @@ -0,0 +1,24 @@ +version: 2 + +seeds: + - name: swell_rsweth_ethereum_withdrawals_seed + config: + column_types: + request_block_time: timestamp + request_tx_hash: varbinary + request_block_number: bigint + request_index: integer + token_id: integer + owner: varbinary + rswETH_amount: double + rswETH_request_rate: double + processed_block_time: timestamp + processed_tx_hash: varbinary + processed_block_number: bigint + processed_index: integer + rswETH_processed_rate: double + claim_block_time: timestamp + claim_tx_hash: varbinary + claim_block_number: bigint + claim_index: integer + ETH_amount: double \ No newline at end of file diff --git a/dbt_subprojects/daily_spellbook/seeds/swell/rsweth/ethereum/swell_rsweth_ethereum_withdrawals_seed.csv b/dbt_subprojects/daily_spellbook/seeds/swell/rsweth/ethereum/swell_rsweth_ethereum_withdrawals_seed.csv new file mode 100644 index 00000000000..c46f8111e3e --- /dev/null +++ b/dbt_subprojects/daily_spellbook/seeds/swell/rsweth/ethereum/swell_rsweth_ethereum_withdrawals_seed.csv @@ -0,0 +1,3 @@ +request_block_time,request_tx_hash,request_block_number,request_index,token_id,owner,rswETH_amount,rswETH_request_rate,processed_block_time,processed_tx_hash,processed_block_number,processed_index,rswETH_processed_rate,claim_block_time,claim_tx_hash,claim_block_number,claim_index,ETH_amount +2024-11-12 06:57:47.000 UTC,0x978742baca60c9f02eec6f8328ee77abb85716eb111d340497315f077f049388,21169958,227,3007,0xc77c6375e9fd581701425a6fb70bf371fb1cba28,500.00,1.023476642102901592,2024-11-21 01:09:47.000 UTC,0x9a0ac005f69cd7df2f10134ae5004a666c0ceb1baf034fdb41460439f02b684d,21232733,167,1.024494608040389206,2024-11-22 02:55:11.000 UTC,0x0fd5c4551eda43ad93f6833e0f5463afac8e88d30e3263cabcf4da9477b81fbf,21240420,274,511.738321051450796 +2024-11-12 06:56:47.000 UTC,0xd0a7cd3f2f67cca12f2fe22ef7a5f0df922c1716d2f894a8c20406fdf01fd9dd,21169953,461,3006,0xc77c6375e9fd581701425a6fb70bf371fb1cba28,500.00,1.023476642102901592,2024-11-21 01:09:47.000 UTC,0x9a0ac005f69cd7df2f10134ae5004a666c0ceb1baf034fdb41460439f02b684d,21232733,167,1.024494608040389206,2024-11-22 02:54:35.000 UTC,0xf4aa4a769d39d6aadc54591eca47f59e688487580ef55f5bf4b21880bce16d3e,21240417,322,511.738321051450796 \ No newline at end of file diff --git a/sources/swell/rsweth/ethereum/swell_rsweth_ethereum_sources.yml b/sources/swell/rsweth/ethereum/swell_rsweth_ethereum_sources.yml new file mode 100644 index 00000000000..d0e05fded47 --- /dev/null +++ b/sources/swell/rsweth/ethereum/swell_rsweth_ethereum_sources.yml @@ -0,0 +1,70 @@ +version: 2 + +sources: + - name: swell_v3_ethereum + description: "swell decoded tables of ethereum contract" + tables: + - name: RswEXIT_evt_WithdrawRequestCreated + description: 'decoded table on `WithdrawRequestCreated` event emitted from RswEXIT contract `0x58749C46Ffe97e4d79508a2C781C440f4756f064`. event emitted when a user initiates a new withdrawal request' + columns: + - &rswexit_contract_address + name: contract_address + description: 'RswExit contract address' + - &evt_tx_hash + name: evt_tx_hash + description: 'transaction hash of event' + - &evt_index + name: evt_index + description: 'index of event within block' + - &evt_block_time + name: evt_block_time + description: 'timestamp of blocktime in UTC' + - &evt_block_number + name: evt_block_number + description: 'block number containing transaction' + - name: lastTokenIdProcessed + description: 'last processed withdrawal' + - &rswexit_owner + name: owner + description: 'owner of rsweth burned' + - &rswexit_tokenid + name: tokenId + description: 'unique id for withdrawal processing' + - name: amount + description: 'amount of rsweth burned' + - name: timestamp + description: 'timestamp of withdrawal request' + - name: rateWhenCreated + description: 'rsweth-eth rate at withdrawal request' + + - name: RswEXIT_evt_WithdrawalsProcessed + description: 'decoded table on `WithdrawalsProcessed` event emitted from RswEXIT contract `0x58749C46Ffe97e4d79508a2C781C440f4756f064`. event emitted when the protocol processes the withdrawal from beaconchain and make it available for user to claim' + columns: + - *rswexit_contract_address + - *evt_tx_hash + - *evt_index + - *evt_block_time + - *evt_block_number + - name: fromTokenId + description: 'first token_id of withdrawals processed in this batch' + - name: toTokenId + description: 'last token_id of withdrawals processed in this batch' + - name: processedRate + description: 'rsweth-eth rate at which withdrawal is processed' + - name: processedExitingETH + description: 'amount of eth exiting in current batch' + - name: processedExitedETH + description: 'amount of eth exited' + + - name: RswEXIT_evt_WithdrawalClaimed + description: 'decoded table on `WithdrawalClaimed` event emitted from RswEXIT contract `0x58749C46Ffe97e4d79508a2C781C440f4756f064`. event emitted when a user claims the ETH withdrawn' + columns: + - *rswexit_contract_address + - *evt_tx_hash + - *evt_index + - *evt_block_time + - *evt_block_number + - *rswexit_owner + - *rswexit_tokenid + - name: exitClaimedETH + description: 'amout of eth claimed' \ No newline at end of file From f8c38c3bbb5b32d7dab6e79dcaa76333a3adf4ca Mon Sep 17 00:00:00 2001 From: whale_hunter <143016036+whalehunting@users.noreply.github.com> Date: Thu, 28 Nov 2024 12:40:29 +0100 Subject: [PATCH 09/18] feat: add bitfoot solana bot trades spellbook (#7202) * feat: add bitfoot solana bot trades spellbook * fix: add correct seed --- .../_sector/dex/bot_trades/solana/_schema.yml | 32 +++- .../solana/dex_solana_bot_trades.sql | 1 + .../platforms/bitfoot_solana_bot_trades.sql | 152 ++++++++++++++++++ .../bitfoot/bitfoot_solana_trades_seed.csv | 21 +++ 4 files changed, 201 insertions(+), 5 deletions(-) create mode 100644 dbt_subprojects/solana/models/_sector/dex/bot_trades/solana/platforms/bitfoot_solana_bot_trades.sql create mode 100644 dbt_subprojects/solana/seeds/bitfoot/bitfoot_solana_trades_seed.csv diff --git a/dbt_subprojects/solana/models/_sector/dex/bot_trades/solana/_schema.yml b/dbt_subprojects/solana/models/_sector/dex/bot_trades/solana/_schema.yml index 6e9faa06c60..c6d9f401458 100644 --- a/dbt_subprojects/solana/models/_sector/dex/bot_trades/solana/_schema.yml +++ b/dbt_subprojects/solana/models/_sector/dex/bot_trades/solana/_schema.yml @@ -454,7 +454,7 @@ models: - check_bot_trades_seed: seed_file: ref('tirador_solana_trades_seed') blockchain: solana - + - name: unibot_solana_bot_trades meta: blockchain: solana @@ -498,7 +498,7 @@ models: - check_bot_trades_seed: seed_file: ref('mev_x_solana_trades_seed') blockchain: solana - + - name: alpha_dex_solana_bot_trades meta: blockchain: solana @@ -578,7 +578,7 @@ models: - check_bot_trades_seed: seed_file: ref('pinkpunk_solana_trades_seed') blockchain: solana - + - name: falcon_solana_bot_trades meta: blockchain: solana @@ -598,7 +598,7 @@ models: - outer_instruction_index - inner_instruction_index - check_bot_trades_seed: - seed_file: ref('falcon_solana_trades_seed') + seed_file: ref('falcon_solana_trades_seed') blockchain: solana - name: bloom_solana_bot_trades @@ -643,4 +643,26 @@ models: - inner_instruction_index - check_bot_trades_seed: seed_file: ref('autosnipe_solana_trades_seed') - blockchain: solana \ No newline at end of file + blockchain: solana + + - name: bitfoot_solana_bot_trades + meta: + blockchain: solana + sector: dex + project: bitfoot + contributors: whale_hunter + config: + tags: ["solana", "dex", "bitfoot", "trades"] + description: > + Bitfoot trades on Solana + data_tests: + - dbt_utils.unique_combination_of_columns: + combination_of_columns: + - blockchain + - tx_id + - tx_index + - outer_instruction_index + - inner_instruction_index + - check_bot_trades_seed: + seed_file: ref('bitfoot_solana_trades_seed') + blockchain: solana diff --git a/dbt_subprojects/solana/models/_sector/dex/bot_trades/solana/dex_solana_bot_trades.sql b/dbt_subprojects/solana/models/_sector/dex/bot_trades/solana/dex_solana_bot_trades.sql index 45a86ca4054..3a46f240904 100644 --- a/dbt_subprojects/solana/models/_sector/dex/bot_trades/solana/dex_solana_bot_trades.sql +++ b/dbt_subprojects/solana/models/_sector/dex/bot_trades/solana/dex_solana_bot_trades.sql @@ -35,6 +35,7 @@ , ref('looter_solana_bot_trades') , ref('wifbot_solana_bot_trades') , ref('autosnipe_solana_bot_trades') + , ref('bitfoot_solana_bot_trades') ] %} {% for bot in solana_trading_bot %} diff --git a/dbt_subprojects/solana/models/_sector/dex/bot_trades/solana/platforms/bitfoot_solana_bot_trades.sql b/dbt_subprojects/solana/models/_sector/dex/bot_trades/solana/platforms/bitfoot_solana_bot_trades.sql new file mode 100644 index 00000000000..4213660df7b --- /dev/null +++ b/dbt_subprojects/solana/models/_sector/dex/bot_trades/solana/platforms/bitfoot_solana_bot_trades.sql @@ -0,0 +1,152 @@ +{{ config( + alias = 'bot_trades', + schema = 'bitfoot_solana', + partition_by = ['block_month'], + materialized = 'incremental', + file_format = 'delta', + incremental_strategy = 'merge', + incremental_predicates = [incremental_predicate('DBT_INTERNAL_DEST.block_time')], + unique_key = ['blockchain', 'tx_id', 'tx_index', 'outer_instruction_index', 'inner_instruction_index'] + ) +}} + +{% set project_start_date = '2024-11-21' %} +{% set fee_receiver_1 = 'BzmpLvrhZHKoXV7CW9F1AVPnie3hNh2JK3BRdHL4Zcya' %} +{% set wsol_token = 'So11111111111111111111111111111111111111112' %} + +WITH + allFeePayments AS ( + SELECT + tx_id, + 'SOL' AS feeTokenType, + balance_change / 1e9 AS fee_token_amount, + '{{wsol_token}}' AS fee_token_mint_address + FROM + {{ source('solana','account_activity') }} + WHERE + {% if is_incremental() %} + {{ incremental_predicate('block_time') }} + {% else %} + block_time >= TIMESTAMP '{{project_start_date}}' + {% endif %} + AND tx_success + AND balance_change > 0 + AND ( + address = '{{fee_receiver_1}}' + ) + ), + botTrades AS ( + SELECT + trades.block_time, + CAST(date_trunc('day', trades.block_time) AS date) AS block_date, + CAST(date_trunc('month', trades.block_time) AS date) AS block_month, + 'solana' AS blockchain, + amount_usd, + IF( + token_sold_mint_address = '{{wsol_token}}', + 'Buy', + 'Sell' + ) AS type, + token_bought_amount, + token_bought_symbol, + token_bought_mint_address AS token_bought_address, + token_sold_amount, + token_sold_symbol, + token_sold_mint_address AS token_sold_address, + fee_token_amount * price AS fee_usd, + fee_token_amount, + IF(feeTokenType = 'SOL', 'SOL', symbol) AS fee_token_symbol, + fee_token_mint_address AS fee_token_address, + project, + version, + token_pair, + project_program_id AS project_contract_address, + trader_id AS user, + trades.tx_id, + tx_index, + outer_instruction_index, + inner_instruction_index + FROM + {{ ref('dex_solana_trades') }} AS trades + JOIN allFeePayments AS feePayments ON trades.tx_id = feePayments.tx_id + LEFT JOIN {{ source('prices', 'usd') }} AS feeTokenPrices ON ( + feeTokenPrices.blockchain = 'solana' + AND fee_token_mint_address = toBase58 (feeTokenPrices.contract_address) + AND date_trunc('minute', block_time) = minute + {% if is_incremental() %} + AND {{ incremental_predicate('minute') }} + {% else %} + AND minute >= TIMESTAMP '{{project_start_date}}' + {% endif %} + ) + JOIN {{ source('solana','transactions') }} AS transactions ON ( + trades.tx_id = id + {% if is_incremental() %} + AND {{ incremental_predicate('transactions.block_time') }} + {% else %} + AND transactions.block_time >= TIMESTAMP '{{project_start_date}}' + {% endif %} + ) + WHERE + trades.trader_id != '{{fee_receiver_1}}' -- Exclude trades signed by FeeWallet + AND transactions.signer != '{{fee_receiver_1}}' -- Exclude trades signed by FeeWallet + {% if is_incremental() %} + AND {{ incremental_predicate('trades.block_time') }} + {% else %} + AND trades.block_time >= TIMESTAMP '{{project_start_date}}' + {% endif %} + ), + highestInnerInstructionIndexForEachTrade AS ( + SELECT + tx_id, + outer_instruction_index, + MAX(inner_instruction_index) AS highestInnerInstructionIndex + FROM + botTrades + GROUP BY + tx_id, + outer_instruction_index + ) +SELECT + block_time, + block_date, + block_month, + 'Bitfoot' as bot, + blockchain, + amount_usd, + type, + token_bought_amount, + token_bought_symbol, + token_bought_address, + token_sold_amount, + token_sold_symbol, + token_sold_address, + fee_usd, + fee_token_amount, + fee_token_symbol, + fee_token_address, + project, + version, + token_pair, + project_contract_address, + user, + botTrades.tx_id, + tx_index, + botTrades.outer_instruction_index, + COALESCE(inner_instruction_index, 0) AS inner_instruction_index, + IF( + inner_instruction_index = highestInnerInstructionIndex, + true, + false + ) AS is_last_trade_in_transaction +FROM + botTrades + JOIN highestInnerInstructionIndexForEachTrade ON ( + botTrades.tx_id = highestInnerInstructionIndexForEachTrade.tx_id + AND botTrades.outer_instruction_index = highestInnerInstructionIndexForEachTrade.outer_instruction_index + ) +ORDER BY + block_time DESC, + tx_index DESC, + outer_instruction_index DESC, + inner_instruction_index DESC diff --git a/dbt_subprojects/solana/seeds/bitfoot/bitfoot_solana_trades_seed.csv b/dbt_subprojects/solana/seeds/bitfoot/bitfoot_solana_trades_seed.csv new file mode 100644 index 00000000000..2dfe16fe5fe --- /dev/null +++ b/dbt_subprojects/solana/seeds/bitfoot/bitfoot_solana_trades_seed.csv @@ -0,0 +1,21 @@ +block_time,block_date,block_month,bot,blockchain,amount_usd,type,token_bought_amount,token_bought_symbol,token_bought_address,token_sold_amount,token_sold_symbol,token_sold_address,fee_usd,fee_token_amount,fee_token_symbol,fee_token_address,project,version,token_pair,project_contract_address,user,tx_id,tx_index,outer_instruction_index,inner_instruction_index,is_last_trade_in_transaction +2024-11-27 11:39:41.000 UTC,2024-11-27,2024-11-01,Bitfoot,solana,11.689425,Buy,709.813803,Aejo,B16bhVYwE9zQVjbYpnnmvfN7qJFDe21nx5iQC5LJpump,0.0495,SOL,So11111111111111111111111111111111111111112,0.078519875,0.0003325,SOL,So11111111111111111111111111111111111111112,raydium,4,SOL-Aejo,7tdNR3TNw13wGDz2PBynb9T6t39oskvzNiTvKCnVN4bw,HZidTvr4nRwhPBJvSb842rtBsedtigFWPMfwZfUSWsA7,52E6dhvq1DCqdjqVAxqJ3CLaJd1qSDMUqAhAzyjB3Gx2U8XWHH3T9YkzzSfGv6YYF5ArePxEDwCpuGncJM3WQ3CU,1097,8,2,true +2024-11-28 04:22:56.000 UTC,2024-11-28,2024-11-01,Bitfoot,solana,6.04981532776,Sell,0.025230692,SOL,So11111111111111111111111111111111111111112,200.217182,TRIP,8JZ2R6dzfCeiAvcJ7H1G6hy8x5pBQryHZcWiVY2spump,0.03720642282,0.000155169,SOL,So11111111111111111111111111111111111111112,raydium,4,TRIP-SOL,58JdqNRYCQthSzKY1rZ9LzaK6UNkKhbQgN5kBtyEGoLL,4KsknATdHdgfU1Wi6kakxPRa2TDg4HCXfoc66b6JXp3M,3av3ivTmU1DZCHG7c5LSMzpg5g4o212opYZmuHDY6UEG5SVf1ZBmSVySZQequCwTTgGrgcu9PhB8PQYvZpbNcQzS,2297,6,1,true +2024-11-23 11:40:07.000 UTC,2024-11-23,2024-11-01,Bitfoot,solana,9.86582145959,Sell,0.038428783,SOL,So11111111111111111111111111111111111111112,1099.306621,Mustard,22Xeo6diWfJrScaoVFzgkwzrCByPukK45fdwkJyrpump,0.06560760823,0.000255551,SOL,So11111111111111111111111111111111111111112,raydium,4,SOL-Mustard,5JbknNv534djG9TeCUMpLevedino2c1WrYB9pqGkyqLe,3VGaybHr8dvZbfTjJrGPLC4WhFUwZgfzXMUHeiymmHRt,2pmYAXnT8hTQzwsfYKVprXvSqcy7aKYedE5AScecSgkngkKmoncXSfM8kwHCYNBgXpvM9ZsB2VS9oK29BABJKSd2,1347,6,1,true +2024-11-25 16:04:04.000 UTC,2024-11-25,2024-11-01,Bitfoot,solana,0.24297570000000002,Buy,17.043396,blob,DgG9sM56ZcVidBV8bNArQPm93a2rmjzHkrrUntGSpump,0.00099,SOL,So11111111111111111111111111111111111111112,0.0015093945000000002,0.00000615,SOL,So11111111111111111111111111111111111111112,raydium,4,SOL-blob,7Y3oLPPrBsXGNYNaZLmhRZExFChXP84z2w1oHBjHTwCa,qjmNJ3PrUjCipauKuVtL1wefYC2kP2qq7PpmUVJdunH,VK5kFKKoSwLTfoU2Ra8ygD1ysh3YbAuN8L34RFDQewdwRGtB7R11Sm9bEWCU3rgM4Ehaj7jRwFx4iM17DmB4AYd,329,8,2,true +2024-11-25 00:40:46.000 UTC,2024-11-25,2024-11-01,Bitfoot,solana,7.53405404689,Buy,737146.667049,FCOCK,D7g7pFmuK5SPwQSvPJHgmg83dDVkSQonGqLz87Lkpump,0.030055667,SOL,So11111111111111111111111111111111111111112,0.05111361836,0.000203908,SOL,So11111111111111111111111111111111111111112,pumpdotfun,1,SOL-FCOCK,3C2YwGLpG6JuKjYJ6NsWAg5HYj7TnuXpGiBTcJzX5VPf,GnHVcSJQCpfiYAid2Vd9B6X12AWavdfuLTz6oMTiqP1A,27nAYJAyFXV3FbrR7rj26F74C9WDhd8ULNxCdnuSsjLpDnTMrcyPANbmWXMqWER2L4B8qkz6XbgxnbqYXZQibXej,1477,4,5,true +2024-11-26 18:15:44.000 UTC,2024-11-26,2024-11-01,Bitfoot,solana,11.6310085968,Sell,0.05097072,SOL,So11111111111111111111111111111111111111112,361734.413897,EDNA,CKrqbYKSUD1NQT4UE2EzVGGJAatvu28u8Ko4RdXypump,0.07369418869,0.000322951,SOL,So11111111111111111111111111111111111111112,pumpdotfun,1,SOL-EDNA,AKfD3c7b9ZQx7PgxbYEVfECnPQSkP16UmsqywAJk1weN,JAZfgjSioeg9CBV6N8L2BWrvLbGrtY5xfbPwvvUYKuFV,QcLdxmZCdmMVn3giuGdCL8HVanXY5PUTGQ3aPVWU4wtASqmv3MxYxky1uvVFBM98Hgr12P9z5QWKdaaRJU51odB,841,4,3,true +2024-11-23 22:52:04.000 UTC,2024-11-23,2024-11-01,Bitfoot,solana,1.2420067715,Sell,0.00487157,SOL,So11111111111111111111111111111111111111112,167202.885023,WINDY,4F1PHzeTp83Vg9nPWiZRYUV8sYHfTLcCDvWc53TJpump,0.007869286699999999,0.000030866,SOL,So11111111111111111111111111111111111111112,pumpdotfun,1,WINDY-SOL,,14nXezfdCTFzJ65C5NBbPvVB13oCE5cH8sX38unVYor1,5A1aCmEo46dX248DsDP9uT8j1Qu5vv4BHD1uyzfedX1TtFTQEhF98WqmcJxWGnA1ccMW8xJXH9N8W7MoGfuGDGCT,472,4,3,true +2024-11-25 05:25:32.000 UTC,2024-11-25,2024-11-01,Bitfoot,solana,12.34330347028,Sell,0.048612908,SOL,So11111111111111111111111111111111111111112,4496.168468,HYPR∞,FEiphCSAqjAxvtyMxj8boFTABrZBFyngfyrZueqeMw3q,0.07591147270000001,0.00029897,SOL,So11111111111111111111111111111111111111112,raydium,4,SOL-HYPR∞,83j8apUzRtpZwPiEdr31TxyGpA1NRLouLXxLMX8DM2iS,DgGzDWQYitL241ejctUaKQmZCRWiepqP67rsJQw9perf,3WA2P1q3BPcnSECH7EzFYiG9PsLvu6FpLH1S9mAUvuRXXFmhHtgBhiKaDNzHuAere8cWj5eLFy7xVcQBxZmkWj9K,1152,6,1,true +2024-11-23 06:29:14.000 UTC,2024-11-23,2024-11-01,Bitfoot,solana,2.03463991424,Sell,0.007840012,SOL,So11111111111111111111111111111111111111112,34.959386,FATHA,EWWDzCwq4UYW3ERTXbdgd6X6sdkKHFMJqRz1ZiFcpump,0.013530334719999999,0.000052136,SOL,So11111111111111111111111111111111111111112,raydium,4,SOL-FATHA,Am1SqWAHhaKWS4H9uwHweeYxCaECR5yDYZiX1jD9RVMP,BperhBq7YoWvWDCQnQrcp55mD6kVDwJ2zQrdSmfwWxYY,3XcpyEkiGpWRLWRsSyyHjmgqHPFk1tL9ASoZcte8wNQK6vZ5mm7xZBPLZdrT3ocHxWa4PLrfFpbpgL16MjWSMuBk,312,6,1,true +2024-11-23 00:02:40.000 UTC,2024-11-23,2024-11-01,Bitfoot,solana,5.72728636454,Sell,0.022308598,SOL,So11111111111111111111111111111111111111112,1841.951928,LEA,8SpPaFLycx897D6sowPZkEkcNdDahzRZb5itr6D8pump,0.03522284254,0.000137198,SOL,So11111111111111111111111111111111111111112,raydium,4,SOL-LEA,4ZF5Dd2KzXFE6ECFpKjcNYh48Ee1UvqiBRsVXk8EBmyT,57HvLnB3QecbktLXvxftJ1dNQ1WXnWs3RaV4mQoMwidh,4p8F6VfbMKqm8YyPgGfy7u9focAihhaVDFEBHPV5dpYyXodHjSFpxwtiBQDavEmzTazzoZLM9w6yJZCesuCYp3Gs,259,6,1,true +2024-11-22 23:19:11.000 UTC,2024-11-22,2024-11-01,Bitfoot,solana,205.32150744042,Sell,0.804614419,SOL,So11111111111111111111111111111111111111112,8657386.523431,Print,2cTbxqtyGr8C18iUPF4ditpjTyQiD5FXFF89bw9epump,1.2501000261000002,0.004898895,SOL,So11111111111111111111111111111111111111112,pumpdotfun,1,SOL-Print,Dt3nPfcQkV5i3xJUSKwox69srxtLBz1UVa9MChurqsez,48D6W1Z2p9FjhcHZi5rSbHuAzbhNHU2w4HeWKneM9NSn,39aJjXEPv9Gu5vzsXCwuEJgZ3Fr9y3UQgkrmoPwMZhas6eJDRYB2JsicR6jEhjhiu5kzwXqb3VCCa7vn2pJGFa2g,1869,4,3,true +2024-11-23 20:12:47.000 UTC,2024-11-23,2024-11-01,Bitfoot,solana,17.04803946804,Sell,0.067060182,SOL,So11111111111111111111111111111111111111112,287334.915975,Wiki,DuEGP63NvXRN1SyonCjsNpdxypMFnYsAzeRJbV9pump,0.113369409,0.00044595,SOL,So11111111111111111111111111111111111111112,raydium,4,Wiki-SOL,69fDvj22wF9DcebHvoD9wMqiFp6ueEGvKtxWcb9esHpm,12ZGFLbq86ryeEfuoeC6T49bZkTUai2EstDx1m81sfnk,5bSHtWqepdHhU14Krbn6wSgW1NX8TXXzfWngXs8R7qPb7S9gzXKFSrLGz4o5ZAbinGpzg5HPX9U2c34aFQ9FSPe9,41,6,1,true +2024-11-26 21:58:56.000 UTC,2024-11-26,2024-11-01,Bitfoot,solana,2.2846230000000003,Buy,63236.629215,AOC,2auaVtspPBW6biCkNw25VhAWmc39CurFSwtf8zRhpump,0.0099,SOL,So11111111111111111111111111111111111111112,0.014192355000000002,0.0000615,SOL,So11111111111111111111111111111111111111112,raydium,4,SOL-AOC,7w5f9fJd7274ADWAsMJ37qe7fy5d3BjNm19X74M4dn45,8RoZvSFwX3zKMHSyKVQ5BwqgGPhfxoeptkgHQy8gi2Ec,3ndjZVkEThqeY8jLZ8hwp9HG7DhQwWxghn4Mup6KSyGmBTaBvaCrwZS8HqdLtAarTN4LW4QsT9BWP2HitweMHeDy,426,9,2,true +2024-11-26 21:45:44.000 UTC,2024-11-26,2024-11-01,Bitfoot,solana,0.9725261359999999,Sell,0.0042236,SOL,So11111111111111111111111111111111111111112,204.969806,JAK,FBbnzHwJ1WHYwP425cqMNb2t7o7sm6AXQjsQ3sZRpump,0.00622438832,0.000027032,SOL,So11111111111111111111111111111111111111112,raydium,4,SOL-JAK,fYxohhysXCovowRnA184Bg5F8hb5EZUQeqFfUNH3SdB,79Ppegxb8QuNzma1KAfNKg26ZcNg1hpdkru7WKMnzZjn,2JqLMdq54vYFJhirQQZNQYSpC2GruQMxULaxcGairzhDBuSk4ES5Q7K1bWwPRr8xcNrHPwnQzNDBneVbkjJ1rYsF,1284,6,1,true +2024-11-23 14:49:56.000 UTC,2024-11-23,2024-11-01,Bitfoot,solana,129.83355,Buy,254676.289256,LOXE,Hdh5UdSUrCRCiqJoYKmgdJ25cubXqpxqkQgkXZdypump,0.495,SOL,So11111111111111111111111111111111111111112,0.8065417500000001,0.003075,SOL,So11111111111111111111111111111111111111112,raydium,4,SOL-LOXE,6dQVmJmwLZTKYwUKft6AHe3KqobkWFde8TbEZCt2dCbs,7zmFhq7LacJFGAH5PwVJrcnhczAAGTYCBa5g4Q5auFtL,z3PYDJuTgfBDbXyWMCaCeo1cWmN7ino5thDfGkZgfon1p2uD86QH3mnVdZJX5oHtGcpABqUt8hmCotW91BqX8Ef,1013,9,2,true +2024-11-25 01:26:39.000 UTC,2024-11-25,2024-11-01,Bitfoot,solana,58.51027910046,Sell,0.237585898,SOL,So11111111111111111111111111111111111111112,6336.104156,OVERSEER,3LP15DWcq1YmVw9gvnmjB1SJGmiScMm4CUX2dYf7HRV5,0.35983814931,0.001461153,SOL,So11111111111111111111111111111111111111112,raydium,4,SOL-OVERSEER,CVB7RXBYVaB6zyrW7edRaMwQo7CrPJ12QsZjgBC6muUB,92w6P8WBtUoA56iEiojd7bRQEtjWEuT5JEGEx24ysr93,4GKcxGm37qNJsg7YuSNZVfa52bCfYyQ6zBF2uFo6QasPCiUbL4gEjyB3osaFYkukhZnH4RZHDLQzgFFUkEMkkUW4,1641,6,1,true +2024-11-25 14:22:24.000 UTC,2024-11-25,2024-11-01,Bitfoot,solana,4.886838,Buy,49122.72145,BURN,7XcpJgx42Ax7oVtg1EbJTf31guAcgVnU7K4B2vG6pump,0.0198,SOL,So11111111111111111111111111111111111111112,0.03159168,0.000128,SOL,So11111111111111111111111111111111111111112,raydium,4,SOL-BURN,FeNMa3en5StkPJYXyv6yUDxbQXoVi3YDQf5whKtokntE,71aUVZrnjwrJSgGYQknJiNVEL3CRPiWNMYXjG7t5sJ9S,5W4m5SYypyyZ6ohaSrHxjkMo7JwzuJ29NCZt6yjbWCfKXzPL8gSbGsv5KuFW1g5q6iF4UEHJRighMfWhysPVsgr,416,9,2,true +2024-11-27 20:19:24.000 UTC,2024-11-27,2024-11-01,Bitfoot,solana,7.938256304599999,Sell,0.033172822,SOL,So11111111111111111111111111111111111111112,1551637.81624,Devika,524V8wH1E9a5uY5swiZMK2sLFRu9JMPTvctrnH17pump,0.07938251040000001,0.000331728,SOL,So11111111111111111111111111111111111111112,raydium,4,SOL-Devika,56v84bhXnpckT6uHzd5hbm2VvMG38oube2n369ro6oEi,HuMWfZnd74V2uv7RosiyK9pBVjzK9UMmrnjKb3Gg4ons,368qxfv512er4fXDCcRkSEjW9BsrWGR1TU6aHPqFUne2indtrHiRrZizn6yEb5QXyXdoforNoE7m2o5LeByew35Q,1228,6,1,true +2024-11-28 03:51:26.000 UTC,2024-11-28,2024-11-01,Bitfoot,solana,0.34270006716,Sell,0.001427679,SOL,So11111111111111111111111111111111111111112,6918.361444,Brooks,2i21MzKS78AJBUhRyXCYQZUhvLdtyC19XzKKBfGypump,0.00342681104,0.000014276,SOL,So11111111111111111111111111111111111111112,raydium,4,SOL-Brooks,D8UZSBfGnUzczobBu2bFUENyRoofNZeDPgSd1JAzoBdL,3jWX6wQQeVtT3yRhKw7n1pnpDZEFwNqfkuzmyFdcUpwV,4rmDWqxyP6cw9iatme7zoSq895Xjk86ffM9SU5NVZp5senxwWeEP6VVbLJHBZ1xaPFypjDmqw13FKt75oSWAGvrj,1314,6,1,true +2024-11-27 19:33:15.000 UTC,2024-11-27,2024-11-01,Bitfoot,solana,11.88,Buy,45555.61198,EFAI,8ZtMMAm42U27efFJ3nPaCvsJ6FGNgG3YJNtLJR93pump,0.0495,SOL,So11111111111111111111111111111111111111112,0.0798,0.0003325,SOL,So11111111111111111111111111111111111111112,raydium,4,SOL-EFAI,3eckL39THW9181eLT6Ry71Nuj34pfctbEs4CHQuouvYJ,Cu6zE46JYtpkaKDk5qKt95vvjJsMzn9byydB6zSbLUMS,DMSFEFYmKaY7BQjmp6qo16aoUaogSKZrFWbtJzYRgjDjDfK6fKerfyS7KQemLeSyzvAyaHgNcbnqAvvftMFUDE8,1971,9,2,true From f14bcb317a81340055ef1e010d32fa297bc08869 Mon Sep 17 00:00:00 2001 From: clizzard <145153355+clizzard7@users.noreply.github.com> Date: Thu, 28 Nov 2024 12:40:38 +0100 Subject: [PATCH 10/18] Update autosnipe_solana_bot_trades.sql (#7199) * Update autosnipe_solana_bot_trades.sql * fixed date * fix: move start date into future to reflect actual start date --------- Co-authored-by: whale_hunter <143016036+whalehunting@users.noreply.github.com> --- .../bot_trades/solana/platforms/autosnipe_solana_bot_trades.sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dbt_subprojects/solana/models/_sector/dex/bot_trades/solana/platforms/autosnipe_solana_bot_trades.sql b/dbt_subprojects/solana/models/_sector/dex/bot_trades/solana/platforms/autosnipe_solana_bot_trades.sql index 97d8d471ae9..d332abc1af8 100644 --- a/dbt_subprojects/solana/models/_sector/dex/bot_trades/solana/platforms/autosnipe_solana_bot_trades.sql +++ b/dbt_subprojects/solana/models/_sector/dex/bot_trades/solana/platforms/autosnipe_solana_bot_trades.sql @@ -10,7 +10,7 @@ ) }} -{% set project_start_date = '2024-11-13' %} +{% set project_start_date = '2024-05-22' %} {% set fee_receiver_1 = 'CWEfC6fLi552zE2KFxhPiBAZUWdT78gMd8NGENik2zfE' %} {% set wsol_token = 'So11111111111111111111111111111111111111112' %} From a9edac4d19695c410e2a4b7c166b174f8f73183b Mon Sep 17 00:00:00 2001 From: senyor-kodi <175722951+senyor-kodi@users.noreply.github.com> Date: Thu, 28 Nov 2024 12:41:04 +0100 Subject: [PATCH 11/18] add sanctum router to solana dex trades (#7188) * add sanctum router to solana dex trades * fix: cast project_program_id as varchar * fix: add time filter for last 14 days to prevent memory spilling issues * fix: fixed typos * fix: cast amount_raw cols to uint256 and drop project_main_id from final select in sanctum_router_base_trades.sql * fix: readded project_main_id col * reverted time filter to project_start_date --- .../_sector/dex/dex_solana_base_trades.sql | 1 + .../sanctum_router_base_trades.sql | 203 ++++++++++++++++++ .../sanctum_router/sanctum_router_trades.sql | 14 ++ .../_sector/dex/sanctum_router/schema.yml | 92 ++++++++ .../_sector/dex/sanctum_router/sources.yml | 19 ++ 5 files changed, 329 insertions(+) create mode 100644 dbt_subprojects/solana/models/_sector/dex/sanctum_router/sanctum_router_base_trades.sql create mode 100644 dbt_subprojects/solana/models/_sector/dex/sanctum_router/sanctum_router_trades.sql create mode 100644 dbt_subprojects/solana/models/_sector/dex/sanctum_router/schema.yml create mode 100644 dbt_subprojects/solana/models/_sector/dex/sanctum_router/sources.yml diff --git a/dbt_subprojects/solana/models/_sector/dex/dex_solana_base_trades.sql b/dbt_subprojects/solana/models/_sector/dex/dex_solana_base_trades.sql index 41a99ad235b..991be84e941 100644 --- a/dbt_subprojects/solana/models/_sector/dex/dex_solana_base_trades.sql +++ b/dbt_subprojects/solana/models/_sector/dex/dex_solana_base_trades.sql @@ -23,6 +23,7 @@ , ref('meteora_v2_solana_base_trades') , ref('goosefx_ssl_v2_solana_base_trades') , ref('pumpdotfun_solana_base_trades') + , ref('sanctum_router_base_trades') ] %} {% for dex in solana_dexes %} diff --git a/dbt_subprojects/solana/models/_sector/dex/sanctum_router/sanctum_router_base_trades.sql b/dbt_subprojects/solana/models/_sector/dex/sanctum_router/sanctum_router_base_trades.sql new file mode 100644 index 00000000000..9f73bb67fff --- /dev/null +++ b/dbt_subprojects/solana/models/_sector/dex/sanctum_router/sanctum_router_base_trades.sql @@ -0,0 +1,203 @@ +{{ + config( + schema = 'sanctum_router', + alias = 'base_trades', + partition_by = ['block_month'], + materialized = 'incremental', + file_format = 'delta', + incremental_strategy = 'merge', + incremental_predicates = [incremental_predicate('DBT_INTERNAL_DEST.block_time')], + unique_key = ['tx_id', 'outer_instruction_index', 'inner_instruction_index', 'tx_index','block_month'], + pre_hook='{{ enforce_join_distribution("PARTITIONED") }}' + ) +}} + +{% set project_start_date = '2023-02-03' %} +{% set dev_start_date = "date_trunc('day', now() - interval '7' day)" %} + +WITH swap_via_stake AS ( + SELECT + call_block_time as block_time + , call_block_slot as block_slot + , call_tx_signer as trader_id + , call_tx_id as tx_id + , call_outer_instruction_index as outer_instruction_index + , COALESCE(call_inner_instruction_index, 0) as inner_instruction_index + , call_tx_index as tx_index + , account_srcTokenFrom as token_sold_vault + , account_destTokenTo as token_bought_vault + , account_srcTokenMint as token_sold_mint_address + , account_destTokenMint as token_bought_mint_address + , bytearray_to_bigint(bytearray_reverse(bytearray_substring(call_data, 2, 8))) AS token_sold_amount_raw + , CASE + WHEN call_outer_executing_account = 'stkitrT1Uoy18Dk1fTrgPw8W6MVzoCfYoAFT4MLsmhq' THEN 'direct' + ELSE call_outer_executing_account + END as trade_source + , 'mintTo' as amount_type + FROM {{ source('sanctum_router_solana', 'stakedex_call_SwapViaStake') }} + WHERE 1=1 + {% if is_incremental() %} + AND {{incremental_predicate('call_block_time')}} + {% else %} + AND call_block_time >= TIMESTAMP '{{project_start_date}}' + {% endif %} +), + +prefund_swap_via_stake AS ( + SELECT + call_block_time as block_time + , call_block_slot as block_slot + , call_tx_signer as trader_id + , call_tx_id as tx_id + , call_outer_instruction_index as outer_instruction_index + , COALESCE(call_inner_instruction_index, 0) as inner_instruction_index + , call_tx_index as tx_index + , account_srcTokenFrom as token_sold_vault + , account_destTokenTo as token_bought_vault + , account_srcTokenMint as token_sold_mint_address + , account_destTokenMint as token_bought_mint_address + , bytearray_to_bigint(bytearray_reverse(bytearray_substring(call_data, 2, 8))) AS token_sold_amount_raw + , CASE + WHEN call_outer_executing_account = 'stkitrT1Uoy18Dk1fTrgPw8W6MVzoCfYoAFT4MLsmhq' THEN 'direct' + ELSE call_outer_executing_account + END as trade_source + , 'transferChecked' as amount_type + FROM {{ source('sanctum_router_solana', 'stakedex_call_PrefundSwapViaStake') }} + WHERE 1=1 + {% if is_incremental() %} + AND {{incremental_predicate('call_block_time')}} + {% else %} + AND call_block_time >= TIMESTAMP '{{project_start_date}}' + {% endif %} +), + +stake_wrapped_sol AS ( + SELECT + call_block_time as block_time + , call_block_slot as block_slot + , call_tx_signer as trader_id + , call_tx_id as tx_id + , call_outer_instruction_index as outer_instruction_index + , COALESCE(call_inner_instruction_index, 0) as inner_instruction_index + , call_tx_index as tx_index + , account_wsolFrom as token_sold_vault + , account_destTokenTo as token_bought_vault + , account_wsolMint as token_sold_mint_address + , account_destTokenMint as token_bought_mint_address + , bytearray_to_bigint(bytearray_reverse(bytearray_substring(call_data, 2, 8))) AS token_sold_amount_raw + , CASE + WHEN call_outer_executing_account = 'stkitrT1Uoy18Dk1fTrgPw8W6MVzoCfYoAFT4MLsmhq' THEN 'direct' + ELSE call_outer_executing_account + END as trade_source + , 'transferChecked' as amount_type + FROM {{ source('sanctum_router_solana', 'stakedex_call_StakeWrappedSol') }} + WHERE 1=1 + {% if is_incremental() %} + AND {{incremental_predicate('call_block_time')}} + {% else %} + AND call_block_time >= TIMESTAMP '{{project_start_date}}' + {% endif %} +), + +withdraw_deposit AS ( + SELECT + w.call_block_time as block_time + , w.call_block_slot as block_slot + , w.call_tx_signer as trader_id + , w.call_tx_id as tx_id + , w.call_outer_instruction_index as outer_instruction_index + , COALESCE(w.call_inner_instruction_index, 0) as inner_instruction_index + , w.call_tx_index as tx_index + , w.account_srcTokenFrom as token_sold_vault + , d.account_destTokenTo as token_bought_vault + , w.account_srcTokenMint as token_sold_mint_address + , d.account_destTokenMint as token_bought_mint_address + , bytearray_to_bigint(bytearray_reverse(bytearray_substring(w.call_data, 2, 8))) AS token_sold_amount_raw + , CASE + WHEN w.call_outer_executing_account = 'stkitrT1Uoy18Dk1fTrgPw8W6MVzoCfYoAFT4MLsmhq' THEN 'direct' + ELSE w.call_outer_executing_account + END as trade_source + , 'transferChecked' as amount_type + FROM {{ source('sanctum_router_solana', 'stakedex_call_PrefundWithdrawStake') }} w + INNER JOIN {{ source('sanctum_router_solana', 'stakedex_call_DepositStake') }} d + ON w.call_tx_id = d.call_tx_id + AND w.call_outer_instruction_index = d.call_outer_instruction_index + AND w.call_inner_instruction_index < d.call_inner_instruction_index + WHERE 1=1 + {% if is_incremental() %} + AND {{incremental_predicate('w.call_block_time')}} + {% else %} + AND w.call_block_time >= TIMESTAMP '{{project_start_date}}' + {% endif %} +), + +all_trades AS ( + SELECT * FROM swap_via_stake + UNION ALL + SELECT * FROM prefund_swap_via_stake + UNION ALL + SELECT * FROM stake_wrapped_sol + UNION ALL + SELECT * FROM withdraw_deposit +), + +token_amounts AS ( + SELECT + ic.tx_id, + ic.outer_instruction_index, + ic.inner_instruction_index, + bytearray_to_bigint(bytearray_reverse(bytearray_substring(ic.data, 2, 8))) AS amount_bought, + ROW_NUMBER() OVER ( + PARTITION BY ic.tx_id, ic.outer_instruction_index + ORDER BY + CASE + WHEN b.amount_type = 'mintTo' THEN ic.inner_instruction_index + ELSE -ic.inner_instruction_index + END + ) as rn + FROM all_trades b + INNER JOIN {{ source('solana','instruction_calls') }} ic + ON ic.tx_id = b.tx_id + AND ic.outer_instruction_index = b.outer_instruction_index + AND ic.block_slot = b.block_slot + WHERE 1=1 + AND ic.executing_account = 'TokenkegQfeZyiNwAJbNbGKPFXCWuBvf9Ss623VQ5DA' + AND ( + (b.amount_type = 'mintTo' AND bytearray_substring(ic.data, 1, 1) = 0x07 AND ELEMENT_AT(ic.account_arguments, 1) = b.token_bought_mint_address) + OR + (b.amount_type = 'transferChecked' AND bytearray_substring(ic.data, 1, 1) = 0x0c AND ELEMENT_AT(ic.account_arguments, 2) = b.token_bought_mint_address AND ELEMENT_AT(ic.account_arguments, 3) = b.token_bought_vault) + ) + {% if is_incremental() %} + AND {{incremental_predicate('ic.block_time')}} + {% else %} + AND ic.block_time >= TIMESTAMP '{{project_start_date}}' + {% endif %} +) + +SELECT + 'solana' as blockchain + , 'sanctum_router' as project + , 1 as version + , CAST(date_trunc('month', b.block_time) AS DATE) as block_month + , b.block_time + , b.block_slot + , b.trade_source + , CAST(t.amount_bought as uint256) as token_bought_amount_raw + , CAST(b.token_sold_amount_raw as uint256) as token_sold_amount_raw + , CAST(NULL as double) as fee_tier + , b.token_bought_mint_address + , b.token_sold_mint_address + , b.token_bought_vault + , b.token_sold_vault + , CAST(NULL as varchar) as project_program_id + , 'stkitrT1Uoy18Dk1fTrgPw8W6MVzoCfYoAFT4MLsmhq' as project_main_id + , b.trader_id + , b.tx_id + , CAST(b.outer_instruction_index as integer) as outer_instruction_index + , CAST(b.inner_instruction_index as integer) as inner_instruction_index + , b.tx_index +FROM all_trades b +INNER JOIN token_amounts t + ON b.tx_id = t.tx_id + AND b.outer_instruction_index = t.outer_instruction_index + AND t.rn = 1 diff --git a/dbt_subprojects/solana/models/_sector/dex/sanctum_router/sanctum_router_trades.sql b/dbt_subprojects/solana/models/_sector/dex/sanctum_router/sanctum_router_trades.sql new file mode 100644 index 00000000000..9580d25e6c4 --- /dev/null +++ b/dbt_subprojects/solana/models/_sector/dex/sanctum_router/sanctum_router_trades.sql @@ -0,0 +1,14 @@ + {{ + config( + schema = 'sanctum_router', + alias = 'trades', + post_hook='{{ expose_spells(\'["solana"]\', + "project", + "sanctum_router", + \'["senyor-kodi"]\') }}' + ) +}} + +-- backwards compatible view so we don't break any user queries +select * from {{ref('dex_solana_trades')}} +where project = 'sanctum_router' and version = 1 \ No newline at end of file diff --git a/dbt_subprojects/solana/models/_sector/dex/sanctum_router/schema.yml b/dbt_subprojects/solana/models/_sector/dex/sanctum_router/schema.yml new file mode 100644 index 00000000000..65523f108c0 --- /dev/null +++ b/dbt_subprojects/solana/models/_sector/dex/sanctum_router/schema.yml @@ -0,0 +1,92 @@ +version: 2 + +models: + - name: sanctum_router_base_trades + meta: + blockchain: solana + sector: dex + project: sanctum + contributors: [ senyor-kodi ] + config: + tags: ['solana','dex'] + description: > + Sanctum Router trades on Solana + data_tests: + - check_columns_solana_dex_trades + - dbt_utils.unique_combination_of_columns: + combination_of_columns: [ 'tx_id', 'outer_instruction_index', 'inner_instruction_index', 'tx_index', 'block_month' ] + + - name: sanctum_router_trades + meta: + blockchain: solana + sector: dex + project: sanctum + contributors: [ senyor-kodi ] + config: + tags: ['solana','dex'] + description: > + Sanctum Router trades on Solana + columns: + - &blockchain + name: blockchain + description: "Blockchain which the DEX is deployed" + - &project + name: project + description: "Project name of the DEX" + - &version + name: version + description: "Version of the contract built and deployed by the DEX project" + - &block_month + name: block_month + description: "UTC event block month of each DEX trade" + - &block_time + name: block_time + description: "UTC event block time of each DEX trade" + - &block_slot + name: block_slot + description: "block slot of each DEX trade" + - &trade_source + name: trade_source + description: "Was the trade a direct call to the dexor did it go through another program like Jupiter (Dex Aggregator)" + - &token_bought_amount_raw + name: token_bought_amount_raw + description: "Raw value of the token bought at time of execution in the original currency" + - &token_sold_amount_raw + name: token_sold_amount_raw + description: "Raw value of the token sold at time of execution in the original currency" + - &fee_tier + name: fee_tier + description: "dexfee tier (fee %)" + - &token_bought_mint_address + name: token_bought_address + description: "token mint address of the token bought" + - &token_sold_mint_address + name: token_sold_address + description: "token mint address of the token sold" + - &token_bought_vault + name: token_bought_vault + description: "token associated address for thedex, of the token bought" + - &token_sold_vault + name: token_sold_vault + description: "token associated address for thedex, of the token sold" + - &project_program_id + name: project_program_id + description: "pool program id of the project" + - &project_main_id + name: project_main_id + description: "main program id of the project" + - &trader_id + name: trader_id + description: "id (address) of trader who purchased a token" + - &tx_id + name: tx_id + description: "Unique transaction id value tied to each transaction on the DEX" + - &outer_instruction_index + name: outer_instruction_index + description: "top level instruction index for a given transaction id" + - &inner_instruction_index + name: inner_instruction_index + description: "inner instruction index for a given transaction id" + - &tx_index + name: tx_index + description: "index of the transaction in the block slot" \ No newline at end of file diff --git a/dbt_subprojects/solana/models/_sector/dex/sanctum_router/sources.yml b/dbt_subprojects/solana/models/_sector/dex/sanctum_router/sources.yml new file mode 100644 index 00000000000..9814e5d2ceb --- /dev/null +++ b/dbt_subprojects/solana/models/_sector/dex/sanctum_router/sources.yml @@ -0,0 +1,19 @@ +version: 2 + +sources: + - name: sanctum_router_solana + description: "Sanctum Router decoded contract calls on Solana" + freshness: # default freshness + warn_after: { count: 12, period: hour } + error_after: { count: 24, period: hour } + tables: + - name: stakedex_call_SwapViaStake + loaded_at_field: call_block_time + - name: stakedex_call_PrefundSwapViaStake + loaded_at_field: call_block_time + - name: stakedex_call_StakeWrappedSol + loaded_at_field: call_block_time + - name: stakedex_call_PrefundWithdrawStake + loaded_at_field: call_block_time + - name: stakedex_call_DepositStake + loaded_at_field: call_block_time From bcb133e1083a668881fc8e3fc634ab9e6687be5e Mon Sep 17 00:00:00 2001 From: whale_hunter <143016036+whalehunting@users.noreply.github.com> Date: Thu, 28 Nov 2024 12:52:13 +0100 Subject: [PATCH 12/18] feat: add banana gun evm bot trades spellbook (#7180) * init * fix: rename tests to data_tests * chore: disable test for now * chore: add base spellbook, remove outcommented test * feat: add blast spellbook * fix: pick tx_hash column from botTrades table * chore: add bot column, change project_start_date * chore: add real start date for blast * chore: move start date to original value * chore: remove miner bribes from spellbook schema * chore: remove blast spellbook, since it has no data yet * fix: add incremental filters * test: add check_bot_trades_seed for dex subproject * fix: remove leading AND in where condition * test: add base seed * fix: cast value column to decimal * test: add correct seed files * fix: force version column to get parsed as string in seed * Revert "fix: force version column to get parsed as string in seed" This reverts commit 7e6e570893d5ccc200bdcc5f2876452c782b8cff. * fix: force seed column as varchar * fix: fix typo * fix: do not cast fee_token_address to varchar * add type info of varbinary column to seed schema * make hash varbinary as well * fix: remove blockchain filter from seed test * Update dbt_subprojects/dex/models/bot_trades/banana_gun/base/banana_gun_base_bot_trades.sql Co-authored-by: 0xRob <83790096+0xRobin@users.noreply.github.com> * perf: add incremental filter to ethereum --------- Co-authored-by: 0xRob <83790096+0xRobin@users.noreply.github.com> --- .../dex/models/bot_trades/_schema.yml | 132 ++++++++++++ .../base/banana_gun_base_bot_trades.sql | 196 +++++++++++++++++ .../banana_gun_ethereum_bot_trades.sql | 198 ++++++++++++++++++ .../models/bot_trades/dex_evm_bot_trades.sql | 57 +++++ .../banana_gun_base_trades_seed.csv | 21 ++ .../banana_gun_ethereum_trades_seed.csv | 21 ++ .../seeds/bot_trades/banana_gun/schema.yml | 24 +++ .../tests/generic/check_bot_trades_seed.sql | 15 ++ 8 files changed, 664 insertions(+) create mode 100644 dbt_subprojects/dex/models/bot_trades/_schema.yml create mode 100644 dbt_subprojects/dex/models/bot_trades/banana_gun/base/banana_gun_base_bot_trades.sql create mode 100644 dbt_subprojects/dex/models/bot_trades/banana_gun/ethereum/banana_gun_ethereum_bot_trades.sql create mode 100644 dbt_subprojects/dex/models/bot_trades/dex_evm_bot_trades.sql create mode 100644 dbt_subprojects/dex/seeds/bot_trades/banana_gun/banana_gun_base_trades_seed.csv create mode 100644 dbt_subprojects/dex/seeds/bot_trades/banana_gun/banana_gun_ethereum_trades_seed.csv create mode 100644 dbt_subprojects/dex/seeds/bot_trades/banana_gun/schema.yml create mode 100644 dbt_subprojects/dex/tests/generic/check_bot_trades_seed.sql diff --git a/dbt_subprojects/dex/models/bot_trades/_schema.yml b/dbt_subprojects/dex/models/bot_trades/_schema.yml new file mode 100644 index 00000000000..2cafe84746b --- /dev/null +++ b/dbt_subprojects/dex/models/bot_trades/_schema.yml @@ -0,0 +1,132 @@ +version: 2 + +models: + - name: dex_evm_bot_trades + meta: + blockchain: ethereum, base, bnb, avalanche_c + sector: dex + contributors: ["whale_hunter"] + config: + tags: ["evm", "dex", "bot", "trades"] + description: > + Trades by trading bots on EVM chains + columns: + - &blockchain + name: blockchain + description: "Blockchain which the DEX is deployed" + - &block_number + name: block_number + description: "Block number that includes the DEX trade" + - &block_time + name: block_time + description: "UTC event block time of each DEX trade" + - &block_date + name: block_date + description: "UTC event block date of each DEX trade" + - &block_month + name: block_month + description: "UTC event block month of each DEX trade" + - &bot + name: bot + description: "Trading bot which executed the trade" + - &amount_usd + name: amount_usd + description: "USD value of the trade at time of execution" + - &type + name: type + description: "Wether the trade is a buy or sell" + - &token_bought_amount + name: token_bought_amount + description: "Value of the token bought at time of execution in the original currency" + - &token_bought_symbol + name: token_bought_symbol + description: "Token symbol for token bought in the trade" + - &token_bought_address + name: token_bought_address + description: "Contract address of the token bought" + - &token_sold_amount + name: token_sold_amount + description: "Value of the token sold at time of execution in the original currency" + - &token_sold_symbol + name: token_sold_symbol + description: "Token symbol for token sold in the trade" + - &token_sold_address + name: token_sold_address + description: "Contract address of the token sold" + - &fee_percentage_fraction + name: fee_percentage_fraction + description: "The trading fee in percent, as a fraction of 100" + - &fee_usd + name: fee_usd + description: "USD value of the fee at time of execution" + - &fee_token_amount + name: fee_token_amount + description: "Value of the fee paid at time of execution in the original currency" + - &fee_token_symbol + name: fee_token_symbol + description: "Token symbol for fee token" + - &fee_token_address + name: fee_token_address + description: "Contract address of the fee token" + - &project + name: project + description: "Project name of the DEX" + - &version + name: version + description: "Version of the contract built and deployed by the DEX project" + - &token_pair + name: token_pair + description: "Token symbol pair for each token involved in the trade" + - &project_contract_address + name: project_contract_address + description: "Project contract address which executed the trade on the blockchain" + - &user + name: user + description: "Address which initiated the trade" + - &tx_hash + name: tx_hash + description: "Unique transaction hash value tied to each transaction on the DEX" + - &evt_index + name: evt_index + description: "Index of the corresponding trade event" + - &is_last_trade_in_transaction + name: is_last_trade_in_transaction + description: "Wether the trade is the last hop of the trade transaction, in case of a multi-hop trade" + + - name: banana_gun_ethereum_bot_trades + meta: + blockchain: ethereum + sector: dex + project: banana_gun + contributors: whale_hunter + config: + tags: ["evm", "dex", "banana_gun", "trades"] + description: > + Banana Gun trades on Ethereum + data_tests: + - dbt_utils.unique_combination_of_columns: + combination_of_columns: + - blockchain + - tx_hash + - evt_index + - check_bot_trades_seed: + seed_file: ref('banana_gun_ethereum_trades_seed') + + - name: banana_gun_base_bot_trades + meta: + blockchain: base + sector: dex + project: banana_gun + contributors: whale_hunter + config: + tags: ["evm", "dex", "banana_gun", "trades"] + description: > + Banana Gun trades on Base + data_tests: + - dbt_utils.unique_combination_of_columns: + combination_of_columns: + - blockchain + - tx_hash + - evt_index + - check_bot_trades_seed: + seed_file: ref('banana_gun_base_trades_seed') diff --git a/dbt_subprojects/dex/models/bot_trades/banana_gun/base/banana_gun_base_bot_trades.sql b/dbt_subprojects/dex/models/bot_trades/banana_gun/base/banana_gun_base_bot_trades.sql new file mode 100644 index 00000000000..6126ae66844 --- /dev/null +++ b/dbt_subprojects/dex/models/bot_trades/banana_gun/base/banana_gun_base_bot_trades.sql @@ -0,0 +1,196 @@ +{{ config( + alias = 'bot_trades', + schema = 'banana_gun_base', + partition_by = ['block_month'], + materialized = 'incremental', + file_format = 'delta', + incremental_strategy = 'merge', + incremental_predicates = [incremental_predicate('DBT_INTERNAL_DEST.block_time')], + unique_key = ['blockchain', 'tx_hash', 'evt_index'] + ) +}} + +{% set project_name = 'Banana Gun' %} +{% set project_start_date = '2024-03-05' %} +{% set blockchain = 'base' %} +{% set bot_deployer_1 = '0x37aAb97476bA8dC785476611006fD5dDA4eed66B' %} +{% set weth = '0x4200000000000000000000000000000000000006' %} +{% set fee_token_symbol = 'ETH' %} + +WITH + botContracts AS ( + SELECT + address + FROM + {{ source('base','creation_traces') }} + WHERE + ( + "from" = {{bot_deployer_1}} + ) + AND block_time >= TIMESTAMP '{{project_start_date}}' + ), + botTrades AS ( + SELECT + trades.block_time, + amount_usd, + IF( + token_sold_address = {{weth}}, + 'Buy', + 'Sell' + ) AS type, + token_bought_amount, + token_bought_symbol, + token_bought_address, + token_sold_amount, + token_sold_symbol, + token_sold_address, + project, + version, + token_pair, + project_contract_address, + tx_from AS user, + tx_to AS bot, + trades.tx_hash, + evt_index + FROM + {{ source('dex', 'trades') }} as trades + JOIN botContracts ON trades.tx_to = botContracts.address + WHERE + trades.blockchain = '{{blockchain}}' + {% if is_incremental() %} + AND {{ incremental_predicate('trades.block_time') }} + {% else %} + AND trades.block_time >= TIMESTAMP '{{project_start_date}}' + {% endif %} + ), + highestEventIndexForEachTrade AS ( + SELECT + tx_hash, + MAX(evt_index) AS highestEventIndex + FROM + botTrades + GROUP BY + tx_hash + ), + botETHDeposits AS ( + SELECT + tx_hash, + block_number, + CAST(value AS DECIMAL (38, 0)) AS deltaGwei, + CAST(value AS DECIMAL (38, 0)) AS depositGwei + FROM + {{ source('base','traces') }} + JOIN botContracts ON to = botContracts.address + WHERE + {% if is_incremental() %} + {{ incremental_predicate('block_time') }} + {% else %} + block_time >= TIMESTAMP '{{project_start_date}}' + {% endif %} + AND value > CAST(0 AS UINT256) + ), + botETHWithdrawals AS ( + SELECT + tx_hash, + block_number, + CAST(value AS DECIMAL (38, 0)) * -1 AS deltaGwei, + 0 AS depositGwei, + block_hash, + to + FROM + {{ source('base','traces') }} + JOIN botContracts ON "from" = botContracts.address + WHERE + {% if is_incremental() %} + {{ incremental_predicate('block_time') }} + {% else %} + block_time >= TIMESTAMP '{{project_start_date}}' + {% endif %} + AND value > CAST(0 AS UINT256) + ), + botEthTransfers AS ( + /* Deposits */ + ( + SELECT + tx_hash, + block_number, + deltaGwei, + depositGwei + FROM + botETHDeposits + ) + UNION ALL + /* Withdrawals */ + ( + SELECT + tx_hash, + block_number, + deltaGwei, + depositGwei + FROM + botETHWithdrawals + ) + ), + botEthDeltas AS ( + SELECT + tx_hash, + block_number, + SUM(deltaGwei) AS feeGwei, + SUM(depositGwei) AS depositGwei + FROM + botEthTransfers + GROUP BY + tx_hash, + block_number + ) +SELECT + block_time, + date_trunc('day', botTrades.block_time) as block_date, + date_trunc('month', botTrades.block_time) as block_month, + '{{project_name}}' as bot, + block_number, + '{{blockchain}}' AS blockchain, + -- Trade + amount_usd, + type, + token_bought_amount, + token_bought_symbol, + token_bought_address, + token_sold_amount, + token_sold_symbol, + token_sold_address, + -- Fees + ROUND( + CAST(feeGwei AS DOUBLE) / CAST(depositGwei AS DOUBLE), + /* Round feePercentage to 0.01% steps */ + 4 + ) AS fee_percentage_fraction, + (feeGwei / 1e18) * price AS fee_usd, + feeGwei / 1e18 fee_token_amount, + '{{fee_token_symbol}}' AS fee_token_symbol, + {{weth}} AS fee_token_address, + -- Dex + project, + version, + token_pair, + project_contract_address, + -- User + user, + botTrades.tx_hash, + evt_index, + IF(evt_index = highestEventIndex, true, false) AS is_last_trade_in_transaction +FROM + botTrades + JOIN highestEventIndexForEachTrade ON botTrades.tx_hash = highestEventIndexForEachTrade.tx_hash + LEFT JOIN botETHDeltas ON botTrades.tx_hash = botETHDeltas.tx_hash + LEFT JOIN {{ source('prices', 'usd') }} ON ( + blockchain = '{{blockchain}}' + AND contract_address = {{weth}} + AND minute = DATE_TRUNC('minute', block_time) + {% if is_incremental() %} + AND {{ incremental_predicate('minute') }} + {% endif %} + ) +ORDER BY + block_time DESC, + evt_index DESC \ No newline at end of file diff --git a/dbt_subprojects/dex/models/bot_trades/banana_gun/ethereum/banana_gun_ethereum_bot_trades.sql b/dbt_subprojects/dex/models/bot_trades/banana_gun/ethereum/banana_gun_ethereum_bot_trades.sql new file mode 100644 index 00000000000..51b2a6ecf64 --- /dev/null +++ b/dbt_subprojects/dex/models/bot_trades/banana_gun/ethereum/banana_gun_ethereum_bot_trades.sql @@ -0,0 +1,198 @@ +{{ config( + alias = 'bot_trades', + schema = 'banana_gun_ethereum', + partition_by = ['block_month'], + materialized = 'incremental', + file_format = 'delta', + incremental_strategy = 'merge', + incremental_predicates = [incremental_predicate('DBT_INTERNAL_DEST.block_time')], + unique_key = ['blockchain', 'tx_hash', 'evt_index'] + ) +}} + +{% set project_name = 'Banana Gun' %} +{% set project_start_date = '2023-05-26' %} +{% set blockchain = 'ethereum' %} +{% set bot_deployer_1 = '0xf414d478934c29d9a80244a3626c681a71e53bb2' %} +{% set bot_deployer_2 = '0x37aAb97476bA8dC785476611006fD5dDA4eed66B' %} +{% set weth = '0xc02aaa39b223fe8d0a0e5c4f27ead9083c756cc2' %} +{% set fee_token_symbol = 'ETH' %} + +WITH + botContracts AS ( + SELECT + address + FROM + {{ source('ethereum','creation_traces') }} + WHERE + ( + "from" = {{bot_deployer_1}} + OR "from" = {{bot_deployer_2}} + ) + AND block_time >= TIMESTAMP '{{project_start_date}}' + ), + botTrades AS ( + SELECT + trades.block_time, + amount_usd, + IF( + token_sold_address = {{weth}}, + 'Buy', + 'Sell' + ) AS type, + token_bought_amount, + token_bought_symbol, + token_bought_address, + token_sold_amount, + token_sold_symbol, + token_sold_address, + project, + version, + token_pair, + project_contract_address, + tx_from AS user, + tx_to AS bot, + trades.tx_hash, + evt_index + FROM + {{ source('dex', 'trades') }} as trades + JOIN botContracts ON trades.tx_to = botContracts.address + WHERE + trades.blockchain = '{{blockchain}}' + {% if is_incremental() %} + AND {{ incremental_predicate('trades.block_time') }} + {% else %} + AND trades.block_time >= TIMESTAMP '{{project_start_date}}' + {% endif %} + ), + highestEventIndexForEachTrade AS ( + SELECT + tx_hash, + MAX(evt_index) AS highestEventIndex + FROM + botTrades + GROUP BY + tx_hash + ), + botETHDeposits AS ( + SELECT + tx_hash, + block_number, + CAST(value AS DECIMAL (38, 0)) AS deltaGwei, + CAST(value AS DECIMAL (38, 0)) AS depositGwei + FROM + {{ source('ethereum','traces') }} + JOIN botContracts ON to = botContracts.address + WHERE + {% if is_incremental() %} + {{ incremental_predicate('block_time') }} + {% else %} + block_time >= TIMESTAMP '{{project_start_date}}' + {% endif %} + AND value > 0 + ), + botETHWithdrawals AS ( + SELECT + tx_hash, + block_number, + CAST(value AS DECIMAL (38, 0)) * -1 AS deltaGwei, + 0 AS depositGwei, + block_hash, + to + FROM + {{ source('ethereum','traces') }} + JOIN botContracts ON "from" = botContracts.address + WHERE + {% if is_incremental() %} + {{ incremental_predicate('block_time') }} + {% else %} + block_time >= TIMESTAMP '{{project_start_date}}' + {% endif %} + AND value > 0 + ), + botEthTransfers AS ( + /* Deposits */ + ( + SELECT + tx_hash, + block_number, + deltaGwei, + depositGwei + FROM + botETHDeposits + ) + UNION ALL + /* Withdrawals */ + ( + SELECT + tx_hash, + block_number, + deltaGwei, + depositGwei + FROM + botETHWithdrawals + ) + ), + botEthDeltas AS ( + SELECT + tx_hash, + block_number, + SUM(deltaGwei) AS feeGwei, + SUM(depositGwei) AS depositGwei + FROM + botEthTransfers + GROUP BY + tx_hash, + block_number + ) +SELECT + block_time, + date_trunc('day', botTrades.block_time) as block_date, + date_trunc('month', botTrades.block_time) as block_month, + '{{project_name}}' as bot, + block_number, + '{{blockchain}}' AS blockchain, + -- Trade + amount_usd, + type, + token_bought_amount, + token_bought_symbol, + token_bought_address, + token_sold_amount, + token_sold_symbol, + token_sold_address, + -- Fees + ROUND( + CAST(feeGwei AS DOUBLE) / CAST(depositGwei AS DOUBLE), + /* Round feePercentage to 0.01% steps */ + 4 + ) AS fee_percentage_fraction, + (feeGwei / 1e18) * price AS fee_usd, + feeGwei / 1e18 fee_token_amount, + '{{fee_token_symbol}}' AS fee_token_symbol, + {{weth}} AS fee_token_address, + -- Dex + project, + version, + token_pair, + project_contract_address, + -- User + user AS user, + botTrades.tx_hash, + evt_index, + IF(evt_index = highestEventIndex, true, false) AS is_last_trade_in_transaction +FROM + botTrades + JOIN highestEventIndexForEachTrade ON botTrades.tx_hash = highestEventIndexForEachTrade.tx_hash + LEFT JOIN botETHDeltas ON botTrades.tx_hash = botETHDeltas.tx_hash + LEFT JOIN {{ source('prices', 'usd') }} ON ( + blockchain = '{{blockchain}}' + AND contract_address = {{weth}} + AND minute = DATE_TRUNC('minute', block_time) + {% if is_incremental() %} + AND {{ incremental_predicate('minute') }} + {% endif %} + ) +ORDER BY + block_time DESC, + evt_index DESC \ No newline at end of file diff --git a/dbt_subprojects/dex/models/bot_trades/dex_evm_bot_trades.sql b/dbt_subprojects/dex/models/bot_trades/dex_evm_bot_trades.sql new file mode 100644 index 00000000000..c339d2e2b55 --- /dev/null +++ b/dbt_subprojects/dex/models/bot_trades/dex_evm_bot_trades.sql @@ -0,0 +1,57 @@ +{{ + config( + schema = 'dex_evm', + alias = 'bot_trades', + materialized = 'view', + post_hook = '{{ expose_spells( + blockchains = \'["ethereum", "base", "blast", "arbitrum", "bnb", "avalanche_c"]\', + spell_type = "sector", + spell_name = "bot_trades", + contributors = \'["whale_hunter"]\') }}' + ) +}} + + + +{% set evm_trading_bots = [ + ref('banana_gun_ethereum_bot_trades') + ,ref('banana_gun_base_bot_trades') +] %} + +{% for bot in evm_trading_bots %} +SELECT block_time, + block_date, + block_month, + bot, + block_number, + blockchain, + -- Trade + amount_usd, + type, + token_bought_amount, + token_bought_symbol, + token_bought_address, + token_sold_amount, + token_sold_symbol, + token_sold_address, + -- Fees + fee_percentage_fraction, + fee_usd, + fee_token_amount, + fee_token_symbol, + fee_token_address, + -- Dex + project, + version, + token_pair, + project_contract_address, + -- User + user, + tx_hash, + evt_index, + is_last_trade_in_transaction +FROM {{ bot }} +{% if not loop.last %} +UNION ALL +{% endif %} +{% endfor %} \ No newline at end of file diff --git a/dbt_subprojects/dex/seeds/bot_trades/banana_gun/banana_gun_base_trades_seed.csv b/dbt_subprojects/dex/seeds/bot_trades/banana_gun/banana_gun_base_trades_seed.csv new file mode 100644 index 00000000000..8f565dddb42 --- /dev/null +++ b/dbt_subprojects/dex/seeds/bot_trades/banana_gun/banana_gun_base_trades_seed.csv @@ -0,0 +1,21 @@ +block_time,block_date,block_month,bot,block_number,blockchain,amount_usd,type,token_bought_amount,token_bought_symbol,token_bought_address,token_sold_amount,token_sold_symbol,token_sold_address,fee_percentage_fraction,fee_usd,fee_token_amount,fee_token_symbol,fee_token_address,project,version,token_pair,project_contract_address,user,tx_hash,evt_index,is_last_trade_in_transaction +2024-09-07 08:46:55.000 UTC,2024-09-07 00:00:00.000 UTC,2024-09-01 00:00:00.000 UTC,Banana Gun,19454734,base,3441.57,Sell,1.5,WETH,0x4200000000000000000000000000000000000006,206629.39352997296,MIGGLES,0xb1a03eda10342529bbf8eb700a06c60441fef25d,0.005,34.4157,0.015,ETH,0x4200000000000000000000000000000000000006,uniswap,2,MIGGLES-WETH,0x17a3ad8c74c4947005afeda9965305ae2eb2518a,0x40237a6d80e9a4d9cd8341c5ad42ce3733b822c1,0xda8352e2a41cc3d5793e2211816d7c234ddad30db2b97860f4b6c26be6d5aa67,7,true +2024-09-01 14:10:41.000 UTC,2024-09-01 00:00:00.000 UTC,2024-09-01 00:00:00.000 UTC,Banana Gun,19205247,base,48.757227722772264,Buy,836176.1213013215,DarkPepePope,0xdc3793eab4681e57b2755783403b3dcc573f220b,0.0198019801980198,WETH,0x4200000000000000000000000000000000000006,0.005,0.48757227722772517,0.000198019801980199,ETH,0x4200000000000000000000000000000000000006,uniswap,2,DarkPepePope-WETH,0x67a45158523835d8477a1877c756c7b8c252d722,0x117c921e599ee1c4f73fe74b08ca584db08475e4,0x25c17e72dc3ede0eec14d4e34b198a7b6854904e8bd862af99814d28e4566204,12,true +2024-09-01 14:10:41.000 UTC,2024-09-01 00:00:00.000 UTC,2024-09-01 00:00:00.000 UTC,Banana Gun,19205247,base,731.358415841584,Buy,538747.542958548,MUNCHER,0xdcce6d3d2f6fe2e9201de68422f7568610b39860,0.297029702970297,WETH,0x4200000000000000000000000000000000000006,0.005,7.313584158415843,0.002970297029702971,ETH,0x4200000000000000000000000000000000000006,uniswap,2,MUNCHER-WETH,0x3be1dd1c8f859fc49f1da72b1d1328b6595a297d,0x56714990cef7019a7ee8a5bc0e8e6ad991aa76e5,0x0ee0458c7dd96ee1125196cbdb414f31de5c2ab6561ec05be72c791359264e3a,5,true +2024-09-10 09:14:03.000 UTC,2024-09-10 00:00:00.000 UTC,2024-09-01 00:00:00.000 UTC,Banana Gun,19585148,base,349.26831683168314,Buy,9105325812.505402,D.O.G.E,0x755b004d7d5dc8fdc92c39333134811e3a69955d,0.1485148514851485,WETH,0x4200000000000000000000000000000000000006,0.005,3.4926831683168333,0.001485148514851486,ETH,0x4200000000000000000000000000000000000006,uniswap,2,D.O.G.E-WETH,0x45b992355d1fa3c5c50ba446c90b578a7b779433,0xaaf1df00167fe59d43c8d1944a6f28aaf8f40f83,0xf09148ffe271bfe5d28c3547362073257ce0d2bb86e6ef788fe6195247441a43,4,true +2024-09-06 15:51:25.000 UTC,2024-09-06 00:00:00.000 UTC,2024-09-01 00:00:00.000 UTC,Banana Gun,19424269,base,112.36625779738371,Sell,0.048872106175385115,WETH,0x4200000000000000000000000000000000000006,1027321.7559139789,UBPS,0x0c90c756350fb803a7d5d9f9ee5ac29e77369973,0.005,1.1236625779738343,0.00048872106175385,ETH,0x4200000000000000000000000000000000000006,uniswap,2,UBPS-WETH,0x4557eacd191e266d4166c603131ef3b006836cae,0x24acc9d9517c1ccfd7ffeaaefbc8666a582fc64b,0xdf164d9670faa4429521a5339a0676d7f47c74817a86fd03cf85fb8e9083757c,9,true +2024-09-05 05:14:37.000 UTC,2024-09-05 00:00:00.000 UTC,2024-09-01 00:00:00.000 UTC,Banana Gun,19361965,base,4.844921559141894,Sell,0.002009432065639148,WETH,0x4200000000000000000000000000000000000006,74784.28391180358,BARIO,0x814fe70e85025bec87d4ad3f3b713bdcaac0579b,0.005,0.04844921559141778,0.000020094320656391,ETH,0x4200000000000000000000000000000000000006,uniswap,3,BARIO-WETH,0x19efcd30370cd4858df84415d9b63eda2048ef27,0x012cd7fb379b72d4c19988d377c3076aa81583a6,0xca1fd9509fd36d1eb4174857b14bfb9dd1fe8ab18fc31c61ad5425301c9b9017,2,true +2024-09-04 00:57:23.000 UTC,2024-09-04 00:00:00.000 UTC,2024-09-01 00:00:00.000 UTC,Banana Gun,19311048,base,0.4824217821782178,Buy,26.822871758695317,MIGGLES,0xb1a03eda10342529bbf8eb700a06c60441fef25d,0.000198019801980198,WETH,0x4200000000000000000000000000000000000006,0.005,0.004824217821782226,0.000001980198019802,ETH,0x4200000000000000000000000000000000000006,uniswap,2,MIGGLES-WETH,0x17a3ad8c74c4947005afeda9965305ae2eb2518a,0xbd5ce37cf9c61d781b28ba8c1d612e12f6507243,0x70166ba1b17ce988996e5240a8fd71021e716b29863236a468c636ededbe5b18,35,true +2024-09-05 16:51:15.000 UTC,2024-09-05 00:00:00.000 UTC,2024-09-01 00:00:00.000 UTC,Banana Gun,19382864,base,141.05584158415843,Buy,16561085558.942457,PEPE,0x52b492a33e447cdb854c7fc19f1e57e8bfa1777d,0.05940594059405941,WETH,0x4200000000000000000000000000000000000006,0.005,1.4105584158415865,0.000594059405940595,ETH,0x4200000000000000000000000000000000000006,uniswap,3,PEPE-WETH,0x0fb597d6cfe5be0d5258a7f017599c2a4ece34c7,0xa7650835f35c4997f93a9e3bb40bc6c0ba8d059d,0x49b82a258a59bd179c4da535038bad40186d59224c431e7fad34a0adf50bf68a,3,true +2024-09-05 22:16:21.000 UTC,2024-09-05 00:00:00.000 UTC,2024-09-01 00:00:00.000 UTC,Banana Gun,19392617,base,117.19306930693071,Buy,8673078.991486527,XRAY,0x3c48ae7b78bf9ba91fc8e5f6f6bf9f089f278328,0.04950495049504951,WETH,0x4200000000000000000000000000000000000006,0.005,1.1719306930693094,0.000495049504950496,ETH,0x4200000000000000000000000000000000000006,uniswap,2,WETH-XRAY,0xb924036f602fa75f0d4ac9545ee4d6d00261c8ad,0xc4dd8211988263c932c0afb59677dd92ab898046,0x6fcd2bc0c758a0aff18c18069a13173f90e7455ed93d8f5317ff62d78dd1cf61,11,true +2024-09-05 21:22:43.000 UTC,2024-09-05 00:00:00.000 UTC,2024-09-01 00:00:00.000 UTC,Banana Gun,19391008,base,235.0237623762376,Buy,17697.68136626327,WELL,0xa88594d404727625a9437c3f886c7643872296ae,0.09900990099009901,WETH,0x4200000000000000000000000000000000000006,0.005,2.350237623762378,0.000990099009900991,ETH,0x4200000000000000000000000000000000000006,aerodrome,1,WELL-WETH,0x89d0f320ac73dd7d9513ffc5bc58d1161452a657,0x56d0ffbb368ecfb8111a6637fc698d6325e8bc01,0x553bf030f289a28cdb96e4da372ebb3ecf8b138062845edc5266494744d894a3,6,true +2024-09-01 16:31:43.000 UTC,2024-09-01 00:00:00.000 UTC,2024-09-01 00:00:00.000 UTC,Banana Gun,19209478,base,0.23501467838723727,Sell,0.000094977299170814,WETH,0x4200000000000000000000000000000000000006,51.4387631,Nermie,0x519a319e9bb1000c504a20485084df2337c4c14d,0.005,0.0023501467838670776,9.49772991706e-7,ETH,0x4200000000000000000000000000000000000006,sushiswap,1,Nermie-WETH,0xeb5b859dcc061e1e567d89c8ea5bacbe570b6c4e,0x2f0fcdfda8b4065a4d266ddc53c077750e3b485f,0xa203f9e3f43c2288aff00fc6a9f2aea8a3dc6f84f2c32abcd70c8d6a272ba743,10,true +2024-09-01 12:09:59.000 UTC,2024-09-01 00:00:00.000 UTC,2024-09-01 00:00:00.000 UTC,Banana Gun,19201626,base,4959.661963579478,Sell,1.9988723188013517,WETH,0x4200000000000000000000000000000000000006,,,0xc53eb9ecc247465e2e1fb16b2ede3a6446053170,0.005,49.59661963579477,0.01998872318801351,ETH,0x4200000000000000000000000000000000000006,uniswap,2,,0xa854492182dcbda450bcd9ee04870818484dd8ad,0x591be26c53aa3ed600c1c994ecbc2deb545bebb4,0x0c581fc5af9d3a53525f5d31fb09fb9d4dbe5772629ed1c5b2879ad6a59ba30c,11,true +2024-09-01 12:09:59.000 UTC,2024-09-01 00:00:00.000 UTC,2024-09-01 00:00:00.000 UTC,Banana Gun,19201626,base,51.84239373333868,Sell,0.020893828356637105,WETH,0x4200000000000000000000000000000000000006,,,0xc53eb9ecc247465e2e1fb16b2ede3a6446053170,0.005,49.59661963579477,0.01998872318801351,ETH,0x4200000000000000000000000000000000000006,uniswap,2,,0xa854492182dcbda450bcd9ee04870818484dd8ad,0x591be26c53aa3ed600c1c994ecbc2deb545bebb4,0x0c581fc5af9d3a53525f5d31fb09fb9d4dbe5772629ed1c5b2879ad6a59ba30c,3,false +2024-09-04 15:30:23.000 UTC,2024-09-04 00:00:00.000 UTC,2024-09-01 00:00:00.000 UTC,Banana Gun,19337238,base,24.098712871287123,Buy,878817.008049761,TRFY,0xa7ebc5f9b67e790ac0b5d24ed0d531d319cea597,0.0099009900990099,WETH,0x4200000000000000000000000000000000000006,0.005,0.2409871287128737,0.0000990099009901,ETH,0x4200000000000000000000000000000000000006,uniswap,2,TRFY-WETH,0x5f27785b5e2c8263bdec279c75245b980099ac9d,0x64c23d3f619d825bab287f3c52a0f39d3a03c263,0x37c292515191685a193c9400f4fed0ecdc3e1e8dc08e2a527f79cbc000cec727,4,true +2024-09-02 16:30:51.000 UTC,2024-09-02 00:00:00.000 UTC,2024-09-01 00:00:00.000 UTC,Banana Gun,19252652,base,33.64625262721495,Sell,0.013366804770162664,WETH,0x4200000000000000000000000000000000000006,1541948.1284249476,BEE,0xfd09f108d1728e6b6ed241ccd254775e322f1ed6,0.005,0.3364625262721479,0.000133668047701626,ETH,0x4200000000000000000000000000000000000006,uniswap,2,BEE-WETH,0x500e45173f466c058e6dcecd15b27a1e091d92d4,0x64c23d3f619d825bab287f3c52a0f39d3a03c263,0x097dd6b26059fce8166f8fea84aaa12ae13b075cf0d077f717cc1bc31e41ec1b,5,true +2024-09-05 21:11:25.000 UTC,2024-09-05 00:00:00.000 UTC,2024-09-01 00:00:00.000 UTC,Banana Gun,19390669,base,117.3920792079208,Buy,5649267.692887611,COFFEE,0xc54b2b7ee65918a8ec08048cd2377998e7415312,0.04950495049504951,WETH,0x4200000000000000000000000000000000000006,0.005,1.1739207920792103,0.000495049504950496,ETH,0x4200000000000000000000000000000000000006,uniswap,2,COFFEE-WETH,0xe28afb7f1cac4bbf87cd9875aa06f470aec965a4,0xe3ea037ca6ea651cc3f476712b30374c2115e74e,0xdaf6c33941140880d5baff25f8c65f1362048575535d31bff39d07a56ac510ae,11,true +2024-09-05 21:11:25.000 UTC,2024-09-05 00:00:00.000 UTC,2024-09-01 00:00:00.000 UTC,Banana Gun,19390669,base,117.3920792079208,Buy,4982612.726137387,COFFEE,0xc54b2b7ee65918a8ec08048cd2377998e7415312,0.04950495049504951,WETH,0x4200000000000000000000000000000000000006,0.005,1.1739207920792103,0.000495049504950496,ETH,0x4200000000000000000000000000000000000006,uniswap,2,COFFEE-WETH,0xe28afb7f1cac4bbf87cd9875aa06f470aec965a4,0xb73bc3772cf29da413966f36fe18a1dd392cf1b6,0xcc475368ad6c4e6e30df134aaad618fc9fcc9101cf01518c2be3d3170ee11d3c,32,true +2024-09-05 21:11:55.000 UTC,2024-09-05 00:00:00.000 UTC,2024-09-01 00:00:00.000 UTC,Banana Gun,19390684,base,117.3920792079208,Buy,4786468.142259231,COFFEE,0xc54b2b7ee65918a8ec08048cd2377998e7415312,0.04950495049504951,WETH,0x4200000000000000000000000000000000000006,0.005,1.1739207920792103,0.000495049504950496,ETH,0x4200000000000000000000000000000000000006,uniswap,2,COFFEE-WETH,0xe28afb7f1cac4bbf87cd9875aa06f470aec965a4,0xe9869a8c089502b8a8b54b53a9df3ce295a4c17b,0x6909a7e16826cf57cbf3b836e57dcb850150f86902c7d62892db55ab02916947,4,true +2024-09-05 21:11:25.000 UTC,2024-09-05 00:00:00.000 UTC,2024-09-01 00:00:00.000 UTC,Banana Gun,19390669,base,117.3920792079208,Buy,5191072.771597835,COFFEE,0xc54b2b7ee65918a8ec08048cd2377998e7415312,0.04950495049504951,WETH,0x4200000000000000000000000000000000000006,0.005,1.1739207920792103,0.000495049504950496,ETH,0x4200000000000000000000000000000000000006,uniswap,2,COFFEE-WETH,0xe28afb7f1cac4bbf87cd9875aa06f470aec965a4,0xfe906d692a47246518fe05cd5852501bfb51f012,0x2935af8e620e15691ca2b9e5dc78d1179dc2b4222d1835a35d5955d3951c7a36,25,true +2024-09-05 21:11:55.000 UTC,2024-09-05 00:00:00.000 UTC,2024-09-01 00:00:00.000 UTC,Banana Gun,19390684,base,117.3920792079208,Buy,4601687.433549198,COFFEE,0xc54b2b7ee65918a8ec08048cd2377998e7415312,0.04950495049504951,WETH,0x4200000000000000000000000000000000000006,0.005,1.1739207920792103,0.000495049504950496,ETH,0x4200000000000000000000000000000000000006,uniswap,2,COFFEE-WETH,0xe28afb7f1cac4bbf87cd9875aa06f470aec965a4,0xfe906d692a47246518fe05cd5852501bfb51f012,0x3f15863dd32a63136646bde0b816fa080f6153c0d6d74d94d7b0bb5fa3d59140,11,true diff --git a/dbt_subprojects/dex/seeds/bot_trades/banana_gun/banana_gun_ethereum_trades_seed.csv b/dbt_subprojects/dex/seeds/bot_trades/banana_gun/banana_gun_ethereum_trades_seed.csv new file mode 100644 index 00000000000..bbaf14cedf0 --- /dev/null +++ b/dbt_subprojects/dex/seeds/bot_trades/banana_gun/banana_gun_ethereum_trades_seed.csv @@ -0,0 +1,21 @@ +block_time,block_date,block_month,bot,block_number,blockchain,amount_usd,type,token_bought_amount,token_bought_symbol,token_bought_address,token_sold_amount,token_sold_symbol,token_sold_address,fee_percentage_fraction,fee_usd,fee_token_amount,fee_token_symbol,fee_token_address,project,version,token_pair,project_contract_address,user,tx_hash,evt_index,is_last_trade_in_transaction +2023-06-19 15:26:59.000 UTC,2023-06-19 00:00:00.000 UTC,2023-06-01 00:00:00.000 UTC,Banana Gun,17514676,ethereum,83.12745837737252,Sell,0.04836365974946039,WETH,0xc02aaa39b223fe8d0a0e5c4f27ead9083c756cc2,401724.712464711,IG,0x1c60343f8ee1db0617097341abf56c308597d9d9,0.005,0.4037242114329775,0.000234887253568174,ETH,0xc02aaa39b223fe8d0a0e5c4f27ead9083c756cc2,uniswap,2,IG-WETH,0xec3615f6cd9f2df234c877e501d53f9eef22fcc4,0x842be3379d61bb4d63e757ab485fee99a6cebecc,0x6173075a5d069d1db4187e145ab35849e0bbc25cc2eb8164d0ec8afd76d27ad7,24,false +2023-06-06 17:22:11.000 UTC,2023-06-06 00:00:00.000 UTC,2023-06-01 00:00:00.000 UTC,Banana Gun,17422854,ethereum,116.46015223455811,Sell,0.06250511334447438,WETH,0xc02aaa39b223fe8d0a0e5c4f27ead9083c756cc2,48700.164228039175,0xC,0x2aa4c7e845ced2cb3180a6584df15efd0d886bdb,0.005,0.5823007611727888,0.000312525566722371,ETH,0xc02aaa39b223fe8d0a0e5c4f27ead9083c756cc2,uniswap,2,0xC-WETH,0x57680221d5f90d871f82f43fdf4824a2717041e7,0xc4fb7ad11fbc1c5733e5f56043f44c690d6c5686,0x7611950dedf03734be5a95a7d5a983bd20edcaeae9151568dc32424b6a61ada5,45,true +2023-06-17 13:13:11.000 UTC,2023-06-17 00:00:00.000 UTC,2023-06-01 00:00:00.000 UTC,Banana Gun,17499751,ethereum,83.27189660506818,Sell,0.04785878709449592,WETH,0xc02aaa39b223fe8d0a0e5c4f27ead9083c756cc2,8710044.64601209,METH,0x2d6cfa8781755c922a14d0e7c111700060490198,0.005,0.41635948302533987,0.000239293935472479,ETH,0xc02aaa39b223fe8d0a0e5c4f27ead9083c756cc2,uniswap,2,METH-WETH,0xcd9e2214ae0449d47f2becd56516d1a497a8c5e7,0xbbd018a45b40bb699df47c7bca87623810a2534b,0x69efeb79af7bc11600b5d11a6a8d85a7949f7695801fe1c648916eef368a0d4e,14,true +2023-06-21 21:56:23.000 UTC,2023-06-21 00:00:00.000 UTC,2023-06-01 00:00:00.000 UTC,Banana Gun,17530850,ethereum,182.88032782937748,Sell,0.09753877587635802,WETH,0xc02aaa39b223fe8d0a0e5c4f27ead9083c756cc2,15122057.069775704,DEN,0xe28901beb0630379ced910053294b6f953309c3a,0.005,0.9144016391468872,0.00048769387938179,ETH,0xc02aaa39b223fe8d0a0e5c4f27ead9083c756cc2,uniswap,3,DEN-WETH,0xe4da8e91076fdd026c5582279268785dd44f0038,0xe12352f93b86b7262099e8039580990c80e83ca5,0xa92bd896bb9c23194180d03fc93ecea65155c42ed1aaaa89737772f8f98ea1c6,12,true +2023-06-19 18:12:47.000 UTC,2023-06-19 00:00:00.000 UTC,2023-06-01 00:00:00.000 UTC,Banana Gun,17515493,ethereum,419.944958659094,Sell,0.24176868838212176,WETH,0xc02aaa39b223fe8d0a0e5c4f27ead9083c756cc2,48012.75194671747,$ELY,0x8ed2fc62d6850eaadcb717465752dab591286839,0.005,2.973855242525905,0.001712093612742825,ETH,0xc02aaa39b223fe8d0a0e5c4f27ead9083c756cc2,uniswap,2,$ELY-WETH,0xeed363729bdb7f9d7b5fce5dbf8bd5cc4fc98f7f,0xc8ef2326093c2491bb474be405fb04f5cb89a7ce,0xd96b1f10f419f83a3653e10d39dcb667f56fb92eeb13ec66e3bae2643311f10d,92,false +2023-06-30 12:38:47.000 UTC,2023-06-30 00:00:00.000 UTC,2023-06-01 00:00:00.000 UTC,Banana Gun,17592143,ethereum,189.009205,Buy,835861314077.003,EPEP,0x975b624dda0903c8100bafccfd537458933d410e,0.0995,WETH,0xc02aaa39b223fe8d0a0e5c4f27ead9083c756cc2,0.005,0.949795,0.0005,ETH,0xc02aaa39b223fe8d0a0e5c4f27ead9083c756cc2,uniswap,2,EPEP-WETH,0x32eabaec2bd26d647b568715a514a76757c86145,0x8b38b7f821b9a7216d45d49047ecfbf80ae3a566,0xfc7d27e1633e07e8581a8c16939936aa446c07a4399ad9f908d563e868b260e3,22,true +2023-06-28 16:03:59.000 UTC,2023-06-28 00:00:00.000 UTC,2023-06-01 00:00:00.000 UTC,Banana Gun,17578886,ethereum,229.26480250785545,Sell,0.12313750900059911,WETH,0xc02aaa39b223fe8d0a0e5c4f27ead9083c756cc2,275400,FIND,0x638df5a6ef481b4e01982ba750d0008c98ed4c24,0.005,1.1463240125392762,0.000615687545002995,ETH,0xc02aaa39b223fe8d0a0e5c4f27ead9083c756cc2,uniswap,2,FIND-WETH,0xd384a0f245d6a54856f924a9708fdde8c7f3d3a9,0x5b46ff3f518d4db5ff50105dfbe917ce79f6c3bb,0x7d6d07a85bdd642e6964552e1f8e72f0a8911e3408b3c9a10072f6928d89a3aa,42,true +2023-06-05 02:01:35.000 UTC,2023-06-05 00:00:00.000 UTC,2023-06-01 00:00:00.000 UTC,Banana Gun,17411242,ethereum,377.23,Buy,2758703618.5463076,SATOSHI,0xf048fdef5f29ca7f4f96a97d534eec1b17c0356f,0.2,WETH,0xc02aaa39b223fe8d0a0e5c4f27ead9083c756cc2,0,0,0,ETH,0xc02aaa39b223fe8d0a0e5c4f27ead9083c756cc2,uniswap,2,SATOSHI-WETH,0x76197528ae1f5586c0d636a8859edb2c5daf7681,0x7fd1fa18d2748a0b39d413724d733d2a24075280,0x0e37882eb552c5f9ff5497a076203a4717654ced04ca8c34adbedd41356d32f5,31,true +2023-06-19 12:05:47.000 UTC,2023-06-19 00:00:00.000 UTC,2023-06-01 00:00:00.000 UTC,Banana Gun,17513684,ethereum,361.039881415359,Sell,0.2091178526463281,WETH,0xc02aaa39b223fe8d0a0e5c4f27ead9083c756cc2,71874339683920.4,SHA,0x95565f27d9b7fcacb04954ca4f24b486a641f001,0.005,1.8051994070767943,0.00104558926323164,ETH,0xc02aaa39b223fe8d0a0e5c4f27ead9083c756cc2,uniswap,2,SHA-WETH,0x81de191d0a57f86901a7aec8d44a0698e0f8fb96,0xe12352f93b86b7262099e8039580990c80e83ca5,0xab8668a0d124e9a2c2a4e01e19476e150007e9ac7f4370339c8ad79f690f8813,22,true +2023-06-25 16:00:23.000 UTC,2023-06-25 00:00:00.000 UTC,2023-06-01 00:00:00.000 UTC,Banana Gun,17557542,ethereum,62.20631360805016,Buy,12000000000,QBERT,0xa09116cfd21f49689c32083cb3a3632d40c6f7fe,0.03293639664106686,WETH,0xc02aaa39b223fe8d0a0e5c4f27ead9083c756cc2,0.0016,0.31103156804025023,0.000164681983205334,ETH,0xc02aaa39b223fe8d0a0e5c4f27ead9083c756cc2,uniswap,2,QBERT-WETH,0x9461485ef8c8c8cdfc910e81e58b03c6b4485553,0xe77fd9756dac9ebfe8a7d9c623d988814987964e,0x41acb6080b560735fd8479e7dc59daf01815c6f9aa6c440ad0ba55aeed653c97,60,true +2023-06-05 01:02:35.000 UTC,2023-06-05 00:00:00.000 UTC,2023-06-01 00:00:00.000 UTC,Banana Gun,17410953,ethereum,266.1148617401801,Sell,0.1412027091472491,WETH,0xc02aaa39b223fe8d0a0e5c4f27ead9083c756cc2,750000,ASUKA,0xfe25f375aa07298f75c39d155855a7c6ecfbbf69,0.005,3.2677701481407015,0.00173390540750211,ETH,0xc02aaa39b223fe8d0a0e5c4f27ead9083c756cc2,uniswap,2,ASUKA-WETH,0xfc93d2b86be176548d2b1b8c007adc85d6c0cc8b,0x9c99d7af32c576eb71dbccc3718be211eb3e14ab,0x263ef72c8cef7859284b4d64f7503ed04e141f79c4482e9fb8360bcbddc79af7,20,false +2023-06-20 15:29:23.000 UTC,2023-06-20 00:00:00.000 UTC,2023-06-01 00:00:00.000 UTC,Banana Gun,17521804,ethereum,56.0700216923059,Sell,0.03259278604695981,WETH,0xc02aaa39b223fe8d0a0e5c4f27ead9083c756cc2,7354.848321544704,0xDeaD,0xdead0f0e934a4137a21811bff798850b0a90dead,0.005,0.2803501084615294,0.000162963930234799,ETH,0xc02aaa39b223fe8d0a0e5c4f27ead9083c756cc2,uniswap,2,0xDeaD-WETH,0xc9a2d5f44b90d957716e656229e142d37270d19a,0x490a028a817a4eb52164c37fda25cff1f1b4b180,0xad85c2bf9d0fc141fe3e44c0f7e69e169c7b9e9bf834c57535d71aa677f296cc,57,true +2023-06-25 19:49:47.000 UTC,2023-06-25 00:00:00.000 UTC,2023-06-01 00:00:00.000 UTC,Banana Gun,17558676,ethereum,141.05486574079083,Sell,0.07459510388525856,WETH,0xc02aaa39b223fe8d0a0e5c4f27ead9083c756cc2,441453.320819254,VisionX,0x8551353a87993c14329df0322f673bca640e3906,0.005,0.7052743287039527,0.000372975519426292,ETH,0xc02aaa39b223fe8d0a0e5c4f27ead9083c756cc2,uniswap,2,VisionX-WETH,0xae5dcd5f4e54998b740eec0af1d8bcbd3ad21b03,0xdee6cfc3c51550e5bd46a4613a09424bc62b6be6,0x96347e0ab77ff0988b616ff584ca8bf2b823ba202e56a1299d45b0e71212a666,20,true +2023-06-23 18:45:23.000 UTC,2023-06-23 00:00:00.000 UTC,2023-06-01 00:00:00.000 UTC,Banana Gun,17544135,ethereum,382.533684871126,Sell,0.2003990260527886,WETH,0xc02aaa39b223fe8d0a0e5c4f27ead9083c756cc2,102707.0809698149,FUMOKILLER,0x9bbe2cebee8c0aba957224e512e4de54bbf79fe6,0.005,1.9126684243556284,0.001001995130263942,ETH,0xc02aaa39b223fe8d0a0e5c4f27ead9083c756cc2,uniswap,2,FUMOKILLER-WETH,0x91459946d9ad8c89fa807488f9e84d83263b5678,0xc4fb7ad11fbc1c5733e5f56043f44c690d6c5686,0x00db9d11fdfc76d624438c577ce6b0734cc50deea875cdb656d730ef036cc1c4,49,true +2023-06-29 20:34:23.000 UTC,2023-06-29 00:00:00.000 UTC,2023-06-01 00:00:00.000 UTC,Banana Gun,17587364,ethereum,106.34892104701531,Sell,0.057466023130925144,WETH,0xc02aaa39b223fe8d0a0e5c4f27ead9083c756cc2,1600000,MrsPepe2.0,0x0c2eee9858623f8e6da32c1c3683ed22ef7be5fc,0.005,0.5317446052350753,0.000287330115654625,ETH,0xc02aaa39b223fe8d0a0e5c4f27ead9083c756cc2,uniswap,2,MrsPepe2.0-WETH,0xeaca43d704daae77aed4722a60c32b5538e4aa1c,0xafabb3a635747797dd05ab662ce5dd070ee28b91,0x2f08c34efc6e3397146aaf70afb65afe4d7d47a6811e2b837df933fea1f308b4,70,true +2023-06-29 12:31:47.000 UTC,2023-06-29 00:00:00.000 UTC,2023-06-01 00:00:00.000 UTC,Banana Gun,17584974,ethereum,153.49601411690773,Sell,0.08200186665504244,WETH,0xc02aaa39b223fe8d0a0e5c4f27ead9083c756cc2,12331187.874773905,YFO,0x1f8c2c3e9a17ecd378911a122e876a43993a5e51,0.005,0.7674800705845383,0.000410009333275212,ETH,0xc02aaa39b223fe8d0a0e5c4f27ead9083c756cc2,uniswap,2,WETH-YFO,0xacbd3bdce79c65250ee53aded4a186d4ca2ace23,0xfcc33fc7ab61cda35b8b8abee0635733b388d1bf,0x4f21c5f745e6945c74d6270f63510139bdffa452b150499cef4c03fcab508aae,142,true +2023-06-20 22:14:11.000 UTC,2023-06-20 00:00:00.000 UTC,2023-06-01 00:00:00.000 UTC,Banana Gun,17523802,ethereum,0.7480057655563455,Sell,0.000420468901031122,WETH,0xc02aaa39b223fe8d0a0e5c4f27ead9083c756cc2,7394.251038698974,RPUNKS,0x340b8776063abfdd5117c35e3a4b228079ea6d90,0.005,0.17150376326334374,0.000096405672499603,ETH,0xc02aaa39b223fe8d0a0e5c4f27ead9083c756cc2,uniswap,2,RPUNKS-WETH,0x7dfb889c888189e91c3ca56a993c55594eee3b61,0x0d87e9d89a0c24f5879ea8ab72590ff5cedf2c9b,0x04858f207f668c2a36221c2bd878680dd4f62e7fc925ffd884c50e8a41c8bd9c,90,false +2023-06-20 15:00:11.000 UTC,2023-06-20 00:00:00.000 UTC,2023-06-01 00:00:00.000 UTC,Banana Gun,17521662,ethereum,171.22855500000003,Buy,2636.3410224539093,0xDeaD,0xdead0f0e934a4137a21811bff798850b0a90dead,0.0995,WETH,0xc02aaa39b223fe8d0a0e5c4f27ead9083c756cc2,0.005,0.860445,0.0005,ETH,0xc02aaa39b223fe8d0a0e5c4f27ead9083c756cc2,uniswap,2,0xDeaD-WETH,0xc9a2d5f44b90d957716e656229e142d37270d19a,0xf5ccaf5dcd921b98c594ccdd3b16435006113cc6,0xf33227d240b85898ce859514f0b0c8c016973a1921ce2cea0cd95a8e251aa04b,274,true +2023-06-21 22:48:59.000 UTC,2023-06-21 00:00:00.000 UTC,2023-06-01 00:00:00.000 UTC,Banana Gun,17531111,ethereum,360.64435523988095,Sell,0.1915335967029475,WETH,0xc02aaa39b223fe8d0a0e5c4f27ead9083c756cc2,261323.81103556513,BCT,0x2281c37cc6e7ac0ded17eb1494f63f94a8ee568a,0.005,1.8032217761994038,0.000957667983514737,ETH,0xc02aaa39b223fe8d0a0e5c4f27ead9083c756cc2,uniswap,2,BCT-WETH,0x244ce07231d5156f22a21212768c1fa5e8d84e1b,0xbd42b534f896448ac6cfa7d8229d5b76d46f1927,0x08e369ddc65e1cfba3f2f4ce79a1a7c375d1511ba7d91dcbe641dfe124e1de87,20,true +2023-06-08 22:40:23.000 UTC,2023-06-08 00:00:00.000 UTC,2023-06-01 00:00:00.000 UTC,Banana Gun,17438595,ethereum,84.80449692865044,Buy,20000000000,TARD,0x4a0386cf685e5e2b73a730a85ac3ee65a62585f5,0.045737914574223326,WETH,0xc02aaa39b223fe8d0a0e5c4f27ead9083c756cc2,0,0,0,ETH,0xc02aaa39b223fe8d0a0e5c4f27ead9083c756cc2,uniswap,2,TARD-WETH,0x01084d791298ed32bde3481d06b1a59d36757897,0xc8ef2326093c2491bb474be405fb04f5cb89a7ce,0x29e5a82febb20b6539ceb635e7d70588d867860456a17fa10739624e28f198d8,61,true diff --git a/dbt_subprojects/dex/seeds/bot_trades/banana_gun/schema.yml b/dbt_subprojects/dex/seeds/bot_trades/banana_gun/schema.yml new file mode 100644 index 00000000000..9e25049c7c2 --- /dev/null +++ b/dbt_subprojects/dex/seeds/bot_trades/banana_gun/schema.yml @@ -0,0 +1,24 @@ +version: 2 + +seeds: + - name: banana_gun_ethereum_trades_seed + config: + column_types: + version: varchar + token_bought_address: varbinary + token_sold_address: varbinary + fee_token_address: varbinary + project_contract_address: varbinary + user: varbinary + tx_hash: varbinary + + - name: banana_gun_base_trades_seed + config: + column_types: + version: varchar + token_bought_address: varbinary + token_sold_address: varbinary + fee_token_address: varbinary + project_contract_address: varbinary + user: varbinary + tx_hash: varbinary diff --git a/dbt_subprojects/dex/tests/generic/check_bot_trades_seed.sql b/dbt_subprojects/dex/tests/generic/check_bot_trades_seed.sql new file mode 100644 index 00000000000..446757bf1ba --- /dev/null +++ b/dbt_subprojects/dex/tests/generic/check_bot_trades_seed.sql @@ -0,0 +1,15 @@ +-- this tests checks a dex bot_trades model for every row in a seed file. +-- actual implementation in macros/test-helpers/check_seed.sql +{% test check_bot_trades_seed(model, seed_file, blockchain=None, filter=None) %} + + {%- if blockchain == 'solana' -%} + {%- set seed_matching_columns = ['blockchain', 'project', 'version', 'tx_id', 'tx_index', 'outer_instruction_index', 'inner_instruction_index'] -%} + {%- else -%} + {%- set seed_matching_columns = ['blockchain', 'project', 'version', 'tx_hash', 'evt_index',] -%} + {%- endif -%} + + {%- set seed_check_columns = ['fee_token_amount', 'fee_token_address', 'project_contract_address', 'token_bought_address', 'token_sold_address', 'user'] -%} + + {{ check_seed_macro(model, seed_file, seed_matching_columns,seed_check_columns,filter) }} + +{% endtest %} \ No newline at end of file From 1d4963bde75da650c2349fa02d252a9a5c91bf5c Mon Sep 17 00:00:00 2001 From: clizzard <145153355+clizzard7@users.noreply.github.com> Date: Thu, 28 Nov 2024 13:03:14 +0100 Subject: [PATCH 13/18] Feat pepeboost evm trades spellbook (#7181) * init * fix: rename tests to data_tests * chore: disable test for now * chore: add base spellbook, remove outcommented test * feat: add blast spellbook * fix: pick tx_hash column from botTrades table * chore: add bot column, change project_start_date * chore: add real start date for blast * chore: move start date to original value * chore: remove miner bribes from spellbook schema * chore: remove blast spellbook, since it has no data yet * fix: add incremental filters * test: add check_bot_trades_seed for dex subproject * fix: remove leading AND in where condition * test: add base seed * fix: cast value column to decimal * test: add correct seed files * add pepeboost evm trades ethereum * fix syntax * fix: force version column to get parsed as string in seed * fix seed * Revert "fix: force version column to get parsed as string in seed" This reverts commit 7e6e570893d5ccc200bdcc5f2876452c782b8cff. * fix: force seed column as varchar * fix: fix typo * add schema * fix: do not cast fee_token_address to varchar * fix varbinary * add type info of varbinary column to seed schema * make hash varbinary as well * fix: remove blockchain filter from seed test * fix: cast all varbinary column explicitly * Update dbt_subprojects/dex/models/bot_trades/banana_gun/base/banana_gun_base_bot_trades.sql Co-authored-by: 0xRob <83790096+0xRobin@users.noreply.github.com> * perf: add incremental filter to ethereum * perf: add incremental filter in join condition --------- Co-authored-by: whalehunting <143016036+whalehunting@users.noreply.github.com> Co-authored-by: 0xRob <83790096+0xRobin@users.noreply.github.com> --- .../dex/models/bot_trades/_schema.yml | 20 ++ .../pepeboost_ethereum_bot_trades.sql | 194 ++++++++++++++++++ .../pepeboost_ethereum_trades_seed.csv | 21 ++ .../dex/seeds/bot_trades/pepeboost/schema.yml | 13 ++ 4 files changed, 248 insertions(+) create mode 100644 dbt_subprojects/dex/models/bot_trades/pepeboost/ethereum/pepeboost_ethereum_bot_trades.sql create mode 100644 dbt_subprojects/dex/seeds/bot_trades/pepeboost/pepeboost_ethereum_trades_seed.csv create mode 100644 dbt_subprojects/dex/seeds/bot_trades/pepeboost/schema.yml diff --git a/dbt_subprojects/dex/models/bot_trades/_schema.yml b/dbt_subprojects/dex/models/bot_trades/_schema.yml index 2cafe84746b..a8b5051de1d 100644 --- a/dbt_subprojects/dex/models/bot_trades/_schema.yml +++ b/dbt_subprojects/dex/models/bot_trades/_schema.yml @@ -130,3 +130,23 @@ models: - evt_index - check_bot_trades_seed: seed_file: ref('banana_gun_base_trades_seed') + + + - name: pepeboost_ethereum_bot_trades + meta: + blockchain: ethereum + sector: dex + project: pepeboost + contributors: whale_hunter + config: + tags: ["evm", "dex", "pepeboost", "trades"] + description: > + Pepeboost trades on Ethereum + data_tests: + - dbt_utils.unique_combination_of_columns: + combination_of_columns: + - blockchain + - tx_hash + - evt_index + - check_bot_trades_seed: + seed_file: ref('pepeboost_ethereum_trades_seed') diff --git a/dbt_subprojects/dex/models/bot_trades/pepeboost/ethereum/pepeboost_ethereum_bot_trades.sql b/dbt_subprojects/dex/models/bot_trades/pepeboost/ethereum/pepeboost_ethereum_bot_trades.sql new file mode 100644 index 00000000000..ce24a17c97e --- /dev/null +++ b/dbt_subprojects/dex/models/bot_trades/pepeboost/ethereum/pepeboost_ethereum_bot_trades.sql @@ -0,0 +1,194 @@ +{{ config( + alias = 'bot_trades', + schema = 'pepeboost_ethereum', + partition_by = ['block_month'], + materialized = 'incremental', + file_format = 'delta', + incremental_strategy = 'merge', + incremental_predicates = [incremental_predicate('DBT_INTERNAL_DEST.block_time')], + unique_key = ['blockchain', 'tx_hash', 'evt_index'] + ) +}} + +{% set project_name = 'Pepeboost' %} +{% set project_start_date = '2023-08-28' %} +{% set blockchain = 'ethereum' %} +{% set bot_deployer_1 = '0xeaBEe4C63bD085Fd906d6be5d387B5EEDfF83919' %} +{% set weth = '0xc02aaa39b223fe8d0a0e5c4f27ead9083c756cc2' %} +{% set fee_token_symbol = 'ETH' %} + +WITH + botContracts AS ( + SELECT + address + FROM + {{ source('ethereum','creation_traces') }} + WHERE + "from" = {{bot_deployer_1}} + AND block_time >= TIMESTAMP '{{project_start_date}}' + ), + botTrades AS ( + SELECT + trades.block_time, + amount_usd, + IF( + token_sold_address = {{weth}}, + 'Buy', + 'Sell' + ) AS type, + token_bought_amount, + token_bought_symbol, + token_bought_address, + token_sold_amount, + token_sold_symbol, + token_sold_address, + project, + version, + token_pair, + project_contract_address, + tx_from AS user, + tx_to AS bot, + trades.tx_hash, + evt_index + FROM + {{ source('dex', 'trades') }} as trades + JOIN botContracts ON trades.tx_to = botContracts.address + WHERE + trades.blockchain = '{{blockchain}}' + {% if is_incremental() %} + AND {{ incremental_predicate('trades.block_time') }} + {% else %} + AND trades.block_time >= TIMESTAMP '{{project_start_date}}' + {% endif %} + ), + highestEventIndexForEachTrade AS ( + SELECT + tx_hash, + MAX(evt_index) AS highestEventIndex + FROM + botTrades + GROUP BY + tx_hash + ), + botETHDeposits AS ( + SELECT + tx_hash, + block_number, + CAST(value AS DECIMAL (38, 0)) AS deltaGwei, + CAST(value AS DECIMAL (38, 0)) AS depositGwei + FROM + {{ source('ethereum','traces') }} + JOIN botContracts ON to = botContracts.address + WHERE + {% if is_incremental() %} + {{ incremental_predicate('block_time') }} + {% else %} + block_time >= TIMESTAMP '{{project_start_date}}' + {% endif %} + AND value > 0 + ), + botETHWithdrawals AS ( + SELECT + tx_hash, + block_number, + CAST(value AS DECIMAL (38, 0)) * -1 AS deltaGwei, + 0 AS depositGwei, + block_hash, + to + FROM + {{ source('ethereum','traces') }} + JOIN botContracts ON "from" = botContracts.address + WHERE + {% if is_incremental() %} + {{ incremental_predicate('block_time') }} + {% else %} + block_time >= TIMESTAMP '{{project_start_date}}' + {% endif %} + AND value > 0 + ), + botEthTransfers AS ( + /* Deposits */ + ( + SELECT + tx_hash, + block_number, + deltaGwei, + depositGwei + FROM + botETHDeposits + ) + UNION ALL + /* Withdrawals */ + ( + SELECT + tx_hash, + block_number, + deltaGwei, + depositGwei + FROM + botETHWithdrawals + ) + ), + botEthDeltas AS ( + SELECT + tx_hash, + block_number, + SUM(deltaGwei) AS feeGwei, + SUM(depositGwei) AS depositGwei + FROM + botEthTransfers + GROUP BY + tx_hash, + block_number + ) +SELECT + block_time, + date_trunc('day', botTrades.block_time) as block_date, + date_trunc('month', botTrades.block_time) as block_month, + '{{project_name}}' as bot, + block_number, + '{{blockchain}}' AS blockchain, + -- Trade + amount_usd, + type, + token_bought_amount, + token_bought_symbol, + token_bought_address, + token_sold_amount, + token_sold_symbol, + token_sold_address, + -- Fees + ROUND( + CAST(feeGwei AS DOUBLE) / CAST(depositGwei AS DOUBLE), + /* Round feePercentage to 0.01% steps */ + 4 + ) AS fee_percentage_fraction, + (feeGwei / 1e18) * price AS fee_usd, + feeGwei / 1e18 fee_token_amount, + '{{fee_token_symbol}}' AS fee_token_symbol, + {{weth}} AS fee_token_address, + -- Dex + project, + version, + token_pair, + project_contract_address, + -- User + user AS user, + botTrades.tx_hash, + evt_index, + IF(evt_index = highestEventIndex, true, false) AS is_last_trade_in_transaction +FROM + botTrades + JOIN highestEventIndexForEachTrade ON botTrades.tx_hash = highestEventIndexForEachTrade.tx_hash + LEFT JOIN botETHDeltas ON botTrades.tx_hash = botETHDeltas.tx_hash + LEFT JOIN {{ source('prices', 'usd') }} ON ( + blockchain = '{{blockchain}}' + AND contract_address = {{weth}} + AND minute = DATE_TRUNC('minute', block_time) + {% if is_incremental() %} + AND {{ incremental_predicate('minute') }} + {% endif %} + ) +ORDER BY + block_time DESC, + evt_index DESC \ No newline at end of file diff --git a/dbt_subprojects/dex/seeds/bot_trades/pepeboost/pepeboost_ethereum_trades_seed.csv b/dbt_subprojects/dex/seeds/bot_trades/pepeboost/pepeboost_ethereum_trades_seed.csv new file mode 100644 index 00000000000..5645bdd3976 --- /dev/null +++ b/dbt_subprojects/dex/seeds/bot_trades/pepeboost/pepeboost_ethereum_trades_seed.csv @@ -0,0 +1,21 @@ +block_time,block_date,block_month,bot,block_number,blockchain,amount_usd,type,token_bought_amount,token_bought_symbol,token_bought_address,token_sold_amount,token_sold_symbol,token_sold_address,fee_percentage_fraction,fee_usd,fee_token_amount,fee_token_symbol,fee_token_address,project,version,token_pair,project_contract_address,user,tx_hash,evt_index,is_last_trade_in_transaction +2024-03-16 11:48:11.000 UTC,2024-03-16 00:00:00.000 UTC,2024-03-01 00:00:00.000 UTC,Pepeboost,19447257,ethereum,1827.9162000000001,Buy,50817275171.0449,HELGA,0x78e3b2ee11950df78a35fd924e92fbb8d1403780,0.495,WETH,0xc02aaa39b223fe8d0a0e5c4f27ead9083c756cc2,0.01,18.463800000000003,0.005,ETH,0xc02aaa39b223fe8d0a0e5c4f27ead9083c756cc2,uniswap,2,HELGA-WETH,0x5484094940909b16fe991d1adad6a40dfef63417,0x85dba58d67e021489ee1f843de7a7e1fc1ec60e7,0x91f963bc342304acdec15f1716199e8f32e6a3e66bdf88a9e9129fa2a21fa563,58,true +2024-03-10 08:09:23.000 UTC,2024-03-10 00:00:00.000 UTC,2024-03-01 00:00:00.000 UTC,Pepeboost,19403410,ethereum,371.51845134960644,Sell,0.09389271523478493,WETH,0xc02aaa39b223fe8d0a0e5c4f27ead9083c756cc2,8895.610034508,TRSCT,0xfe98796e0af4ababd716508429e51ff9ac1bb4d5,0.005,3.715184513496063,0.000938927152347849,ETH,0xc02aaa39b223fe8d0a0e5c4f27ead9083c756cc2,uniswap,2,TRSCT-WETH,0xd4fe3d36d8c405833972987c5fd977345aae2e8a,0x2c32c56471ddae47225ee82658c176945733b180,0x4290ee906089f82d1ba46320d3d2f0737a895009f701afc0d7adfc8e31fbb81d,55,true +2024-03-17 04:57:35.000 UTC,2024-03-17 00:00:00.000 UTC,2024-03-01 00:00:00.000 UTC,Pepeboost,19452334,ethereum,282.385224,Buy,9586055909.065296,Jason,0x5d101234df81ab0dfea96e089e3c3dd8840a6ac9,0.0792,WETH,0xc02aaa39b223fe8d0a0e5c4f27ead9083c756cc2,0.01,2.852376,0.0008,ETH,0xc02aaa39b223fe8d0a0e5c4f27ead9083c756cc2,uniswap,2,Jason-WETH,0x49cb6005bcb0aac703e724abe7e76786e9b46226,0xe8490eefeecdcf6cde83dd58ba084c40d98c6535,0xcd74520b2128a5896bfe82f63154cfa89d2febe57b6de5d78df92c63531a3ffd,104,true +2024-03-04 14:12:35.000 UTC,2024-03-04 00:00:00.000 UTC,2024-03-01 00:00:00.000 UTC,Pepeboost,19362317,ethereum,689.63202,Buy,87639994.84559023,TRUMP INU,0xd3fb8597d260efb2e693efd500d62a330a00f1eb,0.198,WETH,0xc02aaa39b223fe8d0a0e5c4f27ead9083c756cc2,0.01,6.96598,0.002,ETH,0xc02aaa39b223fe8d0a0e5c4f27ead9083c756cc2,uniswap,2,TRUMP INU-WETH,0xa20379e47b2b9b5a02557fd2c862d2915a509bd3,0xf64e814ffad5be14f87a28ab96f9f7112c02d402,0x4835dbed040a104d98fd2e8b8e765e9381dca73c351d28c8b33eb8f6be8106a1,75,true +2024-03-10 07:58:23.000 UTC,2024-03-10 00:00:00.000 UTC,2024-03-01 00:00:00.000 UTC,Pepeboost,19403356,ethereum,979.39215,Buy,710909441.5894133,BOYSCLUB,0x2d9996f3b9d2e73540fdbfdfe81d71e9e08cbf03,0.2475,WETH,0xc02aaa39b223fe8d0a0e5c4f27ead9083c756cc2,0.01,9.89285,0.0025,ETH,0xc02aaa39b223fe8d0a0e5c4f27ead9083c756cc2,uniswap,2,BOYSCLUB-WETH,0x7dbc4253e35f20be7164f0a3e0959c33136d007d,0x422412ee213110bc6a7ef3d1d14237893a862bb2,0xefd57d544fade3d4261a7edf0d4f6c0a752802fe79c06ae11a1b04709892141a,94,true +2024-03-02 11:30:23.000 UTC,2024-03-02 00:00:00.000 UTC,2024-03-01 00:00:00.000 UTC,Pepeboost,19347195,ethereum,676.22544,Buy,5989066734.089885,HELGA,0x78e3b2ee11950df78a35fd924e92fbb8d1403780,0.198,WETH,0xc02aaa39b223fe8d0a0e5c4f27ead9083c756cc2,0.01,6.83056,0.002,ETH,0xc02aaa39b223fe8d0a0e5c4f27ead9083c756cc2,uniswap,2,HELGA-WETH,0x5484094940909b16fe991d1adad6a40dfef63417,0x7523a8fdbcc92143cf523b6d771433369e915722,0x18f32fec182dba1359473a365d08eb6ead9701d38b8ab31b286976aec8582b61,44,true +2024-03-24 03:17:59.000 UTC,2024-03-24 00:00:00.000 UTC,2024-03-01 00:00:00.000 UTC,Pepeboost,19501691,ethereum,1317.00096,Buy,1829.0936718975227,DEAI,0x1495bc9e44af1f8bcb62278d2bec4540cf0c05ea,0.396,WETH,0xc02aaa39b223fe8d0a0e5c4f27ead9083c756cc2,0.01,13.303040000000001,0.004,ETH,0xc02aaa39b223fe8d0a0e5c4f27ead9083c756cc2,uniswap,3,DEAI-WETH,0x1385fc1fe0418ea0b4fcf7adc61fc7535ab7f80d,0x854afe28e369c5c2a11f6db6eb40474715c6f121,0xea9797fd00cee663435ccea89614ec3e15f232c9b71173363583727c4cda4eba,25,true +2024-03-28 04:34:23.000 UTC,2024-03-28 00:00:00.000 UTC,2024-03-01 00:00:00.000 UTC,Pepeboost,19530259,ethereum,788.4641517453738,Sell,0.22455688987963482,WETH,0xc02aaa39b223fe8d0a0e5c4f27ead9083c756cc2,169786.953716372,ZENO,0xd51e4965ad973e8c1e1f22369bb884e6914b012c,0.005,7.884641517453737,0.002245568898796348,ETH,0xc02aaa39b223fe8d0a0e5c4f27ead9083c756cc2,uniswap,2,WETH-ZENO,0xef64da9c4840b2c88b2f73b79db3c4e51e27f53a,0xd11f4f01e19d81f8f5abe1f72b1703abfa6b2bbe,0x7568957888e39759fb13c2f16ea0c6216fcbafe34a51ec6e8f9f051e7b4e2858,29,true +2024-03-16 18:38:23.000 UTC,2024-03-16 00:00:00.000 UTC,2024-03-01 00:00:00.000 UTC,Pepeboost,19449282,ethereum,797.8422523975116,Sell,0.22122464352984395,WETH,0xc02aaa39b223fe8d0a0e5c4f27ead9083c756cc2,3854750.491038198,vitalek,0xf5264e1673c9365e7c5d4d1d8b440bbf131ff435,0.005,7.978422523975114,0.002212246435298439,ETH,0xc02aaa39b223fe8d0a0e5c4f27ead9083c756cc2,uniswap,2,vitalek-WETH,0xc8733a25df841f25afeb97ac77e39909d80d5a37,0x9e2d558fccde1ba893ccd00a967dcb39385cae8f,0x9e43de67645117f2b226bdc4e8487fd840cec5640659a5fec57b982c830810c7,145,true +2024-03-01 15:37:23.000 UTC,2024-03-01 00:00:00.000 UTC,2024-03-01 00:00:00.000 UTC,Pepeboost,19341264,ethereum,530.3106900148174,Sell,0.15543473953989473,WETH,0xc02aaa39b223fe8d0a0e5c4f27ead9083c756cc2,5985742.476313816,VAI,0x0cbda0b3fafc4f3730e2c574f52c01c7d94a8a0c,0.005,5.303106900148173,0.001554347395398947,ETH,0xc02aaa39b223fe8d0a0e5c4f27ead9083c756cc2,uniswap,2,VAI-WETH,0x9021acf529ea6a068188e948891db3450bd9f35b,0x84c54ed3b718abd4fa7890b8ce140de6df4e3559,0xc7952373fbacede7463bf770c785da9a818d4063de3760962015079c33f1e661,57,true +2024-03-10 14:53:35.000 UTC,2024-03-10 00:00:00.000 UTC,2024-03-01 00:00:00.000 UTC,Pepeboost,19405416,ethereum,1776.9845572782094,Sell,0.45657597348347356,WETH,0xc02aaa39b223fe8d0a0e5c4f27ead9083c756cc2,899875.5930412533,vitalek,0xf5264e1673c9365e7c5d4d1d8b440bbf131ff435,0.005,17.769845572782092,0.004565759734834735,ETH,0xc02aaa39b223fe8d0a0e5c4f27ead9083c756cc2,uniswap,2,vitalek-WETH,0xc8733a25df841f25afeb97ac77e39909d80d5a37,0x7523a8fdbcc92143cf523b6d771433369e915722,0x6ae1f85413e33f71f5528b1dd620916b0e1abc5e993ec626ed4b7d6936029c42,80,true +2024-03-10 05:31:23.000 UTC,2024-03-10 00:00:00.000 UTC,2024-03-01 00:00:00.000 UTC,Pepeboost,19402622,ethereum,322.89277658668607,Sell,0.0816455851448454,WETH,0xc02aaa39b223fe8d0a0e5c4f27ead9083c756cc2,608817.981919313,BRETT,0x80ee5c641a8ffc607545219a3856562f56427fe9,0.005,3.2289277658668603,0.000816455851448454,ETH,0xc02aaa39b223fe8d0a0e5c4f27ead9083c756cc2,uniswap,2,BRETT-WETH,0x95f2610d0475d72e75bfcbd8bb7ecafe1f351b8d,0xca1857c70aa23f9b5620dbe2bd71b46cd4cb8386,0x6da3fe9d1d3fe30f4e67ed3392984d620805754c1581a4c06227d1e365a15d56,60,true +2024-03-14 07:10:23.000 UTC,2024-03-14 00:00:00.000 UTC,2024-03-01 00:00:00.000 UTC,Pepeboost,19431678,ethereum,789.6517200000001,Buy,758892204.8401858,SPEPE,0x7dafe897a6ff1d7b0b64f908e60e8665b61d53af,0.198,WETH,0xc02aaa39b223fe8d0a0e5c4f27ead9083c756cc2,0.01,7.97628,0.002,ETH,0xc02aaa39b223fe8d0a0e5c4f27ead9083c756cc2,uniswap,2,SPEPE-WETH,0xf4041d29ad20219188cb6f4a3ea0fd1b8192a856,0xe4fe0f848583560b8d58a11288cf3dcb1d97ba81,0xe3fe2589cfb03be13632d3f0c588b96a73192cf965ea771c254b2b8cb8ff2709,35,true +2024-03-27 06:32:59.000 UTC,2024-03-27 00:00:00.000 UTC,2024-03-01 00:00:00.000 UTC,Pepeboost,19523922,ethereum,7149.4632,Buy,2837722531.63614,GROK1.5,0xa711bcc2b6f5c4fc3dfaccc2a01148765cbbab1c,1.98,WETH,0xc02aaa39b223fe8d0a0e5c4f27ead9083c756cc2,0.01,72.2168,0.02,ETH,0xc02aaa39b223fe8d0a0e5c4f27ead9083c756cc2,uniswap,2,GROK1.5-WETH,0x8a4008961a27752e1c7178b73b9d5fa1d37fee3c,0x4f75a7ca7f6c527d051e400795307e5a395f1b05,0xeca5c9e2e45077a7fe9f4fb19cc4279d8d3556fa3a808c77a822352f9397e56b,46,true +2024-03-07 16:11:35.000 UTC,2024-03-07 00:00:00.000 UTC,2024-03-01 00:00:00.000 UTC,Pepeboost,19384405,ethereum,761.65056,Buy,35389.9155006131,HOSTAI,0x07e128e823d2b9b22edbda43820aa1a72de99613,0.198,WETH,0xc02aaa39b223fe8d0a0e5c4f27ead9083c756cc2,0.01,7.69344,0.002,ETH,0xc02aaa39b223fe8d0a0e5c4f27ead9083c756cc2,uniswap,2,HOSTAI-WETH,0x88ee95da6046bc3256d43b7c6883e61415b6e442,0xc7576de93acff71a4516476431a03a1a7f20f3a6,0xd4439e2a97d08d5ee847dc38b80e8b33957db593d5543c88784f24d266cef0c4,47,true +2024-03-07 19:50:35.000 UTC,2024-03-07 00:00:00.000 UTC,2024-03-01 00:00:00.000 UTC,Pepeboost,19385493,ethereum,4276.941256077766,Sell,1.1036555721767025,WETH,0xc02aaa39b223fe8d0a0e5c4f27ead9083c756cc2,2612.2416695698207,TAS,0x64d5fea7d2d600918b76159285994d6ed218f264,0.005,42.76941256077766,0.011036555721767025,ETH,0xc02aaa39b223fe8d0a0e5c4f27ead9083c756cc2,uniswap,2,TAS-WETH,0x40849deb556ff2c682088a12b0299ad821ae231a,0xa0a8cdf195b8f383339946a6d748ef6b1d55337e,0x410aa5ad1888333775a990fcb0faf140470b0b8e2036d20aa3e0a9b954963790,62,true +2024-03-12 17:37:11.000 UTC,2024-03-12 00:00:00.000 UTC,2024-03-01 00:00:00.000 UTC,Pepeboost,19420533,ethereum,341.20073688283304,Sell,0.08796962241282545,WETH,0xc02aaa39b223fe8d0a0e5c4f27ead9083c756cc2,894722874971.9131,Dinger,0xcd145189a12ccc3f9c622766fb20d00dc2059ead,0.005,3.4120073688283283,0.000879696224128254,ETH,0xc02aaa39b223fe8d0a0e5c4f27ead9083c756cc2,uniswap,2,Dinger-WETH,0x62f0a25f884cc9bf186196cc1cae15a58fce4732,0xccbb78b5b5066128c70310aaf87254e33054711f,0xef54f3fd55bf02228fdd390ffa11f60415c7c37c6375445c45aa4a5fc456c7b5,55,true +2024-03-15 23:39:35.000 UTC,2024-03-15 00:00:00.000 UTC,2024-03-01 00:00:00.000 UTC,Pepeboost,19443665,ethereum,1420.366789210197,Sell,0.380544410177255,WETH,0xc02aaa39b223fe8d0a0e5c4f27ead9083c756cc2,495160811.29605335,GROK1.5,0xa711bcc2b6f5c4fc3dfaccc2a01148765cbbab1c,0.005,14.203667892101969,0.003805444101772549,ETH,0xc02aaa39b223fe8d0a0e5c4f27ead9083c756cc2,uniswap,2,GROK1.5-WETH,0x8a4008961a27752e1c7178b73b9d5fa1d37fee3c,0x6ff42a9d716ffa5f8e234d00486065226e0376d4,0x7f9e9f40792db4c65e763bc10233eaafec8d163783840fcedadfcec35dea2196,23,true +2024-03-07 03:59:59.000 UTC,2024-03-07 00:00:00.000 UTC,2024-03-01 00:00:00.000 UTC,Pepeboost,19380769,ethereum,1311.07284,Buy,1158788900.8645384,DieHarder,0xeea5fb81328343984298819275bda5489e796e62,0.3465,WETH,0xc02aaa39b223fe8d0a0e5c4f27ead9083c756cc2,0.01,13.243160000000001,0.0035,ETH,0xc02aaa39b223fe8d0a0e5c4f27ead9083c756cc2,uniswap,2,DieHarder-WETH,0x70410f43d761c5134fdb6e427beb57c1849bac0f,0x79eb850503df9c01997c4ce4ee35a240cce9a45e,0x5cd2a3a05f9f173704256bad8e47e4aabb8abb3220065870a128002a2b5055d2,64,true +2024-03-18 04:24:23.000 UTC,2024-03-18 00:00:00.000 UTC,2024-03-01 00:00:00.000 UTC,Pepeboost,19459285,ethereum,12.078782121141744,Sell,0.003364600753529773,WETH,0xc02aaa39b223fe8d0a0e5c4f27ead9083c756cc2,845.6214538922925,RAI,0x049c3ee0c38742c1d4904ae957de66d2b240031d,0.005,0.12078782121141482,0.000033646007535297,ETH,0xc02aaa39b223fe8d0a0e5c4f27ead9083c756cc2,uniswap,2,RAI-WETH,0xd12dabe90a1b3dbbf11f257d9d57014ac5ca74cf,0xce23b9958e8e72c716a58041110d05bd705abfff,0xf672df35cd8d7454a7a018da265152a811c2bd05ad21600e1cf9f553e72c5f9e,25,true diff --git a/dbt_subprojects/dex/seeds/bot_trades/pepeboost/schema.yml b/dbt_subprojects/dex/seeds/bot_trades/pepeboost/schema.yml new file mode 100644 index 00000000000..dd18e224376 --- /dev/null +++ b/dbt_subprojects/dex/seeds/bot_trades/pepeboost/schema.yml @@ -0,0 +1,13 @@ +version: 2 + +seeds: + - name: pepeboost_ethereum_trades_seed + config: + column_types: + version: varchar + token_bought_address: varbinary + token_sold_address: varbinary + fee_token_address: varbinary + project_contract_address: varbinary + user: varbinary + tx_hash: varbinary From d8a98843372a68d2a3ffa50f3e96eda83e56fa6b Mon Sep 17 00:00:00 2001 From: bram-vdberg Date: Thu, 28 Nov 2024 13:04:06 +0100 Subject: [PATCH 14/18] Add new addresses for Fractal and Quasilabs (#7185) --- .../cow_protocol/ethereum/cow_protocol_ethereum_solvers.sql | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/dbt_subprojects/hourly_spellbook/models/_project/cow_protocol/ethereum/cow_protocol_ethereum_solvers.sql b/dbt_subprojects/hourly_spellbook/models/_project/cow_protocol/ethereum/cow_protocol_ethereum_solvers.sql index a97515ed8be..84d496300be 100644 --- a/dbt_subprojects/hourly_spellbook/models/_project/cow_protocol/ethereum/cow_protocol_ethereum_solvers.sql +++ b/dbt_subprojects/hourly_spellbook/models/_project/cow_protocol/ethereum/cow_protocol_ethereum_solvers.sql @@ -116,8 +116,11 @@ known_solver_metadata (address, environment, name) as ( (0x2224eAaCC7c2DBf85d5355BAb9E9271e01d30b55, 'prod', 'Sector_Finance'), (0x9DFc9Bb0FfF2dc96728D2bb94eaCee6ba3592351, 'prod', 'Rizzolver'), (0x6bf97aFe2D2C790999cDEd2a8523009eB8a0823f, 'prod', 'Portus'), + (0x95480d3f27658e73b2785d30beb0c847d78294c7, 'prod', 'FractalSolver'), + (0x00806DaA2Cfe49715eA05243FF259DeB195760fC, 'prod', 'Quasilabs'), + (0x7E2eF26AdccB02e57258784957922AEEFEe807e5, 'barn', 'FractalSolver'), (0x5131590ca2E9D3edC182581352b289dcaE83430c, 'barn', 'Portus'), - (0x2a2883ade8ce179265f12fc7b48a4b50b092f1fd, 'barn', 'Fractal'), + (0x2a2883ade8ce179265f12fc7b48a4b50b092f1fd, 'barn', 'FractalSolver'), (0x26B5e3bF135D3Dd05A220508dD61f25BF1A47cBD, 'barn', 'Rizzolver'), (0xA6A871b612bCE899b1CbBad6E545e5e47Da98b87, 'barn', 'Barter'), (0xa08B00576aeE8d8dd960E08298FAc9fD7C756e36, 'barn', 'Apollo'), From 9f37436e1fb9bb9f00279249be15b6871565a141 Mon Sep 17 00:00:00 2001 From: PA <50184410+peyha@users.noreply.github.com> Date: Thu, 28 Nov 2024 13:04:25 +0100 Subject: [PATCH 15/18] feat: add morpho to tokens (#7200) --- .../models/tokens/base/tokens_base_erc20.sql | 101 +++++++++--------- .../tokens/ethereum/tokens_ethereum_erc20.sql | 60 ++++++----- 2 files changed, 82 insertions(+), 79 deletions(-) diff --git a/dbt_subprojects/tokens/models/tokens/base/tokens_base_erc20.sql b/dbt_subprojects/tokens/models/tokens/base/tokens_base_erc20.sql index 027885e76fa..95d8dab348c 100644 --- a/dbt_subprojects/tokens/models/tokens/base/tokens_base_erc20.sql +++ b/dbt_subprojects/tokens/models/tokens/base/tokens_base_erc20.sql @@ -245,58 +245,58 @@ FROM (VALUES , (0x62a52757b580e7fd97203bd0408a7445741b5d5f, 'DT', 18) , (0x6985884c4392d348587b19cb9eaaf157f13271cd, 'ZRO', 18) , (0x64b88c73A5DfA78D1713fE1b4c69a22d7E0faAa7, 'MAV', 18) - , (0x0f1D1b7abAeC1Df25f2C4Db751686FC5233f6D3f, 'IMO', 18) + , (0x0f1D1b7abAeC1Df25f2C4Db751686FC5233f6D3f, 'IMO', 18) , (0x9beec80e62aa257ced8b0edd8692f79ee8783777, 'TIMI', 18) - , (0x013b6451e2b2131052d3473593bd786df5143faf, 'FRENCH', 18) - , (0x31b28012f61fc3600e1c076bafc9fd997fb2da90, 'MRSMIGGLES', 18) - , (0x420b0fa3de2efcf2b2fd04152eb1df36a09717cd, 'KING', 18) + , (0x013b6451e2b2131052d3473593bd786df5143faf, 'FRENCH', 18) + , (0x31b28012f61fc3600e1c076bafc9fd997fb2da90, 'MRSMIGGLES', 18) + , (0x420b0fa3de2efcf2b2fd04152eb1df36a09717cd, 'KING', 18) , (0x190b2aA820495c0e92840E8FA699741976cd6439, 'rETH', 18) , (0x85e90a5430af45776548adb82ee4cd9e33b08077, 'DINO', 18) , (0x7c6b91D9Be155A6Db01f749217d76fF02A7227F2, 'BAL', 18) - , (0xa1e4240C9E9B8db3ba9e7625A9571c3B0FF71988, 'PJ', 18) - , (0x340c070260520ae477b88caa085a33531897145b, '9MM', 18) - , (0x38d513ec43dda20f323f26c7bef74c5cf80b6477, 'CARLO', 18) - , (0x8d3419b9a18651f3926a205ee0b1acea1e7192de, 'LOA', 18) - , (0xed899bfdb28c8ad65307fa40f4acab113ae2e14c, 'ROOST', 18) - , (0x6bd81aad9b25ad1e0b99c47ed01b34eacf4b8be7, 'BODA', 18) - , (0x75570e1189ffc1d63b3417cdf0889f87cd3e9bd1, 'Bunny', 18) - , (0x1c7a460413dd4e964f96d8dfc56e7223ce88cd85, 'SEAM', 18) - , (0x8319767a7b602f88e376368dca1b92d38869b9b4, 'PEACH', 18) - , (0x42069de48741db40aef864f8764432bbccbd0b69, 'BETS', 18) - , (0x1ccb4b14a11e0f2994a7ecbbd4cc69632f4c7c76, 'CCC', 18) - , (0xacd1caef47e4c47bafe8a51b3f4305fc38203b7a, 'LUNE', 18) - , (0xd327d36eb6e1f250d191cd62497d08b4aaa843ce, 'FOMO', 9) - , (0x00e57ec29ef2ba7df07ad10573011647b2366f6d, 'TOSHE', 18) - , (0xf1b4ddf712e108cf43711b1c39f2fddb0d5ce243, 'BASE', 18) - , (0x87c211144b1d9bdaa5a791b8099ea4123dc31d21, 'BCP', 18) - , (0xa5fbc981ebd8f4e791be8bd7a9b7cb144d145bf9, 'MEMEGOD', 18) - , (0x30d19fb77c3ee5cfa97f73d72c6a1e509fa06aef, 'CONDO', 18) - , (0x8b781e0f2967571e96d5d95b3cb61e27785eeaa5, 'BORKIE', 18) - , (0x3159fb5589acd6bf9f82eb0efe8382ed55aed8fd, 'APU', 18) - , (0x029635e83bc26c96006298e05798cdda76a2e7a5, 'SYD', 18) - , (0x314da69de85145fdd5b7580971e9db0388a2cdc4, 'BSW', 18) - , (0x8ad5b9007556749de59e088c88801a3aaa87134b, 'FARTHER', 18) - , (0x64cc19a52f4d631ef5be07947caba14ae00c52eb, 'KIBBLE', 18) - , (0x45096bd2871911ee82a3084d77a01efa3c9c6733, 'BUB', 18) - , (0x44fccd875c8e80b37f38530f434386a6d59141bb, 'DTB', 18) - , (0xfd008f937b4d73eeb00cf74ce90c392be5f07f96, 'MOON', 18) - , (0x160381343365de6d5e7d32172987b3e6bd1cce71, 'Mari', 18) - , (0x546d239032b24eceee0cb05c92fc39090846adc7, 'SEED', 18) - , (0x4c584cbc3a221998dc003349e1c12a4179e97b25, 'BMONEY', 18) - , (0x9721b1ce4ffd8af047bbdfd87e4e20ddc544513a, 'BROS', 18) - , (0xc41ba5737baf6bd0ccd5daf7eee39874e4ad45ff, 'DEGENS', 18) - , (0x10c1b6f768e13c624a4a23337f1a5ba5c9be0e4b, 'WARPIE', 18) - , (0xfd9fa4f785331ce88b5af8994a047ba087c705d8, '$blue', 18) + , (0xa1e4240C9E9B8db3ba9e7625A9571c3B0FF71988, 'PJ', 18) + , (0x340c070260520ae477b88caa085a33531897145b, '9MM', 18) + , (0x38d513ec43dda20f323f26c7bef74c5cf80b6477, 'CARLO', 18) + , (0x8d3419b9a18651f3926a205ee0b1acea1e7192de, 'LOA', 18) + , (0xed899bfdb28c8ad65307fa40f4acab113ae2e14c, 'ROOST', 18) + , (0x6bd81aad9b25ad1e0b99c47ed01b34eacf4b8be7, 'BODA', 18) + , (0x75570e1189ffc1d63b3417cdf0889f87cd3e9bd1, 'Bunny', 18) + , (0x1c7a460413dd4e964f96d8dfc56e7223ce88cd85, 'SEAM', 18) + , (0x8319767a7b602f88e376368dca1b92d38869b9b4, 'PEACH', 18) + , (0x42069de48741db40aef864f8764432bbccbd0b69, 'BETS', 18) + , (0x1ccb4b14a11e0f2994a7ecbbd4cc69632f4c7c76, 'CCC', 18) + , (0xacd1caef47e4c47bafe8a51b3f4305fc38203b7a, 'LUNE', 18) + , (0xd327d36eb6e1f250d191cd62497d08b4aaa843ce, 'FOMO', 9) + , (0x00e57ec29ef2ba7df07ad10573011647b2366f6d, 'TOSHE', 18) + , (0xf1b4ddf712e108cf43711b1c39f2fddb0d5ce243, 'BASE', 18) + , (0x87c211144b1d9bdaa5a791b8099ea4123dc31d21, 'BCP', 18) + , (0xa5fbc981ebd8f4e791be8bd7a9b7cb144d145bf9, 'MEMEGOD', 18) + , (0x30d19fb77c3ee5cfa97f73d72c6a1e509fa06aef, 'CONDO', 18) + , (0x8b781e0f2967571e96d5d95b3cb61e27785eeaa5, 'BORKIE', 18) + , (0x3159fb5589acd6bf9f82eb0efe8382ed55aed8fd, 'APU', 18) + , (0x029635e83bc26c96006298e05798cdda76a2e7a5, 'SYD', 18) + , (0x314da69de85145fdd5b7580971e9db0388a2cdc4, 'BSW', 18) + , (0x8ad5b9007556749de59e088c88801a3aaa87134b, 'FARTHER', 18) + , (0x64cc19a52f4d631ef5be07947caba14ae00c52eb, 'KIBBLE', 18) + , (0x45096bd2871911ee82a3084d77a01efa3c9c6733, 'BUB', 18) + , (0x44fccd875c8e80b37f38530f434386a6d59141bb, 'DTB', 18) + , (0xfd008f937b4d73eeb00cf74ce90c392be5f07f96, 'MOON', 18) + , (0x160381343365de6d5e7d32172987b3e6bd1cce71, 'Mari', 18) + , (0x546d239032b24eceee0cb05c92fc39090846adc7, 'SEED', 18) + , (0x4c584cbc3a221998dc003349e1c12a4179e97b25, 'BMONEY', 18) + , (0x9721b1ce4ffd8af047bbdfd87e4e20ddc544513a, 'BROS', 18) + , (0xc41ba5737baf6bd0ccd5daf7eee39874e4ad45ff, 'DEGENS', 18) + , (0x10c1b6f768e13c624a4a23337f1a5ba5c9be0e4b, 'WARPIE', 18) + , (0xfd9fa4f785331ce88b5af8994a047ba087c705d8, '$blue', 18) , (0x420697291f6ce9fbb34e9feddd61868ca2f81f5c, 'MONEYBEE', 18) - , (0x9a589baa21f0492bd4f35a33915986db56d8eecf, 'CATZ', 18) - , (0x2156006a207a793b4069a2b72be58dc2bd759232, 'COIN', 18) - , (0x17931cfc3217261ce0fa21bb066633c463ed8634, 'BASED', 18) - , (0xb9ea61ed1c9942011e5e44f7fae3efddbf277b1e, 'CHIP', 18) + , (0x9a589baa21f0492bd4f35a33915986db56d8eecf, 'CATZ', 18) + , (0x2156006a207a793b4069a2b72be58dc2bd759232, 'COIN', 18) + , (0x17931cfc3217261ce0fa21bb066633c463ed8634, 'BASED', 18) + , (0xb9ea61ed1c9942011e5e44f7fae3efddbf277b1e, 'CHIP', 18) , (0xea6f7e7e0f46a9e0f4e2048eb129d879f609d632, 'PERCY', 18) - , (0x05ad8adc21778f2a2c6800852f05fb949ef507dc, 'HACHI', 18) - , (0x8fe815417913a93ea99049fc0718ee1647a2a07c, 'XSWAP', 18) - , (0x9260409d4719c235c7c199106219f9616d66bde0, 'ArabWif', 18) - , (0x1c22374032e7e5a1bbde3d943f5deb310db060dd, 'onchain', 18) + , (0x05ad8adc21778f2a2c6800852f05fb949ef507dc, 'HACHI', 18) + , (0x8fe815417913a93ea99049fc0718ee1647a2a07c, 'XSWAP', 18) + , (0x9260409d4719c235c7c199106219f9616d66bde0, 'ArabWif', 18) + , (0x1c22374032e7e5a1bbde3d943f5deb310db060dd, 'onchain', 18) , (0x57d0ffb02f73aa09dd22d7e81d6c0c81054ab5d9, 'DRAGGY', 18) , (0xb59c8912c83157a955f9d715e556257f432c35d7, 'TRUF', 18) , (0x619c4bbbd65f836b78b36cbe781513861d57f39d, 'BWB', 18) @@ -390,12 +390,12 @@ FROM (VALUES , (0xdbcae648ebea8c950b616e10e28da09d1ab4a06d, 'ELIZ', 18) , (0x0a14ef61afb32e5ca672e021784f71705ac14908, 'NULL', 18) , (0x7e72d6410803c40e73806f2a72e3eade5d075cc0, 'MOB', 18) - , (0x24da41bae4d7014ec6faf5403d2db0aa1510358d, 'COPY', 18) + , (0x24da41bae4d7014ec6faf5403d2db0aa1510358d, 'COPY', 18) , (0x02e79d42c3297cd4154a1b52b4a3b27cd75762f1, 'FFTP', 9) , (0xc1bf21674a3d782ee552d835863d065b7a89d619, 'MISHA', 18) - , (0xfc21540d6b89667d167d42086e1feb04da3e9b21, 'INFI', 18) + , (0xfc21540d6b89667d167d42086e1feb04da3e9b21, 'INFI', 18) , (0x9a33406165f562e16c3abd82fd1185482e01b49a, 'TALENT', 18) - , (0xde31526ad9f7e8d271e85677e7ec5882f801a34b, 'First Steps Into Sovereignty', 18) + , (0xde31526ad9f7e8d271e85677e7ec5882f801a34b, 'First Steps Into Sovereignty', 18) , (0xd89cd06d54e5e5af76221c00b60c6b567b17b863, 'TAPPY', 18) , (0x1057cc4ef19cc86a1b034145225f47e354f6f95b, 'FLICKY', 18) , (0x8c3ed1f4d142965d19e0db48ae189dd9448ad95a, 'WOT', 18) @@ -405,5 +405,6 @@ FROM (VALUES , (0x574178357661527601482b79af6bb1ff7cc1306a, '1984', 18) , (0x63e14921ba4546cb15087d574346574eee055812, 'PVP', 18) , (0xd68470491eb513365322810e55e0eba80785dfdd, '243M theft', 18) - , (0xb0505e5a99abd03d94a1169e638b78edfed26ea4, 'uSUI', 18) + , (0xb0505e5a99abd03d94a1169e638b78edfed26ea4, 'uSUI', 18) + , (0xBAa5CC21fd487B8Fcc2F632f3F4E8D37262a0842, 'MORPHO', 18) ) AS temp_table (contract_address, symbol, decimals) diff --git a/dbt_subprojects/tokens/models/tokens/ethereum/tokens_ethereum_erc20.sql b/dbt_subprojects/tokens/models/tokens/ethereum/tokens_ethereum_erc20.sql index 295b395a1d2..10b6ea5c913 100644 --- a/dbt_subprojects/tokens/models/tokens/ethereum/tokens_ethereum_erc20.sql +++ b/dbt_subprojects/tokens/models/tokens/ethereum/tokens_ethereum_erc20.sql @@ -4564,12 +4564,12 @@ FROM (VALUES , ( 0x5983B89FA184f14917013B9C3062afD9434C5b03, 'POWER', 0) , ( 0x988567FE094570cCE1FFdA29D1f2d842B70492be, 'ZERO', 6) , ( 0x10dea67478c5f8c5e2d90e5e9b26dbe60c54d800, 'TAIKO', 18) - , ( 0x80ac24aA929eaF5013f6436cdA2a7ba190f5Cc0b, 'syrupUSDC', 6) - , ( 0xa8258deE2a677874a48F5320670A869D74f0cbC1, 'WAR', 18) + , ( 0x80ac24aA929eaF5013f6436cdA2a7ba190f5Cc0b, 'syrupUSDC', 6) + , ( 0xa8258deE2a677874a48F5320670A869D74f0cbC1, 'WAR', 18) , ( 0xbeef69ac7870777598a04b2bd4771c71212e6abc, 'steakLRT', 18) , ( 0x84631c0d0081fde56deb72f6de77abbbf6a9f93a, 'Re7LRT', 18) , ( 0x7a4effd87c2f3c55ca251080b1343b605f327e3a, 'rstETH', 18) - , ( 0x5fd13359ba15a84b76f7f87568309040176167cd, 'amprETH', 18) + , ( 0x5fd13359ba15a84b76f7f87568309040176167cd, 'amprETH', 18) , ( 0xC8E621909D71F555D2A5A90f68DD9EE0C55514De, 'SATORI', 18) , ( 0x61134511187a9a2df38d10dbe07ba2e8e5563967, 's_aWETH', 18) , ( 0xd1c117319b3595fbc39b471ab1fd485629eb05f2, 'vaETH', 18) @@ -4581,41 +4581,41 @@ FROM (VALUES , ( 0xc14900dfb1aa54e7674e1ecf9ce02b3b35157ba5, 'vaFRAX', 18) , ( 0xce4a49d7ed99c7c8746b713ee2f0c9aa631688d8, 's_aDAI', 18) , ( 0xf591d878608e2e5c7d4f1e499330f4ab9bbae37a, 's_aUSDC', 6) - , ( 0x73a15fed60bf67631dc6cd7bc5b6e8da8190acf5, 'USD0', 18) + , ( 0x73a15fed60bf67631dc6cd7bc5b6e8da8190acf5, 'USD0', 18) , ( 0x2E3CFE45E3EE7C017277f22e35d2f29edC99d570, 'wDAG', 8) , ( 0x7448c7456a97769F6cD04F1E83A4a23cCdC46aBD, 'MAV', 18) , ( 0x63a0964A36c34E81959da5894ad888800e17405b, 'ShezETH', 18) , ( 0xDfFb77dB95E16B791178D28CF994b13E84036076, 'BLU', 18) , ( 0x866a2bf4e572cbcf37d5071a7a58503bfb36be1b, 'M', 6) - , ( 0x20157dbabb84e3bbfe68c349d0d44e48ae7b5ad2, 'dlcBTC', 8) - , ( 0xe1B4d34E8754600962Cd944B535180Bd758E6c2e, 'agETH', 18) - , ( 0x5C5b196aBE0d54485975D1Ec29617D42D9198326, 'stdeUSD', 18) - , ( 0x6dc3ce9c57b20131347fdc9089d740daf6eb34c5, 'balETH', 18) - , ( 0x786f4782d1a5c602ea30bc4a95154110b9e231ea, 'shezBTC', 8) - , ( 0x09db87a538bd693e9d08544577d5ccfaa6373a48, 'ynETH', 18) - , ( 0x3b50805453023a91a8bf641e279401a0b23fa6f9, 'REZ', 18) - , ( 0x7777cec341e7434126864195adef9b05dcc3489c, 'ONI', 18) - , ( 0x3ffeea07a27fab7ad1df5297fa75e77a43cb5790, 'PEIPEI', 18) - , ( 0x00f116ac0c304c570daaa68fa6c30a86a04b5c5f, 'INF', 18) - , ( 0x777be1c6075c20184c4fd76344b7b0b7c858fe6b, 'BAR', 18) - , ( 0xaa32f860756f6dcd5a684dc2aadf6c0921257a9a, 'YUNA', 18) - , ( 0xa045fe936e26e1e1e1fb27c1f2ae3643acde0171, 'KAI', 9) - , ( 0x9e566e28c61690f8afe0468f523e137b1ff29f01, 'GNCAT', 9) - , ( 0x7f911119435d8ded9f018194b4b6661331379a3d, 'FWOG', 18) - , ( 0x6985884c4392d348587b19cb9eaaf157f13271cd, 'ZRO', 18) - , ( 0x7b0df1cd724ec34ec9bc4bd19749b01afb490761, 'KOIN', 9) - , ( 0xb60fdf036f2ad584f79525b5da76c5c531283a1b, 'NEMO', 9) - , ( 0x5b342f03d126314d925fa57a45654f92905e6451, 'MNTA', 18) - , ( 0x8149745670881d99700078ede5903a1a7bebe262, 'PEPAY', 18) - , ( 0xb6f4d11d6c274f8f0a55d101b65223a45a3d1128, 'eBAKE', 9) - , ( 0xb35875b40f5ea5483c84ccd6e4ab0afc42a3c355, 'Nerio', 9) + , ( 0x20157dbabb84e3bbfe68c349d0d44e48ae7b5ad2, 'dlcBTC', 8) + , ( 0xe1B4d34E8754600962Cd944B535180Bd758E6c2e, 'agETH', 18) + , ( 0x5C5b196aBE0d54485975D1Ec29617D42D9198326, 'stdeUSD', 18) + , ( 0x6dc3ce9c57b20131347fdc9089d740daf6eb34c5, 'balETH', 18) + , ( 0x786f4782d1a5c602ea30bc4a95154110b9e231ea, 'shezBTC', 8) + , ( 0x09db87a538bd693e9d08544577d5ccfaa6373a48, 'ynETH', 18) + , ( 0x3b50805453023a91a8bf641e279401a0b23fa6f9, 'REZ', 18) + , ( 0x7777cec341e7434126864195adef9b05dcc3489c, 'ONI', 18) + , ( 0x3ffeea07a27fab7ad1df5297fa75e77a43cb5790, 'PEIPEI', 18) + , ( 0x00f116ac0c304c570daaa68fa6c30a86a04b5c5f, 'INF', 18) + , ( 0x777be1c6075c20184c4fd76344b7b0b7c858fe6b, 'BAR', 18) + , ( 0xaa32f860756f6dcd5a684dc2aadf6c0921257a9a, 'YUNA', 18) + , ( 0xa045fe936e26e1e1e1fb27c1f2ae3643acde0171, 'KAI', 9) + , ( 0x9e566e28c61690f8afe0468f523e137b1ff29f01, 'GNCAT', 9) + , ( 0x7f911119435d8ded9f018194b4b6661331379a3d, 'FWOG', 18) + , ( 0x6985884c4392d348587b19cb9eaaf157f13271cd, 'ZRO', 18) + , ( 0x7b0df1cd724ec34ec9bc4bd19749b01afb490761, 'KOIN', 9) + , ( 0xb60fdf036f2ad584f79525b5da76c5c531283a1b, 'NEMO', 9) + , ( 0x5b342f03d126314d925fa57a45654f92905e6451, 'MNTA', 18) + , ( 0x8149745670881d99700078ede5903a1a7bebe262, 'PEPAY', 18) + , ( 0xb6f4d11d6c274f8f0a55d101b65223a45a3d1128, 'eBAKE', 9) + , ( 0xb35875b40f5ea5483c84ccd6e4ab0afc42a3c355, 'Nerio', 9) , ( 0xf7554eac0bf20d702e69d08c425e817abb976aea, 'MAHA', 18) , ( 0x3742f3fcc56b2d46c7b8ca77c23be60cd43ca80a, 'stAVAIL', 18) , ( 0xd8eb27a94e610a7b859758ed1f2d47ad224bfaaa, 'GOPEPE', 18) , ( 0xf7760c2c0677078b1d1b6b8463f1de1c6c91a9f8, 'PP', 18) , ( 0x78128d17a89e5fe51bd0e757cfbc5703c34c7fb6, '$EWT', 18) , ( 0xf1ff59fa458bee12d6526dbd92290090bf670e29, 'fluffy', 18) - , ( 0xffe9ae172dd1fc9929ba6902873e09fadf81afd7, 'GLOW', 18) + , ( 0xffe9ae172dd1fc9929ba6902873e09fadf81afd7, 'GLOW', 18) , ( 0xb146823fb8ea064d14ba1a52e3e55cde09afff2d, 'EYE', 18) , ( 0x88ce174c655b6d11210a069b2c106632dabdb068, 'YAWN', 18) , ( 0xf6b6235f6725f84457f5d6f0b0aa3c962a079977, 'WBKN', 18) @@ -4643,13 +4643,13 @@ FROM (VALUES , ( 0x0ba45a8b5d5575935b8158a88c631e9f9c95a2e5, 'TRB', 18) , ( 0xf0a949b935e367a94cdfe0f2a54892c2bc7b2131, 'ultraETHs', 18) , ( 0x5e362eb2c0706bd1d134689ec75176018385430b, 'DVstETH', 18) - , ( 0xcbc632833687dacdcc7dfac96f6c5989381f4b47, 'ultraETHs', 18) + , ( 0xcbc632833687dacdcc7dfac96f6c5989381f4b47, 'ultraETHs', 18) , ( 0xa684eaf215ad323452e2b2bf6f817d4aa5c116ab, 'lpETH', 18) , ( 0x100db67f41a2df3c32cc7c0955694b98339b7311, 'MONEY', 18) , ( 0xcbb7c0000ab88b473b1f5afd9ef808440eed33bf, 'cbBTC', 8) , ( 0x66b5228cfd34d9f4d9f03188d67816286c7c0b74, 'VOLT', 18) , ( 0x4d1c297d39c5c1277964d0e3f8aa901493664530, 'PUFER', 18) - , ( 0x460372866fe1448de1549cebdb0539f4075a2aa8, 'SHERK', 9) + , ( 0x460372866fe1448de1549cebdb0539f4075a2aa8, 'SHERK', 9) , ( 0x0a6E7Ba5042B38349e437ec6Db6214AEC7B35676, 'SWELL', 18) , ( 0xf9B24C9364457Ea85792179D285855753549eBAa, 'slisBNB', 18) , ( 0x230ea9aed5d08afdb22cd3c06c47cf24ad501301, 'SPX2.0', 18) @@ -4721,4 +4721,6 @@ FROM (VALUES , ( 0xaf270c38ff895ea3f95ed488ceace2386f038249, 'stataEthDAI', 18) , ( 0x00f2a835758b33f3ac53516ebd69f3dc77b0d152, 'stataEthPYUSD', 6) , ( 0x7cc6694cf75c18d488d16fb4bf3c71a3b31cc7fb, 'stataEthETHx', 18) + , ( 0x58D97B57BB95320F9a05dC918Aef65434969c2B2, 'MORPHO', 18) + , ( 0x9994E35Db50125E0DF82e4c2dde62496CE330999, 'legacyMORPHO', 18) ) AS temp_table (contract_address, symbol, decimals) From 38d67961337dafe3ca6f6f4052826242cfc57257 Mon Sep 17 00:00:00 2001 From: synthquest <112836792+Synthquest@users.noreply.github.com> Date: Thu, 28 Nov 2024 07:04:46 -0500 Subject: [PATCH 16/18] add base cex addresses (#7201) --- .../addresses/chains/cex_evms_addresses.sql | 27 ++++++++++++++++++- 1 file changed, 26 insertions(+), 1 deletion(-) diff --git a/dbt_subprojects/hourly_spellbook/models/_sector/cex/addresses/chains/cex_evms_addresses.sql b/dbt_subprojects/hourly_spellbook/models/_sector/cex/addresses/chains/cex_evms_addresses.sql index dacb8ab17e9..18acb039b82 100644 --- a/dbt_subprojects/hourly_spellbook/models/_sector/cex/addresses/chains/cex_evms_addresses.sql +++ b/dbt_subprojects/hourly_spellbook/models/_sector/cex/addresses/chains/cex_evms_addresses.sql @@ -5,7 +5,7 @@ post_hook='{{ expose_spells(\'["ethereum", "arbitrum", "avalanche_c", "bnb", "fantom", "optimism", "polygon", "zksync", "zora", "celo", "base"]\', "sector", "cex", - \'["hildobby", "soispoke", "web3_data", "msilb7", "Henrystats", "sankinyue"]\') }}')}} + \'["hildobby", "soispoke", "web3_data", "msilb7", "Henrystats", "sankinyue", "synthquest"]\') }}')}} SELECT address, cex_name, distinct_name, added_by, added_date FROM (VALUES @@ -150,6 +150,7 @@ FROM (VALUES , (0x4230c402c08cb66dcf3820649a115e54661fce9d, 'Bybit', 'Bybit 8', 'hildobby', date '2023-11-15') , (0x3d5202a0564de9b05ecd07c955bcca964585ea03, 'Bybit', 'Bybit 9', 'hildobby', date '2024-05-09') , (0x1e32760a3285550278aeafa776e5641bc581c845, 'Bybit', 'Bybit 10', 'hildobby', date '2024-10-06') + , (0x88a1493366d48225fc3cefbdae9ebb23e323ade3, 'Bybit', 'Bybit 11', 'synthquest', date '2024-11-27') , (0xab97925eb84fe0260779f58b7cb08d77dcb1ee2b, 'Bybit', 'Bybit Gas Supplier 1', 'hildobby', date '2023-11-15') , (0xa95b83af96d0b8a90bd507f2bd82ad8f3dbb86bc, 'Bybit', 'Bybit Gas Supplier 2', 'hildobby', date '2023-11-20') , (0xf65d698d18bc37bf36e4c8d4fe4f051ef570e2b6, 'Bybit', 'Bybit Gas Supplier 3', 'hildobby', date '2023-11-23') @@ -260,6 +261,30 @@ FROM (VALUES , (0x3dd87411a3754deea8cc52c4cf57e2fc254924cc, 'Coinbase', 'Coinbase 56', 'hildobby', date '2024-05-04') , (0x441cacfd43856409b163b90e094bb42aeb70a70e, 'Coinbase', 'Coinbase 57', 'hildobby', date '2024-05-25') , (0xa14d57f5ea867572b0d239798d2c1dde13153902, 'Coinbase', 'Coinbase 58', 'hildobby', date '2024-05-25') + , (0x9ebe8ae7dbc0285b04e93dab86a081ca32ccf52e, 'Coinbase', 'Coinbase 59', 'synthquest', date '2024-11-27') + , (0x3dc474a2a65507f32b05c5f80d852515b25b2134, 'Coinbase', 'Coinbase 60', 'synthquest', date '2024-11-27') + , (0xcf63fc571adcec4fd4a750ecacc3af1f5b748101, 'Coinbase', 'Coinbase 61', 'synthquest', date '2024-11-27') + , (0x00aac037c2ca137972f963693d38a57d0e9f7475, 'Coinbase', 'Coinbase 62', 'synthquest', date '2024-11-27') + , (0x60eb0250e3a428a51ccb1e44e0aadbd1fd213ff3, 'Coinbase', 'Coinbase 63', 'synthquest', date '2024-11-27') + , (0x336307f2d8390035ba926a61a86b45ca9dc91e57, 'Coinbase', 'Coinbase 64', 'synthquest', date '2024-11-27') + , (0x57a7560d0ec28065762203c0d633943298eac7c0, 'Coinbase', 'Coinbase 65', 'synthquest', date '2024-11-27') + , (0x0b0a5886664376f59c351ba3f598c8a8b4d0a6f3, 'Coinbase', 'Coinbase 66', 'synthquest', date '2024-11-27') + , (0xee81b5afc73cf528778e0ed98622e434e5efadb4, 'Coinbase', 'Coinbase 67', 'synthquest', date '2024-11-27') + , (0x9b4fc9e22b46487f0810ef5dfa230b9f139e5179, 'Coinbase', 'Coinbase 68', 'synthquest', date '2024-11-27') + , (0x95f90ce2e3abaed29eeebdb42e1fdb146e0f848a, 'Coinbase', 'Coinbase 69', 'synthquest', date '2024-11-27') + , (0x122fdd9fecbc82f7d4237c0549a5057e31c8ef8d, 'Coinbase', 'Coinbase 70', 'synthquest', date '2024-11-27') + , (0x8196f70b2c17ba58d8ef56ad62087ee8231be33a, 'Coinbase', 'Coinbase 71', 'synthquest', date '2024-11-27') + , (0xadbbe373b5b5f72c59c0311cffbded51f0c5f434, 'Coinbase', 'Coinbase 72', 'synthquest', date '2024-11-27') + , (0x67857ee12929e74082f1cae64ef4221830c39113, 'Coinbase', 'Coinbase 73', 'synthquest', date '2024-11-27') + , (0xa2908f1758d1cc3990f4a2da8dea0aa2ecf1b913, 'Coinbase', 'Coinbase 74', 'synthquest', date '2024-11-27') + , (0xd451e3919950963e9c1ca2f78a987dbd7937c0fb, 'Coinbase', 'Coinbase 75', 'synthquest', date '2024-11-27') + , (0x27724b0d4fb98a89a092e6a4adbc09154c182637, 'Coinbase', 'Coinbase 76', 'synthquest', date '2024-11-27') + , (0x7c310a03f4cfa19f7f3d7f36dd3e05828629fa78, 'Coinbase', 'Coinbase 77', 'synthquest', date '2024-11-27') + , (0x21bd501f86a0b5ce0907651df3368da905b300a9, 'Coinbase', 'Coinbase 78', 'synthquest', date '2024-11-27') + , (0x7c41fdced2ea646ed85665d1a9b28e6632b61c41, 'Coinbase', 'Coinbase 79', 'synthquest', date '2024-11-27') + , (0x607094ed3a8361bb5e94dd21bcbef2997b687478, 'Coinbase', 'Coinbase 80', 'synthquest', date '2024-11-27') + , (0x1b4c15b991543a6082213a88276e7a83c9985676, 'Coinbase', 'Coinbase 81', 'synthquest', date '2024-11-27') + , (0xd5c41fd4a31eaaf5559ffcc60ec051fcb8ecc375, 'Coinbase', 'Coinbase 82', 'synthquest', date '2024-11-27') , (0xa090e606e30bd747d4e6245a1517ebe430f0057e, 'Coinbase', 'Coinbase Miscellaneous', 'hildobby', date '2022-08-28') , (0xf6874c88757721a02f47592140905c4336dfbc61, 'Coinbase', 'Coinbase Commerce', 'hildobby', date '2022-08-28') , (0x881d4032abe4188e2237efcd27ab435e81fc6bb1, 'Coinbase', 'Coinbase Commerce 2', 'hildobby', date '2022-08-28') From 8a98cbffed26dd7410ed49c2537951b8a7ff549c Mon Sep 17 00:00:00 2001 From: ppclunghe <90045511+ppclunghe@users.noreply.github.com> Date: Thu, 28 Nov 2024 13:05:32 +0100 Subject: [PATCH 17/18] Lido accounting CSM/stonks update (#7190) * Lido accounting CSM/stonks update * revenue fees fix --- ...ido_accounting_ethereum_other_expenses.sql | 36 ++++++------------- .../lido_accounting_ethereum_other_income.sql | 28 ++++++--------- .../lido_accounting_ethereum_revenue.sql | 10 +++--- 3 files changed, 24 insertions(+), 50 deletions(-) diff --git a/dbt_subprojects/hourly_spellbook/models/_project/lido/accounting/ethereum/lido_accounting_ethereum_other_expenses.sql b/dbt_subprojects/hourly_spellbook/models/_project/lido/accounting/ethereum/lido_accounting_ethereum_other_expenses.sql index 0a2f233dc5d..288a999e1c1 100644 --- a/dbt_subprojects/hourly_spellbook/models/_project/lido/accounting/ethereum/lido_accounting_ethereum_other_expenses.sql +++ b/dbt_subprojects/hourly_spellbook/models/_project/lido/accounting/ethereum/lido_accounting_ethereum_other_expenses.sql @@ -18,13 +18,13 @@ with tokens AS ( select * from (values (0x5A98FcBEA516Cf06857215779Fd812CA3beF1B32), --LDO - (0x6B175474E89094C44Da98b954EedeAC495271d0F), --DAI - (0xA0b86991c6218b36c1d19D4a2e9Eb0cE3606eB48), --USDC + (0x6B175474E89094C44Da98b954EedeAC495271d0F), --DAI + (0xA0b86991c6218b36c1d19D4a2e9Eb0cE3606eB48), --USDC (0xdAC17F958D2ee523a2206206994597C13D831ec7), -- USDT - (0xc02aaa39b223fe8d0a0e5c4f27ead9083c756cc2), --WETH - (0x7D1AfA7B718fb893dB30A3aBc0Cfc608AaCfeBB0), --MATIC - (0xae7ab96520DE3A18E5e111B5EaAb095312D7fE84), --stETH - (0x7f39C581F595B53c5cb19bD0b3f8dA6c935E2Ca0) --wstETH + (0xc02aaa39b223fe8d0a0e5c4f27ead9083c756cc2), --WETH + (0x7D1AfA7B718fb893dB30A3aBc0Cfc608AaCfeBB0), --MATIC + (0xae7ab96520DE3A18E5e111B5EaAb095312D7fE84), --stETH + (0x7f39C581F595B53c5cb19bD0b3f8dA6c935E2Ca0) --wstETH ) as tokens(address)), @@ -99,7 +99,7 @@ dai_referral_payments_addr AS ( steth_referral_payments_addr AS ( SELECT _recipient AS address FROM {{source('lido_ethereum','AllowedRecipientsRegistry_RevShare_evt_RecipientAdded')}} ), -/* + stonks as ( select * from (values ('STETH→DAI', 0x3e2D251275A92a8169A3B17A2C49016e2de492a7), @@ -114,24 +114,7 @@ stonks as ( ) as list(namespace, address) ), -cow_settlement as ( - select * from (values - (0x9008D19f58AAbD9eD0D60971565AA8510560ab41) - ) as list(address) -), -stonks_orders_txns as ( - select evt_tx_hash - from {{source('lido_ethereum', 'steth_evt_Transfer')}} - where "from" in ( - select cast(replace(topic1, 0x000000000000000000000000, 0x) as varbinary) as order_addr - from {{source('ethereum', 'logs')}} l - join stonks s on l.contract_address = s.address - and l.topic0 = 0x96a6d5477fba36522dca4102be8b3785435baf902ef6c4edebcb99850630c75f -- Stonks Deployed - ) - and to in (select address from cow_settlement) -), -*/ other_expenses_txns AS ( SELECT evt_block_time, @@ -160,8 +143,10 @@ other_expenses_txns AS ( SELECT 0x0000000000000000000000000000000000000000 UNION ALL SELECT address FROM diversifications_addresses + UNION ALL + SELECT address FROM stonks ) - -- AND evt_tx_hash NOT IN (select evt_tx_hash from stonks_orders_txns) + UNION ALL --ETH outflow SELECT @@ -199,4 +184,3 @@ other_expenses_txns AS ( FROM other_expenses_txns WHERE contract_address IN (SELECT address FROM tokens) and value != 0 - diff --git a/dbt_subprojects/hourly_spellbook/models/_project/lido/accounting/ethereum/lido_accounting_ethereum_other_income.sql b/dbt_subprojects/hourly_spellbook/models/_project/lido/accounting/ethereum/lido_accounting_ethereum_other_income.sql index 4ab559548b4..255876b67f4 100644 --- a/dbt_subprojects/hourly_spellbook/models/_project/lido/accounting/ethereum/lido_accounting_ethereum_other_income.sql +++ b/dbt_subprojects/hourly_spellbook/models/_project/lido/accounting/ethereum/lido_accounting_ethereum_other_income.sql @@ -16,13 +16,13 @@ with tokens AS ( select * from (values (0x5A98FcBEA516Cf06857215779Fd812CA3beF1B32), --LDO - (0x6B175474E89094C44Da98b954EedeAC495271d0F), --DAI - (0xA0b86991c6218b36c1d19D4a2e9Eb0cE3606eB48), --USDC + (0x6B175474E89094C44Da98b954EedeAC495271d0F), --DAI + (0xA0b86991c6218b36c1d19D4a2e9Eb0cE3606eB48), --USDC (0xdAC17F958D2ee523a2206206994597C13D831ec7), -- USDT - (0xc02aaa39b223fe8d0a0e5c4f27ead9083c756cc2), --WETH - (0x7D1AfA7B718fb893dB30A3aBc0Cfc608AaCfeBB0), --MATIC - (0xae7ab96520DE3A18E5e111B5EaAb095312D7fE84), --stETH - (0x7f39C581F595B53c5cb19bD0b3f8dA6c935E2Ca0) --wstETH + (0xc02aaa39b223fe8d0a0e5c4f27ead9083c756cc2), --WETH + (0x7D1AfA7B718fb893dB30A3aBc0Cfc608AaCfeBB0), --MATIC + (0xae7ab96520DE3A18E5e111B5EaAb095312D7fE84), --stETH + (0x7f39C581F595B53c5cb19bD0b3f8dA6c935E2Ca0) --wstETH ) as tokens(address)), @@ -91,15 +91,6 @@ select * from (values dai_referral_payments_addr AS ( SELECT _recipient AS address FROM {{source('lido_ethereum','AllowedRecipientsRegistry_evt_RecipientAdded')}} - WHERE - ( - NOT EXISTS (SELECT _recipient FROM {{source('lido_ethereum','AllowedRecipientsRegistry_evt_RecipientRemoved')}}) - OR ( - EXISTS (SELECT _recipient FROM {{source('lido_ethereum','AllowedRecipientsRegistry_evt_RecipientRemoved')}}) - AND - _recipient NOT IN (SELECT _recipient FROM {{source('lido_ethereum','AllowedRecipientsRegistry_evt_RecipientRemoved')}}) - ) - ) UNION ALL SELECT 0xaf8aE6955d07776aB690e565Ba6Fbc79B8dE3a5d --rhino ), @@ -107,7 +98,7 @@ dai_referral_payments_addr AS ( steth_referral_payments_addr AS ( SELECT _recipient AS address FROM {{source('lido_ethereum','AllowedRecipientsRegistry_RevShare_evt_RecipientAdded')}} ), -/* + stonks as ( select * from (values ('STETH→DAI', 0x3e2D251275A92a8169A3B17A2C49016e2de492a7), @@ -139,7 +130,7 @@ stonks_orders_txns as ( ) and to in (select address from cow_settlement) ), -*/ + other_income_txns AS ( SELECT evt_block_time, @@ -167,7 +158,7 @@ other_income_txns AS ( UNION ALL SELECT address FROM diversifications_addresses ) - -- AND evt_tx_hash NOT IN (select evt_tx_hash from stonks_orders_txns) + AND evt_tx_hash NOT IN (select evt_tx_hash from stonks_orders_txns) UNION ALL --ETH staked by DAO @@ -251,3 +242,4 @@ SELECT * from ( from_base64(evt_tx_hash) FROM stsol_income ) WHERE amount_token != 0 + diff --git a/dbt_subprojects/hourly_spellbook/models/_project/lido/accounting/ethereum/lido_accounting_ethereum_revenue.sql b/dbt_subprojects/hourly_spellbook/models/_project/lido/accounting/ethereum/lido_accounting_ethereum_revenue.sql index 9847c63cf48..eab910a900a 100644 --- a/dbt_subprojects/hourly_spellbook/models/_project/lido/accounting/ethereum/lido_accounting_ethereum_revenue.sql +++ b/dbt_subprojects/hourly_spellbook/models/_project/lido/accounting/ethereum/lido_accounting_ethereum_revenue.sql @@ -20,6 +20,7 @@ select * from (values (0x3e40d73eb977dc6a537af587d48316fee66e9c8c, 'Aragon'), (0x55032650b14df07b85bF18A3a3eC8E0Af2e028d5, 'NO (Curated)'), (0xaE7B191A31f627b4eB1d4DaC64eaB9976995b433, 'NO (DVT)'), +(0xdA7dE2ECdDfccC6c3AF10108Db212ACBBf9EA83F, 'NO (CSM)'), (0x8B3f33234ABD88493c0Cd28De33D583B70beDe35, 'InsuranceFund') ) as list(address, name) ), @@ -77,9 +78,9 @@ protocol_fee_distribution AS ( 0xae7ab96520DE3A18E5e111B5EaAb095312D7fE84 AS token, lido_rewards AS total, protocol_fee.points AS protocol_fee, - protocol_fee_distribution.insurance_points AS insurance_fee, - protocol_fee_distribution.operators_points AS operators_fee, - protocol_fee_distribution.treasury_points AS treasury_fee, + --protocol_fee_distribution.insurance_points AS insurance_fee, + --protocol_fee_distribution.operators_points AS operators_fee, + --protocol_fee_distribution.treasury_points AS treasury_fee, (1 - protocol_fee.points)*lido_rewards AS depositors_revenue, protocol_fee.points*protocol_fee_distribution.treasury_points*lido_rewards AS treasury_revenue, protocol_fee.points*protocol_fee_distribution.insurance_points*lido_rewards AS insurance_revenue, @@ -96,9 +97,6 @@ protocol_fee_distribution AS ( 0xae7ab96520DE3A18E5e111B5EaAb095312D7fE84 AS token, oracle_txns.treasury_revenue + oracle_txns.operators_revenue + oracle_txns.insurance_revenue AS total, protocol_fee.points AS protocol_fee, - protocol_fee_distribution.insurance_points AS insurance_fee, - protocol_fee_distribution.operators_points AS operators_fee, - protocol_fee_distribution.treasury_points AS treasury_fee, 10*(1 - protocol_fee.points)*(oracle_txns.treasury_revenue + oracle_txns.operators_revenue + oracle_txns.insurance_revenue) AS depositors_revenue, oracle_txns.treasury_revenue AS treasury_revenue, oracle_txns.insurance_revenue AS insurance_revenue, From 4f27a0361469fbd632776fc10b123e7f0b55c83d Mon Sep 17 00:00:00 2001 From: 0xRob <83790096+0xRobin@users.noreply.github.com> Date: Thu, 28 Nov 2024 15:27:15 +0100 Subject: [PATCH 18/18] exclude sanctum (#7206) * exclude sanctum * exclude in jinja --- .../_sector/dex/dex_solana_base_trades.sql | 5 ++- .../sanctum_router_base_trades.sql | 43 ++++++++++--------- 2 files changed, 25 insertions(+), 23 deletions(-) diff --git a/dbt_subprojects/solana/models/_sector/dex/dex_solana_base_trades.sql b/dbt_subprojects/solana/models/_sector/dex/dex_solana_base_trades.sql index 991be84e941..d3dba42ca27 100644 --- a/dbt_subprojects/solana/models/_sector/dex/dex_solana_base_trades.sql +++ b/dbt_subprojects/solana/models/_sector/dex/dex_solana_base_trades.sql @@ -23,9 +23,10 @@ , ref('meteora_v2_solana_base_trades') , ref('goosefx_ssl_v2_solana_base_trades') , ref('pumpdotfun_solana_base_trades') - , ref('sanctum_router_base_trades') ] %} +-- excluded: , ref('sanctum_router_base_trades') + {% for dex in solana_dexes %} SELECT blockchain @@ -52,7 +53,7 @@ SELECT FROM {{ dex }} {% if is_incremental() %} -WHERE +WHERE {{incremental_predicate('block_time')}} {% endif %} {% if not loop.last %} diff --git a/dbt_subprojects/solana/models/_sector/dex/sanctum_router/sanctum_router_base_trades.sql b/dbt_subprojects/solana/models/_sector/dex/sanctum_router/sanctum_router_base_trades.sql index 9f73bb67fff..c7de77c37c3 100644 --- a/dbt_subprojects/solana/models/_sector/dex/sanctum_router/sanctum_router_base_trades.sql +++ b/dbt_subprojects/solana/models/_sector/dex/sanctum_router/sanctum_router_base_trades.sql @@ -1,6 +1,7 @@ {{ config( schema = 'sanctum_router', + tags = ['prod_exclude'], alias = 'base_trades', partition_by = ['block_month'], materialized = 'incremental', @@ -16,7 +17,7 @@ {% set dev_start_date = "date_trunc('day', now() - interval '7' day)" %} WITH swap_via_stake AS ( - SELECT + SELECT call_block_time as block_time , call_block_slot as block_slot , call_tx_signer as trader_id @@ -29,9 +30,9 @@ WITH swap_via_stake AS ( , account_srcTokenMint as token_sold_mint_address , account_destTokenMint as token_bought_mint_address , bytearray_to_bigint(bytearray_reverse(bytearray_substring(call_data, 2, 8))) AS token_sold_amount_raw - , CASE + , CASE WHEN call_outer_executing_account = 'stkitrT1Uoy18Dk1fTrgPw8W6MVzoCfYoAFT4MLsmhq' THEN 'direct' - ELSE call_outer_executing_account + ELSE call_outer_executing_account END as trade_source , 'mintTo' as amount_type FROM {{ source('sanctum_router_solana', 'stakedex_call_SwapViaStake') }} @@ -44,7 +45,7 @@ WITH swap_via_stake AS ( ), prefund_swap_via_stake AS ( - SELECT + SELECT call_block_time as block_time , call_block_slot as block_slot , call_tx_signer as trader_id @@ -57,9 +58,9 @@ prefund_swap_via_stake AS ( , account_srcTokenMint as token_sold_mint_address , account_destTokenMint as token_bought_mint_address , bytearray_to_bigint(bytearray_reverse(bytearray_substring(call_data, 2, 8))) AS token_sold_amount_raw - , CASE + , CASE WHEN call_outer_executing_account = 'stkitrT1Uoy18Dk1fTrgPw8W6MVzoCfYoAFT4MLsmhq' THEN 'direct' - ELSE call_outer_executing_account + ELSE call_outer_executing_account END as trade_source , 'transferChecked' as amount_type FROM {{ source('sanctum_router_solana', 'stakedex_call_PrefundSwapViaStake') }} @@ -72,7 +73,7 @@ prefund_swap_via_stake AS ( ), stake_wrapped_sol AS ( - SELECT + SELECT call_block_time as block_time , call_block_slot as block_slot , call_tx_signer as trader_id @@ -85,9 +86,9 @@ stake_wrapped_sol AS ( , account_wsolMint as token_sold_mint_address , account_destTokenMint as token_bought_mint_address , bytearray_to_bigint(bytearray_reverse(bytearray_substring(call_data, 2, 8))) AS token_sold_amount_raw - , CASE + , CASE WHEN call_outer_executing_account = 'stkitrT1Uoy18Dk1fTrgPw8W6MVzoCfYoAFT4MLsmhq' THEN 'direct' - ELSE call_outer_executing_account + ELSE call_outer_executing_account END as trade_source , 'transferChecked' as amount_type FROM {{ source('sanctum_router_solana', 'stakedex_call_StakeWrappedSol') }} @@ -100,7 +101,7 @@ stake_wrapped_sol AS ( ), withdraw_deposit AS ( - SELECT + SELECT w.call_block_time as block_time , w.call_block_slot as block_slot , w.call_tx_signer as trader_id @@ -113,14 +114,14 @@ withdraw_deposit AS ( , w.account_srcTokenMint as token_sold_mint_address , d.account_destTokenMint as token_bought_mint_address , bytearray_to_bigint(bytearray_reverse(bytearray_substring(w.call_data, 2, 8))) AS token_sold_amount_raw - , CASE + , CASE WHEN w.call_outer_executing_account = 'stkitrT1Uoy18Dk1fTrgPw8W6MVzoCfYoAFT4MLsmhq' THEN 'direct' - ELSE w.call_outer_executing_account + ELSE w.call_outer_executing_account END as trade_source , 'transferChecked' as amount_type FROM {{ source('sanctum_router_solana', 'stakedex_call_PrefundWithdrawStake') }} w INNER JOIN {{ source('sanctum_router_solana', 'stakedex_call_DepositStake') }} d - ON w.call_tx_id = d.call_tx_id + ON w.call_tx_id = d.call_tx_id AND w.call_outer_instruction_index = d.call_outer_instruction_index AND w.call_inner_instruction_index < d.call_inner_instruction_index WHERE 1=1 @@ -133,7 +134,7 @@ withdraw_deposit AS ( all_trades AS ( SELECT * FROM swap_via_stake - UNION ALL + UNION ALL SELECT * FROM prefund_swap_via_stake UNION ALL SELECT * FROM stake_wrapped_sol @@ -142,29 +143,29 @@ all_trades AS ( ), token_amounts AS ( - SELECT + SELECT ic.tx_id, ic.outer_instruction_index, ic.inner_instruction_index, bytearray_to_bigint(bytearray_reverse(bytearray_substring(ic.data, 2, 8))) AS amount_bought, ROW_NUMBER() OVER ( PARTITION BY ic.tx_id, ic.outer_instruction_index - ORDER BY - CASE + ORDER BY + CASE WHEN b.amount_type = 'mintTo' THEN ic.inner_instruction_index ELSE -ic.inner_instruction_index END ) as rn FROM all_trades b - INNER JOIN {{ source('solana','instruction_calls') }} ic - ON ic.tx_id = b.tx_id + INNER JOIN {{ source('solana','instruction_calls') }} ic + ON ic.tx_id = b.tx_id AND ic.outer_instruction_index = b.outer_instruction_index AND ic.block_slot = b.block_slot WHERE 1=1 AND ic.executing_account = 'TokenkegQfeZyiNwAJbNbGKPFXCWuBvf9Ss623VQ5DA' AND ( (b.amount_type = 'mintTo' AND bytearray_substring(ic.data, 1, 1) = 0x07 AND ELEMENT_AT(ic.account_arguments, 1) = b.token_bought_mint_address) - OR + OR (b.amount_type = 'transferChecked' AND bytearray_substring(ic.data, 1, 1) = 0x0c AND ELEMENT_AT(ic.account_arguments, 2) = b.token_bought_mint_address AND ELEMENT_AT(ic.account_arguments, 3) = b.token_bought_vault) ) {% if is_incremental() %} @@ -198,6 +199,6 @@ SELECT , b.tx_index FROM all_trades b INNER JOIN token_amounts t - ON b.tx_id = t.tx_id + ON b.tx_id = t.tx_id AND b.outer_instruction_index = t.outer_instruction_index AND t.rn = 1