From 5ca6a9fa0b54dd4ec961f3a275e89d141dcafaad Mon Sep 17 00:00:00 2001 From: Sebastian Melendez Date: Thu, 4 Apr 2024 15:50:15 -0400 Subject: [PATCH 01/22] updating flows models --- models/dimensions/dim_chain_ids.sql | 28 ++ .../metrics/bridges/fact_bridgehub_flows.sql | 61 +-- models/staging/across/fact_across_flows.sql | 437 ++++++------------ .../fact_arbitrum_one_bridge_flows.sql | 16 +- .../fact_avalanche_bridge_flows.sql | 21 +- .../base_bridge/fact_base_bridge_flows.sql | 20 +- .../fact_optimism_bridge_flows.sql | 20 +- .../fact_polygon_pos_bridge_flows.sql | 33 +- .../fact_starknet_bridge_flows.sql | 20 +- models/staging/synapse/fact_synapse_flows.sql | 77 +-- .../staging/wormhole/fact_wormhole_flows.sql | 14 +- .../fact_zksync_era_bridge_flows.sql | 19 +- 12 files changed, 311 insertions(+), 455 deletions(-) create mode 100644 models/dimensions/dim_chain_ids.sql diff --git a/models/dimensions/dim_chain_ids.sql b/models/dimensions/dim_chain_ids.sql new file mode 100644 index 00000000..78ba7ab1 --- /dev/null +++ b/models/dimensions/dim_chain_ids.sql @@ -0,0 +1,28 @@ +{{ config(materialized="table") }} +select id, chain +from + ( + values + (1, 'ethereum'), + (10, 'optimism'), + (137, 'polygon'), + (288, 'boba'), + (42161, 'arbitrum'), + (8453, 'base'), + (324, 'zksync'), + (59144, 'linea'), + (1313161554, 'aurora'), + (250, 'fantom'), + (1088, 'metis'), + (1284, 'moonbeam'), + (8217, 'klaytn'), + (43114, 'avalanche'), + (1285, 'moonriver'), + (7700, 'canto'), + (2000, 'dogechain'), + (53935, 'dfk'), + (1666600000, 'harmony'), + (56, 'bsc'), + (25, 'cronos'), + (81457, 'blast') + ) as t(id, chain) \ No newline at end of file diff --git a/models/metrics/bridges/fact_bridgehub_flows.sql b/models/metrics/bridges/fact_bridgehub_flows.sql index 99283dce..2be0d77e 100644 --- a/models/metrics/bridges/fact_bridgehub_flows.sql +++ b/models/metrics/bridges/fact_bridgehub_flows.sql @@ -1,31 +1,34 @@ {{ config(materialized="table") }} -select * -from {{ ref("fact_across_flows") }} -union -select * -from {{ ref("fact_synapse_flows") }} -union -select * -from {{ ref("fact_wormhole_flows") }} -union -select * -from {{ ref("fact_arbitrum_one_bridge_flows") }} -union -select * -from {{ ref("fact_avalanche_bridge_flows") }} -union -select * -from {{ ref("fact_base_bridge_flows") }} -union -select * -from {{ ref("fact_optimism_bridge_flows") }} -union -select * -from {{ ref("fact_polygon_pos_bridge_flows") }} -union -select * -from {{ ref("fact_starknet_bridge_flows") }} -union -select * -from {{ ref("fact_zksync_era_bridge_flows") }} +with + daily_data as ( + {{ + dbt_utils.union_relations( + relations=[ + ref("fact_across_flows"), + ref("fact_synapse_flows"), + ref("fact_wormhole_flows"), + ref("fact_arbitrum_one_bridge_flows"), + ref("fact_avalanche_bridge_flows"), + ref("fact_base_bridge_flows"), + ref("fact_optimism_bridge_flows"), + ref("fact_polygon_pos_bridge_flows"), + ref("fact_starknet_bridge_flows"), + ref("fact_zksync_era_bridge_flows"), + ] + ) + }} + ) +select + *, + date::string + || '-' + || app + || '-' + || source_chain + || '-' + || destination_chain + || '-' + || category as unique_id +from daily_data +where date < to_date(sysdate()) diff --git a/models/staging/across/fact_across_flows.sql b/models/staging/across/fact_across_flows.sql index 2c75844a..aefb455e 100644 --- a/models/staging/across/fact_across_flows.sql +++ b/models/staging/across/fact_across_flows.sql @@ -5,345 +5,204 @@ ) }} with + + dim_contracts as ( + select distinct address, chain, category + from {{ ref("dim_contracts_gold") }} + where category is not null and chain is not null + ), + + across_transfers_chain_mapping as ( + select + version, + contract_address, + block_timestamp, + tx_hash, + event_index, + amount, + depositor, + recipient, + destination_chain_id, + destination_token, + origin_chain_id, + destination_token_symbol, + t2.chain as destination_chain, + t3.chain as source_chain, + t4.category as destination_category + from {{ ref("fact_across_transfers") }} t1 + left join {{ ref("dim_chain_ids")}} t2 on t1.destination_chain_id = t2.id + left join {{ ref("dim_chain_ids")}} t3 on t1.origin_chain_id = t3.id + left join dim_contracts t4 on lower(destination_token) = lower(address) and destination_chain = t2.chain + ), + distinct_tokens as ( - select distinct destination_token as token_address - from {{ ref("fact_across_transfers") }} - union all - select distinct input_token as token_address - from {{ ref("fact_across_transfers") }} + select distinct destination_token as token_address, destination_chain as chain + from across_transfers_chain_mapping + where destination_token is not null ), prices as ( select * from ethereum_flipside.price.ez_hourly_token_prices - where token_address in (select * from distinct_tokens) + where token_address in (select token_address from distinct_tokens where chain = 'ethereum') union select * from optimism_flipside.price.ez_hourly_token_prices - where token_address in (select * from distinct_tokens) + where token_address in (select token_address from distinct_tokens where chain = 'optimism') union select * from arbitrum_flipside.price.ez_hourly_token_prices - where token_address in (select * from distinct_tokens) + where token_address in (select token_address from distinct_tokens where chain = 'arbitrum') union select * from polygon_flipside.price.ez_hourly_token_prices - where token_address in (select * from distinct_tokens) + where token_address in (select token_address from distinct_tokens where chain = 'polygon') union select * from base_flipside.price.ez_hourly_token_prices - where token_address in (select * from distinct_tokens) - ), - - zksync_transfers_v2 as ( - select * - from {{ ref("fact_across_transfers") }} - where destination_token_symbol is not null and version = 'v2' + where token_address in (select token_address from distinct_tokens where chain = 'base') ), + dim_zkSync_tokens as ( + select symbol, address, chain, category + from + ( + values + ( + 'DAI', + lower('0x6B175474E89094C44Da98b954EedeAC495271d0F'), + 'ethereum', + 'Stablecoin' + ), + ( + 'USDC', + lower('0xA0b86991c6218b36c1d19D4a2e9Eb0cE3606eB48'), + 'ethereum', + 'Stablecoin' + ), + ( + 'USDC.e', + lower('0xA0b86991c6218b36c1d19D4a2e9Eb0cE3606eB48'), + 'ethereum', + 'Stablecoin' + ), + ( + 'USDT', + lower('0xdAC17F958D2ee523a2206206994597C13D831ec7'), + 'ethereum', + 'Stablecoin' + ), + ( + 'WBTC', + lower('0x2260FAC5E5542a773Aa44fBCfeDf7C193bc2C599'), + 'ethereum', + 'Token' + ), + ( + 'WETH', + lower('0xC02aaA39b223FE8D0A0e5C4F27eAD9083C756Cc2'), + 'ethereum', + 'Token' + ) + ) as t(symbol, address, chain, category) - zksync_transfers_v3 as ( - select * - from {{ ref("fact_across_transfers") }} - where destination_token_symbol is not null and version = 'v3' ), - zksync_tokens as ( - select distinct destination_token as token, destination_token_symbol as symbol - from zksync_transfers_v2 - union - select distinct destination_token as token, destination_token_symbol as symbol - from zksync_transfers_v3 + + zksync_transfers as ( + select + version, + contract_address, + block_timestamp, + tx_hash, + event_index, + amount, + depositor, + recipient, + destination_chain_id, + dim_zkSync_tokens.address as destination_token, + origin_chain_id, + destination_token_symbol, + destination_chain, + source_chain, + dim_zkSync_tokens.category as destination_category + from across_transfers_chain_mapping + left join dim_zkSync_tokens on lower(destination_token_symbol) = lower(symbol) + where destination_token_symbol is not null ), zksync_prices as ( - select hour, symbol, decimals, avg(price) as price + select hour, token_address, decimals, avg(price) as price from ethereum_flipside.price.ez_hourly_token_prices - where - symbol in ( - select distinct destination_token_symbol - from zksync_transfers_v2 - union - select distinct destination_token_symbol - from zksync_transfers_v3 - where symbol is not null - ) + inner join dim_zkSync_tokens on lower(token_address) = lower(address) group by 1, 2, 3 ), - chain_ids as ( - select 'ethereum' as chain, 1 as id - union - select 'optimism' as chain, 10 as id - union - select 'polygon' as chain, 137 as id - union - select 'boba' as chain, 288 as id - union - select 'arbitrum' as chain, 42161 as id - union - select 'base' as chain, 8453 as id - union - select 'zksync' as chain, 324 as id - union - select 'linea' as chain, 59144 as id - - ), - - zksync_volume_and_fees_by_chain_and_symbol_v2 as ( - with - temp as ( - select - date_trunc('hour', block_timestamp) as hour, - origin_chain_id, - destination_chain_id, - destination_token, - sum( - (coalesce(amount, 0) / power(10, p.decimals)) * price - ) as amount_usd, - sum( - ((coalesce(amount, 0) / power(10, p.decimals)) * price) - * (relayer_fee_pct + realized_lp_fee_pct) - ) as fee_usd - from zksync_transfers_v2 t - left join - zksync_prices p - on date_trunc('hour', t.block_timestamp) = p.hour - and t.destination_token_symbol = p.symbol - group by 1, 2, 3, 4 - ) - - select - hour, - c1.chain as source_chain, - c2.chain as destination_chain, - destination_token as token, - coalesce(amount_usd, 0) as amount_usd, - fee_usd - from temp t - left join chain_ids c1 on t.origin_chain_id = c1.id - left join chain_ids c2 on t.destination_chain_id = c2.id - ), - zksync_volume_and_fees_by_chain_and_symbol_v3 as ( - with - temp as ( - select - date_trunc('hour', block_timestamp) as hour, - origin_chain_id, - destination_chain_id, - destination_token, - sum( - (coalesce(amount, 0) / power(10, p1.decimals)) * p1.price - ) as amount_usd, - - sum( - (coalesce(input_amount, 0) / power(10, p2.decimals)) * p2.price - ) as input_amount_usd, - - input_amount_usd - amount_usd as fee_usd - from zksync_transfers_v3 t - left join - zksync_prices p1 - on date_trunc('hour', t.block_timestamp) = p1.hour - and t.destination_token_symbol = p1.symbol - - left join - prices p2 - on date_trunc('hour', t.block_timestamp) = p2.hour - and lower(t.input_token) = lower(p2.token_address) - - group by 1, 2, 3, 4 - ) + zksync_volume_by_chain_and_symbol as ( select - hour, - c1.chain as source_chain, - c2.chain as destination_chain, - destination_token as token, - coalesce(amount_usd, 0) as amount_usd, - fee_usd - from temp t - left join chain_ids c1 on t.origin_chain_id = c1.id - left join chain_ids c2 on t.destination_chain_id = c2.id - ), - - non_zksync_volume_and_fees_by_chain_and_symbol_v1_v2_uba as ( - with - temp as ( - select - date_trunc('hour', block_timestamp) as hour, - origin_chain_id, - destination_chain_id, - destination_token, - sum( - (coalesce(amount, 0) / power(10, p.decimals)) * price - ) as amount_usd, - sum( - ((coalesce(amount, 0) / power(10, p.decimals)) * price) - * (relayer_fee_pct + realized_lp_fee_pct) - ) as fee_usd - from {{ ref("fact_across_transfers") }} t - left join - prices p - on date_trunc('hour', t.block_timestamp) = p.hour - and t.destination_token = p.token_address - where - t.destination_token_symbol is null - and version in ('v1', 'uba', 'v2') - group by 1, 2, 3, 4 - ) - - select - hour, - c1.chain as source_chain, - c2.chain as destination_chain, - destination_token as token, - coalesce(amount_usd, 0) as amount_usd, - fee_usd - from temp t - left join chain_ids c1 on t.origin_chain_id = c1.id - left join chain_ids c2 on t.destination_chain_id = c2.id - ), - - zksync_source_chain_volume_and_fees_v3 as ( - with - temp as ( - select - date_trunc('hour', block_timestamp) as hour, - origin_chain_id, - destination_chain_id, - destination_token, - sum( - (coalesce(amount, 0) / power(10, p1.decimals)) * p1.price - ) as amount_usd, - - sum( - (coalesce(input_amount, 0) / power(10, p2.decimals)) * p2.price - ) as amount_in_usd, - - amount_in_usd - amount_usd as fee_usd - from {{ ref("fact_across_transfers") }} t - left join - prices p1 - on date_trunc('hour', t.block_timestamp) = p1.hour - and t.destination_token = p1.token_address - - left join zksync_tokens zt on lower(t.input_token) = lower(zt.token) - - left join - zksync_prices p2 - on date_trunc('hour', t.block_timestamp) = p2.hour - and zt.symbol = p2.symbol - - where - t.destination_token_symbol is null - and version = 'v3' - and origin_chain_id = 324 - group by 1, 2, 3, 4 - ) + date_trunc('hour', block_timestamp) as hour, + source_chain, + destination_chain, + destination_category, + sum( + coalesce((amount / power(10, p.decimals)) * price, 0) + ) as amount_usd + from zksync_transfers t + left join + zksync_prices p + on date_trunc('hour', t.block_timestamp) = p.hour + and t.destination_token = p.token_address + group by 1, 2, 3, 4 - select - hour, - c1.chain as source_chain, - c2.chain as destination_chain, - destination_token as token, - coalesce(amount_usd, 0) as amount_usd, -- I am here - fee_usd - from temp t - left join chain_ids c1 on t.origin_chain_id = c1.id - left join chain_ids c2 on t.destination_chain_id = c2.id ), - non_zksync_volume_and_fees_by_chain_and_symbol_v3 as ( - with - temp as ( - select - date_trunc('hour', block_timestamp) as hour, - origin_chain_id, - destination_chain_id, - destination_token, - sum( - (coalesce(amount, 0) / power(10, p1.decimals)) * p1.price - ) as amount_usd, - - sum( - (coalesce(input_amount, 0) / power(10, p2.decimals)) * p2.price - ) as amount_in_usd, - - amount_in_usd - amount_usd as fee_usd - from {{ ref("fact_across_transfers") }} t - left join - prices p1 - on date_trunc('hour', t.block_timestamp) = p1.hour - and t.destination_token = p1.token_address - - left join - prices p2 - on date_trunc('hour', t.block_timestamp) = p2.hour - and t.input_token = p2.token_address - - where - t.destination_token_symbol is null - and version = 'v3' - and origin_chain_id != 324 - group by 1, 2, 3, 4 - ) + non_zksync_volume_by_chain_and_symbol as ( + select + date_trunc('hour', block_timestamp) as hour, + source_chain, + destination_chain, + destination_category, + sum( + coalesce((amount / power(10, p.decimals)) * price, 0) + ) as amount_usd + from across_transfers_chain_mapping t + left join + prices p + on date_trunc('hour', t.block_timestamp) = p.hour + and t.destination_token = p.token_address + where + t.destination_token_symbol is null + group by 1, 2, 3, 4 - select - hour, - c1.chain as source_chain, - c2.chain as destination_chain, - destination_token as token, - coalesce(amount_usd, 0) as amount_usd, - fee_usd - from temp t - left join chain_ids c1 on t.origin_chain_id = c1.id - left join chain_ids c2 on t.destination_chain_id = c2.id ), flows_by_token as ( select date_trunc('day', hour) as date, - 'across' as app, source_chain, destination_chain, - token, - sum(amount_usd) as amount_usd, - sum(fee_usd) as fee_usd + coalesce(destination_category, 'Not Categorized') as category, + sum(amount_usd) as amount_usd from ( select * - from non_zksync_volume_and_fees_by_chain_and_symbol_v1_v2_uba - - union - - select * - from zksync_volume_and_fees_by_chain_and_symbol_v2 + from zksync_volume_by_chain_and_symbol union select * - from zksync_volume_and_fees_by_chain_and_symbol_v3 + from non_zksync_volume_by_chain_and_symbol - union - - select * - from non_zksync_volume_and_fees_by_chain_and_symbol_v3 - - union - - select * - from zksync_source_chain_volume_and_fees_v3 ) t - group by 1, 2, 3, 4, 5 + group by 1, 2, 3, 4 ) select - t1.date, - t1.app, - t1.source_chain, - t1.destination_chain, - t2.category, - sum(t1.amount_usd) as amount_usd, - sum(t1.fee_usd) as fee_usd -from flows_by_token t1 -left join {{ ref("dim_contracts_gold") }} t2 on lower(token) = lower(address) -group by date, t1.app, source_chain, destination_chain, category + date, + 'across' as app, + source_chain, + destination_chain, + category, + amount_usd, + null as fee_usd +from flows_by_token order by date desc, source_chain asc + diff --git a/models/staging/arbitrum_one_bridge/fact_arbitrum_one_bridge_flows.sql b/models/staging/arbitrum_one_bridge/fact_arbitrum_one_bridge_flows.sql index 8b166130..d128d102 100644 --- a/models/staging/arbitrum_one_bridge/fact_arbitrum_one_bridge_flows.sql +++ b/models/staging/arbitrum_one_bridge/fact_arbitrum_one_bridge_flows.sql @@ -16,23 +16,26 @@ with where token_address in (select * from distinct_tokens) ), + dim_contracts as ( + select distinct address, chain, category + from {{ ref("dim_contracts_gold") }} + where category is not null and chain is not null + ), + hourly_volume as ( select date_trunc('hour', block_timestamp) as hour, source_chain, destination_chain, - t.token_address, - sum( - (coalesce(amount::bigint, 0) / power(10, coalesce(p.decimals, 18))) - * coalesce(price, 0) - ) as amount_usd + coalesce(c.category, 'Not Categorized') as category, + coalesce(amount::bigint / power(10, p.decimals) * price, 0) as amount_usd from {{ ref("fact_arbitrum_one_bridge_transfers") }} t left join prices p on date_trunc('hour', t.block_timestamp) = p.hour and t.token_address = p.token_address + left join dim_contracts c on lower(t.token_address) = lower(c.address) and c.chain = 'ethereum' --only using l1token where p.symbol != 'ShibDoge' - group by 1, 2, 3, 4 ) select @@ -44,5 +47,4 @@ select sum(amount_usd) as amount_usd, null as fee_usd from hourly_volume -left join {{ ref("dim_contracts_gold") }} t2 on lower(token_address) = lower(t2.address) group by 1, 2, 3, 4, 5 diff --git a/models/staging/avalanche_bridge/fact_avalanche_bridge_flows.sql b/models/staging/avalanche_bridge/fact_avalanche_bridge_flows.sql index 93d6d813..8a2e5ab7 100644 --- a/models/staging/avalanche_bridge/fact_avalanche_bridge_flows.sql +++ b/models/staging/avalanche_bridge/fact_avalanche_bridge_flows.sql @@ -10,26 +10,26 @@ with where token_address in (select * from distinct_tokens) ), + dim_contracts as ( + select distinct address, chain, category + from {{ ref("dim_contracts_gold") }} + where category is not null and chain is not null + ), + hourly_volume as ( select date_trunc('hour', block_timestamp) as hour, source_chain, destination_chain, - t.token_address, - sum( - (coalesce(amount, 0) / power(10, coalesce(p.decimals, 0))) - * coalesce(price, 0) - ) as amount_usd, - sum( - (coalesce(fee, 0) / power(10, coalesce(p.decimals, 0))) - * coalesce(price, 0) - ) as fee_usd + coalesce(c.category, 'Not Categorized') as category, + coalesce((amount / power(10, p.decimals)) * price, 0) as amount_usd, + coalesce((fee / power(10, p.decimals)) * price, 0) as fee_usd from {{ ref("fact_avalanche_bridge_transfers") }} t left join prices p on date_trunc('hour', t.block_timestamp) = p.hour and t.token_address = p.token_address - group by 1, 2, 3, 4 + left join dim_contracts c on lower(t.token_address) = lower(c.address) and c.chain = 'avalanche' --only using l1token ) select @@ -41,5 +41,4 @@ select sum(amount_usd) as amount_usd, sum(fee_usd) as fee_usd from hourly_volume -left join {{ ref("dim_contracts_gold") }} t2 on lower(token_address) = lower(t2.address) group by 1, 2, 3, 4, 5 diff --git a/models/staging/base_bridge/fact_base_bridge_flows.sql b/models/staging/base_bridge/fact_base_bridge_flows.sql index 2b8b207c..fa124b78 100644 --- a/models/staging/base_bridge/fact_base_bridge_flows.sql +++ b/models/staging/base_bridge/fact_base_bridge_flows.sql @@ -1,22 +1,23 @@ with + dim_contracts as ( + select distinct address, chain, category + from {{ ref("dim_contracts_gold") }} + where category is not null and chain is not null + ), + volume_and_fees_by_chain_and_symbol as ( select date_trunc('hour', block_timestamp) as hour, source_chain, destination_chain, - p.symbol, - t.token_address, - sum( - (coalesce(amount, 0) / power(10, coalesce(p.decimals, 18))) - * coalesce(price, 0) - ) as amount_usd, - sum(null) as fee_usd + coalesce(c.category, 'Not Categorized') as category, + coalesce((amount / power(10, p.decimals)) * price, 0) as amount_usd from {{ ref("fact_base_bridge_transfers") }} t left join ethereum_flipside.price.ez_hourly_token_prices p on date_trunc('hour', t.block_timestamp) = p.hour and t.token_address = p.token_address - group by 1, 2, 3, 4, 5 + left join dim_contracts c on lower(t.token_address) = lower(c.address) and c.chain = 'ethereum' --only using l1token ) select @@ -26,8 +27,7 @@ select destination_chain, category, sum(amount_usd) as amount_usd, - sum(fee_usd) as fee_usd + null as fee_usd from volume_and_fees_by_chain_and_symbol -left join {{ ref("dim_contracts_gold") }} t2 on lower(token_address) = lower(t2.address) group by 1, 2, 3, 4, 5 order by date asc, source_chain asc diff --git a/models/staging/optimism_bridge/fact_optimism_bridge_flows.sql b/models/staging/optimism_bridge/fact_optimism_bridge_flows.sql index d097622b..c05f8740 100644 --- a/models/staging/optimism_bridge/fact_optimism_bridge_flows.sql +++ b/models/staging/optimism_bridge/fact_optimism_bridge_flows.sql @@ -1,22 +1,23 @@ with + dim_contracts as ( + select distinct address, chain, category + from {{ ref("dim_contracts_gold") }} + where category is not null and chain is not null + ), + volume_and_fees_by_chain_and_symbol as ( select date_trunc('hour', block_timestamp) as hour, source_chain, destination_chain, - p.symbol, - t.token_address, - sum( - (coalesce(amount, 0) / power(10, coalesce(p.decimals, 0))) - * coalesce(price, 0) - ) as amount_usd, - sum(null) as fee_usd + coalesce(c.category, 'Not Categorized') as category, + coalesce((amount / power(10, p.decimals)) * price, 0) as amount_usd from {{ ref("fact_optimism_bridge_transfers") }} t left join ethereum_flipside.price.ez_hourly_token_prices p on date_trunc('hour', t.block_timestamp) = p.hour and t.token_address = p.token_address - group by 1, 2, 3, 4, 5 + left join dim_contracts c on lower(t.token_address) = lower(c.address) and c.chain = 'ethereum' --only using l1token ) select @@ -26,8 +27,7 @@ select destination_chain, category, sum(amount_usd) as amount_usd, - sum(fee_usd) as fee_usd + null as fee_usd from volume_and_fees_by_chain_and_symbol -left join {{ ref("dim_contracts_gold") }} t2 on lower(token_address) = lower(t2.address) group by 1, 2, 3, 4, 5 order by date asc, source_chain asc diff --git a/models/staging/polygon_pos_bridge/fact_polygon_pos_bridge_flows.sql b/models/staging/polygon_pos_bridge/fact_polygon_pos_bridge_flows.sql index 4efd238b..2b99b2eb 100644 --- a/models/staging/polygon_pos_bridge/fact_polygon_pos_bridge_flows.sql +++ b/models/staging/polygon_pos_bridge/fact_polygon_pos_bridge_flows.sql @@ -1,19 +1,25 @@ with + + dim_contracts as ( + select distinct address, chain, category + from {{ ref("dim_contracts_gold") }} + where category is not null and chain is not null + ), + volume_and_fees_by_chain_and_symbol as ( select date_trunc('hour', block_timestamp) as hour, source_chain, destination_chain, p.symbol, - t.token_address, - (coalesce(amount, 0) / power(10, coalesce(p.decimals, 0))) - * coalesce(price, 0) as amount_usd, - null as fee_usd + coalesce(c.category, 'Not Categorized') as category, + coalesce((amount / power(10, p.decimals)) * price, 0) as amount_usd from {{ ref("fact_polygon_pos_bridge_transfers") }} t left join ethereum_flipside.price.ez_hourly_token_prices p on date_trunc('hour', t.block_timestamp) = p.hour and t.token_address = p.token_address + left join dim_contracts c on lower(t.token_address) = lower(c.address) and c.chain = 'ethereum' --only using l1token ), filtered_volume_and_fees_by_chain_and_symbol as ( @@ -22,17 +28,13 @@ with source_chain, destination_chain, symbol, - token_address, - sum( - case - when (symbol in ('DG', 'BORING', 'TORG')) and amount_usd > 10000000 - then 0 - else amount_usd - end - ) as amount_usd, - sum(fee_usd) as fee_usd + category, + case + when (symbol in ('DG', 'BORING', 'TORG')) and amount_usd > 10000000 + then 0 + else amount_usd + end as amount_usd from volume_and_fees_by_chain_and_symbol - group by 1, 2, 3, 4, 5 ) select @@ -42,8 +44,7 @@ select destination_chain, category, sum(amount_usd) as amount_usd, - sum(fee_usd) as fee_usd + null as fee_usd from filtered_volume_and_fees_by_chain_and_symbol -left join {{ ref("dim_contracts_gold") }} t2 on lower(token_address) = lower(t2.address) group by 1, 2, 3, 4, 5 order by date asc, source_chain asc diff --git a/models/staging/starknet_bridge/fact_starknet_bridge_flows.sql b/models/staging/starknet_bridge/fact_starknet_bridge_flows.sql index 2f2a6f61..d783240b 100644 --- a/models/staging/starknet_bridge/fact_starknet_bridge_flows.sql +++ b/models/staging/starknet_bridge/fact_starknet_bridge_flows.sql @@ -1,22 +1,23 @@ with + dim_contracts as ( + select distinct address, chain, category + from {{ ref("dim_contracts_gold") }} + where category is not null and chain is not null + ), + volume_and_fees_by_chain_and_symbol as ( select date_trunc('hour', block_timestamp) as hour, source_chain, destination_chain, - p.symbol, - t.token_address, - sum( - (coalesce(amount, 0) / power(10, coalesce(p.decimals, 0))) - * coalesce(price, 0) - ) as amount_usd, - sum(null) as fee_usd + coalesce(c.category, 'Not Categorized') as category, + coalesce((amount / power(10, p.decimals)) * price, 0) as amount_usd from {{ ref("fact_starknet_bridge_transfers") }} t left join ethereum_flipside.price.ez_hourly_token_prices p on date_trunc('hour', t.block_timestamp) = p.hour and t.token_address = p.token_address - group by 1, 2, 3, 4, 5 + left join dim_contracts c on lower(t.token_address) = lower(c.address) and c.chain = 'ethereum' --only using l1token ) select @@ -26,8 +27,7 @@ select destination_chain, category, sum(amount_usd) as amount_usd, - sum(fee_usd) as fee_usd + null as fee_usd from volume_and_fees_by_chain_and_symbol -left join {{ ref("dim_contracts_gold") }} t2 on lower(token_address) = lower(t2.address) group by 1, 2, 3, 4, 5 order by date asc, source_chain asc diff --git a/models/staging/synapse/fact_synapse_flows.sql b/models/staging/synapse/fact_synapse_flows.sql index 46207ed4..ea60bb13 100644 --- a/models/staging/synapse/fact_synapse_flows.sql +++ b/models/staging/synapse/fact_synapse_flows.sql @@ -276,77 +276,38 @@ with ), final_combined as ( - select *, coalesce(usd_value, gecko_usd_value) as final_usd_value + select *, coalesce(usd_value, gecko_usd_value, 0) as final_usd_value from combined left join gecko_prices using (synapse_tx_hash) ), - chain_ids as ( - select 1 as id, 'ethereum' as chain - union - select 1313161554 as id, 'aurora' as chain - union - select 10 as id, 'optimism' as chain - union - select 137 as id, 'polygon' as chain - union - select 42161 as id, 'arbitrum' as chain - union - select 250 as id, 'fantom' as chain - union - select 1088 as id, 'metis' as chain - union - select 1284 as id, 'moonbeam' as chain - union - select 8217 as id, 'klaytn' as chain - union - select 43114 as id, 'avalanche' as chain - union - select 1285 as id, 'moonriver' as chain - union - select 7700 as id, 'canto' as chain - union - select 2000 as id, 'dogechain' as chain - union - select 53935 as id, 'dfk' as chain - union - select 1666600000 as id, 'harmony' as chain - union - select 56 as id, 'bsc' as chain - union - select 25 as id, 'cronos' as chain - union - select 8453 as id, 'base' as chain - union - select 288 as id, 'boba' as chain - union - select 81457 as id, 'blast' as chain + dim_contracts as ( + select distinct address, chain, category + from {{ ref("dim_contracts_gold") }} + where category is not null and chain is not null ), flows_by_chain_id as ( select date_trunc('day', origin_block_timestamp) as date, - origin_chain_id, - destination_chain_id, - category, - sum(coalesce(final_usd_value, 0)) as amount_usd, - null as fee_usd - from final_combined - left join - {{ ref("dim_contracts_gold") }} t2 - on lower(destination_token_address) = lower(t2.address) - group by 1, 2, 3, 4 + t2.chain as source_chain, + t3.chain as destination_chain, + coalesce(t4.category, 'Not Categorized') as category, + final_usd_value as amount_usd + from final_combined t1 + left join {{ ref("dim_chain_ids")}} t2 on t1.origin_chain_id = t2.id + left join {{ ref("dim_chain_ids")}} t3 on t1.destination_chain_id = t3.id + left join dim_contracts t4 on lower(destination_token_address) = lower(t4.address) and t3.chain = t4.chain ) select date, - t2.chain as source_chain, 'synapse' as app, - t3.chain as destination_chain, + source_chain, + destination_chain, category, - amount_usd, - fee_usd -from flows_by_chain_id t1 -left join chain_ids t2 on t1.origin_chain_id = t2.id -left join chain_ids t3 on t1.destination_chain_id = t3.id + sum(amount_usd) as amount_usd, + null as fee_usd +from flows_by_chain_id +group by 1, 2, 3, 4, 5 order by date desc, source_chain asc diff --git a/models/staging/wormhole/fact_wormhole_flows.sql b/models/staging/wormhole/fact_wormhole_flows.sql index fac96027..c46a0dc9 100644 --- a/models/staging/wormhole/fact_wormhole_flows.sql +++ b/models/staging/wormhole/fact_wormhole_flows.sql @@ -68,20 +68,24 @@ with ) as t(id, chain) ), + dim_contracts as ( + select distinct address, chain, category + from {{ ref("dim_contracts_gold") }} + where category is not null and chain is not null + ), + volume_by_chain_and_symbol as ( select date_trunc('day', timestamp) as date, c1.chain as source_chain, c2.chain as destination_chain, - symbol, - category, + coalesce(t2.category, t3.category, 'Not Categorized') as category, amount_usd from transfers t left join chain_ids c1 on t.from_chain = c1.id left join chain_ids c2 on t.to_chain = c2.id - left join - {{ ref("dim_contracts_gold") }} t2 - on lower(t.token_address) = lower(t2.address) + left join dim_contracts t2 on lower(t.token_address) = lower(t2.address) and c1.chain = t2.chain + left join dim_contracts t3 on lower(t.token_address) = lower(t3.address) and c2.chain = t3.chain ) -- note source and destination chain may be null if not provided above diff --git a/models/staging/zksync_era_bridge/fact_zksync_era_bridge_flows.sql b/models/staging/zksync_era_bridge/fact_zksync_era_bridge_flows.sql index c3b3cb6b..20a11fc1 100644 --- a/models/staging/zksync_era_bridge/fact_zksync_era_bridge_flows.sql +++ b/models/staging/zksync_era_bridge/fact_zksync_era_bridge_flows.sql @@ -1,22 +1,22 @@ with + dim_contracts as ( + select distinct address, chain, category + from {{ ref("dim_contracts_gold") }} + where category is not null and chain is not null + ), volume_and_fees_by_chain_and_symbol as ( select date_trunc('hour', block_timestamp) as hour, source_chain, destination_chain, - p.symbol, - t.token_address, - sum( - (coalesce(amount, 0) / power(10, coalesce(p.decimals, 18))) - * coalesce(price, 0) - ) as amount_usd, - sum(null) as fee_usd + coalesce(c.category, 'Not Categorized') as category, + coalesce((amount / power(10, p.decimals)) * price, 0) as amount_usd from {{ ref("fact_zksync_era_bridge_transfers") }} t left join ethereum_flipside.price.ez_hourly_token_prices p on date_trunc('hour', t.block_timestamp) = p.hour and t.token_address = p.token_address - group by 1, 2, 3, 4, 5 + left join dim_contracts c on lower(t.token_address) = lower(c.address) and c.chain = 'ethereum' --only using l1token ) select @@ -26,8 +26,7 @@ select destination_chain, category, sum(amount_usd) as amount_usd, - sum(fee_usd) as fee_usd + null as fee_usd from volume_and_fees_by_chain_and_symbol -left join {{ ref("dim_contracts_gold") }} t2 on lower(token_address) = lower(t2.address) group by 1, 2, 3, 4, 5 order by date asc, source_chain asc From 527872d06a401a0d211b41511fe17820f12aab77 Mon Sep 17 00:00:00 2001 From: Son Do Date: Fri, 5 Apr 2024 11:56:15 -0400 Subject: [PATCH 02/22] Add Sui Contracts --- models/staging/sui/dim_sui_contracts.sql | 26 +++++++++++++++++++ models/staging/sui/dim_sui_namespaces.sql | 10 +++++++ .../staging/sui/fact_sui_contracts_silver.sql | 24 +++++++++++++++++ 3 files changed, 60 insertions(+) create mode 100644 models/staging/sui/dim_sui_contracts.sql create mode 100644 models/staging/sui/dim_sui_namespaces.sql create mode 100644 models/staging/sui/fact_sui_contracts_silver.sql diff --git a/models/staging/sui/dim_sui_contracts.sql b/models/staging/sui/dim_sui_contracts.sql new file mode 100644 index 00000000..ca2cae71 --- /dev/null +++ b/models/staging/sui/dim_sui_contracts.sql @@ -0,0 +1,26 @@ +{{config(materialized="incremental", unique_key=["package_id", "version"])}} +with +sui_contracts_sigma_over as ( + select + package_id as address, + package_name as name, + coalesce(overwrite.overwrite_namespace, overwrite_w_exisiting_namespace, namespace) as namespace, + friendly_name, + project_img as icon, + category, + sub_category, + 'sui' as chain + from {{ ref("fact_sui_contracts_silver") }} as sui_contracts full join {{ source("SIGMA", "sui_overwrite_namespace") }} overwrite + on sui_contracts.namespace = overwrite.sui_namespace +) +select + address, + name, + namespace, + friendly_name, + icon, + category, + sub_category, + chain +from sui_contracts_sigma_over +where address is not null diff --git a/models/staging/sui/dim_sui_namespaces.sql b/models/staging/sui/dim_sui_namespaces.sql new file mode 100644 index 00000000..875eaa87 --- /dev/null +++ b/models/staging/sui/dim_sui_namespaces.sql @@ -0,0 +1,10 @@ +{{config(materialized="incremental", unique_key=["namespace"])}} + +select + namespace, + max(friendly_name) as friendly_name, + max(sub_category) as sub_category, + max(category) as category, + max(icon) as icon +from {{ ref("dim_sui_contracts") }} as sui_contracts +group by namespace diff --git a/models/staging/sui/fact_sui_contracts_silver.sql b/models/staging/sui/fact_sui_contracts_silver.sql new file mode 100644 index 00000000..0e00304e --- /dev/null +++ b/models/staging/sui/fact_sui_contracts_silver.sql @@ -0,0 +1,24 @@ +{{ config(materialized="incremental", unique_key=["package_id", "timestamp", "version"]) }} + +with + max_extraction as ( + select max(extraction_date) as max_date + from {{ source("PROD_LANDING", "raw_sui_contracts") }} + ), + sui_data as ( + select parse_json(source_json) as data + from {{ source("PROD_LANDING", "raw_sui_contracts") }} + where extraction_date = (select max_date from max_extraction) + ) + select + value:"packageId"::string as package_id, + value:"creator"::string as creator_address, + value:"packageName"::string as package_name, + lower(replace(value:"projectName"::string, ' ', '_')) as namespace, + value:"projectName"::string as friendly_name, + value:"projectImg"::string as project_img, + date(to_timestamp(value:"timestamp"::number / 1000)) as timestamp, + value:"transactions" as transactions, + value:"version"::number as version + from sui_data, lateral flatten(input => data) + order by transactions desc From 5d9a1e97adde1c597dd99070108f99374d135b0f Mon Sep 17 00:00:00 2001 From: Son Do Date: Fri, 5 Apr 2024 11:58:10 -0400 Subject: [PATCH 03/22] Updated dim apps silver --- models/dimensions/apps/dim_apps_silver.sql | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/models/dimensions/apps/dim_apps_silver.sql b/models/dimensions/apps/dim_apps_silver.sql index f17cbeca..dc6734b3 100644 --- a/models/dimensions/apps/dim_apps_silver.sql +++ b/models/dimensions/apps/dim_apps_silver.sql @@ -56,15 +56,16 @@ with select coalesce( - sigma_app_tagged.namespace, dune_namespace.namespace, flipside.namespace + sigma_app_tagged.namespace, sui.namespace, dune_namespace.namespace, flipside.namespace ) as namespace, coalesce( sigma_app_tagged.friendly_name, + sui.friendly_name, initcap(replace(dune_namespace.namespace, '_', ' ')), flipside.friendly_name ) as friendly_name, - coalesce(sigma_app_tagged.sub_category, flipside.sub_category) as sub_category, - coalesce(sigma_app_tagged.category, flipside.category) as category, + coalesce(sigma_app_tagged.sub_category, sui.sub_category, flipside.sub_category) as sub_category, + coalesce(sigma_app_tagged.category, sui.category, flipside.category) as category, artemis_id, coingecko_id, ecosystem_id, @@ -72,7 +73,7 @@ select parent_app, coalesce(visibility, 1) as visibility, symbol, - icon + coalesce(sigma_app_tagged.icon, sui.icon) as icon from sigma_app_tagged full join {{ ref("dim_dune_namespaces") }} as dune_namespace @@ -80,7 +81,11 @@ full join full join {{ ref("dim_flipside_namespaces") }} as flipside on sigma_app_tagged.namespace = flipside.namespace +full join + {{ ref("dim_sui_namespaces") }} as sui + on sigma_app_tagged.namespace = sui.namespace where sigma_app_tagged.namespace is not null or dune_namespace.namespace is not null or flipside.namespace is not null + or sui.namespace is not null From a89ed2631f72536a73159f4bf044150a4dd8e92d Mon Sep 17 00:00:00 2001 From: Son Do Date: Fri, 5 Apr 2024 11:59:22 -0400 Subject: [PATCH 04/22] Updated dim contracts silver --- models/dimensions/contracts/dim_contracts_silver.sql | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/models/dimensions/contracts/dim_contracts_silver.sql b/models/dimensions/contracts/dim_contracts_silver.sql index 4ae561e1..6c08ec98 100644 --- a/models/dimensions/contracts/dim_contracts_silver.sql +++ b/models/dimensions/contracts/dim_contracts_silver.sql @@ -24,6 +24,9 @@ with union select lower(address) address, chain from {{ ref("dim_scanner_contracts") }} + union + select lower(address) address, chain + from {{ ref("dim_sui_contracts") }} ), distinct_contract as (select address, chain from contracts group by address, chain), contract_waterfall as ( @@ -34,6 +37,7 @@ with user_sub.namespace, scanner.namespace, dune.namespace, + sui.namespace, flipside_near.namespace, flipside_sol.namespace, null @@ -42,6 +46,7 @@ with user_sub.name, scanner.name, dune.name, + sui.name, flipside_near.name, flipside_sol.name, null @@ -73,6 +78,9 @@ with {{ ref("dim_scanner_contracts") }} as scanner on lower(dc.address) = lower(scanner.address) and dc.chain = scanner.chain + left join {{ ref("dim_sui_contracts") }} sui + on lower(dc.address) = lower(sui.address) + and dc.chain = sui.chain ), contracts_to_parent_app as ( select From 1608b17e887fc0897e130a3cf6e7ae5b1d754dc0 Mon Sep 17 00:00:00 2001 From: Son Do Date: Fri, 5 Apr 2024 13:06:38 -0400 Subject: [PATCH 05/22] Added Sources --- models/staging/sui/__sui__sources.yml | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/models/staging/sui/__sui__sources.yml b/models/staging/sui/__sui__sources.yml index c64bd428..016169e4 100644 --- a/models/staging/sui/__sui__sources.yml +++ b/models/staging/sui/__sui__sources.yml @@ -1,6 +1,12 @@ sources: + - name: SIGMA + schema: PROD + database: SIGMA + tables: + - name: sui_overwrite_namespace - name: PROD_LANDING schema: PROD_LANDING database: LANDING_DATABASE tables: - name: raw_sui_data + - name: raw_sui_contracts From ca3108fb4d596b522913691f4f840e68bf992adf Mon Sep 17 00:00:00 2001 From: Son Do Date: Fri, 5 Apr 2024 17:36:39 -0400 Subject: [PATCH 06/22] Fix Sui incremental build --- models/staging/sui/dim_sui_contracts.sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/models/staging/sui/dim_sui_contracts.sql b/models/staging/sui/dim_sui_contracts.sql index ca2cae71..5e80f172 100644 --- a/models/staging/sui/dim_sui_contracts.sql +++ b/models/staging/sui/dim_sui_contracts.sql @@ -1,4 +1,4 @@ -{{config(materialized="incremental", unique_key=["package_id", "version"])}} +{{config(materialized="incremental", unique_key=["address"])}} with sui_contracts_sigma_over as ( select From ff63f278ab0620d1349f973a00632e46bacf4f42 Mon Sep 17 00:00:00 2001 From: Sebastian Melendez Date: Sun, 7 Apr 2024 18:27:34 -0400 Subject: [PATCH 07/22] updating stride code --- .../stride/fact_stride_daa_gas_usd_txns.sql | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-) diff --git a/models/staging/stride/fact_stride_daa_gas_usd_txns.sql b/models/staging/stride/fact_stride_daa_gas_usd_txns.sql index d0df056d..f176aba5 100644 --- a/models/staging/stride/fact_stride_daa_gas_usd_txns.sql +++ b/models/staging/stride/fact_stride_daa_gas_usd_txns.sql @@ -1,6 +1,11 @@ -select t1.date, daa, gas_usd, txns, t1.chain -from {{ ref("fact_stride_daa") }} t1 -join {{ ref("fact_stride_txns") }} t2 on t1.date = t2.date -join {{ ref("fact_stride_gas_usd") }} t3 on t1.date = t3.date +select + t1.date, + daa, + gas_usd, + txns, + 'stride' as chain +from {{ ref("fact_stride_txns") }} t1 +left join {{ ref("fact_stride_daa") }} t2 on t1.date = t2.date +left join {{ ref("fact_stride_gas_usd") }} t3 on t1.date = t3.date where t1.date < to_date(sysdate()) -order by date +order by date desc From 1a47ba95e1f2721001afd3b314f24fccc2242926 Mon Sep 17 00:00:00 2001 From: Son Do Date: Mon, 8 Apr 2024 14:58:13 -0700 Subject: [PATCH 08/22] Fixed Tron BAM where gas is none --- .../all_chains/all_chains_gas_dau_txns_by_category.sql | 2 +- .../all_chains/all_chains_gas_dau_txns_by_namespace.sql | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/models/bam_models/all_chains/all_chains_gas_dau_txns_by_category.sql b/models/bam_models/all_chains/all_chains_gas_dau_txns_by_category.sql index 92165ab1..815e1665 100644 --- a/models/bam_models/all_chains/all_chains_gas_dau_txns_by_category.sql +++ b/models/bam_models/all_chains/all_chains_gas_dau_txns_by_category.sql @@ -30,4 +30,4 @@ where ) and category is not null and app is null - and gas is not null + and txns is not null diff --git a/models/bam_models/all_chains/all_chains_gas_dau_txns_by_namespace.sql b/models/bam_models/all_chains/all_chains_gas_dau_txns_by_namespace.sql index 83672e79..d8d6313d 100644 --- a/models/bam_models/all_chains/all_chains_gas_dau_txns_by_namespace.sql +++ b/models/bam_models/all_chains/all_chains_gas_dau_txns_by_namespace.sql @@ -34,7 +34,7 @@ with 'tron' ) and app is not null - and gas is not null + and txns is not null ), namespace_coingecko as ( select From c676192c505486bbd6866144fb0435499d1f55b7 Mon Sep 17 00:00:00 2001 From: Sebastian Melendez Date: Mon, 8 Apr 2024 19:25:58 -0400 Subject: [PATCH 09/22] adding stablecoin contracts as well as celo transfer events --- macros/decode/decode_events.sql | 5 +- macros/decode/transfer_events.sql | 21 + .../contracts/dim_contract_abis.sql | 630 +++++++++++++++++- .../dimensions/events/dim_events_silver.sql | 22 +- .../fact_celo_stablecoin_contracts.sql | 9 + .../celo/fact_celo_token_transfers.sql | 3 + 6 files changed, 681 insertions(+), 9 deletions(-) create mode 100644 macros/decode/transfer_events.sql create mode 100644 models/metrics/stablecoins/contracts/fact_celo_stablecoin_contracts.sql create mode 100644 models/staging/celo/fact_celo_token_transfers.sql diff --git a/macros/decode/decode_events.sql b/macros/decode/decode_events.sql index 71bdcadb..0b26c14d 100644 --- a/macros/decode/decode_events.sql +++ b/macros/decode/decode_events.sql @@ -7,11 +7,12 @@ contract_address, origin_from_address, origin_to_address, - status, + status as tx_status, + t1.topic_zero, t2.event_name, {{ target.schema }}.decode_evm_event_log( event_data, t2.event_info - ) as decoded_event + ) as decoded_log from {{ ref("fact_" ~ chain ~ "_events") }} t1 inner join {{ ref("dim_events_silver") }} t2 on t1.topic_zero = t2.topic_zero {% if is_incremental() %} diff --git a/macros/decode/transfer_events.sql b/macros/decode/transfer_events.sql new file mode 100644 index 00000000..c5606050 --- /dev/null +++ b/macros/decode/transfer_events.sql @@ -0,0 +1,21 @@ +{% macro transfer_events(chain) %} + select + block_timestamp, + block_number, + transaction_hash, + event_index, + origin_from_address, + origin_to_address, + contract_address, + decoded_log:"from"::string as "from", + decoded_log:"to"::string as "to", + decoded_log:"value"::string as amount, + tx_status + from {{ ref("fact_" ~ chain ~ "_decoded_events") }} + where topic_zero = '0xddf252ad1be2c89b69c2b068fc378daa952ba7f163c4a11628f55a4df523b3ef' + {% if is_incremental() %} + and + block_timestamp + >= (select dateadd('day', -3, max(block_timestamp)) from {{ this }}) + {% endif %} +{% endmacro %} diff --git a/models/dimensions/contracts/dim_contract_abis.sql b/models/dimensions/contracts/dim_contract_abis.sql index 9b44b2b6..1a751146 100644 --- a/models/dimensions/contracts/dim_contract_abis.sql +++ b/models/dimensions/contracts/dim_contract_abis.sql @@ -1,6 +1,6 @@ {{ config(materialized="table") }} -select null as contract_address, 'ERC20_Standard' as abi_type, parse_json('[ +select null as contract_address, null as chain, 'ERC20_Standard' as abi_type, parse_json('[ { "constant": true, "inputs": [], @@ -223,3 +223,631 @@ select null as contract_address, 'ERC20_Standard' as abi_type, parse_json('[ } ] ') as abi + +union all +select '0x48065fbBE25f71C9282ddf5e1cD6D6A887483D5e' as contract_address, 'celo' as chain, 'Contract' as abi_type, parse_json('[ + { + "anonymous": false, + "inputs": [ + { + "indexed": true, + "internalType": "address", + "name": "owner", + "type": "address" + }, + { + "indexed": true, + "internalType": "address", + "name": "spender", + "type": "address" + }, + { + "indexed": false, + "internalType": "uint256", + "name": "value", + "type": "uint256" + } + ], + "name": "Approval", + "type": "event" + }, + { + "anonymous": false, + "inputs": [ + { + "indexed": true, + "internalType": "address", + "name": "authorizer", + "type": "address" + }, + { + "indexed": true, + "internalType": "bytes32", + "name": "nonce", + "type": "bytes32" + } + ], + "name": "AuthorizationCanceled", + "type": "event" + }, + { + "anonymous": false, + "inputs": [ + { + "indexed": true, + "internalType": "address", + "name": "authorizer", + "type": "address" + }, + { + "indexed": true, + "internalType": "bytes32", + "name": "nonce", + "type": "bytes32" + } + ], + "name": "AuthorizationUsed", + "type": "event" + }, + { + "anonymous": false, + "inputs": [ + { + "indexed": true, + "internalType": "address", + "name": "_user", + "type": "address" + } + ], + "name": "BlockPlaced", + "type": "event" + }, + { + "anonymous": false, + "inputs": [ + { + "indexed": true, + "internalType": "address", + "name": "_user", + "type": "address" + } + ], + "name": "BlockReleased", + "type": "event" + }, + { + "anonymous": false, + "inputs": [ + { + "indexed": true, + "internalType": "address", + "name": "_blockedUser", + "type": "address" + }, + { + "indexed": false, + "internalType": "uint256", + "name": "_balance", + "type": "uint256" + } + ], + "name": "DestroyedBlockedFunds", + "type": "event" + }, + { + "anonymous": false, + "inputs": [ + { + "indexed": true, + "internalType": "address", + "name": "feeCurrencyWrapper", + "type": "address" + } + ], + "name": "LogSetFeeCurrencyWrapper", + "type": "event" + }, + { + "anonymous": false, + "inputs": [ + { + "indexed": true, + "internalType": "address", + "name": "_destination", + "type": "address" + }, + { + "indexed": false, + "internalType": "uint256", + "name": "_amount", + "type": "uint256" + } + ], + "name": "Mint", + "type": "event" + }, + { + "anonymous": false, + "inputs": [ + { + "indexed": true, + "internalType": "address", + "name": "previousOwner", + "type": "address" + }, + { + "indexed": true, + "internalType": "address", + "name": "newOwner", + "type": "address" + } + ], + "name": "OwnershipTransferred", + "type": "event" + }, + { + "anonymous": false, + "inputs": [ + { + "indexed": false, + "internalType": "uint256", + "name": "_amount", + "type": "uint256" + } + ], + "name": "Redeem", + "type": "event" + }, + { + "anonymous": false, + "inputs": [ + { + "indexed": true, + "internalType": "address", + "name": "from", + "type": "address" + }, + { + "indexed": true, + "internalType": "address", + "name": "to", + "type": "address" + }, + { + "indexed": false, + "internalType": "uint256", + "name": "value", + "type": "uint256" + } + ], + "name": "Transfer", + "type": "event" + }, + { + "inputs": [], + "name": "CANCEL_AUTHORIZATION_TYPEHASH", + "outputs": [{ "internalType": "bytes32", "name": "", "type": "bytes32" }], + "stateMutability": "view", + "type": "function" + }, + { + "inputs": [], + "name": "DOMAIN_SEPARATOR", + "outputs": [{ "internalType": "bytes32", "name": "", "type": "bytes32" }], + "stateMutability": "view", + "type": "function" + }, + { + "inputs": [], + "name": "RECEIVE_WITH_AUTHORIZATION_TYPEHASH", + "outputs": [{ "internalType": "bytes32", "name": "", "type": "bytes32" }], + "stateMutability": "view", + "type": "function" + }, + { + "inputs": [], + "name": "TRANSFER_WITH_AUTHORIZATION_TYPEHASH", + "outputs": [{ "internalType": "bytes32", "name": "", "type": "bytes32" }], + "stateMutability": "view", + "type": "function" + }, + { + "inputs": [ + { "internalType": "address", "name": "_user", "type": "address" } + ], + "name": "addToBlockedList", + "outputs": [], + "stateMutability": "nonpayable", + "type": "function" + }, + { + "inputs": [ + { "internalType": "address", "name": "owner", "type": "address" }, + { "internalType": "address", "name": "spender", "type": "address" } + ], + "name": "allowance", + "outputs": [{ "internalType": "uint256", "name": "", "type": "uint256" }], + "stateMutability": "view", + "type": "function" + }, + { + "inputs": [ + { "internalType": "address", "name": "spender", "type": "address" }, + { "internalType": "uint256", "name": "amount", "type": "uint256" } + ], + "name": "approve", + "outputs": [{ "internalType": "bool", "name": "", "type": "bool" }], + "stateMutability": "nonpayable", + "type": "function" + }, + { + "inputs": [ + { "internalType": "address", "name": "authorizer", "type": "address" }, + { "internalType": "bytes32", "name": "nonce", "type": "bytes32" } + ], + "name": "authorizationState", + "outputs": [{ "internalType": "bool", "name": "", "type": "bool" }], + "stateMutability": "view", + "type": "function" + }, + { + "inputs": [ + { "internalType": "address", "name": "account", "type": "address" } + ], + "name": "balanceOf", + "outputs": [{ "internalType": "uint256", "name": "", "type": "uint256" }], + "stateMutability": "view", + "type": "function" + }, + { + "inputs": [ + { "internalType": "address", "name": "authorizer", "type": "address" }, + { "internalType": "bytes32", "name": "nonce", "type": "bytes32" }, + { "internalType": "uint8", "name": "v", "type": "uint8" }, + { "internalType": "bytes32", "name": "r", "type": "bytes32" }, + { "internalType": "bytes32", "name": "s", "type": "bytes32" } + ], + "name": "cancelAuthorization", + "outputs": [], + "stateMutability": "nonpayable", + "type": "function" + }, + { + "inputs": [ + { "internalType": "address", "name": "authorizer", "type": "address" }, + { "internalType": "bytes32", "name": "nonce", "type": "bytes32" }, + { "internalType": "bytes", "name": "signature", "type": "bytes" } + ], + "name": "cancelAuthorization", + "outputs": [], + "stateMutability": "nonpayable", + "type": "function" + }, + { + "inputs": [ + { + "internalType": "address", + "name": "refundRecipient", + "type": "address" + }, + { "internalType": "address", "name": "tipRecipient", "type": "address" }, + { "internalType": "address", "name": "", "type": "address" }, + { + "internalType": "address", + "name": "baseFeeRecipient", + "type": "address" + }, + { "internalType": "uint256", "name": "refundAmount", "type": "uint256" }, + { "internalType": "uint256", "name": "tipAmount", "type": "uint256" }, + { "internalType": "uint256", "name": "", "type": "uint256" }, + { "internalType": "uint256", "name": "baseFeeAmount", "type": "uint256" } + ], + "name": "creditGasFees", + "outputs": [], + "stateMutability": "nonpayable", + "type": "function" + }, + { + "inputs": [ + { "internalType": "address", "name": "from", "type": "address" }, + { "internalType": "uint256", "name": "value", "type": "uint256" } + ], + "name": "debitGasFees", + "outputs": [], + "stateMutability": "nonpayable", + "type": "function" + }, + { + "inputs": [], + "name": "decimals", + "outputs": [{ "internalType": "uint8", "name": "", "type": "uint8" }], + "stateMutability": "view", + "type": "function" + }, + { + "inputs": [ + { "internalType": "address", "name": "spender", "type": "address" }, + { + "internalType": "uint256", + "name": "subtractedValue", + "type": "uint256" + } + ], + "name": "decreaseAllowance", + "outputs": [{ "internalType": "bool", "name": "", "type": "bool" }], + "stateMutability": "nonpayable", + "type": "function" + }, + { + "inputs": [ + { "internalType": "address", "name": "_blockedUser", "type": "address" } + ], + "name": "destroyBlockedFunds", + "outputs": [], + "stateMutability": "nonpayable", + "type": "function" + }, + { + "inputs": [], + "name": "feeCurrencyWrapper", + "outputs": [{ "internalType": "address", "name": "", "type": "address" }], + "stateMutability": "view", + "type": "function" + }, + { + "inputs": [ + { "internalType": "address", "name": "spender", "type": "address" }, + { "internalType": "uint256", "name": "addedValue", "type": "uint256" } + ], + "name": "increaseAllowance", + "outputs": [{ "internalType": "bool", "name": "", "type": "bool" }], + "stateMutability": "nonpayable", + "type": "function" + }, + { + "inputs": [ + { "internalType": "string", "name": "_name", "type": "string" }, + { "internalType": "string", "name": "_symbol", "type": "string" }, + { "internalType": "uint8", "name": "_decimals", "type": "uint8" } + ], + "name": "initialize", + "outputs": [], + "stateMutability": "nonpayable", + "type": "function" + }, + { + "inputs": [{ "internalType": "address", "name": "", "type": "address" }], + "name": "isBlocked", + "outputs": [{ "internalType": "bool", "name": "", "type": "bool" }], + "stateMutability": "view", + "type": "function" + }, + { + "inputs": [{ "internalType": "address", "name": "", "type": "address" }], + "name": "isTrusted", + "outputs": [{ "internalType": "bool", "name": "", "type": "bool" }], + "stateMutability": "view", + "type": "function" + }, + { + "inputs": [ + { "internalType": "address", "name": "_destination", "type": "address" }, + { "internalType": "uint256", "name": "_amount", "type": "uint256" } + ], + "name": "mint", + "outputs": [], + "stateMutability": "nonpayable", + "type": "function" + }, + { + "inputs": [ + { + "internalType": "address[]", + "name": "_recipients", + "type": "address[]" + }, + { "internalType": "uint256[]", "name": "_values", "type": "uint256[]" } + ], + "name": "multiTransfer", + "outputs": [], + "stateMutability": "nonpayable", + "type": "function" + }, + { + "inputs": [], + "name": "name", + "outputs": [{ "internalType": "string", "name": "", "type": "string" }], + "stateMutability": "view", + "type": "function" + }, + { + "inputs": [ + { "internalType": "address", "name": "owner", "type": "address" } + ], + "name": "nonces", + "outputs": [{ "internalType": "uint256", "name": "", "type": "uint256" }], + "stateMutability": "view", + "type": "function" + }, + { + "inputs": [], + "name": "owner", + "outputs": [{ "internalType": "address", "name": "", "type": "address" }], + "stateMutability": "view", + "type": "function" + }, + { + "inputs": [ + { "internalType": "address", "name": "owner_", "type": "address" }, + { "internalType": "address", "name": "spender", "type": "address" }, + { "internalType": "uint256", "name": "value", "type": "uint256" }, + { "internalType": "uint256", "name": "deadline", "type": "uint256" }, + { "internalType": "bytes", "name": "signature", "type": "bytes" } + ], + "name": "permit", + "outputs": [], + "stateMutability": "nonpayable", + "type": "function" + }, + { + "inputs": [ + { "internalType": "address", "name": "owner_", "type": "address" }, + { "internalType": "address", "name": "spender", "type": "address" }, + { "internalType": "uint256", "name": "value", "type": "uint256" }, + { "internalType": "uint256", "name": "deadline", "type": "uint256" }, + { "internalType": "uint8", "name": "v", "type": "uint8" }, + { "internalType": "bytes32", "name": "r", "type": "bytes32" }, + { "internalType": "bytes32", "name": "s", "type": "bytes32" } + ], + "name": "permit", + "outputs": [], + "stateMutability": "nonpayable", + "type": "function" + }, + { + "inputs": [ + { "internalType": "address", "name": "from", "type": "address" }, + { "internalType": "address", "name": "to", "type": "address" }, + { "internalType": "uint256", "name": "value", "type": "uint256" }, + { "internalType": "uint256", "name": "validAfter", "type": "uint256" }, + { "internalType": "uint256", "name": "validBefore", "type": "uint256" }, + { "internalType": "bytes32", "name": "nonce", "type": "bytes32" }, + { "internalType": "bytes", "name": "signature", "type": "bytes" } + ], + "name": "receiveWithAuthorization", + "outputs": [], + "stateMutability": "nonpayable", + "type": "function" + }, + { + "inputs": [ + { "internalType": "address", "name": "from", "type": "address" }, + { "internalType": "address", "name": "to", "type": "address" }, + { "internalType": "uint256", "name": "value", "type": "uint256" }, + { "internalType": "uint256", "name": "validAfter", "type": "uint256" }, + { "internalType": "uint256", "name": "validBefore", "type": "uint256" }, + { "internalType": "bytes32", "name": "nonce", "type": "bytes32" }, + { "internalType": "uint8", "name": "v", "type": "uint8" }, + { "internalType": "bytes32", "name": "r", "type": "bytes32" }, + { "internalType": "bytes32", "name": "s", "type": "bytes32" } + ], + "name": "receiveWithAuthorization", + "outputs": [], + "stateMutability": "nonpayable", + "type": "function" + }, + { + "inputs": [ + { "internalType": "uint256", "name": "_amount", "type": "uint256" } + ], + "name": "redeem", + "outputs": [], + "stateMutability": "nonpayable", + "type": "function" + }, + { + "inputs": [ + { "internalType": "address", "name": "_user", "type": "address" } + ], + "name": "removeFromBlockedList", + "outputs": [], + "stateMutability": "nonpayable", + "type": "function" + }, + { + "inputs": [], + "name": "renounceOwnership", + "outputs": [], + "stateMutability": "nonpayable", + "type": "function" + }, + { + "inputs": [ + { + "internalType": "address", + "name": "_feeCurrencyWrapper", + "type": "address" + } + ], + "name": "setFeeCurrencyWrapper", + "outputs": [], + "stateMutability": "nonpayable", + "type": "function" + }, + { + "inputs": [], + "name": "symbol", + "outputs": [{ "internalType": "string", "name": "", "type": "string" }], + "stateMutability": "view", + "type": "function" + }, + { + "inputs": [], + "name": "totalSupply", + "outputs": [{ "internalType": "uint256", "name": "", "type": "uint256" }], + "stateMutability": "view", + "type": "function" + }, + { + "inputs": [ + { "internalType": "address", "name": "recipient", "type": "address" }, + { "internalType": "uint256", "name": "amount", "type": "uint256" } + ], + "name": "transfer", + "outputs": [{ "internalType": "bool", "name": "", "type": "bool" }], + "stateMutability": "nonpayable", + "type": "function" + }, + { + "inputs": [ + { "internalType": "address", "name": "_sender", "type": "address" }, + { "internalType": "address", "name": "_recipient", "type": "address" }, + { "internalType": "uint256", "name": "_amount", "type": "uint256" } + ], + "name": "transferFrom", + "outputs": [{ "internalType": "bool", "name": "", "type": "bool" }], + "stateMutability": "nonpayable", + "type": "function" + }, + { + "inputs": [ + { "internalType": "address", "name": "newOwner", "type": "address" } + ], + "name": "transferOwnership", + "outputs": [], + "stateMutability": "nonpayable", + "type": "function" + }, + { + "inputs": [ + { "internalType": "address", "name": "from", "type": "address" }, + { "internalType": "address", "name": "to", "type": "address" }, + { "internalType": "uint256", "name": "value", "type": "uint256" }, + { "internalType": "uint256", "name": "validAfter", "type": "uint256" }, + { "internalType": "uint256", "name": "validBefore", "type": "uint256" }, + { "internalType": "bytes32", "name": "nonce", "type": "bytes32" }, + { "internalType": "bytes", "name": "signature", "type": "bytes" } + ], + "name": "transferWithAuthorization", + "outputs": [], + "stateMutability": "nonpayable", + "type": "function" + }, + { + "inputs": [ + { "internalType": "address", "name": "from", "type": "address" }, + { "internalType": "address", "name": "to", "type": "address" }, + { "internalType": "uint256", "name": "value", "type": "uint256" }, + { "internalType": "uint256", "name": "validAfter", "type": "uint256" }, + { "internalType": "uint256", "name": "validBefore", "type": "uint256" }, + { "internalType": "bytes32", "name": "nonce", "type": "bytes32" }, + { "internalType": "uint8", "name": "v", "type": "uint8" }, + { "internalType": "bytes32", "name": "r", "type": "bytes32" }, + { "internalType": "bytes32", "name": "s", "type": "bytes32" } + ], + "name": "transferWithAuthorization", + "outputs": [], + "stateMutability": "nonpayable", + "type": "function" + } +]') as abi \ No newline at end of file diff --git a/models/dimensions/events/dim_events_silver.sql b/models/dimensions/events/dim_events_silver.sql index a6ea4e7d..50d7e5e9 100644 --- a/models/dimensions/events/dim_events_silver.sql +++ b/models/dimensions/events/dim_events_silver.sql @@ -1,7 +1,17 @@ {{ config(materialized="table", unique_key="topic_zero", sort="topic_zero") }} -select - value:"name"::string as event_name, - value as event_info, - {{ target.schema }}.event_info_to_keccak_event_signature(event_info) as topic_zero -from {{ ref("dim_contract_abis") }}, lateral flatten(input => abi) as f -where value:"type" = 'event' +with +event_signatures as ( + select + value:"name"::string as event_name, + value as event_info, + {{ target.schema }}.event_info_to_keccak_event_signature(value) as topic_zero, + row_number() over (partition by topic_zero order by event_name) as event_id + from {{ ref("dim_contract_abis") }}, lateral flatten(input => abi) as f + where value:"type" = 'event' +) +select + event_name, + event_info, + topic_zero +from event_signatures +where event_id = 1 \ No newline at end of file diff --git a/models/metrics/stablecoins/contracts/fact_celo_stablecoin_contracts.sql b/models/metrics/stablecoins/contracts/fact_celo_stablecoin_contracts.sql new file mode 100644 index 00000000..681194d4 --- /dev/null +++ b/models/metrics/stablecoins/contracts/fact_celo_stablecoin_contracts.sql @@ -0,0 +1,9 @@ +{{ config(materialized="table") }} +select symbol, contract_address, num_decimals, coingecko_id, initial_supply +from + ( + values + ('USDC', '0xcebA9300f2b948710d2653dD7B07f33A8B32118C', 6, 'usd-coin', 0), + ('USDT', '0x48065fbBE25f71C9282ddf5e1cD6D6A887483D5e', 6, 'tether', 0), + ('cUSD', '0x765DE816845861e75A25fCA122bb6898B8B1282a', 18, 'celo-dollar', 0), + ) as results(symbol, contract_address, num_decimals, coingecko_id, initial_supply) diff --git a/models/staging/celo/fact_celo_token_transfers.sql b/models/staging/celo/fact_celo_token_transfers.sql new file mode 100644 index 00000000..a985cb9c --- /dev/null +++ b/models/staging/celo/fact_celo_token_transfers.sql @@ -0,0 +1,3 @@ +{{ config(materialized="incremental", unique_key=["transaction_hash", "event_index"]) }} + +{{ transfer_events("celo") }} \ No newline at end of file From 8b585a2787801a86dae9f66a0baf6f052422ff4e Mon Sep 17 00:00:00 2001 From: Son Do Date: Tue, 9 Apr 2024 17:12:41 -0700 Subject: [PATCH 10/22] Added Retention for bam chains except solana --- macros/retention/get_cohort_retention.sql | 79 +++++++++++++++++++ models/data_hubs/fact_retention_datahub.sql | 17 ++++ .../arbitrum/core/ez_arbitrum_retention.sql | 11 +++ .../avalanche/core/ez_avalanche_retention.sql | 11 +++ .../projects/base/core/ez_base_retention.sql | 11 +++ models/projects/bsc/core/ez_bsc_retention.sql | 11 +++ .../ethereum/core/ez_ethereum_retention.sql | 11 +++ .../projects/near/core/ez_near_retention.sql | 11 +++ .../optimism/core/ez_optimism_retention.sql | 11 +++ .../polygon/core/ez_polygon_retention.sql | 11 +++ .../projects/tron/core/ez_tron_retention.sql | 11 +++ 11 files changed, 195 insertions(+) create mode 100644 macros/retention/get_cohort_retention.sql create mode 100644 models/data_hubs/fact_retention_datahub.sql create mode 100644 models/projects/arbitrum/core/ez_arbitrum_retention.sql create mode 100644 models/projects/avalanche/core/ez_avalanche_retention.sql create mode 100644 models/projects/base/core/ez_base_retention.sql create mode 100644 models/projects/bsc/core/ez_bsc_retention.sql create mode 100644 models/projects/ethereum/core/ez_ethereum_retention.sql create mode 100644 models/projects/near/core/ez_near_retention.sql create mode 100644 models/projects/optimism/core/ez_optimism_retention.sql create mode 100644 models/projects/polygon/core/ez_polygon_retention.sql create mode 100644 models/projects/tron/core/ez_tron_retention.sql diff --git a/macros/retention/get_cohort_retention.sql b/macros/retention/get_cohort_retention.sql new file mode 100644 index 00000000..d1ba344e --- /dev/null +++ b/macros/retention/get_cohort_retention.sql @@ -0,0 +1,79 @@ +{% macro get_cohort_retention(chain) %} + with +min_date as ( + select + date_trunc('month', min(raw_date)) as first_month + from {{ ref( "ez_" ~ chain ~ "_transactions") }} +), +base_table as( + select + date_trunc('month', raw_date) as base_month, + from_address as address + from {{ ref( "ez_" ~ chain ~ "_transactions") }} + group by 1,2 +), +-- from user_cohorts onward, should be able to reuse ctes on any chain + +-- grab each users first action timestamp on chain +user_cohorts as ( + select + address, + min(base_month) as cohort_month + from + base_table + group by 1 +), + +--compute cohort size per distinct cohort period +cohort_size as ( + select + cohort_month, + count(distinct(address)) as cohort_size + from + user_cohorts + group by 1 +), + +-- determine if/ when users came back for another interaction based on period number +following_months as ( + select + bt.address, + timestampdiff(month, uc.cohort_month, bt.base_month) as month_number + from + base_table as bt + inner join user_cohorts as uc on bt.address = uc.address + where + bt.base_month > uc.cohort_month + group by + bt.address, + month_number +), +--aggregate and calcualte the retained user amount per time period per cohort +retention_data as( + select + uc.cohort_month as cohort_month, + fm.month_number, + count(distinct(fm.address)) as retained_user_count + from + following_months as fm + inner join user_cohorts as uc on fm.address = uc.address + where + cohort_month >= (select first_month from min_date) -- Grab data from the beginning of the month 24 months ago + group by + uc.cohort_month, + fm.month_number +) +select + '{{ chain }}' as chain, + r.cohort_month, + c.cohort_size, + r.month_number, + round(r.retained_user_count::numeric / c.cohort_size::numeric , 2) as retention_ratio +from + retention_data as r + inner join cohort_size as c on r.cohort_month = c.cohort_month +order by + r.cohort_month, + r.month_number + +{% endmacro %} \ No newline at end of file diff --git a/models/data_hubs/fact_retention_datahub.sql b/models/data_hubs/fact_retention_datahub.sql new file mode 100644 index 00000000..c908bcd2 --- /dev/null +++ b/models/data_hubs/fact_retention_datahub.sql @@ -0,0 +1,17 @@ +{{ config(materialized="table", snowflake_warehouse="RETENTION") }} + +{{ + dbt_utils.union_relations( + relations=[ + ref("ez_arbitrum_retention"), + ref("ez_avalanche_retention"), + ref("ez_base_retention"), + ref("ez_bsc_retention"), + ref("ez_ethereum_retention"), + ref("ez_near_retention"), + ref("ez_optimism_retention"), + ref("ez_polygon_retention"), + ref("ez_tron_retention") + ], + ) +}} \ No newline at end of file diff --git a/models/projects/arbitrum/core/ez_arbitrum_retention.sql b/models/projects/arbitrum/core/ez_arbitrum_retention.sql new file mode 100644 index 00000000..80e4ffb6 --- /dev/null +++ b/models/projects/arbitrum/core/ez_arbitrum_retention.sql @@ -0,0 +1,11 @@ +{{ + config( + materialized="table", + snowflake_warehouse="arbitrum", + database="arbitrum", + schema="core", + alias="ez_cohort_retention", + ) + }} + +{{ get_cohort_retention("arbitrum") }} \ No newline at end of file diff --git a/models/projects/avalanche/core/ez_avalanche_retention.sql b/models/projects/avalanche/core/ez_avalanche_retention.sql new file mode 100644 index 00000000..d1083696 --- /dev/null +++ b/models/projects/avalanche/core/ez_avalanche_retention.sql @@ -0,0 +1,11 @@ +{{ + config( + materialized="table", + snowflake_warehouse="avalanche", + database="avalanche", + schema="core", + alias="ez_cohort_retention", + ) + }} + +{{ get_cohort_retention("avalanche") }} \ No newline at end of file diff --git a/models/projects/base/core/ez_base_retention.sql b/models/projects/base/core/ez_base_retention.sql new file mode 100644 index 00000000..5fb388bf --- /dev/null +++ b/models/projects/base/core/ez_base_retention.sql @@ -0,0 +1,11 @@ +{{ + config( + materialized="table", + snowflake_warehouse="base", + database="base", + schema="core", + alias="ez_cohort_retention", + ) + }} + +{{ get_cohort_retention("base") }} \ No newline at end of file diff --git a/models/projects/bsc/core/ez_bsc_retention.sql b/models/projects/bsc/core/ez_bsc_retention.sql new file mode 100644 index 00000000..f2c8f5f6 --- /dev/null +++ b/models/projects/bsc/core/ez_bsc_retention.sql @@ -0,0 +1,11 @@ +{{ + config( + materialized="table", + snowflake_warehouse="bsc", + database="bsc", + schema="core", + alias="ez_cohort_retention", + ) + }} + +{{ get_cohort_retention("bsc") }} \ No newline at end of file diff --git a/models/projects/ethereum/core/ez_ethereum_retention.sql b/models/projects/ethereum/core/ez_ethereum_retention.sql new file mode 100644 index 00000000..bad9865a --- /dev/null +++ b/models/projects/ethereum/core/ez_ethereum_retention.sql @@ -0,0 +1,11 @@ +{{ + config( + materialized="table", + snowflake_warehouse="ethereum", + database="ethereum", + schema="core", + alias="ez_cohort_retention", + ) + }} + +{{ get_cohort_retention("ethereum") }} \ No newline at end of file diff --git a/models/projects/near/core/ez_near_retention.sql b/models/projects/near/core/ez_near_retention.sql new file mode 100644 index 00000000..0a7a63f2 --- /dev/null +++ b/models/projects/near/core/ez_near_retention.sql @@ -0,0 +1,11 @@ +{{ + config( + materialized="table", + snowflake_warehouse="near", + database="near", + schema="core", + alias="ez_cohort_retention", + ) + }} + +{{ get_cohort_retention("near") }} \ No newline at end of file diff --git a/models/projects/optimism/core/ez_optimism_retention.sql b/models/projects/optimism/core/ez_optimism_retention.sql new file mode 100644 index 00000000..1a78ff2d --- /dev/null +++ b/models/projects/optimism/core/ez_optimism_retention.sql @@ -0,0 +1,11 @@ +{{ + config( + materialized="table", + snowflake_warehouse="optimism", + database="optimism", + schema="core", + alias="ez_cohort_retention", + ) + }} + +{{ get_cohort_retention("optimism") }} \ No newline at end of file diff --git a/models/projects/polygon/core/ez_polygon_retention.sql b/models/projects/polygon/core/ez_polygon_retention.sql new file mode 100644 index 00000000..e7db17d2 --- /dev/null +++ b/models/projects/polygon/core/ez_polygon_retention.sql @@ -0,0 +1,11 @@ +{{ + config( + materialized="table", + snowflake_warehouse="polygon", + database="polygon", + schema="core", + alias="ez_cohort_retention", + ) + }} + +{{ get_cohort_retention("polygon") }} \ No newline at end of file diff --git a/models/projects/tron/core/ez_tron_retention.sql b/models/projects/tron/core/ez_tron_retention.sql new file mode 100644 index 00000000..cd31cd2d --- /dev/null +++ b/models/projects/tron/core/ez_tron_retention.sql @@ -0,0 +1,11 @@ +{{ + config( + materialized="table", + snowflake_warehouse="tron", + database="tron", + schema="core", + alias="ez_cohort_retention", + ) + }} + +{{ get_cohort_retention("tron") }} \ No newline at end of file From 004ab07da9e105fb42c344289cc2fc42c8c92a9b Mon Sep 17 00:00:00 2001 From: Son Do Date: Wed, 10 Apr 2024 12:49:38 -0700 Subject: [PATCH 11/22] Added Unique ID for retention datahub --- models/data_hubs/fact_retention_datahub.sql | 46 ++++++++++++++------- 1 file changed, 31 insertions(+), 15 deletions(-) diff --git a/models/data_hubs/fact_retention_datahub.sql b/models/data_hubs/fact_retention_datahub.sql index c908bcd2..b253b32e 100644 --- a/models/data_hubs/fact_retention_datahub.sql +++ b/models/data_hubs/fact_retention_datahub.sql @@ -1,17 +1,33 @@ {{ config(materialized="table", snowflake_warehouse="RETENTION") }} -{{ - dbt_utils.union_relations( - relations=[ - ref("ez_arbitrum_retention"), - ref("ez_avalanche_retention"), - ref("ez_base_retention"), - ref("ez_bsc_retention"), - ref("ez_ethereum_retention"), - ref("ez_near_retention"), - ref("ez_optimism_retention"), - ref("ez_polygon_retention"), - ref("ez_tron_retention") - ], - ) -}} \ No newline at end of file +with retention_data as ( + {{ + dbt_utils.union_relations( + relations=[ + ref("ez_arbitrum_retention"), + ref("ez_avalanche_retention"), + ref("ez_base_retention"), + ref("ez_bsc_retention"), + ref("ez_ethereum_retention"), + ref("ez_near_retention"), + ref("ez_optimism_retention"), + ref("ez_polygon_retention"), + ref("ez_tron_retention") + ], + ) + }} +) +SELECT + concat( + coalesce(cast(chain as string), '_this_is_null_'), + '|', + coalesce(cast(cohort_month as string), '_this_is_null_'), + '|', + coalesce(cast(month_number as string), '_this_is_null_') + ) as unique_id, + chain, + cohort_month, + cohort_size, + month_number, + retention_ratio +FROM retention_data From 41da8fd7c76cc59483599e59d2975d07223c4bdc Mon Sep 17 00:00:00 2001 From: Sebastian Melendez Date: Thu, 11 Apr 2024 14:11:37 -0400 Subject: [PATCH 12/22] creating celo stablecoin tables --- .../address_credits_quicknode.sql | 22 +++++++++++++++++ .../address_debits_quicknode.sql | 22 +++++++++++++++++ macros/agg_chain_stablecoin_breakdown.sql | 1 + macros/agg_chain_stablecoin_transfers.sql | 24 ++++++++++++------- macros/decode/decode_events.sql | 3 ++- ...r_events.sql => token_transfer_events.sql} | 11 +++++---- .../dim_celo_current_balances.sql | 4 ++++ .../agg_celo_stablecoin_breakdown_daily.sql | 4 ++++ .../agg_celo_stablecoin_breakdown_monthly.sql | 4 ++++ .../agg_celo_stablecoin_breakdown_weekly.sql | 4 ++++ .../fact_celo_stablecoin_contracts.sql | 1 + .../metrics/agg_celo_stablecoin_metrics.sql | 2 ++ models/staging/celo/__celo__sources.yml | 5 ++++ .../fact_celo_address_balances_by_token.sql | 12 ++++++++++ .../fact_celo_address_credit_by_token.sql | 9 +++++++ .../celo/fact_celo_address_debit_by_token.sql | 9 +++++++ .../celo/fact_celo_token_transfers.sql | 2 +- 17 files changed, 124 insertions(+), 15 deletions(-) create mode 100644 macros/address_balances/address_credits_quicknode.sql create mode 100644 macros/address_balances/address_debits_quicknode.sql rename macros/decode/{transfer_events.sql => token_transfer_events.sql} (65%) create mode 100644 models/dimensions/current_balances/dim_celo_current_balances.sql create mode 100644 models/metrics/stablecoins/breakdowns/daily/agg_celo_stablecoin_breakdown_daily.sql create mode 100644 models/metrics/stablecoins/breakdowns/monthly/agg_celo_stablecoin_breakdown_monthly.sql create mode 100644 models/metrics/stablecoins/breakdowns/weekly/agg_celo_stablecoin_breakdown_weekly.sql create mode 100644 models/metrics/stablecoins/metrics/agg_celo_stablecoin_metrics.sql create mode 100644 models/staging/celo/fact_celo_address_balances_by_token.sql create mode 100644 models/staging/celo/fact_celo_address_credit_by_token.sql create mode 100644 models/staging/celo/fact_celo_address_debit_by_token.sql diff --git a/macros/address_balances/address_credits_quicknode.sql b/macros/address_balances/address_credits_quicknode.sql new file mode 100644 index 00000000..ba30e124 --- /dev/null +++ b/macros/address_balances/address_credits_quicknode.sql @@ -0,0 +1,22 @@ +-- This model currently does not take into account native tokens. +-- We do not have access to traces from quicknode yet. +-- Only use ("celo") address balances for stablecoin data +{% macro address_credits_quicknode(chain) %} +select + to_address as address, + contract_address, + block_timestamp, + cast(amount as float) as credit, + null as credit_usd, + transaction_hash as tx_hash, + null as trace_index, + event_index +from {{ ref("fact_" ~ chain ~ "_token_transfers") }} +where + to_address <> lower('0x0000000000000000000000000000000000000000') + and to_date(block_timestamp) < to_date(sysdate()) + {% if is_incremental() %} + and block_timestamp + >= (select dateadd('day', -3, max(block_timestamp)) from {{ this }}) + {% endif %} +{% endmacro %} diff --git a/macros/address_balances/address_debits_quicknode.sql b/macros/address_balances/address_debits_quicknode.sql new file mode 100644 index 00000000..a4e0fbc2 --- /dev/null +++ b/macros/address_balances/address_debits_quicknode.sql @@ -0,0 +1,22 @@ +-- This model currently does not take into account native tokens. +-- We do not have access to traces from quicknode yet. +-- Only use ("celo") address balances for stablecoin data +{% macro address_debits_quicknode(chain) %} +select + from_address as address, + contract_address, + block_timestamp, + cast(amount * -1 as float) as debit, + null as debit_usd, + transaction_hash as tx_hash, + null as trace_index, + event_index +from {{ ref("fact_" ~ chain ~ "_token_transfers")}} +where + to_date(block_timestamp) < to_date(sysdate()) + and from_address <> lower('0x0000000000000000000000000000000000000000') + {% if is_incremental() %} + and block_timestamp + >= (select dateadd('day', -3, max(block_timestamp)) from {{ this }}) + {% endif %} +{% endmacro %} \ No newline at end of file diff --git a/macros/agg_chain_stablecoin_breakdown.sql b/macros/agg_chain_stablecoin_breakdown.sql index fc87f912..e6741cff 100644 --- a/macros/agg_chain_stablecoin_breakdown.sql +++ b/macros/agg_chain_stablecoin_breakdown.sql @@ -6,6 +6,7 @@ select dateadd({{ granularity }}, -1, max(block_timestamp)) {% if chain in ("tron") %} from tron_allium.assets.trc20_token_transfers {% elif chain in ("solana") %} from solana_flipside.core.fact_transfers + {% elif chain in ("celo") %} from {{ref("fact_" ~ chain ~ "_decoded_events")}} {% else %} from {{ chain }}_flipside.core.ez_decoded_event_logs {% endif %} ) diff --git a/macros/agg_chain_stablecoin_transfers.sql b/macros/agg_chain_stablecoin_transfers.sql index 32c94d6e..6c274402 100644 --- a/macros/agg_chain_stablecoin_transfers.sql +++ b/macros/agg_chain_stablecoin_transfers.sql @@ -80,11 +80,9 @@ -- Notably, we do NOT use the Mint / Burn events here because the -- basic IERC20 interface does not require them to be implemented coalesce( - decoded_log: - from - , - -- DAI on ETH Mainnet does not follow the IERC20 interface - decoded_log:src + decoded_log:from, + -- DAI on ETH Mainnet does not follow the IERC20 interface + decoded_log:src ) as from_address, coalesce(decoded_log:to, decoded_log:dst) as to_address, from_address = '0x0000000000000000000000000000000000000000' @@ -104,20 +102,28 @@ case when not is_mint and not is_burn then amount else 0 end as transfer_volume, - ez_decoded_event_logs.contract_address, + t1.contract_address, fact_{{ chain }}_stablecoin_contracts.symbol - from {{ chain }}_flipside.core.ez_decoded_event_logs + {% if chain in ("celo")%} + from {{ref("fact_" ~ chain ~ "_decoded_events")}} t1 + {% else %} + from {{ chain }}_flipside.core.ez_decoded_event_logs t1 + {% endif %} join fact_{{ chain }}_stablecoin_contracts - on lower(ez_decoded_event_logs.contract_address) + on lower(t1.contract_address) = lower(fact_{{ chain }}_stablecoin_contracts.contract_address) where - lower(ez_decoded_event_logs.contract_address) in ( + lower(t1.contract_address) in ( select lower(contract_address) from fact_{{ chain }}_stablecoin_contracts ) -- DO NOT include mint / burn events here - they will be duped and event_name in ('Transfer', 'Issue', 'Redeem') + {% if chain in ("celo") %} + and tx_status = 1 + {% else %} and tx_status = 'SUCCESS' + {% endif %} {% endif %} {% endmacro %} diff --git a/macros/decode/decode_events.sql b/macros/decode/decode_events.sql index 0b26c14d..4ecbba9f 100644 --- a/macros/decode/decode_events.sql +++ b/macros/decode/decode_events.sql @@ -12,7 +12,8 @@ t2.event_name, {{ target.schema }}.decode_evm_event_log( event_data, t2.event_info - ) as decoded_log + ) as decoded_log, + data from {{ ref("fact_" ~ chain ~ "_events") }} t1 inner join {{ ref("dim_events_silver") }} t2 on t1.topic_zero = t2.topic_zero {% if is_incremental() %} diff --git a/macros/decode/transfer_events.sql b/macros/decode/token_transfer_events.sql similarity index 65% rename from macros/decode/transfer_events.sql rename to macros/decode/token_transfer_events.sql index c5606050..26b0454e 100644 --- a/macros/decode/transfer_events.sql +++ b/macros/decode/token_transfer_events.sql @@ -1,4 +1,4 @@ -{% macro transfer_events(chain) %} +{% macro token_transfer_events(chain) %} select block_timestamp, block_number, @@ -7,12 +7,15 @@ origin_from_address, origin_to_address, contract_address, - decoded_log:"from"::string as "from", - decoded_log:"to"::string as "to", - decoded_log:"value"::string as amount, + decoded_log:"from"::string as from_address, + decoded_log:"to"::string as to_address, + try_to_number(decoded_log:"value"::string) as amount, tx_status from {{ ref("fact_" ~ chain ~ "_decoded_events") }} where topic_zero = '0xddf252ad1be2c89b69c2b068fc378daa952ba7f163c4a11628f55a4df523b3ef' + and data != '0x' + and amount is not null + and tx_status = 1 {% if is_incremental() %} and block_timestamp diff --git a/models/dimensions/current_balances/dim_celo_current_balances.sql b/models/dimensions/current_balances/dim_celo_current_balances.sql new file mode 100644 index 00000000..6630963e --- /dev/null +++ b/models/dimensions/current_balances/dim_celo_current_balances.sql @@ -0,0 +1,4 @@ +-- depends_on: {{ ref("fact_celo_address_balances_by_token") }} +{{ config(materialized="incremental", unique_key=["address", "contract_address"]) }} + +{{ current_balances("celo") }} diff --git a/models/metrics/stablecoins/breakdowns/daily/agg_celo_stablecoin_breakdown_daily.sql b/models/metrics/stablecoins/breakdowns/daily/agg_celo_stablecoin_breakdown_daily.sql new file mode 100644 index 00000000..e9f088c5 --- /dev/null +++ b/models/metrics/stablecoins/breakdowns/daily/agg_celo_stablecoin_breakdown_daily.sql @@ -0,0 +1,4 @@ +-- depends_on: {{ ref('fact_celo_stablecoin_contracts') }} +-- depends_on: {{ ref('fact_celo_address_balances_by_token') }} +{{ config(materialized="table", snowflake_warehouse="STABLECOIN_LG_2") }} +{{ agg_chain_stablecoin_breakdown("celo", "day") }} diff --git a/models/metrics/stablecoins/breakdowns/monthly/agg_celo_stablecoin_breakdown_monthly.sql b/models/metrics/stablecoins/breakdowns/monthly/agg_celo_stablecoin_breakdown_monthly.sql new file mode 100644 index 00000000..d497dffb --- /dev/null +++ b/models/metrics/stablecoins/breakdowns/monthly/agg_celo_stablecoin_breakdown_monthly.sql @@ -0,0 +1,4 @@ +-- depends_on: {{ ref('fact_celo_stablecoin_contracts') }} +-- depends_on: {{ ref('fact_celo_address_balances_by_token') }} +{{ config(materialized="table", snowflake_warehouse="STABLECOIN_LG_2") }} +{{ agg_chain_stablecoin_breakdown("celo", "month") }} diff --git a/models/metrics/stablecoins/breakdowns/weekly/agg_celo_stablecoin_breakdown_weekly.sql b/models/metrics/stablecoins/breakdowns/weekly/agg_celo_stablecoin_breakdown_weekly.sql new file mode 100644 index 00000000..2b5e9b32 --- /dev/null +++ b/models/metrics/stablecoins/breakdowns/weekly/agg_celo_stablecoin_breakdown_weekly.sql @@ -0,0 +1,4 @@ +-- depends_on: {{ ref('fact_celo_stablecoin_contracts') }} +-- depends_on: {{ ref('fact_celo_address_balances_by_token') }} +{{ config(materialized="table", snowflake_warehouse="STABLECOIN_LG_2") }} +{{ agg_chain_stablecoin_breakdown("celo", "week") }} diff --git a/models/metrics/stablecoins/contracts/fact_celo_stablecoin_contracts.sql b/models/metrics/stablecoins/contracts/fact_celo_stablecoin_contracts.sql index 681194d4..f0451dae 100644 --- a/models/metrics/stablecoins/contracts/fact_celo_stablecoin_contracts.sql +++ b/models/metrics/stablecoins/contracts/fact_celo_stablecoin_contracts.sql @@ -6,4 +6,5 @@ from ('USDC', '0xcebA9300f2b948710d2653dD7B07f33A8B32118C', 6, 'usd-coin', 0), ('USDT', '0x48065fbBE25f71C9282ddf5e1cD6D6A887483D5e', 6, 'tether', 0), ('cUSD', '0x765DE816845861e75A25fCA122bb6898B8B1282a', 18, 'celo-dollar', 0), + ('cEUR', '0xD8763CBa276a3738E6DE85b4b3bF5FDed6D6cA73', 18, 'celo-euro', 0) ) as results(symbol, contract_address, num_decimals, coingecko_id, initial_supply) diff --git a/models/metrics/stablecoins/metrics/agg_celo_stablecoin_metrics.sql b/models/metrics/stablecoins/metrics/agg_celo_stablecoin_metrics.sql new file mode 100644 index 00000000..42bdbbd3 --- /dev/null +++ b/models/metrics/stablecoins/metrics/agg_celo_stablecoin_metrics.sql @@ -0,0 +1,2 @@ +-- depends_on: {{ ref('fact_celo_stablecoin_contracts') }} +{{ config(materialized="table") }} {{ agg_chain_stablecoin_metrics("celo") }} diff --git a/models/staging/celo/__celo__sources.yml b/models/staging/celo/__celo__sources.yml index 2bb13b7d..f852b613 100644 --- a/models/staging/celo/__celo__sources.yml +++ b/models/staging/celo/__celo__sources.yml @@ -4,3 +4,8 @@ sources: database: LANDING_DATABASE tables: - name: raw_celo_blocks_receipts + - name: BALANCES + schema: PROD + database: PC_DBT_DB + tables: + - name: dim_celo_current_balances diff --git a/models/staging/celo/fact_celo_address_balances_by_token.sql b/models/staging/celo/fact_celo_address_balances_by_token.sql new file mode 100644 index 00000000..f228c6f5 --- /dev/null +++ b/models/staging/celo/fact_celo_address_balances_by_token.sql @@ -0,0 +1,12 @@ +-- depends_on: {{ ref("fact_celo_address_credit_by_token") }} +-- depends_on: {{ ref("fact_celo_address_debit_by_token") }} +-- depends_on: {{ source("BALANCES", "dim_celo_current_balances") }} +{{ + config( + materialized="incremental", + unique_key=["address", "contract_address", "block_timestamp"], + snowflake_warehouse="BALANCES_LG", + ) +}} + +{{ address_balances("celo") }} \ No newline at end of file diff --git a/models/staging/celo/fact_celo_address_credit_by_token.sql b/models/staging/celo/fact_celo_address_credit_by_token.sql new file mode 100644 index 00000000..c8a0bb50 --- /dev/null +++ b/models/staging/celo/fact_celo_address_credit_by_token.sql @@ -0,0 +1,9 @@ +{{ + config( + materialized="table", + unique_key=["tx_hash", "event_index", "trace_index"], + snowflake_warehouse="BALANCES_MD", + ) +}} + +{{ address_credits_quicknode("celo") }} \ No newline at end of file diff --git a/models/staging/celo/fact_celo_address_debit_by_token.sql b/models/staging/celo/fact_celo_address_debit_by_token.sql new file mode 100644 index 00000000..d9910ca3 --- /dev/null +++ b/models/staging/celo/fact_celo_address_debit_by_token.sql @@ -0,0 +1,9 @@ +{{ + config( + materialized="table", + unique_key=["tx_hash", "event_index", "trace_index"], + snowflake_warehouse="BALANCES_MD", + ) +}} + +{{ address_debits_quicknode("celo") }} \ No newline at end of file diff --git a/models/staging/celo/fact_celo_token_transfers.sql b/models/staging/celo/fact_celo_token_transfers.sql index a985cb9c..e70bc87e 100644 --- a/models/staging/celo/fact_celo_token_transfers.sql +++ b/models/staging/celo/fact_celo_token_transfers.sql @@ -1,3 +1,3 @@ {{ config(materialized="incremental", unique_key=["transaction_hash", "event_index"]) }} -{{ transfer_events("celo") }} \ No newline at end of file +{{ token_transfer_events("celo") }} \ No newline at end of file From 647ebdd7f41cecd6f0dd2f1bf6a6ddba69581424 Mon Sep 17 00:00:00 2001 From: Sebastian Melendez Date: Thu, 11 Apr 2024 17:24:23 -0400 Subject: [PATCH 13/22] adding nft trading volume --- .../cardano/core/ez_cardano_metrics.sql | 7 ++-- models/staging/cardano/__cardano__sources.yml | 1 + .../_test_fact_cardano_nft_trading_volume.yml | 32 +++++++++++++++++++ .../fact_cardano_nft_trading_volume.sql | 15 +++++++++ 4 files changed, 53 insertions(+), 2 deletions(-) create mode 100644 models/staging/cardano/_test_fact_cardano_nft_trading_volume.yml create mode 100644 models/staging/cardano/fact_cardano_nft_trading_volume.sql diff --git a/models/projects/cardano/core/ez_cardano_metrics.sql b/models/projects/cardano/core/ez_cardano_metrics.sql index 45bc88f3..d3f0203d 100644 --- a/models/projects/cardano/core/ez_cardano_metrics.sql +++ b/models/projects/cardano/core/ez_cardano_metrics.sql @@ -33,7 +33,8 @@ with ), price_data as ({{ get_coingecko_metrics("cardano") }}), defillama_data as ({{ get_defillama_metrics("cardano") }}), - github_data as ({{ get_github_metrics("cardano") }}) + github_data as ({{ get_github_metrics("cardano") }}), + nft_metrics as ({{ get_nft_metrics("cardano") }}) select fundamental_data.date, fundamental_data.chain, @@ -49,9 +50,11 @@ select weekly_commits_core_ecosystem, weekly_commits_sub_ecosystem, weekly_developers_core_ecosystem, - weekly_developers_sub_ecosystem + weekly_developers_sub_ecosystem, + nft_trading_volume from fundamental_data left join price_data on fundamental_data.date = price_data.date left join defillama_data on fundamental_data.date = defillama_data.date left join github_data on fundamental_data.date = github_data.date +left join nft_metrics on fundamental_data.date = nft_metrics.date where fundamental_data.date < to_date(sysdate()) diff --git a/models/staging/cardano/__cardano__sources.yml b/models/staging/cardano/__cardano__sources.yml index 013e897d..68a0b00a 100644 --- a/models/staging/cardano/__cardano__sources.yml +++ b/models/staging/cardano/__cardano__sources.yml @@ -6,3 +6,4 @@ sources: - name: raw_cardano_daa - name: raw_cardano_fees - name: raw_cardano_txns + - name: raw_cardano_nft_trading_volume diff --git a/models/staging/cardano/_test_fact_cardano_nft_trading_volume.yml b/models/staging/cardano/_test_fact_cardano_nft_trading_volume.yml new file mode 100644 index 00000000..2df77d2f --- /dev/null +++ b/models/staging/cardano/_test_fact_cardano_nft_trading_volume.yml @@ -0,0 +1,32 @@ +models: + - name: fact_cardano_nft_trading_volume + tests: + - "dbt_expectations.expect_table_row_count_to_be_between:": + min_value: 1 + max_value: 1000000 + columns: + - name: CHAIN + tests: + - not_null + - dbt_expectations.expect_column_to_exist + - name: nft_trading_volume + tests: + - not_null + - dbt_expectations.expect_column_to_exist + - dbt_expectations.expect_column_values_to_be_within_n_moving_stdevs: + date_column_name: DATE + period: day + lookback_periods: 1 + trend_periods: 14 + test_periods: 28 + sigma_threshold: 3 + take_logs: true + severity: warn + - name: DATE + tests: + - not_null + - dbt_expectations.expect_column_to_exist + - dbt_expectations.expect_row_values_to_have_recent_data: + datepart: day + interval: 3 + severity: warn diff --git a/models/staging/cardano/fact_cardano_nft_trading_volume.sql b/models/staging/cardano/fact_cardano_nft_trading_volume.sql new file mode 100644 index 00000000..31ed2c9f --- /dev/null +++ b/models/staging/cardano/fact_cardano_nft_trading_volume.sql @@ -0,0 +1,15 @@ +{{ config(materialized="table") }} + +with + max_extraction as ( + select max(extraction_date) as max_date + from {{ source("PROD_LANDING", "raw_cardano_nft_trading_volume") }} + ) +select + to_date(value:"date"::string) as date, + value:"totalPriceUSD"::float as nft_trading_volume, + 'cardano' as chain +from + {{ source("PROD_LANDING", "raw_cardano_nft_trading_volume") }}, + lateral flatten(input => parse_json(source_json)) +where extraction_date = (select max_date from max_extraction) From f1994b63ef3bdd1e6b27f5e5e4213311b2c3d498 Mon Sep 17 00:00:00 2001 From: Alex Weseley Date: Tue, 16 Apr 2024 15:11:43 -0400 Subject: [PATCH 14/22] helium fee tables silver + test files --- .../_test_fact_helium_network_fees_silver.yml | 18 +++++++++ .../_test_fact_helium_onboard_fees_silver.yml | 18 +++++++++ .../fact_helium_network_fees_silver.sql | 20 ++++++++++ .../fact_helium_onboard_fees_silver.sql | 37 +++++++++++++++++++ 4 files changed, 93 insertions(+) create mode 100644 models/projects/helium/_test_fact_helium_network_fees_silver.yml create mode 100644 models/projects/helium/_test_fact_helium_onboard_fees_silver.yml create mode 100644 models/projects/helium/fact_helium_network_fees_silver.sql create mode 100644 models/projects/helium/fact_helium_onboard_fees_silver.sql diff --git a/models/projects/helium/_test_fact_helium_network_fees_silver.yml b/models/projects/helium/_test_fact_helium_network_fees_silver.yml new file mode 100644 index 00000000..44c32fe0 --- /dev/null +++ b/models/projects/helium/_test_fact_helium_network_fees_silver.yml @@ -0,0 +1,18 @@ +models: + - name: fact_helium_network_fees_silver + tests: + - "dbt_expectations.expect_table_row_count_to_be_between:": + min_value: 1 + max_value: 1000000 + columns: + - name: DATE + tests: + - not_null + - dbt_expectations.expect_column_to_exist + - dbt_expectations.expect_row_values_to_have_recent_data: + datepart: day + interval: 3 + - name: NETWORK_FEES + tests: + - not_null + - dbt_expectations.expect_column_to_exist \ No newline at end of file diff --git a/models/projects/helium/_test_fact_helium_onboard_fees_silver.yml b/models/projects/helium/_test_fact_helium_onboard_fees_silver.yml new file mode 100644 index 00000000..62337ba1 --- /dev/null +++ b/models/projects/helium/_test_fact_helium_onboard_fees_silver.yml @@ -0,0 +1,18 @@ +models: + - name: fact_helium_onboard_fees_silver + tests: + - "dbt_expectations.expect_table_row_count_to_be_between:": + min_value: 1 + max_value: 1000000 + columns: + - name: DATE + tests: + - not_null + - dbt_expectations.expect_column_to_exist + - dbt_expectations.expect_row_values_to_have_recent_data: + datepart: day + interval: 3 + - name: ONBOARD_FEES + tests: + - not_null + - dbt_expectations.expect_column_to_exist \ No newline at end of file diff --git a/models/projects/helium/fact_helium_network_fees_silver.sql b/models/projects/helium/fact_helium_network_fees_silver.sql new file mode 100644 index 00000000..a84dc40f --- /dev/null +++ b/models/projects/helium/fact_helium_network_fees_silver.sql @@ -0,0 +1,20 @@ +with program as +( + SELECT + * + FROM + solana_flipside.core.fact_events + WHERE + program_id = 'credMBJhYFzfn7NxBMdU4aUqFggAjgztaCcv2Fo6fPT' + AND SUCCEEDED = 'TRUE' + order by 1 desc +) +SELECT + date(block_timestamp) as date, + SUM(GET_PATH(inner_instruction, 'instructions[0]:parsed:info:amount'))::INT/1e5 AS network_fees +FROM program p, + LATERAL flatten(input => p.signers) a +WHERE a.value in ('mobMc6Q18xT78cRiExgVbhadEKHiTUm6ZDnSZn2cU8h', 'iotdbYfWqPiDa5MruziFHxBsjHamKuEnMhkPFoW4mKE') +and date < to_date(sysdate()) +group by 1 +order by 1 desc diff --git a/models/projects/helium/fact_helium_onboard_fees_silver.sql b/models/projects/helium/fact_helium_onboard_fees_silver.sql new file mode 100644 index 00000000..42dbef47 --- /dev/null +++ b/models/projects/helium/fact_helium_onboard_fees_silver.sql @@ -0,0 +1,37 @@ +with events as( + SELECT + * + FROM + solana_flipside.core.fact_events e + WHERE program_id = 'hemjuPXBpNvggtaUnN1MwT3wrdhttKEfosTcc2P9Pg8' + AND SUCCEEDED = 'TRUE' + AND ( + GET_PATH(inner_instruction, 'instructions[3]:parsed:info:amount') IS NOT NULL OR + GET_PATH(inner_instruction, 'instructions[5]:parsed:info:amount') IS NOT NULL OR + GET_PATH(inner_instruction, 'instructions[7]:parsed:info:amount') IS NOT NULL + ) + ORDER BY + block_timestamp DESC +), +all_burns AS( + SELECT + block_timestamp, + tx_id, + CASE + WHEN GET_PATH(inner_instruction, 'instructions[3]:parsed:info:amount') IS NOT NULL -- 3.4 + THEN CAST(GET_PATH(inner_instruction, 'instructions[3]:parsed:info:amount') AS NUMBER) + WHEN GET_PATH(inner_instruction, 'instructions[5]:parsed:info:amount') IS NOT NULL -- 3.6 + THEN CAST(GET_PATH(inner_instruction, 'instructions[5]:parsed:info:amount') AS NUMBER) + WHEN GET_PATH(inner_instruction, 'instructions[7]:parsed:info:amount') IS NOT NULL -- 3.8 (very rare) + THEN CAST(GET_PATH(inner_instruction, 'instructions[7]:parsed:info:amount') AS NUMBER) + END AS dc_burned + FROM + events +) +SELECT + DATE(block_timestamp) as date, + SUM(dc_burned)*1e-5 as onboard_fees +FROM all_burns +where date < to_date(sysdate()) +GROUP BY 1 +ORDER BY 1 desc From ca94f822d394fe19e21000c5a7209cd8048ee0d6 Mon Sep 17 00:00:00 2001 From: Alex Weseley Date: Tue, 16 Apr 2024 15:12:37 -0400 Subject: [PATCH 15/22] ignore .vscode folder with personal color settings --- .gitignore | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.gitignore b/.gitignore index 3cb1f6f6..522c74e8 100644 --- a/.gitignore +++ b/.gitignore @@ -222,4 +222,6 @@ Temporary Items # iCloud generated files *.icloud -dbt_scripts/grant_permissions.sql \ No newline at end of file +dbt_scripts/grant_permissions.sql + +.vscode \ No newline at end of file From d57e8851aacfb030b98c11068dad689c469a8de3 Mon Sep 17 00:00:00 2001 From: Alex Weseley Date: Tue, 16 Apr 2024 15:36:46 -0400 Subject: [PATCH 16/22] add helium total demand side fees + test file --- .../helium/_test_fact_helium_fees_silver.yml | 18 ++++++++++++++++++ .../helium/fact_helium_fees_silver.sql | 6 ++++++ 2 files changed, 24 insertions(+) create mode 100644 models/projects/helium/_test_fact_helium_fees_silver.yml create mode 100644 models/projects/helium/fact_helium_fees_silver.sql diff --git a/models/projects/helium/_test_fact_helium_fees_silver.yml b/models/projects/helium/_test_fact_helium_fees_silver.yml new file mode 100644 index 00000000..2a58260f --- /dev/null +++ b/models/projects/helium/_test_fact_helium_fees_silver.yml @@ -0,0 +1,18 @@ +models: + - name: fact_helium_fees_silver + tests: + - "dbt_expectations.expect_table_row_count_to_be_between:": + min_value: 1 + max_value: 1000000 + columns: + - name: DATE + tests: + - not_null + - dbt_expectations.expect_column_to_exist + - dbt_expectations.expect_row_values_to_have_recent_data: + datepart: day + interval: 3 + - name: FEES + tests: + - not_null + - dbt_expectations.expect_column_to_exist \ No newline at end of file diff --git a/models/projects/helium/fact_helium_fees_silver.sql b/models/projects/helium/fact_helium_fees_silver.sql new file mode 100644 index 00000000..019c43c4 --- /dev/null +++ b/models/projects/helium/fact_helium_fees_silver.sql @@ -0,0 +1,6 @@ +SELECT o.date, o.onboard_fees + n.network_fees as fees +FROM +{{ ref("fact_helium_onboard_fees_silver") }} o +LEFT JOIN +{{ ref("fact_helium_network_fees_silver")}} n +ON o.date = n.date \ No newline at end of file From cfab7160277e2ac749111fd10892bd6071e4a7c9 Mon Sep 17 00:00:00 2001 From: Alex Weseley Date: Tue, 16 Apr 2024 15:51:00 -0400 Subject: [PATCH 17/22] add helium revenue --- .../_test_fact_helium_revenue_silver.yml | 22 +++++++++ .../helium/fact_helium_revenue_silver.sql | 48 +++++++++++++++++++ 2 files changed, 70 insertions(+) create mode 100644 models/projects/helium/_test_fact_helium_revenue_silver.yml create mode 100644 models/projects/helium/fact_helium_revenue_silver.sql diff --git a/models/projects/helium/_test_fact_helium_revenue_silver.yml b/models/projects/helium/_test_fact_helium_revenue_silver.yml new file mode 100644 index 00000000..eea4dbed --- /dev/null +++ b/models/projects/helium/_test_fact_helium_revenue_silver.yml @@ -0,0 +1,22 @@ +models: + - name: fact_helium_revenue_silver + tests: + - "dbt_expectations.expect_table_row_count_to_be_between:": + min_value: 1 + max_value: 1000000 + columns: + - name: DATE + tests: + - not_null + - dbt_expectations.expect_column_to_exist + - dbt_expectations.expect_row_values_to_have_recent_data: + datepart: day + interval: 3 + - name: HNT_BURNED + tests: + - not_null + - dbt_expectations.expect_column_to_exist + - name: REVENUE + tests: + - not_null + - dbt_expectations.expect_column_to_exist \ No newline at end of file diff --git a/models/projects/helium/fact_helium_revenue_silver.sql b/models/projects/helium/fact_helium_revenue_silver.sql new file mode 100644 index 00000000..403a308e --- /dev/null +++ b/models/projects/helium/fact_helium_revenue_silver.sql @@ -0,0 +1,48 @@ +WITH program AS ( + SELECT + *, + GET_PATH(inner_instruction, 'instructions[1]:parsed:info:amount') AS inst1_amount, + GET_PATH(inner_instruction, 'instructions[3]:parsed:info:amount') AS inst3_amount, + GET_PATH(inner_instruction, 'instructions[0]:parsed:info:amount') AS inst0_amount, + GET_PATH(inner_instruction, 'instructions[2]:parsed:info:amount') AS inst2_amount, + GET_PATH(inner_instruction, 'instructions[5]:parsed:info:amount') AS inst5_amount, + GET_PATH(inner_instruction, 'instructions[7]:parsed:info:amount') AS inst7_amount + FROM + solana_flipside.core.fact_events + WHERE + program_id = 'credMBJhYFzfn7NxBMdU4aUqFggAjgztaCcv2Fo6fPT' AND + SUCCEEDED = 'TRUE' +), +consolidated AS ( + SELECT + block_timestamp, + lower(tx_id) AS tx_id, + CASE + WHEN inst1_amount IS NOT NULL AND inst3_amount IS NOT NULL THEN inst1_amount::INT/1e8 + WHEN inst0_amount IS NOT NULL AND inst2_amount IS NOT NULL THEN inst0_amount::INT/1e8 + WHEN inst5_amount IS NOT NULL AND inst7_amount IS NOT NULL THEN inst5_amount::INT/1e8 + ELSE NULL + END AS hnt_burned, + CASE + WHEN inst1_amount IS NOT NULL AND inst3_amount IS NOT NULL THEN inst3_amount::INT/1e5 + WHEN inst0_amount IS NOT NULL AND inst2_amount IS NOT NULL THEN inst2_amount::INT/1e5 + WHEN inst5_amount IS NOT NULL AND inst7_amount IS NOT NULL THEN inst7_amount::INT/1e5 + ELSE NULL + END AS dc_minted + FROM + program + WHERE + (inst1_amount IS NOT NULL AND inst3_amount IS NOT NULL) OR + (inst0_amount IS NOT NULL AND inst2_amount IS NOT NULL) OR + (inst5_amount IS NOT NULL AND inst7_amount IS NOT NULL) +) +SELECT + date(block_timestamp) AS date, + SUM(hnt_burned) AS hnt_burned, + SUM(dc_minted) AS revenue +FROM + consolidated +GROUP BY + 1 +ORDER BY + 1 DESC From 4e00a98465aee788ba24d8ed8b159d2091a7bb75 Mon Sep 17 00:00:00 2001 From: Alex Weseley Date: Tue, 16 Apr 2024 15:57:17 -0400 Subject: [PATCH 18/22] add helium to assets.csv --- assets.csv | 1 + 1 file changed, 1 insertion(+) diff --git a/assets.csv b/assets.csv index 8df542d5..45583cae 100644 --- a/assets.csv +++ b/assets.csv @@ -21,6 +21,7 @@ ETHEREUM_FLIPSIDE FLOW_FLIPSIDE GEODNET GNOSIS_FLIPSIDE +HELIUM INJECTIVE LANDING_DATABASE NEAR From e3e372e7b0b6b50fa0f0c327af7357796fd880b5 Mon Sep 17 00:00:00 2001 From: Alex Weseley Date: Tue, 16 Apr 2024 16:24:41 -0400 Subject: [PATCH 19/22] helium add chain and protocol columns --- .../projects/helium/_test_fact_helium_fees_silver.yml | 10 +++++++++- .../helium/_test_fact_helium_network_fees_silver.yml | 10 +++++++++- .../helium/_test_fact_helium_onboard_fees_silver.yml | 10 +++++++++- .../helium/_test_fact_helium_revenue_silver.yml | 10 +++++++++- models/projects/helium/fact_helium_fees_silver.sql | 6 +++++- .../helium/fact_helium_network_fees_silver.sql | 4 +++- .../helium/fact_helium_onboard_fees_silver.sql | 4 +++- models/projects/helium/fact_helium_revenue_silver.sql | 4 +++- 8 files changed, 50 insertions(+), 8 deletions(-) diff --git a/models/projects/helium/_test_fact_helium_fees_silver.yml b/models/projects/helium/_test_fact_helium_fees_silver.yml index 2a58260f..62c378d5 100644 --- a/models/projects/helium/_test_fact_helium_fees_silver.yml +++ b/models/projects/helium/_test_fact_helium_fees_silver.yml @@ -15,4 +15,12 @@ models: - name: FEES tests: - not_null - - dbt_expectations.expect_column_to_exist \ No newline at end of file + - dbt_expectations.expect_column_to_exist + - name: CHAIN + tests: + - not_null + - dbt_expectations.expect_column_to_exist + - name: PROTOCOL + tests: + - not_null + - dbt_expectations.expect_column_to_exist \ No newline at end of file diff --git a/models/projects/helium/_test_fact_helium_network_fees_silver.yml b/models/projects/helium/_test_fact_helium_network_fees_silver.yml index 44c32fe0..a5d45004 100644 --- a/models/projects/helium/_test_fact_helium_network_fees_silver.yml +++ b/models/projects/helium/_test_fact_helium_network_fees_silver.yml @@ -15,4 +15,12 @@ models: - name: NETWORK_FEES tests: - not_null - - dbt_expectations.expect_column_to_exist \ No newline at end of file + - dbt_expectations.expect_column_to_exist + - name: CHAIN + tests: + - not_null + - dbt_expectations.expect_column_to_exist + - name: PROTOCOL + tests: + - not_null + - dbt_expectations.expect_column_to_exist \ No newline at end of file diff --git a/models/projects/helium/_test_fact_helium_onboard_fees_silver.yml b/models/projects/helium/_test_fact_helium_onboard_fees_silver.yml index 62337ba1..2db6dd52 100644 --- a/models/projects/helium/_test_fact_helium_onboard_fees_silver.yml +++ b/models/projects/helium/_test_fact_helium_onboard_fees_silver.yml @@ -15,4 +15,12 @@ models: - name: ONBOARD_FEES tests: - not_null - - dbt_expectations.expect_column_to_exist \ No newline at end of file + - dbt_expectations.expect_column_to_exist + - name: CHAIN + tests: + - not_null + - dbt_expectations.expect_column_to_exist + - name: PROTOCOL + tests: + - not_null + - dbt_expectations.expect_column_to_exist \ No newline at end of file diff --git a/models/projects/helium/_test_fact_helium_revenue_silver.yml b/models/projects/helium/_test_fact_helium_revenue_silver.yml index eea4dbed..7b35d6d5 100644 --- a/models/projects/helium/_test_fact_helium_revenue_silver.yml +++ b/models/projects/helium/_test_fact_helium_revenue_silver.yml @@ -19,4 +19,12 @@ models: - name: REVENUE tests: - not_null - - dbt_expectations.expect_column_to_exist \ No newline at end of file + - dbt_expectations.expect_column_to_exist + - name: CHAIN + tests: + - not_null + - dbt_expectations.expect_column_to_exist + - name: PROTOCOL + tests: + - not_null + - dbt_expectations.expect_column_to_exist \ No newline at end of file diff --git a/models/projects/helium/fact_helium_fees_silver.sql b/models/projects/helium/fact_helium_fees_silver.sql index 019c43c4..951ce86d 100644 --- a/models/projects/helium/fact_helium_fees_silver.sql +++ b/models/projects/helium/fact_helium_fees_silver.sql @@ -1,4 +1,8 @@ -SELECT o.date, o.onboard_fees + n.network_fees as fees +SELECT + o.date, + o.onboard_fees + n.network_fees as fees, + o.chain, + o.protocol FROM {{ ref("fact_helium_onboard_fees_silver") }} o LEFT JOIN diff --git a/models/projects/helium/fact_helium_network_fees_silver.sql b/models/projects/helium/fact_helium_network_fees_silver.sql index a84dc40f..b018e576 100644 --- a/models/projects/helium/fact_helium_network_fees_silver.sql +++ b/models/projects/helium/fact_helium_network_fees_silver.sql @@ -11,7 +11,9 @@ with program as ) SELECT date(block_timestamp) as date, - SUM(GET_PATH(inner_instruction, 'instructions[0]:parsed:info:amount'))::INT/1e5 AS network_fees + SUM(GET_PATH(inner_instruction, 'instructions[0]:parsed:info:amount'))::INT/1e5 AS network_fees, + 'solana' AS chain, + 'helium' AS protocol FROM program p, LATERAL flatten(input => p.signers) a WHERE a.value in ('mobMc6Q18xT78cRiExgVbhadEKHiTUm6ZDnSZn2cU8h', 'iotdbYfWqPiDa5MruziFHxBsjHamKuEnMhkPFoW4mKE') diff --git a/models/projects/helium/fact_helium_onboard_fees_silver.sql b/models/projects/helium/fact_helium_onboard_fees_silver.sql index 42dbef47..f415aa34 100644 --- a/models/projects/helium/fact_helium_onboard_fees_silver.sql +++ b/models/projects/helium/fact_helium_onboard_fees_silver.sql @@ -30,7 +30,9 @@ all_burns AS( ) SELECT DATE(block_timestamp) as date, - SUM(dc_burned)*1e-5 as onboard_fees + SUM(dc_burned)*1e-5 as onboard_fees, + 'solana' AS chain, + 'helium' AS protocol FROM all_burns where date < to_date(sysdate()) GROUP BY 1 diff --git a/models/projects/helium/fact_helium_revenue_silver.sql b/models/projects/helium/fact_helium_revenue_silver.sql index 403a308e..99004007 100644 --- a/models/projects/helium/fact_helium_revenue_silver.sql +++ b/models/projects/helium/fact_helium_revenue_silver.sql @@ -39,7 +39,9 @@ consolidated AS ( SELECT date(block_timestamp) AS date, SUM(hnt_burned) AS hnt_burned, - SUM(dc_minted) AS revenue + SUM(dc_minted) AS revenue, + 'solana' AS chain, + 'helium' AS protocol FROM consolidated GROUP BY From 0ff40845b4968da892f5651959f710c306912fda Mon Sep 17 00:00:00 2001 From: Alex Weseley Date: Tue, 16 Apr 2024 17:13:21 -0400 Subject: [PATCH 20/22] add helium ez_metrics --- models/projects/helium/ez_helium_metrics.sql | 34 ++++++++++++++++++++ 1 file changed, 34 insertions(+) create mode 100644 models/projects/helium/ez_helium_metrics.sql diff --git a/models/projects/helium/ez_helium_metrics.sql b/models/projects/helium/ez_helium_metrics.sql new file mode 100644 index 00000000..8e762079 --- /dev/null +++ b/models/projects/helium/ez_helium_metrics.sql @@ -0,0 +1,34 @@ +{{ + config( + materialized="table", + snowflake_warehouse="HELIUM", + database="helium", + schema="core", + alias="ez_metrics", + ) +}} + +with + revenue_data as ( + select date, revenue, chain, protocol + from {{ ref("fact_helium_revenue_silver") }} + ), + fees_data as( + select date, fees, chain, protocol + from {{ ref("fact_helium_fees_silver") }} + ), + price_data as ({{ get_coingecko_metrics("helium") }}) +select + revenue_data.date, + revenue_data.chain, + revenue_data.protocol, + revenue_data.revenue, + fees_data.fees, + price_data.price, + price_data.market_cap, + price_data.fdmc +from revenue_data +left join price_data on revenue_data.date = price_data.date +left join fees_data on fees_data.date = revenue_data.date +where revenue_data.date < to_date(sysdate()) +order by 1 desc From 2ab1133dfa36e71e19f9837749668c112874f472 Mon Sep 17 00:00:00 2001 From: Sebastian Melendez Date: Wed, 17 Apr 2024 10:22:00 -0400 Subject: [PATCH 21/22] updating cosmos models --- models/staging/cosmoshub/fact_cosmoshub_daa.sql | 5 ++--- .../cosmoshub/fact_cosmoshub_fees_and_revenue.sql | 9 +++++---- models/staging/cosmoshub/fact_cosmoshub_txns.sql | 3 +-- 3 files changed, 8 insertions(+), 9 deletions(-) diff --git a/models/staging/cosmoshub/fact_cosmoshub_daa.sql b/models/staging/cosmoshub/fact_cosmoshub_daa.sql index c057eed4..44c9762e 100644 --- a/models/staging/cosmoshub/fact_cosmoshub_daa.sql +++ b/models/staging/cosmoshub/fact_cosmoshub_daa.sql @@ -6,10 +6,9 @@ with ) select date(value:date) as date, - value:"DAU"::int as daa, - value as source, + value:"daa"::int as daa, 'cosmoshub' as chain from {{ source("PROD_LANDING", "raw_cosmoshub_daa") }}, lateral flatten(input => parse_json(source_json)) -where extraction_date = (select max_date from max_extraction) +where extraction_date = (select max_date from max_extraction) and date(value:date) is not null diff --git a/models/staging/cosmoshub/fact_cosmoshub_fees_and_revenue.sql b/models/staging/cosmoshub/fact_cosmoshub_fees_and_revenue.sql index f2bb1e84..332d4cb8 100644 --- a/models/staging/cosmoshub/fact_cosmoshub_fees_and_revenue.sql +++ b/models/staging/cosmoshub/fact_cosmoshub_fees_and_revenue.sql @@ -6,7 +6,7 @@ with ), cosmoshub_prices as ({{ get_coingecko_price_with_latest("cosmos") }}), raw as ( - select date(value:date) as date, value:"fees"::int as fees, value as source + select date(value:date) as date, value:"fee"::float as fees, value:"fee_currency"::string as fee_currency from {{ source("PROD_LANDING", "raw_cosmoshub_fees") }}, lateral flatten(input => parse_json(source_json)) @@ -15,8 +15,9 @@ with select raw.date, 'cosmoshub' as chain, - fees as gas, - fees * coalesce(price, 0) as gas_usd, + sum(fees) as gas, + sum(fees * coalesce(price, 0)) as gas_usd, gas_usd * .02 as revenue from raw -left join cosmoshub_prices on raw.date = cosmoshub_prices.date +left join cosmoshub_prices on raw.date = cosmoshub_prices.date and fee_currency = 'uatom' +group by 1 diff --git a/models/staging/cosmoshub/fact_cosmoshub_txns.sql b/models/staging/cosmoshub/fact_cosmoshub_txns.sql index 876be2f6..5f159725 100644 --- a/models/staging/cosmoshub/fact_cosmoshub_txns.sql +++ b/models/staging/cosmoshub/fact_cosmoshub_txns.sql @@ -7,9 +7,8 @@ with select date(value:date) as date, value:txns::int as txns, - value as source, 'cosmoshub' as chain from {{ source("PROD_LANDING", "raw_cosmoshub_txns") }}, lateral flatten(input => parse_json(source_json)) -where extraction_date = (select max_date from max_extraction) +where extraction_date = (select max_date from max_extraction) and date(value:date) is not null From ce35c78cdf1f94fb168e27835ed16122dc7c7d23 Mon Sep 17 00:00:00 2001 From: Sebastian Melendez Date: Wed, 17 Apr 2024 10:27:14 -0400 Subject: [PATCH 22/22] fixing bug in gold tables --- models/staging/cosmoshub/fact_cosmoshub_daa_gold.sql | 2 +- models/staging/cosmoshub/fact_cosmoshub_txns_gold.sql | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/models/staging/cosmoshub/fact_cosmoshub_daa_gold.sql b/models/staging/cosmoshub/fact_cosmoshub_daa_gold.sql index 9b881629..226fc3fc 100644 --- a/models/staging/cosmoshub/fact_cosmoshub_daa_gold.sql +++ b/models/staging/cosmoshub/fact_cosmoshub_daa_gold.sql @@ -1,3 +1,3 @@ {{ config(materialized="table") }} -select date, daa, chain, source +select date, daa, chain from {{ ref("fact_cosmoshub_daa") }} diff --git a/models/staging/cosmoshub/fact_cosmoshub_txns_gold.sql b/models/staging/cosmoshub/fact_cosmoshub_txns_gold.sql index 766ca038..0348200a 100644 --- a/models/staging/cosmoshub/fact_cosmoshub_txns_gold.sql +++ b/models/staging/cosmoshub/fact_cosmoshub_txns_gold.sql @@ -1,3 +1,3 @@ {{ config(materialized="table") }} -select date, txns, chain, source +select date, txns, chain from {{ ref("fact_cosmoshub_txns") }}