Merge pull request #87 from Artemis-xyz/add-lst

Add Binance Staked ETH (+macros to get the rest more easily!)

macros/forward_fill.sql

@@ -0,0 +1,40 @@
+{% macro forward_fill(date_col, value_col, table_name) %}
+(
+    with results as (
+        select 
+            {{ date_col }} as date,
+            {{ value_col }} as value 
+        from {{ table_name }}
+    ),
+    date_range as (
+        select 
+            -1 + row_number() over (order by 0) as i, 
+            start_date + i as date
+        from (
+            select 
+                min(date)::date as start_date, 
+                current_date() as end_date
+            from results
+        ) 
+        join table(generator(rowcount => 10000)) x
+        qualify i < 1 + end_date - start_date
+    ),
+    full_range_data as (
+        select
+            coalesce(date_range.date, results.date) as date,
+            value
+        from results
+        full join date_range on results.date = date_range.date
+    ),
+    forward_fill_data as (
+        select
+            date,
+            coalesce(
+                value,
+                lag(value) ignore nulls over (order by date)
+            ) as value
+        from full_range_data
+    )
+    select * from forward_fill_data
+)
+{% endmacro %}

macros/lst/calc_staked_eth.sql

@@ -0,0 +1,38 @@
+{% macro calc_staked_eth(table_name) %}
+with
+    temp as (
+        select
+            f.date,
+            f.total_supply as num_staked_eth,
+            f.total_supply * p.shifted_token_price_usd as amount_staked_usd,
+            p.shifted_token_price_usd
+        from {{ ref(table_name) }} f
+        join pc_dbt_db.prod.fact_coingecko_token_date_adjusted_gold as p
+            on f.date = p.date
+        where p.coingecko_id = 'ethereum'
+        order by date desc
+    )
+select
+    t.date,
+    t.num_staked_eth,
+    t.amount_staked_usd,
+    case
+        when t.date = (select min(date) from {{ ref(table_name) }})
+        then 0
+        else t.num_staked_eth - lag(t.num_staked_eth, 1) over (order by t.date)
+    end as num_staked_eth_net_change,  
+    case
+        when t.date = (select min(date) from {{ ref(table_name) }})
+        then 0
+        else
+            (t.num_staked_eth - lag(t.num_staked_eth, 1) over (order by t.date)) * (
+                (
+                    t.shifted_token_price_usd
+                    + lag(t.shifted_token_price_usd, 1) over (order by t.date)
+                )
+                / 2
+            )
+    end as amount_staked_usd_net_change  
+from temp t
+order by date desc
+{% endmacro %}

macros/transform/raw_partioned_array_to_fact_table_many_columns.sql

@@ -0,0 +1,64 @@
+{% macro raw_partitioned_array_to_fact_table_many_columns(
+    raw_table_name, date_column_name, value_columns
+) %}
+    -- Data has the following structure:
+    -- {
+    -- "extraction_date": "2021-09-01T00:00:00.000Z",
+    -- "source_json": [
+    -- {
+    -- "date_time": "2021-09-01T00:00:00.000Z",
+    -- "DAU": 0,
+    -- "MAU": 0
+    -- },
+    -- {
+    -- "date_time": "2021-09-02T00:00:00.000Z",
+    -- "DAU": 0,
+    -- "MAU": 0
+    -- },
+    -- {
+    -- "date_time": "2021-09-03T00:00:00.000Z",
+    -- "DAU": 0,
+    -- "MAU": 0
+    -- },
+    -- ]
+    -- }
+    -- Any arbitrary row will NOT contain the full history of the data, so we need to
+    -- extract the data for each date and then break ties by extraction_date
+
+    with
+        dates as (
+            select
+                extraction_date,
+                flat_json.value:"{{date_column_name}}"::timestamp as date
+            from
+                {{ raw_table_name }},
+                lateral flatten(input => parse_json(source_json)) as flat_json
+            group by date, extraction_date
+        ),
+        max_extraction_per_day as (
+            select date, max(extraction_date) as extraction_date
+            from dates
+            group by date
+            order by date
+        ),
+        flattened_json as (
+            select
+                extraction_date,
+                flat_json.value:"{{date_column_name}}"::timestamp as date,
+                {% for column in value_columns %}
+                flat_json.value:"{{ column }}" as {{ column }}{% if not loop.last %},{% endif %}
+                {% endfor %}
+            from
+                {{ raw_table_name }},
+                lateral flatten(input => parse_json(source_json)) as flat_json
+        ),
+        map_reduce_json as (
+            select t1.*
+            from flattened_json t1
+            left join max_extraction_per_day t2 on t1.date = t2.date
+            where t1.extraction_date = t2.extraction_date
+        )
+    select *
+    from map_reduce_json
+    where date < to_date(sysdate())
+{% endmacro %}

models/data_hubs/fact_daily_app_datahub.sql

@@ -41,6 +41,7 @@ with
                     ref("fact_venus_lending_bsc_gold"),
                     ref("fact_lido_staked_eth_count_with_USD_and_change_gold"),
                     ref("fact_meth_staked_eth_count_with_USD_and_change_gold"),
+                    ref("fact_binance_staked_eth_count_with_usd_and_change"),
                     ref("fact_rocketpool_staked_eth_count_with_USD_and_change_gold"),
                     ref("fact_frax_staked_eth_count_with_USD_and_change_gold"),
                     ref("fact_stakewise_staked_eth_count_with_USD_and_change_gold"),

models/staging/binance_staked_eth/__binance_staked_eth__sources.yml

@@ -0,0 +1,7 @@
+sources:
+  - name: PROD_LANDING
+    schema: PROD_LANDING
+    database: LANDING_DATABASE
+    tables:
+      - name: raw_beth_staked_count_ethereum
+      - name: raw_beth_staked_count_bsc

models/staging/binance_staked_eth/fact_binance_staked_eth_count.sql

@@ -0,0 +1,36 @@
+{{ config(materialized="table") }}
+-- depends_on: {{ source("PROD_LANDING", "raw_beth_staked_count_ethereum") }}
+-- depends_on: {{ source("PROD_LANDING", "raw_beth_staked_count_bsc") }}
+with raw_eth_supply as (
+    {{ raw_partitioned_array_to_fact_table_many_columns('landing_database.prod_landing.raw_beth_staked_count_ethereum', 'date', ['totalSupply', 'exchangeRate']) }}
+),
+eth_supply as (
+    select 
+        date,
+        totalSupply * exchangeRate as total_supply
+    from raw_eth_supply
+),
+eth_supply_forward_filled as (
+    {{ forward_fill('date', 'total_supply', 'eth_supply') }}
+),
+raw_bsc_supply as (
+    {{ raw_partitioned_array_to_fact_table_many_columns('landing_database.prod_landing.raw_beth_staked_count_bsc', 'date', ['totalSupply', 'exchangeRate']) }}
+),
+bsc_supply as (
+    select 
+        date,
+        totalSupply * exchangeRate as total_supply
+    from raw_bsc_supply
+),
+bsc_supply_forward_filled as (
+    {{ forward_fill('date', 'total_supply', 'bsc_supply') }}
+)
+select
+    coalesce(eth_supply_forward_filled.date, bsc_supply_forward_filled.date) as date,
+    coalesce(eth_supply_forward_filled.value, 0) + 
+    coalesce(bsc_supply_forward_filled.value, 0) 
+        as total_supply
+from eth_supply_forward_filled
+full join bsc_supply_forward_filled 
+    on 
+eth_supply_forward_filled.date = bsc_supply_forward_filled.date

models/staging/binance_staked_eth/fact_binance_staked_eth_count_with_usd_and_change.sql

@@ -0,0 +1,6 @@
+{{ config(materialized="table") }}
+
+select *, 'ethereum' as chain, 'binance-staked-eth' as app, 'DeFi' as category 
+from (
+    {{ calc_staked_eth('fact_binance_staked_eth_count') }}
+)