Updated Ton to use Raw Transaction data from snowflake instead of dir…

…ectly to ton Apps

models/projects/ton/core/ez_ton_metrics.sql

@@ -14,19 +14,14 @@ with
             date,
             txns as transaction_nodes
         from {{ ref("fact_ton_daa_txns_gas_gas_usd_revenue_revenue_native") }}
-    ), ton_app_daa as (
+    ), ton_apps_fundamental_data as (
         select 
             date,
-            daa as dau
-        from {{ ref("fact_ton_app_daa") }}
-    ),
-    ton_app_txns_fees as (
-        select 
-            date,
-            txns,
-            fees_native,
-            avg_txn_fee_native,
-        from {{ ref("fact_ton_app_fees_txns") }}
+            , dau
+            , fees_native
+            , txns
+            , avg_txn_fee_native
+        from {{ ref("fact_ton_fundamental_metrics") }}
     ),
     price_data as ({{ get_coingecko_metrics("the-open-network") }}),
     defillama_data as ({{ get_defillama_metrics("ton") }}),
@@ -38,7 +33,7 @@ with
     ),
     github_data as ({{ get_github_metrics("ton") }})
 select
-    ton_app_daa.date,
+    ton.date,
     'ton' as chain,
     transaction_nodes,
     price,
@@ -57,11 +52,10 @@ select
     fees_native / 2 as revenue_native,
     (fees_native / 2) * price as revenue,
     avg_txn_fee_native * price as avg_txn_fee
-from ton_app_daa
-left join price_data on ton_app_daa.date = price_data.date
-left join defillama_data on ton_app_daa.date = defillama_data.date
-left join github_data on ton_app_daa.date = github_data.date
-left join dex_data on ton_app_daa.date = dex_data.date
-left join fundamental_data on ton_app_daa.date = fundamental_data.date
-left join ton_app_txns_fees on ton_app_daa.date = ton_app_txns_fees.date
-where ton_app_daa.date < to_date(sysdate())
+from ton_apps_fundamental_data as ton
+left join price_data on ton.date = price_data.date
+left join defillama_data on ton.date = defillama_data.date
+left join github_data on ton.date = github_data.date
+left join dex_data on ton.date = dex_data.date
+left join fundamental_data on ton.date = fundamental_data.date
+where ton.date < to_date(sysdate())

models/staging/ton/__ton__sources.yml

@@ -13,3 +13,4 @@ sources:
       - name: raw_ton_tokens_openleague
       - name: raw_ton_app_daa
       - name: raw_ton_app_txns_fees
+      - name: raw_ton_transactions

models/staging/ton/fact_ton_fundamental_metrics.sql

@@ -0,0 +1,46 @@
+{{ config(
+    materialized="incremental",
+    unique_key="date",
+) }}
+
+with flatten_ton_transaction as (
+    select
+        true = all(array_agg(success)) as success
+        , trace_id
+        , max(block_timestamp::date) as date
+        , min_by(transaction_account_interfaces, lt)[1] as interface
+        , min_by(transaction_account, lt) as first_account
+        , min_by(transaction_account_workchain, lt) as workchain
+        , sum(total_fees) as transaction_fees
+    from {{ ref('fact_ton_transactions') }}
+    group by trace_id
+), txns as (
+    SELECT
+        date
+        , count(*) as txns
+        , (sum(transaction_fees) / POWER(10, 9)) / count(*) as avg_txn_fee_native
+    from flatten_ton_transaction
+    where success and workchain <> -1
+    group by date
+), fees as (
+    SELECT
+        date
+        , sum(transaction_fees) / POWER(10, 9) as fees_native
+    FROM flatten_ton_transaction
+    GROUP by date
+),
+dau as (
+    select
+        date
+        , count(distinct first_account) as dau
+    from flatten_ton_transaction
+    where success and interface like 'wallet_v%'
+)
+SELECT
+    coalesce(fees.date, txns.date, dau.date) as date
+    , dau
+    , fees_native
+    , txns
+    , avg_txn_fee_native
+FROM fees left join txns on fees.date = txns.date
+left join dau on fees.date = dau.date

models/staging/ton/fact_ton_transactions.sql

@@ -0,0 +1,67 @@
+{{
+    config(
+        materialized="incremental",
+        unique_key="tx_id",
+    )
+}}
+with raw_data as (
+    select
+        extraction_date
+        , parse_json(source_json) as source_json
+        , parse_json(source_json):"decoded_hash"::string as tx_hash
+        , parse_json(source_json):"tx_id"::string as tx_id
+        , TO_TIMESTAMP(parse_json(source_json):"utime") as block_timestamp
+        , parse_json(source_json):"total_fees"::bigint as total_fees
+        , parse_json(source_json):"storage_ph_fees_collected"::bigint as storage_ph_fees_collected
+        , parse_json(source_json):"success"::boolean as success
+        , parse_json(source_json):"in_msg"::string as in_msg
+        , parse_json(source_json):"lt"::bigint as lt
+        , parse_json(source_json):"transaction_account"::string as transaction_account
+        , parse_json(source_json):"transaction_account_workchain"::int as transaction_account_workchain
+        , parse_json(source_json):"transaction_account_interfaces"::array as transaction_account_interfaces
+        , parse_json(source_json):"from_address"::string as from_address
+        , parse_json(source_json):"from_workchain"::int as from_workchain
+        , parse_json(source_json):"from_interfaces"::array as from_interfaces
+        , parse_json(source_json):"to_address"::string as to_address
+        , parse_json(source_json):"to_workchain"::int as to_workchain
+        , parse_json(source_json):"to_interfaces"::array as to_interfaces
+        , parse_json(source_json):"trace_id"::string as trace_id
+        , parse_json(source_json):"op_code"::string as op_code
+        , parse_json(parse_json(source_json):"decoded_body") as decoded_body
+        , parse_json(source_json):"init"::string as init
+        , parse_json(source_json):"value"::bigint as value
+        , parse_json(source_json):"msg_type"::string as msg_type
+        , parse_json(source_json):"init_interfaces"::array as init_interfaces
+    from
+        {{ source("PROD_LANDING", "raw_ton_transactions") }}
+    {% if is_incremental() %}
+        where TO_TIMESTAMP(parse_json(source_json):"utime") > (select dateadd('day', -3, max(block_timestamp)) from {{ this }})
+    {% endif %}
+)
+select 
+    max_by(tx_hash, extraction_date) as tx_hash
+    , tx_id
+    , max_by(block_timestamp, extraction_date) as block_timestamp
+    , max_by(total_fees, extraction_date) as total_fees
+    , max_by(storage_ph_fees_collected, extraction_date) as storage_ph_fees_collected
+    , max_by(success, extraction_date) as success
+    , max_by(in_msg, extraction_date) as in_msg
+    , max_by(lt, extraction_date) as lt
+    , max_by(transaction_account, extraction_date) as transaction_account
+    , max_by(transaction_account_workchain, extraction_date) as transaction_account_workchain
+    , max_by(transaction_account_interfaces, extraction_date) as transaction_account_interfaces
+    , max_by(from_address, extraction_date) as from_address
+    , max_by(from_workchain, extraction_date) as from_workchain
+    , max_by(from_interfaces, extraction_date) as from_interfaces
+    , max_by(to_address, extraction_date) as to_address
+    , max_by(to_workchain, extraction_date) as to_workchain
+    , max_by(to_interfaces, extraction_date) as to_interfaces
+    , max_by(trace_id, extraction_date) as trace_id
+    , max_by(op_code, extraction_date) as op_code
+    , max_by(decoded_body, extraction_date) as decoded_body
+    , max_by(init, extraction_date) as init
+    , max_by(value, extraction_date) as value
+    , max_by(msg_type, extraction_date) as msg_type
+    , max_by(init_interfaces, extraction_date) as init_interfaces
+from raw_data
+group by tx_id