From 01be16ef076dddc51fd609295c4f682546ec0d37 Mon Sep 17 00:00:00 2001 From: Horace Liu Date: Thu, 19 Dec 2024 18:46:01 -0500 Subject: [PATCH] Add raw application and category tables --- .../dimensions/apps/dim_dune_namespaces.sql | 6 +++- .../apps/dim_flipside_namespaces.sql | 33 ++++++++++++------- .../dim_dune_contracts_post_sigma.sql | 3 +- .../labeling/dim_all_apps_silver.sql | 25 ++++++++++++++ .../labeling/dim_all_categores_silver.sql | 22 +++++++++++++ models/staging/sui/dim_sui_namespaces.sql | 5 +-- 6 files changed, 79 insertions(+), 15 deletions(-) diff --git a/models/dimensions/apps/dim_dune_namespaces.sql b/models/dimensions/apps/dim_dune_namespaces.sql index d6c816aa..d2a2b20e 100644 --- a/models/dimensions/apps/dim_dune_namespaces.sql +++ b/models/dimensions/apps/dim_dune_namespaces.sql @@ -1 +1,5 @@ -select distinct namespace as namespace from {{ ref("dim_dune_contracts_post_sigma") }} +select + namespace, + max(last_updated) as last_updated +from {{ ref("dim_dune_contracts_post_sigma") }} +group by namespace \ No newline at end of file diff --git a/models/dimensions/apps/dim_flipside_namespaces.sql b/models/dimensions/apps/dim_flipside_namespaces.sql index ed1c8327..af01f0c2 100644 --- a/models/dimensions/apps/dim_flipside_namespaces.sql +++ b/models/dimensions/apps/dim_flipside_namespaces.sql @@ -7,7 +7,8 @@ with namespace, initcap(replace(replace(namespace, '-', ' '), '_', ' ')) as friendly_name, max(category) as category, - max(sub_category) as sub_category + max(sub_category) as sub_category, + max(last_updated) as last_updated from {{ ref("dim_flipside_contracts") }} where chain = 'arbitrum' and namespace is not null and category not in ('NFT', 'Token', 'ERC_1155') @@ -19,7 +20,8 @@ with namespace, initcap(replace(replace(namespace, '-', ' '), '_', ' ')) as friendly_name, max(category) as category, - max(sub_category) as sub_category + max(sub_category) as sub_category, + max(last_updated) as last_updated from {{ ref("dim_flipside_contracts") }} where chain = 'avalanche' and namespace is not null and category not in ('NFT', 'Token', 'ERC_1155') @@ -31,7 +33,8 @@ with namespace, initcap(replace(replace(namespace, '-', ' '), '_', ' ')) as friendly_name, max(category) as category, - max(sub_category) as sub_category + max(sub_category) as sub_category, + max(last_updated) as last_updated from {{ ref("dim_flipside_contracts") }} where chain = 'base' and namespace is not null and category not in ('NFT', 'Token', 'ERC_1155') @@ -43,7 +46,8 @@ with namespace, initcap(replace(replace(namespace, '-', ' '), '_', ' ')) as friendly_name, max(category) as category, - max(sub_category) as sub_category + max(sub_category) as sub_category, + max(last_updated) as last_updated from {{ ref("dim_flipside_contracts") }} where chain = 'bsc' and namespace is not null and category not in ('NFT', 'Token', 'ERC_1155') @@ -55,7 +59,8 @@ with namespace, initcap(replace(replace(namespace, '-', ' '), '_', ' ')) as friendly_name, max(category) as category, - max(sub_category) as sub_category + max(sub_category) as sub_category, + max(last_updated) as last_updated from {{ ref("dim_flipside_contracts") }} where chain = 'ethereum' and namespace is not null and category not in ('NFT', 'Token', 'ERC_1155') @@ -67,7 +72,8 @@ with namespace, initcap(replace(replace(namespace, '-', ' '), '_', ' ')) as friendly_name, max(category) as category, - max(sub_category) as sub_category + max(sub_category) as sub_category, + max(last_updated) as last_updated from {{ ref("dim_flipside_contracts") }} where chain = 'polygon' and namespace is not null and category not in ('NFT', 'Token', 'ERC_1155') @@ -79,7 +85,8 @@ with namespace, initcap(replace(replace(namespace, '-', ' '), '_', ' ')) as friendly_name, max(category) as category, - max(sub_category) as sub_category + max(sub_category) as sub_category, + max(last_updated) as last_updated from {{ ref("dim_flipside_contracts") }} where chain = 'optimism' and namespace is not null and category not in ('NFT', 'Token', 'ERC_1155') @@ -91,7 +98,8 @@ with namespace, initcap(replace(replace(namespace, '-', ' '), '_', ' ')) as friendly_name, max(category) as category, - max(sub_category) as sub_category + max(sub_category) as sub_category, + max(last_updated) as last_updated from {{ ref("dim_flipside_contracts") }} where chain = 'sei' and namespace is not null group by namespace @@ -102,7 +110,8 @@ with namespace, initcap(replace(replace(namespace, '-', ' '), '_', ' ')) as friendly_name, max(category) as category, - max(sub_category) as sub_category + max(sub_category) as sub_category, + max(last_updated) as last_updated from {{ ref("dim_flipside_contracts") }} where chain = 'near' and namespace is not null group by namespace @@ -114,7 +123,8 @@ with namespace, initcap(replace(replace(namespace, '-', ' '), '_', ' ')) as friendly_name, max(category) as category, - null as sub_category + null as sub_category, + max(last_updated) as last_updated from {{ ref("dim_flipside_contracts") }} where chain = 'solana' @@ -127,6 +137,7 @@ select namespace, coalesce(max_by(friendly_name, namespace), max(friendly_name)) as friendly_name, coalesce(max_by(category, namespace), max(category)) as category, - coalesce(max_by(sub_category, namespace), max(sub_category)) as sub_category + coalesce(max_by(sub_category, namespace), max(sub_category)) as sub_category, + max(last_updated) as last_updated from flipside_namespace group by namespace diff --git a/models/dimensions/contracts/dim_dune_contracts_post_sigma.sql b/models/dimensions/contracts/dim_dune_contracts_post_sigma.sql index bb7376a9..9b9bee45 100644 --- a/models/dimensions/contracts/dim_dune_contracts_post_sigma.sql +++ b/models/dimensions/contracts/dim_dune_contracts_post_sigma.sql @@ -3,7 +3,8 @@ select dune.address, coalesce(sigma.override_namespace, dune.namespace) as namespace, dune.name, - dune.chain + dune.chain, + dune.last_updated as last_updated from {{ ref("dim_dune_contracts") }} as dune left join {{ source("SIGMA", "sigma_tagged_dune_contracts") }} as sigma diff --git a/models/dimensions/labeling/dim_all_apps_silver.sql b/models/dimensions/labeling/dim_all_apps_silver.sql index e69de29b..cff7f66f 100644 --- a/models/dimensions/labeling/dim_all_apps_silver.sql +++ b/models/dimensions/labeling/dim_all_apps_silver.sql @@ -0,0 +1,25 @@ +{{ + config( + materialized="incremental", + unique_key=["application", "source"], + incremental_strategy="merge", + ) +}} + +-- there are technically two more tables: +-- dim_new_apps_post_sigma and dim_apps_post_sigma +-- but they can be handled later +with unioned_table as ( + SELECT namespace as application, 'dune' AS source, last_updated + FROM {{ ref("dim_dune_namespaces") }} where namespace is not null + UNION + SELECT namespace as application, 'sui' AS source, last_updated + FROM {{ ref("dim_sui_namespaces") }} where namespace is not null + UNION + SELECT namespace as application, 'flipside' AS source, last_updated + FROM {{ ref("dim_flipside_namespaces") }} where namespace is not null +) +select * from unioned_table +{% if is_incremental() %} + where last_updated > (SELECT MAX(last_updated) FROM {{ this }}) +{% endif %} \ No newline at end of file diff --git a/models/dimensions/labeling/dim_all_categores_silver.sql b/models/dimensions/labeling/dim_all_categores_silver.sql index e69de29b..b5551c9a 100644 --- a/models/dimensions/labeling/dim_all_categores_silver.sql +++ b/models/dimensions/labeling/dim_all_categores_silver.sql @@ -0,0 +1,22 @@ +{{ + config( + materialized="incremental", + unique_key=["category", "sub_category", "source"], + incremental_strategy="merge", + ) +}} + +-- there are technically two more tables: +-- dim_new_apps_post_sigma and dim_apps_post_sigma +-- but they can be handled later +with unioned_table as ( + SELECT category, sub_category, 'sui' AS source, last_updated + FROM {{ ref("dim_sui_namespaces") }} where namespace is not null + UNION + SELECT category, sub_category, 'flipside' AS source, last_updated + FROM {{ ref("dim_flipside_namespaces") }} where namespace is not null +) +select * from unioned_table +{% if is_incremental() %} + where last_updated > (SELECT MAX(last_updated) FROM {{ this }}) +{% endif %} \ No newline at end of file diff --git a/models/staging/sui/dim_sui_namespaces.sql b/models/staging/sui/dim_sui_namespaces.sql index 875eaa87..eaeb7433 100644 --- a/models/staging/sui/dim_sui_namespaces.sql +++ b/models/staging/sui/dim_sui_namespaces.sql @@ -5,6 +5,7 @@ select max(friendly_name) as friendly_name, max(sub_category) as sub_category, max(category) as category, - max(icon) as icon -from {{ ref("dim_sui_contracts") }} as sui_contracts + max(icon) as icon, + max(last_updated) as last_updated +from {{ ref("dim_sui_namespaces") }} as sui_contracts group by namespace