From dffb01fd34fb32207cfbc59f8533d7b5e42b5096 Mon Sep 17 00:00:00 2001 From: Andrew <47720952+andrewhong5297@users.noreply.github.com> Date: Sun, 14 May 2023 10:34:54 -0400 Subject: [PATCH 01/10] needs to use {{this}} format for incremental --- models/solana_utils/solana_utils_daily_balances.sql | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/models/solana_utils/solana_utils_daily_balances.sql b/models/solana_utils/solana_utils_daily_balances.sql index a293a18e900..e103696da53 100644 --- a/models/solana_utils/solana_utils_daily_balances.sql +++ b/models/solana_utils/solana_utils_daily_balances.sql @@ -20,11 +20,12 @@ WITH , cast(post_balance as double)/1e9 as sol_balance --lamport -> sol , post_token_balance as token_balance --tokens are already correct decimals in this table , token_balance_owner + , block_time as last_updated_at , row_number() OVER (partition by address, date_trunc('day', block_time) order by block_slot desc) as latest_balance FROM {{ source('solana','account_activity') }} WHERE tx_success {% if is_incremental() %} - AND block_time >= date_trunc("day", now() - interval '1 day') + AND block_time > (SELECT max(last_updated_at) FROM {{this}}) {% endif %} ) @@ -35,6 +36,7 @@ SELECT , token_mint_address , token_balance , token_balance_owner + , last_updated_at , now() as updated_at FROM updated_balances WHERE latest_balance = 1 \ No newline at end of file From bebf854f91248bf71497e9eadbcb99528c674752 Mon Sep 17 00:00:00 2001 From: Andrew <47720952+andrewhong5297@users.noreply.github.com> Date: Sun, 14 May 2023 10:36:16 -0400 Subject: [PATCH 02/10] should always be next day only --- models/solana_utils/solana_utils_daily_balances.sql | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/models/solana_utils/solana_utils_daily_balances.sql b/models/solana_utils/solana_utils_daily_balances.sql index e103696da53..d8f8769b261 100644 --- a/models/solana_utils/solana_utils_daily_balances.sql +++ b/models/solana_utils/solana_utils_daily_balances.sql @@ -20,12 +20,11 @@ WITH , cast(post_balance as double)/1e9 as sol_balance --lamport -> sol , post_token_balance as token_balance --tokens are already correct decimals in this table , token_balance_owner - , block_time as last_updated_at , row_number() OVER (partition by address, date_trunc('day', block_time) order by block_slot desc) as latest_balance FROM {{ source('solana','account_activity') }} WHERE tx_success {% if is_incremental() %} - AND block_time > (SELECT max(last_updated_at) FROM {{this}}) + AND block_time > (SELECT max(day) FROM {{this}}) {% endif %} ) @@ -36,7 +35,6 @@ SELECT , token_mint_address , token_balance , token_balance_owner - , last_updated_at , now() as updated_at FROM updated_balances WHERE latest_balance = 1 \ No newline at end of file From 3945b01d509bc6cda59dbcd9f60de9d535004939 Mon Sep 17 00:00:00 2001 From: Huang Geyang Date: Mon, 15 May 2023 12:44:23 +0800 Subject: [PATCH 03/10] Add uniqueness test --- models/solana_utils/solana_utils_schema.yml | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/models/solana_utils/solana_utils_schema.yml b/models/solana_utils/solana_utils_schema.yml index 06e0efd86df..291d2352f1b 100644 --- a/models/solana_utils/solana_utils_schema.yml +++ b/models/solana_utils/solana_utils_schema.yml @@ -30,6 +30,12 @@ models: tags: ['solana','balances'] description: > get the daily balances for each address, from which we can materialize a latest balance later on. + tests: + - dbt_utils.unique_combination_of_columns: + combination_of_columns: + - token_mint_address + - address + - day columns: - &day name: day From 7afffa38f12d9e6b3db930dcdbed14f70e07fb2e Mon Sep 17 00:00:00 2001 From: Andrew <47720952+andrewhong5297@users.noreply.github.com> Date: Tue, 16 May 2023 19:40:38 -0400 Subject: [PATCH 04/10] keep block_slot --- models/solana_utils/solana_utils_daily_balances.sql | 1 + 1 file changed, 1 insertion(+) diff --git a/models/solana_utils/solana_utils_daily_balances.sql b/models/solana_utils/solana_utils_daily_balances.sql index d8f8769b261..7ccc678f9d0 100644 --- a/models/solana_utils/solana_utils_daily_balances.sql +++ b/models/solana_utils/solana_utils_daily_balances.sql @@ -20,6 +20,7 @@ WITH , cast(post_balance as double)/1e9 as sol_balance --lamport -> sol , post_token_balance as token_balance --tokens are already correct decimals in this table , token_balance_owner + , block_slot , row_number() OVER (partition by address, date_trunc('day', block_time) order by block_slot desc) as latest_balance FROM {{ source('solana','account_activity') }} WHERE tx_success From 5ff9b392b40d2572ce9c2e90e6615148fd8c8d18 Mon Sep 17 00:00:00 2001 From: Andrew <47720952+andrewhong5297@users.noreply.github.com> Date: Sat, 20 May 2023 14:15:57 -0400 Subject: [PATCH 05/10] test 30 --- models/solana_utils/solana_utils_daily_balances.sql | 1 + 1 file changed, 1 insertion(+) diff --git a/models/solana_utils/solana_utils_daily_balances.sql b/models/solana_utils/solana_utils_daily_balances.sql index 7ccc678f9d0..58bb855a7c9 100644 --- a/models/solana_utils/solana_utils_daily_balances.sql +++ b/models/solana_utils/solana_utils_daily_balances.sql @@ -24,6 +24,7 @@ WITH , row_number() OVER (partition by address, date_trunc('day', block_time) order by block_slot desc) as latest_balance FROM {{ source('solana','account_activity') }} WHERE tx_success + AND block_time > now() - interval '30' day {% if is_incremental() %} AND block_time > (SELECT max(day) FROM {{this}}) {% endif %} From 710f4613dfd342c45f342398d12df969dfce49dd Mon Sep 17 00:00:00 2001 From: Huang Geyang Date: Sun, 21 May 2023 03:07:19 +0800 Subject: [PATCH 06/10] Try to fix the incremental merge checking bug --- models/solana_utils/solana_utils_daily_balances.sql | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/models/solana_utils/solana_utils_daily_balances.sql b/models/solana_utils/solana_utils_daily_balances.sql index 58bb855a7c9..6998d064806 100644 --- a/models/solana_utils/solana_utils_daily_balances.sql +++ b/models/solana_utils/solana_utils_daily_balances.sql @@ -4,7 +4,7 @@ materialized='incremental', file_format = 'delta', incremental_strategy='merge', - unique_key = ['token_mint_address', 'address','day'], + unique_key = ['token_mint_address', 'address', 'day'], post_hook='{{ expose_spells(\'["solana"]\', "sector", "solana_utils", @@ -16,7 +16,7 @@ WITH SELECT address , date_trunc('day', block_time) as day - , token_mint_address + , coalesce(token_mint_address, '') as token_mint_address , cast(post_balance as double)/1e9 as sol_balance --lamport -> sol , post_token_balance as token_balance --tokens are already correct decimals in this table , token_balance_owner From 53e7fede2d0185fab26971dbc81f14745c298518 Mon Sep 17 00:00:00 2001 From: Andrew <47720952+andrewhong5297@users.noreply.github.com> Date: Tue, 23 May 2023 14:58:42 -0400 Subject: [PATCH 07/10] remove 30 day limit --- models/solana_utils/solana_utils_daily_balances.sql | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/models/solana_utils/solana_utils_daily_balances.sql b/models/solana_utils/solana_utils_daily_balances.sql index 6998d064806..ee113cc8b1e 100644 --- a/models/solana_utils/solana_utils_daily_balances.sql +++ b/models/solana_utils/solana_utils_daily_balances.sql @@ -24,7 +24,6 @@ WITH , row_number() OVER (partition by address, date_trunc('day', block_time) order by block_slot desc) as latest_balance FROM {{ source('solana','account_activity') }} WHERE tx_success - AND block_time > now() - interval '30' day {% if is_incremental() %} AND block_time > (SELECT max(day) FROM {{this}}) {% endif %} @@ -39,4 +38,4 @@ SELECT , token_balance_owner , now() as updated_at FROM updated_balances -WHERE latest_balance = 1 \ No newline at end of file +WHERE latest_balance = 1 From f85eaa29ef95cabcbd0b59204fd7debb320e0740 Mon Sep 17 00:00:00 2001 From: Andrew <47720952+andrewhong5297@users.noreply.github.com> Date: Tue, 23 May 2023 17:58:15 -0400 Subject: [PATCH 08/10] add space --- models/solana_utils/solana_utils_daily_balances.sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/models/solana_utils/solana_utils_daily_balances.sql b/models/solana_utils/solana_utils_daily_balances.sql index ee113cc8b1e..6445f32f61f 100644 --- a/models/solana_utils/solana_utils_daily_balances.sql +++ b/models/solana_utils/solana_utils_daily_balances.sql @@ -25,7 +25,7 @@ WITH FROM {{ source('solana','account_activity') }} WHERE tx_success {% if is_incremental() %} - AND block_time > (SELECT max(day) FROM {{this}}) + AND block_time > (SELECT max(day) FROM {{this}}) {% endif %} ) From 9b83a827e00f2dab2850cbe221251ecbd79b0263 Mon Sep 17 00:00:00 2001 From: Huang Geyang Date: Wed, 24 May 2023 21:39:03 +0800 Subject: [PATCH 09/10] Add not_null tests --- models/solana_utils/solana_utils_daily_balances.sql | 2 +- models/solana_utils/solana_utils_schema.yml | 6 ++++++ 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/models/solana_utils/solana_utils_daily_balances.sql b/models/solana_utils/solana_utils_daily_balances.sql index 6445f32f61f..ee113cc8b1e 100644 --- a/models/solana_utils/solana_utils_daily_balances.sql +++ b/models/solana_utils/solana_utils_daily_balances.sql @@ -25,7 +25,7 @@ WITH FROM {{ source('solana','account_activity') }} WHERE tx_success {% if is_incremental() %} - AND block_time > (SELECT max(day) FROM {{this}}) + AND block_time > (SELECT max(day) FROM {{this}}) {% endif %} ) diff --git a/models/solana_utils/solana_utils_schema.yml b/models/solana_utils/solana_utils_schema.yml index 291d2352f1b..2aef825c5fe 100644 --- a/models/solana_utils/solana_utils_schema.yml +++ b/models/solana_utils/solana_utils_schema.yml @@ -17,8 +17,12 @@ models: columns: - &token_mint_address name: token_mint_address + tests: + - not_null - &address name: address + tests: + - not_null - &updated_at name: updated_at @@ -39,6 +43,8 @@ models: columns: - &day name: day + tests: + - not_null - *token_mint_address - *address - &sol_balance From 7d38cf07135647f2f9a901a2e50c97290413aed4 Mon Sep 17 00:00:00 2001 From: Huang Geyang Date: Sun, 28 May 2023 00:30:08 +0800 Subject: [PATCH 10/10] Remove not_null test for latest_balance --- models/solana_utils/solana_utils_schema.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/models/solana_utils/solana_utils_schema.yml b/models/solana_utils/solana_utils_schema.yml index 2aef825c5fe..0e81252cea9 100644 --- a/models/solana_utils/solana_utils_schema.yml +++ b/models/solana_utils/solana_utils_schema.yml @@ -67,7 +67,7 @@ models: - *address - *sol_balance - *token_balance - - *token_mint_address + - name: token_mint_address - *token_balance_owner - *updated_at