From 88ed3ef9704d9ea6f272c26e4902cf6c7d44a5ab Mon Sep 17 00:00:00 2001 From: Alex Kan <29241719+akan72@users.noreply.github.com> Date: Wed, 6 Nov 2024 19:16:42 -0500 Subject: [PATCH 1/6] Fix fact_jupiter_perps_txs model --- models/staging/jupiter/fact_jupiter_perps_txs.sql | 12 +----------- 1 file changed, 1 insertion(+), 11 deletions(-) diff --git a/models/staging/jupiter/fact_jupiter_perps_txs.sql b/models/staging/jupiter/fact_jupiter_perps_txs.sql index 72d74fee..249905e1 100644 --- a/models/staging/jupiter/fact_jupiter_perps_txs.sql +++ b/models/staging/jupiter/fact_jupiter_perps_txs.sql @@ -53,17 +53,7 @@ with hex_cte as ( {{ hex_to_base58("SUBSTRING(hex_data,456+1,64)") }} as owner, -- owner/trader address {{ hex_to_base58("SUBSTRING(hex_data,243,64)") }} as mint, -- Price calculation with dynamic offset - {{ big_endian_hex_to_decimal("SUBSTRING(hex_data,619+1,16)") }}/1e6 as price, - SUBSTRING( - hex_data, - 619 -- Base offset to price - + (CASE WHEN SUBSTRING(hex_data, 618, 1) = '1' - THEN 16 - ELSE 0 - END) -- Add offset if transferToken present - , 16 - ) - )/1e6 as price, + {{ big_endian_hex_to_decimal("SUBSTRING(hex_data,619+1,16)") }}/1e6 as price, hex_data FROM hex_cte WHERE substring(hex_data,1+16,16) = '409c2b4a6d83107f' -- DecreasePosition From 6d920aa03ad5abfc98d544fc68f297ed896fe4c5 Mon Sep 17 00:00:00 2001 From: Alex Weseley Date: Wed, 6 Nov 2024 23:16:46 -0500 Subject: [PATCH 2/6] jupiter tx dynamic offset --- models/staging/jupiter/fact_jupiter_perps_txs.sql | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/models/staging/jupiter/fact_jupiter_perps_txs.sql b/models/staging/jupiter/fact_jupiter_perps_txs.sql index 249905e1..fe3a0a73 100644 --- a/models/staging/jupiter/fact_jupiter_perps_txs.sql +++ b/models/staging/jupiter/fact_jupiter_perps_txs.sql @@ -53,7 +53,13 @@ with hex_cte as ( {{ hex_to_base58("SUBSTRING(hex_data,456+1,64)") }} as owner, -- owner/trader address {{ hex_to_base58("SUBSTRING(hex_data,243,64)") }} as mint, -- Price calculation with dynamic offset - {{ big_endian_hex_to_decimal("SUBSTRING(hex_data,619+1,16)") }}/1e6 as price, + {{ big_endian_hex_to_decimal( + "SUBSTRING( + hex_data, + 619 + (CASE WHEN SUBSTRING(hex_data, 618, 1) = '1' THEN 16 ELSE 0 END), + 16 + )" + ) }}/1e6 as price, hex_data FROM hex_cte WHERE substring(hex_data,1+16,16) = '409c2b4a6d83107f' -- DecreasePosition From 6ce18123c1326765b592ef2e87f6f56766a21e47 Mon Sep 17 00:00:00 2001 From: Alex Weseley Date: Thu, 7 Nov 2024 18:15:58 -0500 Subject: [PATCH 3/6] dydx pricing models --- .../raw/ez_dydx_v4_price_performance.sql | 33 ++++++++++++ models/staging/dydx/__dydx__sources.yml | 2 + .../staging/dydx/dim_dydx_perps_markets.sql | 32 +++++++++++ .../dydx/fact_dydx_v4_perps_prices.sql | 54 +++++++++++++++++++ 4 files changed, 121 insertions(+) create mode 100644 models/projects/dydx_v4/raw/ez_dydx_v4_price_performance.sql create mode 100644 models/staging/dydx/dim_dydx_perps_markets.sql create mode 100644 models/staging/dydx/fact_dydx_v4_perps_prices.sql diff --git a/models/projects/dydx_v4/raw/ez_dydx_v4_price_performance.sql b/models/projects/dydx_v4/raw/ez_dydx_v4_price_performance.sql new file mode 100644 index 00000000..c1b8c53e --- /dev/null +++ b/models/projects/dydx_v4/raw/ez_dydx_v4_price_performance.sql @@ -0,0 +1,33 @@ +{{ + config( + materialized = "table", + snowflake_warehouse = "DYDX", + database = "dydx_v4", + schema = "raw", + alias = "ez_price_performance" + ) +}} + +with prices as ( + select + date_trunc('hour', block_timestamp) as hour + , symbol + , price + , CASE + when date_part('DOW', convert_timezone('UTC', 'America/New_York', block_timestamp)) IN (0, 6) then 'FALSE' + when convert_timezone('UTC', 'America/New_York', block_timestamp)::time between '09:00:00' and '15:59:59' then 'TRUE' + else 'FALSE' + END AS nyc_operating_hours + from {{ref('fact_dydx_v4_perps_prices')}} t1 +) + +select + hour + , symbol + , max(price) as high + , min(price) as low + , avg(price) as average + , median(price) as median + , nyc_operating_hours +from prices +group by hour, symbol, nyc_operating_hours \ No newline at end of file diff --git a/models/staging/dydx/__dydx__sources.yml b/models/staging/dydx/__dydx__sources.yml index 5fca0228..17a6ce41 100644 --- a/models/staging/dydx/__dydx__sources.yml +++ b/models/staging/dydx/__dydx__sources.yml @@ -9,3 +9,5 @@ sources: - name: raw_dydx_v4_txn_and_trading_fees - name: raw_dydx_v4_unique_traders - name: raw_dydx_v4_fees + - name: raw_dydx_v4_perpetuals_markets_metadata + - name: raw_dydx_v4_price \ No newline at end of file diff --git a/models/staging/dydx/dim_dydx_perps_markets.sql b/models/staging/dydx/dim_dydx_perps_markets.sql new file mode 100644 index 00000000..214d9c9e --- /dev/null +++ b/models/staging/dydx/dim_dydx_perps_markets.sql @@ -0,0 +1,32 @@ +{{ + config({ + "materialized": "table" + }) +}} + +with max_extraction as ( + SELECT + max(extraction_date) as max_extraction + FROM + {{ source("PROD_LANDING", "raw_dydx_v4_perpetuals_markets_metadata") }} +), +latest_data as ( + SELECT + parse_json(source_json) as data + FROM + {{ source("PROD_LANDING", "raw_dydx_v4_perpetuals_markets_metadata") }} + WHERE + extraction_date = ( + select + max_extraction + from + max_extraction + ) +) +select + value:ticker::string as ticker + , value:clobPairId::number as clob_pair_id + , value:atomicResolution::number as atomic_resolution +from + latest_data + , lateral flatten (input => data:markets) \ No newline at end of file diff --git a/models/staging/dydx/fact_dydx_v4_perps_prices.sql b/models/staging/dydx/fact_dydx_v4_perps_prices.sql new file mode 100644 index 00000000..0e82517c --- /dev/null +++ b/models/staging/dydx/fact_dydx_v4_perps_prices.sql @@ -0,0 +1,54 @@ +{{ + config({ + "materialized": "incremental", + "snowflake_warehouse": "DYDX", + }) +}} + +with all_dates as ( + select + extraction_date, + TO_TIMESTAMP_NTZ(value:block_timestamp)::date as date + from {{ source("PROD_LANDING", "raw_dydx_v4_price") }}, + lateral flatten (input => parse_json(source_json)) as flat_json + {% if is_incremental() %} + where TO_TIMESTAMP_NTZ(value:block_timestamp)::date >= (select max(block_timestamp) from {{ this }}) + {% endif %} +) +, dates as ( + SELECT + MAX(extraction_date) as max_extraction, + date + FROM all_dates + GROUP BY 2 +) +, flattened_json as ( + SELECT + TO_TIMESTAMP_NTZ(value:block_timestamp)::timestamp as block_timestamp, + value:tx_hash::string as tx_hash, + value:E::number as E, + value:G::number as G, + value:I::number as I, + value:perpetual_id::number as perpetual_id, + extraction_date + FROM {{ source("PROD_LANDING", "raw_dydx_v4_price") }} raw, + LATERAL FLATTEN (input => PARSE_JSON(source_json)) AS flat_json + {% if is_incremental() %} + where TO_TIMESTAMP_NTZ(value:block_timestamp)::date >= (select max(block_timestamp) from {{ this }}) + {% endif %} +) +select + f.block_timestamp, + f.tx_hash, + f.perpetual_id, + left(ticker, length(ticker)-4) as symbol, + abs((f.E + f.G) / f.I) * POW(10, (-6 - m.atomic_resolution)) as price, -- based on equation found here https://docs.dydx.exchange/api_integration-guides/how_to_interpret_block_data_for_trades + f.E, + f.G, + f.I, + m.clob_pair_id, + m.atomic_resolution +from flattened_json f +join dates d on d.max_extraction = f.extraction_date AND d.date = date(f.block_timestamp) +left join {{ ref("dim_dydx_perps_markets") }} m on f.perpetual_id = m.clob_pair_id +where m.ticker like '%USD%' -- in case they add non-USD pairs From 54df09409780b8220bd7828c28bb8de217b676f0 Mon Sep 17 00:00:00 2001 From: Alex Weseley Date: Fri, 8 Nov 2024 11:19:00 -0500 Subject: [PATCH 4/6] dydx trade level data exposed to prod_raw --- .../dydx_v4/raw/fact_dydx_v4_perps_token_prices.sql | 11 +++++++++++ 1 file changed, 11 insertions(+) create mode 100644 models/projects/dydx_v4/raw/fact_dydx_v4_perps_token_prices.sql diff --git a/models/projects/dydx_v4/raw/fact_dydx_v4_perps_token_prices.sql b/models/projects/dydx_v4/raw/fact_dydx_v4_perps_token_prices.sql new file mode 100644 index 00000000..b0d35aa2 --- /dev/null +++ b/models/projects/dydx_v4/raw/fact_dydx_v4_perps_token_prices.sql @@ -0,0 +1,11 @@ +{{ + config( + materialized = "table", + snowflake_warehouse = "DYDX", + database = "dydx_v4", + schema = "raw", + alias = "fact_perps_token_prices" + ) +}} + +select * from {{ ref('fact_dydx_v4_perps_prices') }} \ No newline at end of file From 5350c5d71ff3749adbd451c5a217e455ff11e4bb Mon Sep 17 00:00:00 2001 From: Alex Weseley Date: Fri, 8 Nov 2024 11:39:41 -0500 Subject: [PATCH 5/6] nyc operating hours macro --- macros/is_nyc_operating_hours.sql | 7 +++++++ .../projects/dydx_v4/raw/ez_dydx_v4_price_performance.sql | 6 +----- .../jupiter/raw/ez_jupiter_perps_price_performance.sql | 5 +---- .../projects/uniswap/raw/ez_uniswap_price_performance.sql | 6 +----- 4 files changed, 10 insertions(+), 14 deletions(-) create mode 100644 macros/is_nyc_operating_hours.sql diff --git a/macros/is_nyc_operating_hours.sql b/macros/is_nyc_operating_hours.sql new file mode 100644 index 00000000..29260e71 --- /dev/null +++ b/macros/is_nyc_operating_hours.sql @@ -0,0 +1,7 @@ +{% macro is_nyc_operating_hours(timestamp_column) %} + case + when date_part('DOW', convert_timezone('UTC', 'America/New_York', {{ timestamp_column }})) in (0, 6) then false + when convert_timezone('UTC', 'America/New_York', {{ timestamp_column }})::time between '09:00:00' and '15:59:59' then true + else false + end +{% endmacro %} \ No newline at end of file diff --git a/models/projects/dydx_v4/raw/ez_dydx_v4_price_performance.sql b/models/projects/dydx_v4/raw/ez_dydx_v4_price_performance.sql index c1b8c53e..c1b9b046 100644 --- a/models/projects/dydx_v4/raw/ez_dydx_v4_price_performance.sql +++ b/models/projects/dydx_v4/raw/ez_dydx_v4_price_performance.sql @@ -13,11 +13,7 @@ with prices as ( date_trunc('hour', block_timestamp) as hour , symbol , price - , CASE - when date_part('DOW', convert_timezone('UTC', 'America/New_York', block_timestamp)) IN (0, 6) then 'FALSE' - when convert_timezone('UTC', 'America/New_York', block_timestamp)::time between '09:00:00' and '15:59:59' then 'TRUE' - else 'FALSE' - END AS nyc_operating_hours + , {{ is_nyc_operating_hours(hour) }} as nyc_operating_hours from {{ref('fact_dydx_v4_perps_prices')}} t1 ) diff --git a/models/projects/jupiter/raw/ez_jupiter_perps_price_performance.sql b/models/projects/jupiter/raw/ez_jupiter_perps_price_performance.sql index 8c4825df..9d006f08 100644 --- a/models/projects/jupiter/raw/ez_jupiter_perps_price_performance.sql +++ b/models/projects/jupiter/raw/ez_jupiter_perps_price_performance.sql @@ -15,10 +15,7 @@ with agg as ( agg.mint, m.symbol, CASE - when date_part('DOW', convert_timezone('UTC', 'America/New_York', block_timestamp)) IN (0, 6) then 'FALSE' - when convert_timezone('UTC', 'America/New_York', block_timestamp)::time between '09:00:00' and '15:59:59' then 'TRUE' - else 'FALSE' - END AS nyc_operating_hours + , {{ is_nyc_operating_hours(hour) }} as nyc_operating_hours FROM {{ ref('fact_jupiter_perps_txs') }} agg LEFT JOIN solana_flipside.price.ez_asset_metadata m ON m.token_address = agg.mint ) diff --git a/models/projects/uniswap/raw/ez_uniswap_price_performance.sql b/models/projects/uniswap/raw/ez_uniswap_price_performance.sql index 2e382289..b78ac405 100644 --- a/models/projects/uniswap/raw/ez_uniswap_price_performance.sql +++ b/models/projects/uniswap/raw/ez_uniswap_price_performance.sql @@ -19,11 +19,7 @@ prices as ( date_trunc('hour', block_timestamp) as hour , unwrapped_symbol as symbol , price - , CASE - when date_part('DOW', convert_timezone('UTC', 'America/New_York', block_timestamp)) IN (0, 6) then 'FALSE' - when convert_timezone('UTC', 'America/New_York', block_timestamp)::time between '09:00:00' and '15:59:59' then 'TRUE' - else 'FALSE' - END AS nyc_operating_hours + , {{ is_nyc_operating_hours(hour) }} as nyc_operating_hours from {{ref('fact_uniswap_dex_token_prices')}} t1 inner join tracked_metadata on lower(t1.token_address) = lower(tracked_metadata.contract_address) From bd60c94a476e1299dfb9d209db1c9fbb548a4876 Mon Sep 17 00:00:00 2001 From: Alex Weseley Date: Fri, 8 Nov 2024 12:32:50 -0500 Subject: [PATCH 6/6] aligning perps token prices tables names + macro fix --- .../dydx_v4/raw/ez_dydx_v4_price_performance.sql | 3 ++- .../dydx_v4/raw/fact_dydx_v4_perps_token_prices.sql | 10 +++++++++- .../jupiter/raw/ez_jupiter_perps_price_performance.sql | 3 +-- .../jupiter/raw/fact_jupiter_perps_token_prices.sql | 10 +++++----- .../uniswap/raw/ez_uniswap_price_performance.sql | 2 +- 5 files changed, 18 insertions(+), 10 deletions(-) diff --git a/models/projects/dydx_v4/raw/ez_dydx_v4_price_performance.sql b/models/projects/dydx_v4/raw/ez_dydx_v4_price_performance.sql index c1b9b046..217f2c52 100644 --- a/models/projects/dydx_v4/raw/ez_dydx_v4_price_performance.sql +++ b/models/projects/dydx_v4/raw/ez_dydx_v4_price_performance.sql @@ -13,8 +13,9 @@ with prices as ( date_trunc('hour', block_timestamp) as hour , symbol , price - , {{ is_nyc_operating_hours(hour) }} as nyc_operating_hours + , {{ is_nyc_operating_hours('hour') }} as nyc_operating_hours from {{ref('fact_dydx_v4_perps_prices')}} t1 + WHERE symbol in ('BTC', 'ETH') ) select diff --git a/models/projects/dydx_v4/raw/fact_dydx_v4_perps_token_prices.sql b/models/projects/dydx_v4/raw/fact_dydx_v4_perps_token_prices.sql index b0d35aa2..781a88a4 100644 --- a/models/projects/dydx_v4/raw/fact_dydx_v4_perps_token_prices.sql +++ b/models/projects/dydx_v4/raw/fact_dydx_v4_perps_token_prices.sql @@ -8,4 +8,12 @@ ) }} -select * from {{ ref('fact_dydx_v4_perps_prices') }} \ No newline at end of file +select + block_timestamp + , tx_hash + , 'dydx_v4' as chain + , 'dydx_v4' as app + , symbol + , price + , null as token_address +from {{ ref('fact_dydx_v4_perps_prices') }} diff --git a/models/projects/jupiter/raw/ez_jupiter_perps_price_performance.sql b/models/projects/jupiter/raw/ez_jupiter_perps_price_performance.sql index 9d006f08..a1db3eac 100644 --- a/models/projects/jupiter/raw/ez_jupiter_perps_price_performance.sql +++ b/models/projects/jupiter/raw/ez_jupiter_perps_price_performance.sql @@ -14,8 +14,7 @@ with agg as ( agg.price, agg.mint, m.symbol, - CASE - , {{ is_nyc_operating_hours(hour) }} as nyc_operating_hours + {{ is_nyc_operating_hours('hour') }} as nyc_operating_hours FROM {{ ref('fact_jupiter_perps_txs') }} agg LEFT JOIN solana_flipside.price.ez_asset_metadata m ON m.token_address = agg.mint ) diff --git a/models/projects/jupiter/raw/fact_jupiter_perps_token_prices.sql b/models/projects/jupiter/raw/fact_jupiter_perps_token_prices.sql index 1a44904b..2a5051e9 100644 --- a/models/projects/jupiter/raw/fact_jupiter_perps_token_prices.sql +++ b/models/projects/jupiter/raw/fact_jupiter_perps_token_prices.sql @@ -10,11 +10,11 @@ SELECT block_timestamp, - tx_id, + tx_id as tx_hash, chain, app, - mint, + symbol, price, - size_usd, - fee_usd -FROM {{ ref('fact_jupiter_perps_txs') }} \ No newline at end of file + mint as token_address +FROM {{ ref('fact_jupiter_perps_txs') }} t +LEFT JOIN solana_flipside.price.ez_asset_metadata m ON m.token_address = t.mint \ No newline at end of file diff --git a/models/projects/uniswap/raw/ez_uniswap_price_performance.sql b/models/projects/uniswap/raw/ez_uniswap_price_performance.sql index b78ac405..478a49e2 100644 --- a/models/projects/uniswap/raw/ez_uniswap_price_performance.sql +++ b/models/projects/uniswap/raw/ez_uniswap_price_performance.sql @@ -19,7 +19,7 @@ prices as ( date_trunc('hour', block_timestamp) as hour , unwrapped_symbol as symbol , price - , {{ is_nyc_operating_hours(hour) }} as nyc_operating_hours + , {{ is_nyc_operating_hours('hour') }} as nyc_operating_hours from {{ref('fact_uniswap_dex_token_prices')}} t1 inner join tracked_metadata on lower(t1.token_address) = lower(tracked_metadata.contract_address)