Update Git Commit History

.pre-commit-config.yaml

@@ -0,0 +1,9 @@
+---
+repos:
+  - repo: local
+    hooks:
+      - id: sqlfluff-changed-files
+        name: sqlfluff-changed-files
+        entry: ./dbt_scripts/run_sqlfluff_on_changed_files.sh
+        language: script
+        files: \.sql$

.sqlfluff

@@ -0,0 +1,25 @@
+# .sqlfluff configuration file
+[sqlfluff]
+dialect = snowflake
+templater = jinja
+
+# General linting rules
+[sqlfluff:rules]
+max_line_length = 100  # Set maximum line length
+tab_space_size = 4  # Define tab space size
+
+# Specific rule configurations
+[sqlfluff:rules:L003]  # Indentation should be a multiple of 4 spaces
+tab_space_size = 4
+
+[sqlfluff:rules:capitalisation.keywords]
+capitalisation_policy = upper
+
+[sqlfluff:rules:capitalisation.identifiers]  # Inconsistent capitalisation of unquoted identifiers
+extended_capitalisation_policy = consistent
+
+[sqlfluff:rules:references.consistent]  # Avoid using aliases in join clauses
+single_table_references = consistent
+
+[sqlfluff:layout:type:comma]
+line_position = leading

README.md

@@ -24,7 +24,7 @@ Welcome to the Artemis DBT repository!
 
 ### What is this?
 
-This is the home of all of the data-munging business logic that ultimately powers a variety of the data in the Artemis application suite, including the [Terminal](https://app.artemis.xyz/), [Sheets](https://www.artemis.xyz/sheets), and [Snowflake](https://www.artemis.xyz/datashare) integration.
+This is the home of all of the data-munging business logic that ultimately powers a variety of the data in the Artemis application suite, including the [Terminal](https://app.artemisanalytics.com/), [Sheets](https://www.artemis.xyz/sheets), and [Snowflake](https://www.artemis.xyz/datashare) integration.
 
 ### Who is this for?
 
@@ -56,8 +56,9 @@ There are two ways to get help:
 ## Environment Setup
 
 1. Fork this repository (button towards the top right)
-2. Write SQL model changes
-3. Open a PR and view results of your changes directly in the Github Actions - more on this in [adding a new metric](#adding-a-new-metric).
+2. Run `source syncenv` to download dependencies
+3. Write SQL model changes
+4. Open a PR and view results of your changes directly in the Github Actions - more on this in [adding a new metric](#adding-a-new-metric).
 
 ## System Design
 
@@ -71,7 +72,7 @@ Fact tables are then combined into `ez_asset_metrics` tables that are piped into
 
 **BEFORE** adding metrics for a protocol, you must create the asset first.
 
-For example, let's say a user wants to add GEODNET fees to our [DePin dashboard.](https://app.artemis.xyz/sectors?tab=dePin). They must first add the [GEODNET](https://geodnet.com/) asset first by completing the following steps:
+For example, let's say a user wants to add GEODNET fees to our [DePin dashboard.](https://app.artemisanalytics.com/sectors?tab=dePin). They must first add the [GEODNET](https://geodnet.com/) asset first by completing the following steps:
 
 - [ ] Fork this repository
 - [ ] Add GEODNET to the `databases.csv` file

databases.csv

@@ -12,24 +12,37 @@ AVALANCHE_FLIPSIDE
 AXELAR_FLIPSIDE
 BASE
 BASE_FLIPSIDE
+BEAM
 BEDROCK
 BITCOIN
 BITCOIN_FLIPSIDE
 BLAST
 BLAST_FLIPSIDE
+BLUR
 BSC
 BSC_FLIPSIDE
+BUSD
 CARDANO
 CCTP
 CELESTIA
 CELO
+CEUR
+CHAINLINK
+CKES
 COMMON
 COMPOUND
 COSMOSHUB
+CREAL
+CUSD
+DAI
+DEXALOT
+DEFILLAMA
+DFK
 EIGENPIE
 ETHEREUM
 ETHEREUM_FLIPSIDE
 ETHERFI
+EURC
 FANTOM
 FLOW
 FLOW_FLIPSIDE
@@ -40,8 +53,10 @@ IMMUTABLE_X
 INTERNET_COMPUTER
 INJECTIVE
 JUPITER
+KAICHING
 KELP_DAO
 LANDING_DATABASE
+MAKER
 NEAR
 NEAR_FLIPSIDE
 OPTIMISM
@@ -54,6 +69,7 @@ POLYGON
 POLYGON_FLIPSIDE
 POLYGON_ZK
 PUFFER_FINANCE
+PYUSD
 RAINBOW_BRIDGE
 RENZO_PROTOCOL
 SEI
@@ -68,5 +84,10 @@ SWELL
 SUI
 TON
 TRON
-ZKSYNC
 UNISWAP
+USDC
+USDE
+USDGLO
+USDP
+USDT
+ZKSYNC

dbt_project.yml

@@ -38,6 +38,139 @@ models:
         +snowflake_warehouse: "DAILY_DEFILLAMA"
     bam_models:
       +snowflake_warehouse: "DAILY_BAM"
+    staging:
+      acala:
+        +snowflake_warehouse: "ACALA"
+      across:
+        +snowflake_warehouse: "ACROSS"
+      akash:
+        +snowflake_warehouse: "AKASH"
+      aptos:
+        +snowflake_warehouse: "APTOS"
+      arbitrum:
+        +snowflake_warehouse: "ARBITRUM"
+      avalanche:
+        +snowflake_warehouse: "AVALANCHE"
+      axelar:
+        +snowflake_warehouse: "AXELAR"
+      base:
+        +snowflake_warehouse: "BASE"
+      beam:
+        +snowflake_warehouse: "BEAM"
+      benqi:
+        +snowflake_warehouse: "BENQI_FINANCE"
+      bitcoin:
+        +snowflake_warehouse: "BITCOIN"
+      blast:
+        +snowflake_warehouse: "BLAST"
+      blur:
+        +snowflake_warehouse: "BLUR"
+      cardano:
+        +snowflake_warehouse: "CARDANO"
+      cctp:
+        +snowflake_warehouse: "CCTP"
+      celestia:
+        +snowflake_warehouse: "CELESTIA"
+      celo:
+        +snowflake_warehouse: "CELO"
+      coinbase:
+        +snowflake_warehouse: "COINBASE"
+      compound:
+        +snowflake_warehouse: "COMPOUND"
+      cosmoshub:
+        +snowflake_warehouse: "COSMOSHUB"
+      curve:
+        +snowflake_warehouse: "CURVE"
+      dexalot:
+        +snowflake_warehouse: "DEXALOT"
+      dfk:
+        +snowflake_warehouse: "DFK"
+      ethereum:
+        +snowflake_warehouse: "ETHEREUM"
+      fantom:
+        +snowflake_warehouse: "FANTOM"
+      flow:
+        +snowflake_warehouse: "FLOW"
+      frax:
+        +snowflake_warehouse: "FRAX"
+      fuse:
+        +snowflake_warehouse: "FUSE"
+      gnosis:
+        +snowflake_warehouse: "GNOSIS"
+      immutable_x:
+        +snowflake_warehouse: "IMMUTABLE_X"
+      injective:
+        +snowflake_warehouse: "INJECTIVE"
+      lido:
+        +snowflake_warehouse: "LIDO"
+      linea:
+        +snowflake_warehouse: "LINEA"
+      mantle:
+        +snowflake_warehouse: "mantle"
+      moonwell:
+        +snowflake_warehouse: "MOONWELL"
+      multiversx:
+        +snowflake_warehouse: "MULTIVERSX"
+      optimism:
+        +snowflake_warehouse: "OPTIMISM"
+      osmosis:
+        +snowflake_warehouse: "OSMOSIS"
+      pancakeswap:
+        +snowflake_warehouse: "PANCAKESWAP_SM"
+      polkadot:
+        +snowflake_warehouse: "POLKADOT"
+      polygon:
+        +snowflake_warehouse: "POLYGON"
+      near:
+        +snowflake_warehouse: "NEAR"
+      quickswap:
+        +snowflake_warehouse: "QUICKSWAP"
+      radiant:
+        +snowflake_warehouse: "RADIANT"
+      rainbow_bridge:
+        +snowflake_warehouse: "RAINBOW_BRIDGE"
+      rocketpool:
+        +snowflake_warehouse: "ROCKETPOOL"
+      scroll:
+        +snowflake_warehouse: "SCROLL"
+      seamless:
+        +snowflake_warehouse: "SEAMLESSPROTOCOL"
+      sei:
+        +snowflake_warehouse: "SEI"
+      solana:
+        +snowflake_warehouse: "SOLANA"
+      sonne:
+        +snowflake_warehouse: "SONNE_FINANCE"
+      spark:
+        +snowflake_warehouse: "SPARK"
+      stacks:
+        +snowflake_warehouse: "STACKS"
+      stader:
+        +snowflake_warehouse: "STADER"
+      stakewise:
+        +snowflake_warehouse: "STAKEWISE"
+      stride:
+        +snowflake_warehouse: "STRIDE"
+      sushiswap:
+        +snowflake_warehouse: "SUSHISWAP_SM"
+      ton:
+        +snowflake_warehouse: "TON"
+      trader_joe:
+        +snowflake_warehouse: "TRADER_JOE"
+      tron:
+        +snowflake_warehouse: "TRON"
+      uniswap:
+        +snowflake_warehouse: "UNISWAP_SM"
+      uwu_lend:
+        +snowflake_warehouse: "UWULEND"
+      venus:
+        +snowflake_warehouse: "VENUS"
+      zcash:
+        +snowflake_warehouse: "ZCASH"
+      zksync:
+        +snowflake_warehouse: "ZKSYNC"
+      zora:
+        +snowflake_warehouse: "ZORA"
 
 vars:
   dbt_date:time_zone: "Etc/UTC"

dbt_scripts/create_fidelity_queries.sh

@@ -0,0 +1,52 @@
+# Assumtions:
+# 1. Everything is stored in the projects folder not staging
+
+# Check if exactly one argument is provided
+if [ "$#" -ne 1 ]; then
+    echo "Usage: $0 <project_name>"
+    exit 1
+fi
+
+PROJECT_NAME="$1"
+
+INPUT_FOLDER="models/projects/$PROJECT_NAME"
+OUTPUT_FOLDER="models/_complete_asset_models/$PROJECT_NAME"
+
+mkdir -p "$OUTPUT_FOLDER"
+
+BACKUP_FILE="materialized_backup.txt"
+
+echo "Backing up original materialized settings with filenames..."
+grep -r 'materialized="' "$INPUT_FOLDER" | sed 's/^\(.*\):.*materialized="\(.*\)".*/\1:\2/' > "$BACKUP_FILE"
+
+echo "Changing all materialized settings to ephemeral..."
+find "$INPUT_FOLDER" -type f -name "*.sql" | while read -r file; do
+    sed -i '' 's/materialized="table"/materialized="ephemeral"/' "$file"
+    sed -i '' 's/materialized="incremental"/materialized="ephemeral"/' "$file"
+done
+
+CORE_FOLDER_PATH="$INPUT_FOLDER/core"
+echo "Running dbt compile for each file..."
+find "$CORE_FOLDER_PATH" -type f -name "*.sql" | while read -r file; do
+    base_filename=$(basename "$file" .sql)
+
+    output_file="$OUTPUT_FOLDER/${base_filename}__complete__.sql"
+
+    dbt compile --select "$file" > "$output_file"
+    # Remove unwanted log lines from the first 7 lines of the output file
+    head -n 7 "$output_file" | sed '/^\[0m/d' | sed '/^Found /d' | sed '/^Concurrency: /d' > "${output_file}.tmp"
+    tail -n +8 "$output_file" >> "${output_file}.tmp"
+    mv "${output_file}.tmp" "$output_file"
+
+    echo "Compiled and created: $output_file"
+done
+
+#Restore the original `materialized` settings
+echo "Restoring original materialized settings..."
+while IFS=: read -r filepath materialized_value; do
+    sed -i '' "s/materialized=\"ephemeral\"/materialized=\"$materialized_value\"/" "$filepath"
+done < "$BACKUP_FILE"
+
+Clean up the backup file
+rm "$BACKUP_FILE"
+echo "All files restored to original materialized settings."

dbt_scripts/run_sqlfluff_on_changed_files.sh

@@ -0,0 +1,19 @@
+#!/bin/bash
+# Get the list of changed SQL files
+CHANGED_FILES=$(git diff --cached --name-only --diff-filter=ACM | grep '\.sql$')
+
+if [ -z "$CHANGED_FILES" ]; then
+  echo "No changed SQL files to lint"
+  exit 0
+fi
+
+# Run SQLFluff on each changed SQL file
+for file in $CHANGED_FILES; do
+  sqlfluff fix "$file"
+  if [ $? -ne 0 ]; then
+    echo "SQLFluff found issues in $file"
+    exit 1
+  fi
+done
+
+exit 0