chore: housekeeping in dbt project (et/somenergia-jardiner!156)

* fix: add dbt-postgres explictly to dbt-deps group * ci: ignore build cache during pre-releases * fix: temporary disable test to allow merging * ci: update paths in dbt release task * chore: remove dbt-core and dbt-adapters and let dbt-postgres handle dependencies * chore: add dbt-adapters to deps explicitly * chore: update dbt-core manually required by dbt 1.8 see dbt-labs/dbt-core#10135 * ref: add some extra commands to makefile * chore: update dbt deps * chore: remove dbt-osmosis, upgrade deps once more * ref: set elementary tests as warnings * chore: run pre-commit on failing task * chore: remove references to non-existing dirs in dbt_project.yml they were polluting the logs with warning * chore: move existing tests to data_tests directory * chore: update python and pre-commit dependencies * chore: rename tests to data_tests in dbt 1.8 The `tests` config has been renamed to `data_tests`. Please see https://docs.getdbt.com/docs/build/data-tests#new-data_tests-syntax for more * ref: move dbt settings from profiles to dbt_project as per warnings in versions 1.8.1
Som-Energia · Sep 10, 2024 · dfab8b8 · dfab8b8
1 parent de12acc
commit dfab8b8
Show file tree

Hide file tree

Showing 26 changed files with 837 additions and 827 deletions.
diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
@@ -16,7 +16,7 @@ stages: # List of stages for jobs, and their order of execution
   - deploy
 
 variables:
-  CI_PIPELINE_DEBUG_FLAG: $CI_PIPELINE_DEBUG_FLAG
+  CI_PIPELINE_DEBUG_FLAG: ${CI_PIPELINE_DEBUG_FLAG-false}
   GIT_CLONE_PATH: $CI_BUILDS_DIR/$CI_CONCURRENT_ID/$CI_PROJECT_PATH # https://docs.gitlab.com/ee/ci/runners/configure_runners.html#handling-concurrency
 
   SOMENERGIA_REGISTRY: $SOM_HARBOR_DADES_URL
@@ -96,7 +96,7 @@ build-builder-image-pre-release:
   stage: build
   before_script: *docker-login-before-script
   script:
-    - docker build --pull -t $IMAGE_NAME_MAIN_PRE_RELEASE --progress=plain --target=main -f docker/main/Dockerfile .
+    - docker build --pull -t $IMAGE_NAME_MAIN_PRE_RELEASE --progress=plain --target=builder --no-cache -f docker/main/Dockerfile .
     - docker push $IMAGE_NAME_MAIN_PRE_RELEASE
   rules:
     - if: $CI_COMMIT_REF_NAME == $CI_DEFAULT_BRANCH && $CI_PIPELINE_SOURCE == "push"
@@ -118,7 +118,7 @@ build-main-image-pre-release:
   stage: build
   before_script: *docker-login-before-script
   script:
-    - docker build --pull -t $IMAGE_NAME_MAIN_PRE_RELEASE --progress=plain --target=main -f docker/main/Dockerfile .
+    - docker build --pull -t $IMAGE_NAME_MAIN_PRE_RELEASE --progress=plain --target=main --no-cache -f docker/main/Dockerfile .
     - docker push $IMAGE_NAME_MAIN_PRE_RELEASE
   rules:
     - if: $CI_COMMIT_REF_NAME == $CI_DEFAULT_BRANCH && $CI_PIPELINE_SOURCE == "push"
@@ -140,7 +140,7 @@ build-dbt-deps-image-pre-release:
   stage: build
   before_script: *docker-login-before-script
   script:
-    - docker build --pull -t $IMAGE_NAME_DBT_DEPS_PRE_RELEASE --progress=plain --target=dbt-deps -f docker/main/Dockerfile .
+    - docker build --pull -t $IMAGE_NAME_DBT_DEPS_PRE_RELEASE --progress=plain --target=dbt-deps --no-cache -f docker/main/Dockerfile .
     - docker push $IMAGE_NAME_DBT_DEPS_PRE_RELEASE
   rules:
     - if: $CI_COMMIT_REF_NAME == $CI_DEFAULT_BRANCH && $CI_PIPELINE_SOURCE == "push"
@@ -178,6 +178,11 @@ dbt-build:
     - echo "retrieving dbt artifacts from previous succesful job"
     # gitlab sometimes fails silently when artifact is invalid or not found, https://gitlab.com/gitlab-org/gitlab/-/issues/29118
     - export RESULT=$(curl --output /dev/null --silent --write-out "%{content_type}" "$DBT_MANIFEST_ARTIFACT_URL")
+    - |
+      if [ "$CI_PIPELINE_DEBUG_FLAG" = "true" ]; then
+          echo "Sleeping for 1 hour to allow debugging"
+          sleep 3600
+      fi
     - |
       echo "Artifact content type: ${RESULT}"
       if [ "$RESULT" = "application/zip" ]; then
@@ -219,13 +224,12 @@ release-main-image:
   script:
     - docker pull $IMAGE_NAME_MAIN_PRE_RELEASE
     - docker tag $IMAGE_NAME_MAIN_PRE_RELEASE $IMAGE_NAME_MAIN_RELEASE
-    # https://blog.container-solutions.com/tagging-docker-images-the-right-way
-    - docker tag $IMAGE_NAME_MAIN_RELEASE $IMAGE_NAME_MAIN_SHA_RELEASE
-    - docker push $IMAGE_NAME_MAIN_SHA_RELEASE
     - docker push $IMAGE_NAME_MAIN_RELEASE
   rules:
     # run only on main branch after build-main-image-pre-release is successful
     - if: $CI_COMMIT_REF_NAME == $CI_DEFAULT_BRANCH && $CI_PIPELINE_SOURCE == "push"
+      changes:
+        paths: *main-build-changes-paths
     - if: $CI_PIPELINE_SOURCE == "merge_request_event"
       when: manual
       allow_failure: true
@@ -343,8 +347,6 @@ release-dbt-deps-image:
   script:
     - docker pull $IMAGE_NAME_DBT_DEPS_PRE_RELEASE
     - docker tag $IMAGE_NAME_DBT_DEPS_PRE_RELEASE $IMAGE_NAME_DBT_DEPS_RELEASE
-    - docker tag $IMAGE_NAME_DBT_DEPS_PRE_RELEASE $IMAGE_NAME_DBT_DEPS_SHA_RELEASE
-    - docker push $IMAGE_NAME_DBT_DEPS_SHA_RELEASE
     - docker push $IMAGE_NAME_DBT_DEPS_RELEASE
   tags:
     - somenergia-et

diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml
@@ -1,7 +1,7 @@
 ---
 repos:
   - repo: https://github.com/pre-commit/pre-commit-hooks
-    rev: v4.4.0
+    rev: v4.6.0
     hooks:
       - id: check-yaml
         args: ["--unsafe"]
@@ -10,22 +10,22 @@ repos:
       - id: name-tests-test
         args: ["--pytest-test-first"]
   - repo: https://github.com/psf/black
-    rev: 23.7.0
+    rev: 24.8.0
     hooks:
       - id: black
   - repo: https://github.com/pycqa/isort
-    rev: 5.12.0
+    rev: 5.13.2
     hooks:
       - id: isort
         name: isort (python)
         args: ["--profile", "black", "--filter-files"]
   - repo: https://github.com/charliermarsh/ruff-pre-commit
     # Ruff version.
-    rev: "v0.0.283"
+    rev: "v0.6.3"
     hooks:
       - id: ruff
   - repo: https://github.com/sqlfluff/sqlfluff
-    rev: 2.3.5
+    rev: 3.1.1
     hooks:
       # enable sqlfluff-fix will modify sql files.
       - id: sqlfluff-fix

diff --git a/Makefile b/Makefile
@@ -9,15 +9,23 @@ mkdocs_compose_file		:= docker-compose.mkdocs.yml
 help: ## Print this help
 	@grep -E '^[0-9a-zA-Z_\-\.]+:.*?## .*$$' Makefile | awk 'BEGIN {FS = ":.*?## "}; {printf "\033[36m%-30s\033[0m %s\n", $$1, $$2}'
 
-sh: ## run a shell in the container
-	@docker compose run --rm -it --entrypoint sh main
+
+# ---------------------------------------------------------------------------- #
+#                         docker image with target main                        #
+# ---------------------------------------------------------------------------- #
+
+main.bash: ## run a bash shell in the container main
+	@docker compose run --rm -it --entrypoint bash main
 
 main.build: ## build docker image
 	@docker compose -f $(main_compose_file)  build main --progress=plain
 
 main.push: ## push docker image to registry
 	@docker compose -f $(main_compose_file)  push main
 
+main_dev.bash: ## run a bash shell in the container main_dev
+	@docker compose run --rm -it --entrypoint bash dev
+
 main_dev.build: ## build docker image for development
 	@docker compose -f $(main_compose_file)  build dev --progress=plain
 

diff --git a/dags/dset_materialize_dag.py b/dags/dset_materialize_dag.py
@@ -53,9 +53,11 @@ def dbapi_to_dict(dbapi: str):
     return {
         "provider": parsed_string.scheme,
         "user": parsed_string.username,
-        "password": urllib.parse.unquote(parsed_string.password)
-        if parsed_string.password
-        else None,
+        "password": (
+            urllib.parse.unquote(parsed_string.password)
+            if parsed_string.password
+            else None
+        ),
         "host": parsed_string.hostname,
         "port": parsed_string.port,
         "database": parsed_string.path[1:],

diff --git a/dags/plant_production_datasets_dag.py b/dags/plant_production_datasets_dag.py
@@ -44,9 +44,11 @@ def dbapi_to_dict(dbapi: str):
     return {
         "provider": parsed_string.scheme,
         "user": parsed_string.username,
-        "password": urllib.parse.unquote(parsed_string.password)
-        if parsed_string.password
-        else None,
+        "password": (
+            urllib.parse.unquote(parsed_string.password)
+            if parsed_string.password
+            else None
+        ),
         "host": parsed_string.hostname,
         "port": parsed_string.port,
         "database": parsed_string.path[1:],

diff --git a/dbt_jardiner/config/profiles.yml b/dbt_jardiner/config/profiles.yml
@@ -12,7 +12,8 @@ dbt_jardiner:
             dbname: "{{ env_var('DBNAME') }}"
             schema: dbt_prod
 
-        prod: # used in production
+        # used in production
+        prod:
             type: postgres
             threads: 1
             host: "{{ env_var('DBHOST') }}"
@@ -22,7 +23,8 @@ dbt_jardiner:
             dbname: "{{ env_var('DBNAME') }}"
             schema: dbt_prod
 
-        pre: # pre-production, not used at the moment
+        # pre-production, not used at the moment
+        pre:
             type: postgres
             threads: 1
             host: "{{ env_var('DBHOST') }}"
@@ -32,7 +34,8 @@ dbt_jardiner:
             dbname: "{{ env_var('DBNAME') }}"
             schema: dbt_pre
 
-        test: # used during CI
+        # used during CI
+        test:
             type: postgres
             threads: 1
             host: "{{ env_var('DBHOST') }}"
@@ -42,7 +45,8 @@ dbt_jardiner:
             dbname: "{{ env_var('DBNAME') }}"
             schema: dbt_test
 
-        dev: # used during local development
+        # used during local development
+        dev:
             type: postgres
             threads: 1
             host: "{{ env_var('DBHOST') }}"
@@ -101,5 +105,3 @@ elementary:
             keepalives_idle: 0  # default 0 seconds
             connect_timeout: 10  # default 10 seconds
 
-config:
-    send_anonymous_usage_stats: false
diff --git a/dbt_jardiner/tests/.gitkeep → dbt_jardiner/data_tests/.gitkeep b/dbt_jardiner/tests/.gitkeep → dbt_jardiner/data_tests/.gitkeep
diff --git a/..._active_power__missing_phase_last_day.sql → ..._active_power__missing_phase_last_day.sql b/..._active_power__missing_phase_last_day.sql → ..._active_power__missing_phase_last_day.sql
diff --git a/...et_gaps_per_day_and_signal_last_month.sql → ...et_gaps_per_day_and_signal_last_month.sql b/...et_gaps_per_day_and_signal_last_month.sql → ...et_gaps_per_day_and_signal_last_month.sql
diff --git a/...ave_references_in_signal_denormalized.sql → ...ave_references_in_signal_denormalized.sql b/...ave_references_in_signal_denormalized.sql → ...ave_references_in_signal_denormalized.sql
diff --git a/...et_raw_uuids_not_empty_or_null_3_days.sql → ...et_raw_uuids_not_empty_or_null_3_days.sql b/...et_raw_uuids_not_empty_or_null_3_days.sql → ...et_raw_uuids_not_empty_or_null_3_days.sql
diff --git a/...t/test_dset_signals_receiver_all_time.sql → ...t/test_dset_signals_receiver_all_time.sql b/...t/test_dset_signals_receiver_all_time.sql → ...t/test_dset_signals_receiver_all_time.sql
diff --git a/.../test_dset_signals_receiver_last_hour.sql → .../test_dset_signals_receiver_last_hour.sql b/.../test_dset_signals_receiver_last_hour.sql → .../test_dset_signals_receiver_last_hour.sql
diff --git a/...er/tests/dset/test_gapfill_5min_nrows.sql → ...ta_tests/dset/test_gapfill_5min_nrows.sql b/...er/tests/dset/test_gapfill_5min_nrows.sql → ...ta_tests/dset/test_gapfill_5min_nrows.sql
diff --git a/...dset/test_repeated_device_uuid_errors.sql → ...dset/test_repeated_device_uuid_errors.sql b/...dset/test_repeated_device_uuid_errors.sql → ...dset/test_repeated_device_uuid_errors.sql
@@ -1,3 +1,5 @@
+{{ config(enabled=false) }}
+
 -- tests pairs device_uuid, plant, device, device_type, device_parent as unique
 with
 unique_device_per_plant as (

diff --git a/...nal_is_correctly_converted_to_numeric.sql → ...nal_is_correctly_converted_to_numeric.sql b/...nal_is_correctly_converted_to_numeric.sql → ...nal_is_correctly_converted_to_numeric.sql
diff --git a/...et/test_solargis_temperature_registry.sql → ...et/test_solargis_temperature_registry.sql b/...et/test_solargis_temperature_registry.sql → ...et/test_solargis_temperature_registry.sql
diff --git a/...jardiner/tests/dset/test_spine_hourly.sql → ...ner/data_tests/dset/test_spine_hourly.sql b/...jardiner/tests/dset/test_spine_hourly.sql → ...ner/data_tests/dset/test_spine_hourly.sql
diff --git a/dbt_jardiner/dbt_project.yml b/dbt_jardiner/dbt_project.yml
@@ -26,6 +26,10 @@ clean-targets: # directories to be removed by `dbt clean`
   - "target"
   - "dbt_packages"
 
+flags:
+  send_anonymous_usage_stats: false
+  require_explicit_package_overrides_for_builtin_materializations: false
+
 # Configuring models
 # Full documentation: https://docs.getdbt.com/docs/configuring-models
 
@@ -45,23 +49,18 @@ models:
           +tags: dset
         simel:
           +grants:
-            select: ['mercat_electric']
+            select: ["mercat_electric"]
       intermediate:
         +docs:
           node_color: "#d05808"
         dset:
           +tags: dset
         simel:
           +grants:
-            select: ['mercat_electric']
+            select: ["mercat_electric"]
       marts:
         +docs:
           node_color: "goldenrod"
-        dset:
-          +tags: dset
-        simel:
-          +grants:
-            select: ['mercat_electric']
       observability:
         +docs:
           node_color: "#c3c557"
@@ -84,16 +83,11 @@ models:
     ## To disable elementary for dev, uncomment this:
     # enabled: "{{ target.name in ['prod','analytics'] }}"
 
-  re_data:
-    +docs:
-      show: false
-      node_color: "lightblue"
-
 snapshots:
   +docs:
     node_color: "#e7bca1"
 
-tests:
+data_tests:
   +docs:
     node_color: "#84b4bc"
 
@@ -103,4 +97,3 @@ seeds:
       +tags: jardiner
     legacy:
       +tags: legacy
-
diff --git a/dbt_jardiner/models/jardiner/intermediate/meteologica/_int_meteologica__models.yml b/dbt_jardiner/models/jardiner/intermediate/meteologica/_int_meteologica__models.yml
@@ -5,7 +5,7 @@ models:
     description: energy forecast of the previous day. Discards SomRenovables plants.
     config:
       tags: meteologica
-    tests:
+    data_tests:
       - dbt_utils.unique_combination_of_columns:
           config:
             severity: warn

diff --git a/dbt_jardiner/models/jardiner/marts/_marts__models.yaml b/dbt_jardiner/models/jardiner/marts/_marts__models.yaml
@@ -39,28 +39,28 @@ models:
     description: Conté les lectures crues que venen de la producció horària de les plantes
     columns:
       - name: hora_inici
-        tests:
+        data_tests:
           - not_null
       - name: hora_final
-        tests:
+        data_tests:
           - not_null
       - name: nom_planta
-        tests:
+        data_tests:
           - not_null
       - name: tecnologia
-        tests:
+        data_tests:
           - not_null
       - name: energia_instantania_inversor_kwh
       - name: energia_exportada_instantania_comptador_kwh
-        tests:
+        data_tests:
           - dbt_utils.expression_is_true:
               expression: ">= energia_instantania_inversor_kwh"
               config:
                 severity: warn
                 where: "hora_inici > (now() at time zone 'Europe/Madrid')::date - interval '7 days'"
                 tags: source
       - name: energia_perduda_inversor_a_comptador
-        tests:
+        data_tests:
           - dbt_utils.accepted_range:
               max_value: 1
               min_value: 0
@@ -79,7 +79,7 @@ models:
       - name: irradiation_wh_m2
       - name: irradiacio_satellit_wh_m2
       - name: temperatura_modul_avg_c
-        tests:
+        data_tests:
           - dbt_utils.not_null_proportion:
               description: //TODO get the right proportion or whatever
               error_if: ">=7"
@@ -93,7 +93,7 @@ models:
                 enabled: false
                 where: "hora_inici > (now() at time zone 'Europe/Madrid')::date - interval '7 days'"
       - name: pr_horari
-        tests:
+        data_tests:
           - dbt_utils.accepted_range:
               max_value: 1
               min_value: 0

diff --git a/dbt_jardiner/models/jardiner/observability/dset/_obs_dset__models.yml b/dbt_jardiner/models/jardiner/observability/dset/_obs_dset__models.yml
@@ -13,7 +13,7 @@ models:
       - name: device_uuid
       - name: device_parent
       - name: signal_uuid
-        tests:
+        data_tests:
           - not_null:
               severity: warn
               config:
@@ -34,7 +34,7 @@ models:
           Valor del senyal en el moment de la lectura. El valor del senyal per
           les mètriques à testejat dins d'uns rangs, triat manualment observant
           valors actuals Enable tests when signals are more estable
-        tests:
+        data_tests:
           - dbt_utils.accepted_range:
               name: accepted_range_intensity_instantaneous_last_7_days
               max_value: 500

diff --git a/dbt_jardiner/models/jardiner/raw/dset/_raw_dset__models.yaml b/dbt_jardiner/models/jardiner/raw/dset/_raw_dset__models.yaml
@@ -43,6 +43,8 @@ models:
               column_anomalies:
                 - null_count
                 - null_percent
+              config:
+                severity: warn
       - name: signal_value
         description: Valor del senyal observat. Camp definit per partner DSET
         tests:
@@ -51,6 +53,8 @@ models:
               column_anomalies:
                 - null_count
                 - null_percent
+              config:
+                severity: warn
   - name: raw_dset_responses__last_signal_reading
     description: >
       Conté l'últim batch de dades, descartant ts i value. Es fa servir per obtenir signal_last_ts

diff --git a/dbt_jardiner/packages.yml b/dbt_jardiner/packages.yml
@@ -1,6 +1,6 @@
 packages:
   - package: dbt-labs/dbt_utils
-    version: [">=1.0.0", "<1.1.0"]
+    version: [">=1.3.0", "<2.0.0"]
   - package: elementary-data/elementary
-    version: 0.14.1
+    version: [">=0.14.1", "<0.17.0"]
     ## Docs: https://docs.elementary-data.com