new: add device_parent_uuid into pipe and filters out deactivated sig…

…nals (et/somenergia-jardiner!120)

Closes #96

Merge branch 'fix/let-snapshot-add-new-columns' into 'main'

.gitlab-ci.yml

@@ -48,7 +48,7 @@ include:
 
 pre-commit:
   variables:
-    SKIP: "sqlfluff-lint"
+    SKIP: "sqlfluff-fix,sqlfluff-lint"
   before_script:
     - git config --global --add safe.directory "$CI_PROJECT_DIR"
   tags:

.pre-commit-config.yaml

@@ -28,8 +28,8 @@ repos:
     rev: 2.3.5
     hooks:
       # enable sqlfluff-fix will modify sql files.
-      # - id: sqlfluff-fix
-      #  additional_dependencies: ["dbt-postgres", "sqlfluff-templater-dbt"]
+      - id: sqlfluff-fix
+        additional_dependencies: ["dbt-postgres", "sqlfluff-templater-dbt"]
       - id: sqlfluff-lint
         additional_dependencies: ["dbt-postgres", "sqlfluff-templater-dbt"]
 fail_fast: false

Makefile

@@ -85,5 +85,8 @@ dbt_docs.logs: ## show the logs of the dbt-docs container
 local.re_data_models.dev: ## Run re_data models
 	@(cd dbt_jardiner && dbt run --target dev --models package:re_data)
 
-changelog: ## generate changelog
+changelog.docker: ## generate changelog usind docker image
 	@docker compose -f $(app_compose_file) --env-file $(app_compose_env_file) run --rm -it --entrypoint git-changelog app-dev
+
+changelog: ## generate changelog
+	@git-changelog

dbt_jardiner/models/jardiner/intermediate/dset/int_dset_last_signal__from_latest_batch_dset_last.sql

@@ -1,36 +1,37 @@
 {{ config(materialized="view") }}
 
 with last_dset_batch as (
-    select
-        case
-            when signal_uuid ~ e'^[[:xdigit:]]{8}-([[:xdigit:]]{4}-){3}[[:xdigit:]]{12}$' then signal_uuid::uuid -- noqa: LT01
-        end as signal_uuid,
-        signal_uuid as signal_uuid_raw,
-        {{ dbt_utils.star(from=ref("raw_dset_responses__last_signal_reading"), except=['signal_uuid']) }}
-    from {{ ref("raw_dset_responses__last_signal_reading") }}
+  select
+    case
+      when signal_uuid ~ e'^[[:xdigit:]]{8}-([[:xdigit:]]{4}-){3}[[:xdigit:]]{12}$' then signal_uuid::uuid -- noqa: LT01
+    end as signal_uuid,
+    signal_uuid as signal_uuid_raw,
+    {{ dbt_utils.star(from=ref("raw_dset_responses__last_signal_reading"), except=['signal_uuid']) }}
+  from {{ ref("raw_dset_responses__last_signal_reading") }}
 )
 select
-    dset.signal_last_ts,
-    dset.signal_last_value,
-    dset.group_name as dset_plant_name,
-    plants.plant_name,
-    dset.queried_at,
-    metadata.signal_name,
-    metadata.metric_name,
-    metadata.device_name,
-    metadata.device_type,
-    metadata.plant_uuid,
-    metadata.device_uuid,
-    metadata.device_parent,
-    metadata.signal_uuid,
-    dset.signal_id,
-    dset.signal_tz,
-    dset.signal_code,
-    dset.signal_type,
-    dset.signal_unit
+  dset.signal_last_ts,
+  dset.signal_last_value,
+  dset.group_name as dset_plant_name,
+  plants.plant_name,
+  dset.queried_at,
+  metadata.signal_name,
+  metadata.metric_name,
+  metadata.device_name,
+  metadata.device_type,
+  metadata.plant_uuid,
+  metadata.device_uuid,
+  metadata.device_parent,
+  metadata.device_parent_uuid,
+  metadata.signal_uuid,
+  dset.signal_id,
+  dset.signal_tz,
+  dset.signal_code,
+  dset.signal_type,
+  dset.signal_unit
 from last_dset_batch as dset
-full outer join
-  {{ ref("raw_gestio_actius__signal_denormalized") }} as metadata
-  using (signal_uuid)
-left join {{ ref("int_gda_plants__plants_catalog") }} as plants
-  using (plant_uuid)
+  full outer join
+    {{ ref("raw_gestio_actius__signal_denormalized") }} as metadata
+    using (signal_uuid)
+  left join {{ ref("int_gda_plants__plants_catalog") }} as plants
+    using (plant_uuid)

dbt_jardiner/models/jardiner/intermediate/dset/int_dset_responses__spined_metadata.sql

@@ -15,7 +15,8 @@ spined_expected_signals as (
     metadata.signal_name,
     metadata.signal_uuid,
     metadata.metric_name,
-    metadata.device_parent
+    metadata.device_parent,
+    metadata.device_parent_uuid
   from
     spina5m
     {# dm_plants is the SSOT of the plants names #}
@@ -42,6 +43,7 @@ spined_dset as (
     spined.signal_name,
     spined.metric_name,
     spined.device_parent,
+    spined.device_parent_uuid,
     valors.signal_value,
     valors.group_name,
     valors.signal_id,

dbt_jardiner/models/jardiner/intermediate/gestio_actius/int_signal_device_relation__distinct_devices.sql

@@ -1,5 +1,5 @@
 {{ config(materialized="table") }}
 
-select distinct plant_name, plant_uuid, device_name, device_uuid, device_type, device_parent
+select distinct plant_name, plant_uuid, device_name, device_uuid, device_type, device_parent, device_parent_uuid
 from {{ ref('raw_gestio_actius__signal_denormalized') }}
 order by plant_name, device_name

dbt_jardiner/models/jardiner/marts/dm_dset_last_reading__from_signal_last_ts.sql

@@ -1,19 +1,20 @@
 {{ config(materialized="view") }}
 
 select
-    plant_name as nom_planta,
-    dset_plant_name as nom_planta_dset,
-    signal_last_ts as ultim_ts,
-    signal_last_value as ultim_valor,
-    queried_at as peticionat_a,
-    signal_name as nom_senyal,
-    metric_name as nom_metrica,
-    signal_unit as unitat,
-    device_name as nom_aparell,
-    device_type as tipus_aparell,
-    device_parent as aparell_pare,
-    signal_uuid as uuid_senyal,
-    signal_id as id_senyal_dset,
-    signal_code as codi_senyal_dset,
-    signal_type as tipus_senyal
-from {{ ref("int_dset_last_signal__from_latest_batch_dset_last") }}
+  plant_name as nom_planta,
+  dset_plant_name as nom_planta_dset,
+  signal_last_ts as ultim_ts,
+  signal_last_value as ultim_valor,
+  queried_at as peticionat_a,
+  signal_name as nom_senyal,
+  metric_name as nom_metrica,
+  signal_unit as unitat,
+  device_name as nom_aparell,
+  device_type as tipus_aparell,
+  device_parent as aparell_pare,
+  device_parent_uuid as uuid_aparell_pare,
+  signal_uuid as uuid_senyal,
+  signal_id as id_senyal_dset,
+  signal_code as codi_senyal_dset,
+  signal_type as tipus_senyal
+from {{ ref("int_dset_last_signal__from_latest_batch_dset_last") }}

dbt_jardiner/models/jardiner/marts/dm_metrics_production_5min.sql

@@ -10,7 +10,8 @@ select
   signal_unit as unitat_senyal,
   metric_name as nom_metrica,
   signal_name as senyal,
-  device_parent as nom_aparell_pare
+  device_parent as nom_aparell_pare,
+  device_parent_uuid as uuid_aparell_pare
 from {{ ref("int_dset_responses__values_incremental") }}
 where
   (

dbt_jardiner/models/jardiner/observability/dset/obs_dset_responses__with_signal_metadata.sql

@@ -4,35 +4,36 @@
 {# TODO one metric could have different units, we could standardize here
 e.g. signal_value * (case when signal_unit = 'kwh' then 1000 else 1)#}
 with
-    base as (
-        select
-            metadata.plant_uuid,
-            metadata.plant_name,
-            metadata.signal_name,
-            metadata.metric_name,
-            metadata.device_name,
-            metadata.device_type,
-            metadata.device_uuid,
-            metadata.device_parent,
-            metadata.signal_uuid,
-            valors.group_name,
-            valors.signal_id,
-            valors.signal_tz,
-            valors.signal_code,
-            valors.signal_type,
-            valors.signal_unit,
-            valors.signal_last_ts,
-            valors.signal_frequency,
-            valors.signal_is_virtual,
-            valors.signal_last_value,
-            valors.queried_at,
-            valors.ts,
-            valors.signal_value as signal_value
-        from {{ ref("raw_gestio_actius__signal_denormalized") }} as metadata
-        left join
-            {{ ref("int_dset_responses__materialized") }} as valors
-            on metadata.signal_uuid = valors.signal_uuid
-    )
+base as (
+  select
+    metadata.plant_uuid,
+    metadata.plant_name,
+    metadata.signal_name,
+    metadata.metric_name,
+    metadata.device_name,
+    metadata.device_type,
+    metadata.device_uuid,
+    metadata.device_parent,
+    metadata.device_parent_uuid,
+    metadata.signal_uuid,
+    valors.group_name,
+    valors.signal_id,
+    valors.signal_tz,
+    valors.signal_code,
+    valors.signal_type,
+    valors.signal_unit,
+    valors.signal_last_ts,
+    valors.signal_frequency,
+    valors.signal_is_virtual,
+    valors.signal_last_value,
+    valors.queried_at,
+    valors.ts,
+    valors.signal_value as signal_value
+  from {{ ref("raw_gestio_actius__signal_denormalized") }} as metadata
+    left join
+      {{ ref("int_dset_responses__materialized") }} as valors
+      on metadata.signal_uuid = valors.signal_uuid
+)
 
 select *
 from base

dbt_jardiner/models/jardiner/raw/gestio_actius/_raw_gestio_actius__models.yaml

@@ -97,7 +97,22 @@ models:
         description:
           Imputat per nosaltres que es correspon amb la columna homònima de la
           taula d'aparell que correspongui ja sigui inverter, string, sensor, plant, etc.
+      - name: is_enabled
+        description: >
+          Boolean que indica si el senyal està habilitat o no. Si no ho està no es
+          processarà.
+        tests:
+          - accepted_values:
+              values:
+                [true, false]
+      - name: device_parent_uuid
+        description: UUID del parent device, si n'hi ha
     tests:
+      - dbt_utils.expression_is_true:
+            # XOR, one of the two can't be null
+            expression: "device_parent_uuid is not null"
+            config:
+              where: "device_parent is not null"
       - dbt_utils.unique_combination_of_columns:
           config:
             severity: error

dbt_jardiner/models/jardiner/raw/gestio_actius/raw_gestio_actius__signal_denormalized.sql

@@ -17,10 +17,13 @@ base as (
     device_type::text,
     device_uuid::uuid,
     trim(device_parent::text) as device_parent,
+    device_parent_uuid::uuid,
     signal_uuid::uuid,
+    coalesce(is_enabled = 'VERDADERO', false) as is_enabled,
     inserted_at::timestamptz,
     updated_at::timestamptz
   from fresh
 )
 
 select * from base
+where is_enabled

dbt_jardiner/snapshots/_jardiner__snapshots_sources.yaml

@@ -43,3 +43,7 @@ sources:
             description: "Insertion timestamp in the database, in UTC"
           - name: updated_at
             description: "Update timestamp in the database, in UTC"
+          - name: is_enabled
+            description: "Boolean. If the signal is enabled or not"
+          - name: device_parent_uuid
+            description: "UUID of the parent device, version 4"

dbt_jardiner/tests/dset/test_repeated_device_uuid_errors.sql

@@ -1,28 +1,28 @@
 -- tests pairs device_uuid, plant, device, device_type, device_parent as unique
 with
-    unique_device_per_plant as (
-        select distinct device_uuid, plant_uuid, plant_name, device_name, device_type, device_parent
-        from {{ ref("raw_gestio_actius__signal_denormalized") }}
-        order by plant_name, device_name
-    ),
+unique_device_per_plant as (
+  select distinct device_uuid, plant_uuid, plant_name, device_name, device_type, device_parent, device_parent_uuid
+  from {{ ref("raw_gestio_actius__signal_denormalized") }}
+  order by plant_name, device_name
+),
 
-    duplicated_devices as (
-        select udpp.device_uuid, count(*) as n_duplicates
-        from unique_device_per_plant as udpp
-        group by udpp.device_uuid
-        having count(*) > 1
-    ),
+duplicated_devices as (
+  select udpp.device_uuid, count(*) as n_duplicates
+  from unique_device_per_plant as udpp
+  group by udpp.device_uuid
+  having count(*) > 1
+),
 
-    joined as (
-        select expected.*, duplicated_devices.n_duplicates
-        from {{ ref("raw_gestio_actius__signal_denormalized") }} as expected
-        left join duplicated_devices on expected.device_uuid = duplicated_devices.device_uuid
-    ),
+joined as (
+  select expected.*, duplicated_devices.n_duplicates
+  from {{ ref("raw_gestio_actius__signal_denormalized") }} as expected
+    left join duplicated_devices on expected.device_uuid = duplicated_devices.device_uuid
+),
 
-    filtered as (
-        select *
-        from joined
-        where n_duplicates is not null
-    )
+filtered as (
+  select *
+  from joined
+  where n_duplicates is not null
+)
 
 select * from filtered