diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 8e5f9c57..45af020b 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -48,7 +48,7 @@ include: pre-commit: variables: - SKIP: "sqlfluff-lint" + SKIP: "sqlfluff-fix,sqlfluff-lint" before_script: - git config --global --add safe.directory "$CI_PROJECT_DIR" tags: diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index c3771e29..23cc08ce 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -28,8 +28,8 @@ repos: rev: 2.3.5 hooks: # enable sqlfluff-fix will modify sql files. - # - id: sqlfluff-fix - # additional_dependencies: ["dbt-postgres", "sqlfluff-templater-dbt"] + - id: sqlfluff-fix + additional_dependencies: ["dbt-postgres", "sqlfluff-templater-dbt"] - id: sqlfluff-lint additional_dependencies: ["dbt-postgres", "sqlfluff-templater-dbt"] fail_fast: false diff --git a/Makefile b/Makefile index b852cea2..2df0065b 100644 --- a/Makefile +++ b/Makefile @@ -85,5 +85,8 @@ dbt_docs.logs: ## show the logs of the dbt-docs container local.re_data_models.dev: ## Run re_data models @(cd dbt_jardiner && dbt run --target dev --models package:re_data) -changelog: ## generate changelog +changelog.docker: ## generate changelog usind docker image @docker compose -f $(app_compose_file) --env-file $(app_compose_env_file) run --rm -it --entrypoint git-changelog app-dev + +changelog: ## generate changelog + @git-changelog diff --git a/dbt_jardiner/models/jardiner/intermediate/dset/int_dset_last_signal__from_latest_batch_dset_last.sql b/dbt_jardiner/models/jardiner/intermediate/dset/int_dset_last_signal__from_latest_batch_dset_last.sql index fef2dbdc..49de7a08 100644 --- a/dbt_jardiner/models/jardiner/intermediate/dset/int_dset_last_signal__from_latest_batch_dset_last.sql +++ b/dbt_jardiner/models/jardiner/intermediate/dset/int_dset_last_signal__from_latest_batch_dset_last.sql @@ -1,36 +1,37 @@ {{ config(materialized="view") }} with last_dset_batch as ( - select - case - when signal_uuid ~ e'^[[:xdigit:]]{8}-([[:xdigit:]]{4}-){3}[[:xdigit:]]{12}$' then signal_uuid::uuid -- noqa: LT01 - end as signal_uuid, - signal_uuid as signal_uuid_raw, - {{ dbt_utils.star(from=ref("raw_dset_responses__last_signal_reading"), except=['signal_uuid']) }} - from {{ ref("raw_dset_responses__last_signal_reading") }} + select + case + when signal_uuid ~ e'^[[:xdigit:]]{8}-([[:xdigit:]]{4}-){3}[[:xdigit:]]{12}$' then signal_uuid::uuid -- noqa: LT01 + end as signal_uuid, + signal_uuid as signal_uuid_raw, + {{ dbt_utils.star(from=ref("raw_dset_responses__last_signal_reading"), except=['signal_uuid']) }} + from {{ ref("raw_dset_responses__last_signal_reading") }} ) select - dset.signal_last_ts, - dset.signal_last_value, - dset.group_name as dset_plant_name, - plants.plant_name, - dset.queried_at, - metadata.signal_name, - metadata.metric_name, - metadata.device_name, - metadata.device_type, - metadata.plant_uuid, - metadata.device_uuid, - metadata.device_parent, - metadata.signal_uuid, - dset.signal_id, - dset.signal_tz, - dset.signal_code, - dset.signal_type, - dset.signal_unit + dset.signal_last_ts, + dset.signal_last_value, + dset.group_name as dset_plant_name, + plants.plant_name, + dset.queried_at, + metadata.signal_name, + metadata.metric_name, + metadata.device_name, + metadata.device_type, + metadata.plant_uuid, + metadata.device_uuid, + metadata.device_parent, + metadata.device_parent_uuid, + metadata.signal_uuid, + dset.signal_id, + dset.signal_tz, + dset.signal_code, + dset.signal_type, + dset.signal_unit from last_dset_batch as dset -full outer join - {{ ref("raw_gestio_actius__signal_denormalized") }} as metadata - using (signal_uuid) -left join {{ ref("int_gda_plants__plants_catalog") }} as plants - using (plant_uuid) + full outer join + {{ ref("raw_gestio_actius__signal_denormalized") }} as metadata + using (signal_uuid) + left join {{ ref("int_gda_plants__plants_catalog") }} as plants + using (plant_uuid) diff --git a/dbt_jardiner/models/jardiner/intermediate/dset/int_dset_responses__spined_metadata.sql b/dbt_jardiner/models/jardiner/intermediate/dset/int_dset_responses__spined_metadata.sql index 0450f474..8e76ea3b 100644 --- a/dbt_jardiner/models/jardiner/intermediate/dset/int_dset_responses__spined_metadata.sql +++ b/dbt_jardiner/models/jardiner/intermediate/dset/int_dset_responses__spined_metadata.sql @@ -15,7 +15,8 @@ spined_expected_signals as ( metadata.signal_name, metadata.signal_uuid, metadata.metric_name, - metadata.device_parent + metadata.device_parent, + metadata.device_parent_uuid from spina5m {# dm_plants is the SSOT of the plants names #} @@ -42,6 +43,7 @@ spined_dset as ( spined.signal_name, spined.metric_name, spined.device_parent, + spined.device_parent_uuid, valors.signal_value, valors.group_name, valors.signal_id, diff --git a/dbt_jardiner/models/jardiner/intermediate/gestio_actius/int_signal_device_relation__distinct_devices.sql b/dbt_jardiner/models/jardiner/intermediate/gestio_actius/int_signal_device_relation__distinct_devices.sql index 0f4fd679..427ab1a3 100644 --- a/dbt_jardiner/models/jardiner/intermediate/gestio_actius/int_signal_device_relation__distinct_devices.sql +++ b/dbt_jardiner/models/jardiner/intermediate/gestio_actius/int_signal_device_relation__distinct_devices.sql @@ -1,5 +1,5 @@ {{ config(materialized="table") }} -select distinct plant_name, plant_uuid, device_name, device_uuid, device_type, device_parent +select distinct plant_name, plant_uuid, device_name, device_uuid, device_type, device_parent, device_parent_uuid from {{ ref('raw_gestio_actius__signal_denormalized') }} order by plant_name, device_name diff --git a/dbt_jardiner/models/jardiner/marts/dm_dset_last_reading__from_signal_last_ts.sql b/dbt_jardiner/models/jardiner/marts/dm_dset_last_reading__from_signal_last_ts.sql index 5513a001..a7a05f6b 100644 --- a/dbt_jardiner/models/jardiner/marts/dm_dset_last_reading__from_signal_last_ts.sql +++ b/dbt_jardiner/models/jardiner/marts/dm_dset_last_reading__from_signal_last_ts.sql @@ -1,19 +1,20 @@ {{ config(materialized="view") }} select - plant_name as nom_planta, - dset_plant_name as nom_planta_dset, - signal_last_ts as ultim_ts, - signal_last_value as ultim_valor, - queried_at as peticionat_a, - signal_name as nom_senyal, - metric_name as nom_metrica, - signal_unit as unitat, - device_name as nom_aparell, - device_type as tipus_aparell, - device_parent as aparell_pare, - signal_uuid as uuid_senyal, - signal_id as id_senyal_dset, - signal_code as codi_senyal_dset, - signal_type as tipus_senyal -from {{ ref("int_dset_last_signal__from_latest_batch_dset_last") }} \ No newline at end of file + plant_name as nom_planta, + dset_plant_name as nom_planta_dset, + signal_last_ts as ultim_ts, + signal_last_value as ultim_valor, + queried_at as peticionat_a, + signal_name as nom_senyal, + metric_name as nom_metrica, + signal_unit as unitat, + device_name as nom_aparell, + device_type as tipus_aparell, + device_parent as aparell_pare, + device_parent_uuid as uuid_aparell_pare, + signal_uuid as uuid_senyal, + signal_id as id_senyal_dset, + signal_code as codi_senyal_dset, + signal_type as tipus_senyal +from {{ ref("int_dset_last_signal__from_latest_batch_dset_last") }} diff --git a/dbt_jardiner/models/jardiner/marts/dm_metrics_production_5min.sql b/dbt_jardiner/models/jardiner/marts/dm_metrics_production_5min.sql index 700f2854..d0f151b2 100644 --- a/dbt_jardiner/models/jardiner/marts/dm_metrics_production_5min.sql +++ b/dbt_jardiner/models/jardiner/marts/dm_metrics_production_5min.sql @@ -10,7 +10,8 @@ select signal_unit as unitat_senyal, metric_name as nom_metrica, signal_name as senyal, - device_parent as nom_aparell_pare + device_parent as nom_aparell_pare, + device_parent_uuid as uuid_aparell_pare from {{ ref("int_dset_responses__values_incremental") }} where ( diff --git a/dbt_jardiner/models/jardiner/observability/dset/obs_dset_responses__with_signal_metadata.sql b/dbt_jardiner/models/jardiner/observability/dset/obs_dset_responses__with_signal_metadata.sql index 212bf5c5..d239bbcb 100644 --- a/dbt_jardiner/models/jardiner/observability/dset/obs_dset_responses__with_signal_metadata.sql +++ b/dbt_jardiner/models/jardiner/observability/dset/obs_dset_responses__with_signal_metadata.sql @@ -4,35 +4,36 @@ {# TODO one metric could have different units, we could standardize here e.g. signal_value * (case when signal_unit = 'kwh' then 1000 else 1)#} with - base as ( - select - metadata.plant_uuid, - metadata.plant_name, - metadata.signal_name, - metadata.metric_name, - metadata.device_name, - metadata.device_type, - metadata.device_uuid, - metadata.device_parent, - metadata.signal_uuid, - valors.group_name, - valors.signal_id, - valors.signal_tz, - valors.signal_code, - valors.signal_type, - valors.signal_unit, - valors.signal_last_ts, - valors.signal_frequency, - valors.signal_is_virtual, - valors.signal_last_value, - valors.queried_at, - valors.ts, - valors.signal_value as signal_value - from {{ ref("raw_gestio_actius__signal_denormalized") }} as metadata - left join - {{ ref("int_dset_responses__materialized") }} as valors - on metadata.signal_uuid = valors.signal_uuid - ) +base as ( + select + metadata.plant_uuid, + metadata.plant_name, + metadata.signal_name, + metadata.metric_name, + metadata.device_name, + metadata.device_type, + metadata.device_uuid, + metadata.device_parent, + metadata.device_parent_uuid, + metadata.signal_uuid, + valors.group_name, + valors.signal_id, + valors.signal_tz, + valors.signal_code, + valors.signal_type, + valors.signal_unit, + valors.signal_last_ts, + valors.signal_frequency, + valors.signal_is_virtual, + valors.signal_last_value, + valors.queried_at, + valors.ts, + valors.signal_value as signal_value + from {{ ref("raw_gestio_actius__signal_denormalized") }} as metadata + left join + {{ ref("int_dset_responses__materialized") }} as valors + on metadata.signal_uuid = valors.signal_uuid +) select * from base diff --git a/dbt_jardiner/models/jardiner/raw/gestio_actius/_raw_gestio_actius__models.yaml b/dbt_jardiner/models/jardiner/raw/gestio_actius/_raw_gestio_actius__models.yaml index b762209f..a4e07fcf 100644 --- a/dbt_jardiner/models/jardiner/raw/gestio_actius/_raw_gestio_actius__models.yaml +++ b/dbt_jardiner/models/jardiner/raw/gestio_actius/_raw_gestio_actius__models.yaml @@ -97,7 +97,22 @@ models: description: Imputat per nosaltres que es correspon amb la columna homònima de la taula d'aparell que correspongui ja sigui inverter, string, sensor, plant, etc. + - name: is_enabled + description: > + Boolean que indica si el senyal està habilitat o no. Si no ho està no es + processarà. + tests: + - accepted_values: + values: + [true, false] + - name: device_parent_uuid + description: UUID del parent device, si n'hi ha tests: + - dbt_utils.expression_is_true: + # XOR, one of the two can't be null + expression: "device_parent_uuid is not null" + config: + where: "device_parent is not null" - dbt_utils.unique_combination_of_columns: config: severity: error diff --git a/dbt_jardiner/models/jardiner/raw/gestio_actius/raw_gestio_actius__signal_denormalized.sql b/dbt_jardiner/models/jardiner/raw/gestio_actius/raw_gestio_actius__signal_denormalized.sql index 0e4e59a6..b9ff5e1f 100644 --- a/dbt_jardiner/models/jardiner/raw/gestio_actius/raw_gestio_actius__signal_denormalized.sql +++ b/dbt_jardiner/models/jardiner/raw/gestio_actius/raw_gestio_actius__signal_denormalized.sql @@ -17,10 +17,13 @@ base as ( device_type::text, device_uuid::uuid, trim(device_parent::text) as device_parent, + device_parent_uuid::uuid, signal_uuid::uuid, + coalesce(is_enabled = 'VERDADERO', false) as is_enabled, inserted_at::timestamptz, updated_at::timestamptz from fresh ) select * from base +where is_enabled diff --git a/dbt_jardiner/snapshots/_jardiner__snapshots_sources.yaml b/dbt_jardiner/snapshots/_jardiner__snapshots_sources.yaml index ef764f17..b3d954c3 100644 --- a/dbt_jardiner/snapshots/_jardiner__snapshots_sources.yaml +++ b/dbt_jardiner/snapshots/_jardiner__snapshots_sources.yaml @@ -43,3 +43,7 @@ sources: description: "Insertion timestamp in the database, in UTC" - name: updated_at description: "Update timestamp in the database, in UTC" + - name: is_enabled + description: "Boolean. If the signal is enabled or not" + - name: device_parent_uuid + description: "UUID of the parent device, version 4" diff --git a/dbt_jardiner/tests/dset/test_repeated_device_uuid_errors.sql b/dbt_jardiner/tests/dset/test_repeated_device_uuid_errors.sql index c6118691..3ea9c7f9 100644 --- a/dbt_jardiner/tests/dset/test_repeated_device_uuid_errors.sql +++ b/dbt_jardiner/tests/dset/test_repeated_device_uuid_errors.sql @@ -1,28 +1,28 @@ -- tests pairs device_uuid, plant, device, device_type, device_parent as unique with - unique_device_per_plant as ( - select distinct device_uuid, plant_uuid, plant_name, device_name, device_type, device_parent - from {{ ref("raw_gestio_actius__signal_denormalized") }} - order by plant_name, device_name - ), +unique_device_per_plant as ( + select distinct device_uuid, plant_uuid, plant_name, device_name, device_type, device_parent, device_parent_uuid + from {{ ref("raw_gestio_actius__signal_denormalized") }} + order by plant_name, device_name +), - duplicated_devices as ( - select udpp.device_uuid, count(*) as n_duplicates - from unique_device_per_plant as udpp - group by udpp.device_uuid - having count(*) > 1 - ), +duplicated_devices as ( + select udpp.device_uuid, count(*) as n_duplicates + from unique_device_per_plant as udpp + group by udpp.device_uuid + having count(*) > 1 +), - joined as ( - select expected.*, duplicated_devices.n_duplicates - from {{ ref("raw_gestio_actius__signal_denormalized") }} as expected - left join duplicated_devices on expected.device_uuid = duplicated_devices.device_uuid - ), +joined as ( + select expected.*, duplicated_devices.n_duplicates + from {{ ref("raw_gestio_actius__signal_denormalized") }} as expected + left join duplicated_devices on expected.device_uuid = duplicated_devices.device_uuid +), - filtered as ( - select * - from joined - where n_duplicates is not null - ) +filtered as ( + select * + from joined + where n_duplicates is not null +) select * from filtered