Data Quality Daily Row Check Mart Model

transform/dbt_project.yml

@@ -101,6 +101,8 @@ models:
       geo:
         +schema: geo
         +post-hook: "{{ unload_relation_as_geojson(strip_leading_words=1) }}"
+      quality:
+        +schema: quality
 
 on-run-start:
   - "{{create_udfs()}}"

transform/models/marts/quality/_quality.yml

@@ -0,0 +1,36 @@
+version: 2
+
+models:
+  - name: quality__row_count_summary
+    description: |
+      This model counts the number of ML and HV detectors across multiple models on a daily basis
+      to ensure that the detector count is consistent between all of the models. The models
+      where detector counts are being evaluated are:
+      - int_diagnostics__detector_status model
+      - int_clearinghouse__detector_agg_five_minutes_with_missing_rows
+      - int_imputation__detector_imputed_agg_five_minutes
+      - int_performance__detector_metrics_agg_five_minutes
+      The daily detector counts for these models should match. As an additional aggregation
+      the number of good, bad, and total detectors is calculated in this model as well to monitor
+      for any changes that should be investigated. The number of detectors and their health
+      should not vary drastically between days so this model can help identify anamolies.
+    columns:
+      - name: SAMPLE_DATE
+        description: |
+          The date associated with the aggregate counts down in the 5 models to be compared.
+      - name: ML_HV_DETECTOR_STATUS_TOTAL_COUNT
+        description: Counts the number of distinct ML and HV detectors on a daily basis
+      - name: ML_HV_CLEARINGHOUSE_DETECTOR_COUNT
+        description: Counts the number of distinct ML and HV detectors on a daily basis
+      - name: ML_HV_IMPUTATION_DETECTOR_COUNT
+        description: Counts the number of distinct ML and HV detectors on a daily basis
+      - name: ML_HV_PERFORMANCE_DETECTOR_COUNT
+        description: Counts the number of distinct ML and HV detectors on a daily basis
+      - name: GOOD_STATUS_COUNT
+        description: |
+          Counts the total number of detectors with a status of Good on a daily basis.
+      - name: BAD_STATUS_COUNT
+        description: |
+          Counts the total number of detectors with a status other than Good on a daily basis.
+      - name: ALL_DETECTOR_STATUS_TOTAL_COUNT
+        description: Counts the total number of rows/detectors on a daily basis

transform/models/marts/quality/quality__row_count_summary.sql

@@ -0,0 +1,80 @@
+with
+
+ML_HV_DETECTOR_STATUS_DAILY_COUNT as (
+    /*
+    * This CTE returns the number of total rows created daily in
+    * the int_diagnostics__detector_status model. This count should be
+    * to checked against the daily detector counts for the following models:
+    * - int_clearinghouse__detector_agg_five_minutes_with_missing_rows
+    * - int_imputation__detector_imputed_agg_five_minutes
+    * - int_performance__detector_metrics_agg_five_minutes
+    * The daily detector counts for these models should match for
+    */
+    select
+        SAMPLE_DATE,
+        count(*) as ML_HV_DETECTOR_STATUS_TOTAL_COUNT
+    from {{ ref('int_diagnostics__detector_status') }}
+    where STATION_TYPE = 'ML' or STATION_TYPE = 'HV'
+    group by SAMPLE_DATE
+),
+
+-- Clearinghouse Detector Count per Day
+ML_HV_CLEARINGHOUSE_DETECTOR_DAILY_COUNT as (
+    select
+        SAMPLE_DATE,
+        count(distinct DETECTOR_ID) as ML_HV_CLEARINGHOUSE_DETECTOR_COUNT
+    from {{ ref('int_clearinghouse__detector_agg_five_minutes_with_missing_rows') }}
+    where STATION_TYPE = 'ML' or STATION_TYPE = 'HV'
+    group by SAMPLE_DATE
+),
+
+-- Imputation Detector Count per Day
+ML_HV_IMPUTATION_DETECTOR_DAILY_COUNT as (
+    select
+        SAMPLE_DATE,
+        count(distinct DETECTOR_ID) as ML_HV_IMPUTATION_DETECTOR_COUNT
+    from {{ ref('int_imputation__detector_imputed_agg_five_minutes') }}
+    group by SAMPLE_DATE
+),
+
+-- Performance Detector Count per Day
+ML_HV_PERFORMANCE_DETECTOR_DAILY_COUNT as (
+    select
+        SAMPLE_DATE,
+        count(distinct DETECTOR_ID) as ML_HV_PERFORMANCE_DETECTOR_COUNT
+    from {{ ref('int_performance__detector_metrics_agg_five_minutes') }}
+    group by SAMPLE_DATE
+),
+
+-- Returns count for all station types per Day
+DETECTOR_STATUS_COUNT as (
+    select
+        SAMPLE_DATE,
+        count_if(STATUS = 'Good') as GOOD_STATUS_COUNT,
+        count_if(STATUS != 'Good') as BAD_STATUS_COUNT,
+        count(*) as ALL_DETECTOR_STATUS_TOTAL_COUNT
+    from {{ ref('int_diagnostics__detector_status') }}
+    group by SAMPLE_DATE
+),
+
+DAILY_DETECTOR_COUNT_CHECK as (
+    select
+        MHDSDC.*,
+        CDDC.ML_HV_CLEARINGHOUSE_DETECTOR_COUNT,
+        IDDC.ML_HV_IMPUTATION_DETECTOR_COUNT,
+        PDDC.ML_HV_PERFORMANCE_DETECTOR_COUNT,
+        DSC.GOOD_STATUS_COUNT,
+        DSC.BAD_STATUS_COUNT,
+        DSC.ALL_DETECTOR_STATUS_TOTAL_COUNT
+    from ML_HV_DETECTOR_STATUS_DAILY_COUNT as MHDSDC
+    left join ML_HV_CLEARINGHOUSE_DETECTOR_DAILY_COUNT as CDDC
+        on MHDSDC.SAMPLE_DATE = CDDC.SAMPLE_DATE
+    left join ML_HV_IMPUTATION_DETECTOR_DAILY_COUNT as IDDC
+        on MHDSDC.SAMPLE_DATE = IDDC.SAMPLE_DATE
+    left join ML_HV_PERFORMANCE_DETECTOR_DAILY_COUNT as PDDC
+        on MHDSDC.SAMPLE_DATE = PDDC.SAMPLE_DATE
+    left join DETECTOR_STATUS_COUNT as DSC
+        on MHDSDC.SAMPLE_DATE = DSC.SAMPLE_DATE
+)
+
+select * from DAILY_DETECTOR_COUNT_CHECK