Merge pull request #4 from datacoves/feature/mayrapena1324

feature/mayrapena1324

.github/workflows/pull_request_build.yml

@@ -91,7 +91,9 @@ jobs:
         run: "dbt-coves generate docs --merge-deferred --state logs"
 
       - name: Run governance checks
-        run: "pre-commit run --from-ref origin/${{ github.event.pull_request.base.ref }} --to-ref HEAD"
+        # run: "pre-commit run --from-ref origin/${{ github.event.pull_request.base.ref }} --to-ref HEAD"
+        run: "pre-commit run --all"
+
 
       ##### Real dbt run given that we passed governance checks
       - name: Run dbt build slim mode

.github/workflows/push-to-main.yml

@@ -160,4 +160,3 @@ jobs:
 
       - name: Drop PR database
         run: "dbt --no-write-json run-operation drop_recreate_db --args '{db_name: ${{env.DATACOVES__MAIN__DATABASE}}, recreate: False}'"  # yamllint disable-line rule:line-length
-

.pre-commit-config.yaml

@@ -11,8 +11,7 @@ repos:
       - id: check-script-ref-and-source
       - id: check-model-has-description
       - id: check-model-has-properties-file
-      # - id: check-model-has-all-columns
-      # - id: check-database-casing-consistency
+      - id: check-model-has-all-columns
         always_run: true
 
   - repo: https://github.com/sqlfluff/sqlfluff

orchestrate/dags/sample_dag.py

@@ -16,10 +16,10 @@
     tags=["version_1"],
     catchup=False,
 )
-def daily_run():
+def sample_dag():
     run_dbt = DatacovesDbtOperator(
-        task_id="run_dbt", bash_command="dbt source freshness && dbt build"
+        task_id="run_dbt", bash_command="dbt debug"
     )
 
 
-dag = daily_run()
+dag = sample_dag()

orchestrate/dag_yml_definitions/sample_dag.yml → orchestrate/dags_yml_definitions/sample_dag.yml

@@ -14,4 +14,4 @@ nodes:
   run_dbt:
     type: task
     operator: operators.datacoves.dbt.DatacovesDbtOperator
-    bash_command: "dbt source freshness && dbt build"
+    bash_command: "dbt debug"

transform/.dbt_coves/config.yml

@@ -24,20 +24,20 @@ generate:
 
     # UNCOMMENT THE FOLLOWING LINES TO ENABLE AIRFLOW DAGS GENERATION
     # BASED ON AIRBYTE AND FIVETRAN CONNECTIONS
-    # generators_params:
-    #   AirbyteDbtGenerator:
-    #     host: "{{ env_var('DATACOVES__AIRBYTE_HOST_NAME') }}"
-    #     port: "{{ env_var('DATACOVES__AIRBYTE_PORT') }}"
-    #     airbyte_conn_id: airbyte_connection
+    generators_params:
+      AirbyteDbtGenerator:
+        host: "{{ env_var('DATACOVES__AIRBYTE_HOST_NAME') }}"
+        port: "{{ env_var('DATACOVES__AIRBYTE_PORT') }}"
+        airbyte_conn_id: airbyte_connection
 
-    #     dbt_project_path: "{{ env_var('DATACOVES__DBT_HOME') }}"
-    #     run_dbt_compile: false
-    #     run_dbt_deps: false
+        dbt_project_path: "{{ env_var('DATACOVES__DBT_HOME') }}"
+        run_dbt_compile: false
+        run_dbt_deps: false
 
-    #   AirbyteGenerator:
-    #     host: "{{ env_var('DATACOVES__AIRBYTE_HOST_NAME') }}"
-    #     port: "{{ env_var('DATACOVES__AIRBYTE_PORT') }}"
-    #     airbyte_conn_id: airbyte_connection
+      AirbyteGenerator:
+        host: "{{ env_var('DATACOVES__AIRBYTE_HOST_NAME') }}"
+        port: "{{ env_var('DATACOVES__AIRBYTE_PORT') }}"
+        airbyte_conn_id: airbyte_connection
 
     #   FivetranDbtGenerator:
     #     api_key: "{{ env_var('DATACOVES__FIVETRAN_API_KEY') }}"
@@ -55,11 +55,11 @@ generate:
     #       - fivetran-connection-id-2
 
 # UNCOMMENT THE FOLLOWING LINES TO ENABLE AIRBYTE EXTRACTION
-# extract:
-#   airbyte:
-#     path: /config/workspace/load/airbyte
-#     host: "{{ env_var('DATACOVES__AIRBYTE_HOST_NAME') }}"
-#     port: "{{ env_var('DATACOVES__AIRBYTE_PORT') }}"
+extract:
+  airbyte:
+    path: /config/workspace/load/airbyte
+    host: "{{ env_var('DATACOVES__AIRBYTE_HOST_NAME') }}"
+    port: "{{ env_var('DATACOVES__AIRBYTE_PORT') }}"
 
 #   fivetran:
 #     path: /config/workspace/extract/fivetran
@@ -74,6 +74,7 @@ generate:
 #     path: /config/workspace/load/fivetran
 #     run_connection_tests: true
 
+
 blue_green:
   prod_db_env_var: DATACOVES__MAIN__DATABASE # This holds the name of the production database --- no default
   # staging_database: STAGING_DB # Optional name you want to give to the staging database --- no default

transform/models/L1_staging/country_data/_country_data.yml

@@ -3,6 +3,8 @@ version: 2
 sources:
   - name: COUNTRY_DATA
     database: RAW
+    tags:
+      - daily_run_airbyte
     tables:
       - name: COUNTRY_POPULATIONS
         description: 'Raw population information from Github Datasets repository'