test: implement framework for integration tests (DNA-19347: DNA-19404)

.gitignore

@@ -0,0 +1,14 @@
+# compiled code & logs
+**/target/
+**/logs/
+# package folder
+**/dbt_packages/
+# dbt folders that are not used
+**/snapshots/
+**/analysis/
+**/data/
+# user specific file
+**/.user.yml
+
+# MacOS specific files
+.DS_Store

.pipelines/azure-pipelines-integration-tests.yml

@@ -0,0 +1,96 @@
+name: Integration tests
+
+resources:
+  repositories:
+    - repository: toggleKeyVaultRepo
+      endpoint: UiPath
+      type: github
+      name: UiPath/AzurePipelinesTemplates
+      ref: refs/tags/uipath.kv-access.1.2.7
+
+    - repository: testTransformations
+      endpoint: UiPath
+      type: github
+      name: UiPath/ProcessMining-framework-resources
+      ref: refs/tags/test_transformations.1.1.1
+
+# Trigger the pipeline for all PRs we open, but do not start as long as there is no PR.
+trigger:
+  none
+
+pr:
+  branches:
+    include:
+    - '*'
+
+pool:
+  vmImage: ubuntu-latest
+
+jobs:
+  - job: get_keyvault_secrets
+    displayName: Get keyvault secrets
+    steps:
+      - template: ./.pipelines/templates/get_keyvault_secrets.job.yml@testTransformations
+        parameters:
+          serviceConnection: $(serviceConnection)
+          keyVaultName: $(keyVaultName)
+
+  - job: integration_tests
+    displayName: Integration tests
+    dependsOn: get_keyvault_secrets
+    condition: eq(dependencies.get_keyvault_secrets.result, 'Succeeded')
+    variables:
+        DBT_SQL_SERVER_SERVER: $[ dependencies.get_keyvault_secrets.outputs['set_variables.DBT_SQL_SERVER_SERVER'] ]
+        DBT_SQL_SERVER_USER: $[ dependencies.get_keyvault_secrets.outputs['set_variables.DBT_SQL_SERVER_USER'] ]
+        DBT_SQL_SERVER_PASSWORD: $[ dependencies.get_keyvault_secrets.outputs['set_variables.DBT_SQL_SERVER_PASSWORD'] ]
+        DBT_SQL_SERVER_DATABASE: $[ dependencies.get_keyvault_secrets.outputs['set_variables.DBT_SQL_SERVER_DATABASE'] ]
+        DBT_SNOWFLAKE_ACCOUNT: $[ dependencies.get_keyvault_secrets.outputs['set_variables.DBT_SNOWFLAKE_ACCOUNT'] ]
+        DBT_SNOWFLAKE_USER: $[ dependencies.get_keyvault_secrets.outputs['set_variables.DBT_SNOWFLAKE_USER'] ]
+        DBT_SNOWFLAKE_PASSWORD: $[ dependencies.get_keyvault_secrets.outputs['set_variables.DBT_SNOWFLAKE_PASSWORD'] ]
+        DBT_SNOWFLAKE_ROLE: $[ dependencies.get_keyvault_secrets.outputs['set_variables.DBT_SNOWFLAKE_ROLE'] ]
+        DBT_SNOWFLAKE_DATABASE: $[ dependencies.get_keyvault_secrets.outputs['set_variables.DBT_SNOWFLAKE_DATABASE'] ]
+        DBT_SNOWFLAKE_WAREHOUSE: $[ dependencies.get_keyvault_secrets.outputs['set_variables.DBT_SNOWFLAKE_WAREHOUSE'] ]
+    steps:
+      - checkout: self
+        path: self
+        displayName: Checkout current repository
+
+      - bash: |
+          echo "##vso[task.setvariable variable=DBT_SCHEMA]pm_utils_integration_tests"
+          echo "##vso[task.setvariable variable=dbtProjectPath]$(Agent.BuildDirectory)/self/integration_tests"
+        displayName: Set variables
+
+      - bash: |
+          python --version
+
+          sudo apt-get install git libpq-dev python-dev python3-pip
+          sudo apt-get remove python-cffi
+          sudo pip install --upgrade cffi
+
+          python3 -m venv dbt-env
+          source dbt-env/bin/activate
+
+          pip install dbt-sqlserver==1.4.3
+          pip install dbt-snowflake==1.4.3
+          dbt --version
+        displayName: Install dbt
+
+      - bash: |
+          source dbt-env/bin/activate
+          cd $(dbtProjectPath)
+          dbt deps
+        displayName: Install dbt dependencies
+
+      - bash: |
+          source dbt-env/bin/activate
+          cd $(dbtProjectPath)
+          dbt build --profiles-dir $(Agent.BuildDirectory)/self/.pipelines --profile default -t sqlserver-ci \
+            --vars '{"schema_sources": "$(DBT_SCHEMA_SOURCES)", "DBT_SQL_SERVER_SERVER": "$(DBT_SQL_SERVER_SERVER)", "DBT_SQL_SERVER_USER": "$(DBT_SQL_SERVER_USER)", "DBT_SQL_SERVER_PASSWORD": "$(DBT_SQL_SERVER_PASSWORD)", "DBT_SQL_SERVER_DATABASE": "$(DBT_SQL_SERVER_DATABASE)", "DBT_SCHEMA": "$(DBT_SCHEMA)"}'
+        displayName: Test (SQL Server)
+
+      - bash: |
+          source dbt-env/bin/activate
+          cd $(dbtProjectPath)
+          dbt build --profiles-dir $(Agent.BuildDirectory)/self/.pipelines --profile default -t snowflake-ci \
+            --vars '{"schema_sources": "$(DBT_SCHEMA_SOURCES)", "DBT_SNOWFLAKE_ACCOUNT": "$(DBT_SNOWFLAKE_ACCOUNT)", "DBT_SNOWFLAKE_USER": "$(DBT_SNOWFLAKE_USER)", "DBT_SNOWFLAKE_PASSWORD": "$(DBT_SNOWFLAKE_PASSWORD)", "DBT_SNOWFLAKE_ROLE": "$(DBT_SNOWFLAKE_ROLE)", "DBT_SNOWFLAKE_DATABASE": "$(DBT_SNOWFLAKE_DATABASE)", "DBT_SNOWFLAKE_WAREHOUSE": "$(DBT_SNOWFLAKE_WAREHOUSE)", "DBT_SCHEMA": "$(DBT_SCHEMA)"}'
+        displayName: Test (Snowflake)

.pipelines/profiles.yml

@@ -0,0 +1,27 @@
+default:
+  outputs:
+
+    sqlserver-ci:
+      type: sqlserver
+      driver: 'ODBC Driver 17 for SQL Server'
+      port: 1433
+      server: "{{ var('DBT_SQL_SERVER_SERVER') }}"
+      user: "{{ var('DBT_SQL_SERVER_USER') }}"
+      password: "{{ var('DBT_SQL_SERVER_PASSWORD') }}"
+      database: "{{ var('DBT_SQL_SERVER_DATABASE') }}"
+      schema: "{{ var('DBT_SCHEMA') }}"
+      encrypt: true
+      trust_cert: true
+
+    snowflake-ci:
+      type: snowflake
+      account: "{{ var('DBT_SNOWFLAKE_ACCOUNT') }}"
+      user: "{{ var('DBT_SNOWFLAKE_USER') }}"
+      password: "{{ var('DBT_SNOWFLAKE_PASSWORD') }}"
+      role: "{{ var('DBT_SNOWFLAKE_ROLE') }}"
+      database: "{{ var('DBT_SNOWFLAKE_DATABASE') }}"
+      warehouse: "{{ var('DBT_SNOWFLAKE_WAREHOUSE') }}"
+      schema: "{{ var('DBT_SCHEMA') }}"
+      threads: 10
+
+  target: snowflake-ci

integration_tests/dbt_project.yml

@@ -0,0 +1,5 @@
+name: 'pm_utils_integration_tests'
+version: '1.0.0'
+config-version: 2
+
+profile: 'integration_tests'

integration_tests/macros/tests.sql

@@ -0,0 +1,3 @@
+{% test equal_value(model, actual, expected) %}
+    select * from {{ model }} where {{ actual }} != {{ expected }}
+{% endtest %}

integration_tests/models/schema.yml

@@ -0,0 +1,11 @@
+version: 2
+
+models:
+  - name: test_concat
+    tests:
+      - equal_value:
+          actual: '"Two_text_fields"'
+          expected: '"Two_text_fields_expected"'
+      - equal_value:
+          actual: '"One_null"'
+          expected: '"One_null_expected"'

integration_tests/models/test_concat.sql

@@ -0,0 +1,16 @@
+with Input_data as (
+    select
+        'A' as "Column_A",
+        'B' as "Column_B",
+        null as "Column_C"
+)
+
+select
+    {# Concatenate two text fields (basic scenario) #}
+    {{ pm_utils.concat('"Column_A"', '"Column_B"') }} as "Two_text_fields",
+    'AB' as "Two_text_fields_expected",
+
+    {# Concatenate two fields of which one is null #}
+    {{ pm_utils.concat('"Column_A"', '"Column_C"') }} as "One_null",
+    'A' as "One_null_expected"
+from Input_data

integration_tests/packages.yml

@@ -0,0 +1,2 @@
+packages:
+  - local: ../