From f573066f1a372488b487ea1c32ba52bd0863db37 Mon Sep 17 00:00:00 2001 From: Celine Verhoef Date: Fri, 6 Oct 2023 16:39:49 +0200 Subject: [PATCH 1/4] test: implement initial testing framework --- .gitignore | 14 ++++++++++++++ integration_tests/dbt_project.yml | 5 +++++ integration_tests/macros/tests.sql | 3 +++ integration_tests/models/schema.yml | 11 +++++++++++ integration_tests/models/test_concat.sql | 16 ++++++++++++++++ integration_tests/packages.yml | 2 ++ 6 files changed, 51 insertions(+) create mode 100644 .gitignore create mode 100644 integration_tests/dbt_project.yml create mode 100644 integration_tests/macros/tests.sql create mode 100644 integration_tests/models/schema.yml create mode 100644 integration_tests/models/test_concat.sql create mode 100644 integration_tests/packages.yml diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..a5c830a --- /dev/null +++ b/.gitignore @@ -0,0 +1,14 @@ +# compiled code & logs +**/target/ +**/logs/ +# package folder +**/dbt_packages/ +# dbt folders that are not used +**/snapshots/ +**/analysis/ +**/data/ +# user specific file +**/.user.yml + +# MacOS specific files +.DS_Store diff --git a/integration_tests/dbt_project.yml b/integration_tests/dbt_project.yml new file mode 100644 index 0000000..8158b70 --- /dev/null +++ b/integration_tests/dbt_project.yml @@ -0,0 +1,5 @@ +name: 'pm_utils_integration_tests' +version: '1.0.0' +config-version: 2 + +profile: 'integration_tests' diff --git a/integration_tests/macros/tests.sql b/integration_tests/macros/tests.sql new file mode 100644 index 0000000..e65a640 --- /dev/null +++ b/integration_tests/macros/tests.sql @@ -0,0 +1,3 @@ +{% test equal_value(model, actual, expected) %} + select * from {{ model }} where {{ actual }} != {{ expected }} +{% endtest %} diff --git a/integration_tests/models/schema.yml b/integration_tests/models/schema.yml new file mode 100644 index 0000000..232ff2b --- /dev/null +++ b/integration_tests/models/schema.yml @@ -0,0 +1,11 @@ +version: 2 + +models: + - name: test_concat + tests: + - equal_value: + actual: '"Two_text_fields"' + expected: '"Two_text_fields_expected"' + - equal_value: + actual: '"One_null"' + expected: '"One_null_expected"' diff --git a/integration_tests/models/test_concat.sql b/integration_tests/models/test_concat.sql new file mode 100644 index 0000000..cc8a71b --- /dev/null +++ b/integration_tests/models/test_concat.sql @@ -0,0 +1,16 @@ +with Input_data as ( + select + 'A' as "Column_A", + 'B' as "Column_B", + null as "Column_C" +) + +select + {# Concatenate two text fields (basic scenario) #} + {{ pm_utils.concat('"Column_A"', '"Column_B"') }} as "Two_text_fields", + 'AB' as "Two_text_fields_expected", + + {# Concatenate two fields of which one is null #} + {{ pm_utils.concat('"Column_A"', '"Column_C"') }} as "One_null", + 'A' as "One_null_expected" +from Input_data diff --git a/integration_tests/packages.yml b/integration_tests/packages.yml new file mode 100644 index 0000000..4a6b9c1 --- /dev/null +++ b/integration_tests/packages.yml @@ -0,0 +1,2 @@ +packages: + - local: ../ From 9093d48d3a472e11999e2c3f206f60177b248d38 Mon Sep 17 00:00:00 2001 From: Celine Verhoef Date: Fri, 6 Oct 2023 16:14:43 +0200 Subject: [PATCH 2/4] test: implement azure pipeline --- .../azure-pipelines-integration-tests.yml | 97 +++++++++++++++++++ .pipelines/profiles.yml | 27 ++++++ 2 files changed, 124 insertions(+) create mode 100644 .pipelines/azure-pipelines-integration-tests.yml create mode 100644 .pipelines/profiles.yml diff --git a/.pipelines/azure-pipelines-integration-tests.yml b/.pipelines/azure-pipelines-integration-tests.yml new file mode 100644 index 0000000..098eec8 --- /dev/null +++ b/.pipelines/azure-pipelines-integration-tests.yml @@ -0,0 +1,97 @@ +name: Integration tests + +resources: + repositories: + - repository: toggleKeyVaultRepo + endpoint: UiPath + type: github + name: UiPath/AzurePipelinesTemplates + ref: refs/tags/uipath.kv-access.1.2.7 + + - repository: testTransformations + endpoint: UiPath + type: github + name: UiPath/ProcessMining-framework-resources + ref: refs/tags/test_transformations.1.1.1 + +# Trigger the pipeline for all PRs we open, but do not start as long as there is no PR. +trigger: + none + +pr: + branches: + include: + - '*' + +pool: + vmImage: ubuntu-latest + +jobs: + - job: get_keyvault_secrets + displayName: Get keyvault secrets + steps: + - template: ./.pipelines/templates/get_keyvault_secrets.job.yml@testTransformations + parameters: + serviceConnection: AzureDevTestEASubscription-ProcessMining-apps-team-development + keyVaultName: pmc-dev-config-kv + + - job: integration_tests + displayName: Integration tests + dependsOn: get_keyvault_secrets + condition: eq(dependencies.get_keyvault_secrets.result, 'Succeeded') + variables: + SNOWSQL_PWD_input: $[ dependencies.get_keyvault_secrets.outputs['set_variables.SNOWSQL_PWD_input'] ] + DBT_SQL_SERVER_SERVER: $[ dependencies.get_keyvault_secrets.outputs['set_variables.DBT_SQL_SERVER_SERVER'] ] + DBT_SQL_SERVER_USER: $[ dependencies.get_keyvault_secrets.outputs['set_variables.DBT_SQL_SERVER_USER'] ] + DBT_SQL_SERVER_PASSWORD: $[ dependencies.get_keyvault_secrets.outputs['set_variables.DBT_SQL_SERVER_PASSWORD'] ] + DBT_SQL_SERVER_DATABASE: $[ dependencies.get_keyvault_secrets.outputs['set_variables.DBT_SQL_SERVER_DATABASE'] ] + DBT_SNOWFLAKE_ACCOUNT: $[ dependencies.get_keyvault_secrets.outputs['set_variables.DBT_SNOWFLAKE_ACCOUNT'] ] + DBT_SNOWFLAKE_USER: $[ dependencies.get_keyvault_secrets.outputs['set_variables.DBT_SNOWFLAKE_USER'] ] + DBT_SNOWFLAKE_PASSWORD: $[ dependencies.get_keyvault_secrets.outputs['set_variables.DBT_SNOWFLAKE_PASSWORD'] ] + DBT_SNOWFLAKE_ROLE: $[ dependencies.get_keyvault_secrets.outputs['set_variables.DBT_SNOWFLAKE_ROLE'] ] + DBT_SNOWFLAKE_DATABASE: $[ dependencies.get_keyvault_secrets.outputs['set_variables.DBT_SNOWFLAKE_DATABASE'] ] + DBT_SNOWFLAKE_WAREHOUSE: $[ dependencies.get_keyvault_secrets.outputs['set_variables.DBT_SNOWFLAKE_WAREHOUSE'] ] + steps: + - checkout: self + path: self + displayName: Checkout current repository + + - bash: | + echo "##vso[task.setvariable variable=DBT_SCHEMA]pm_utils_integration_tests" + echo "##vso[task.setvariable variable=dbtProjectPath]$(Agent.BuildDirectory)/self/integration_tests" + displayName: Set variables + + - bash: | + python --version + + sudo apt-get install git libpq-dev python-dev python3-pip + sudo apt-get remove python-cffi + sudo pip install --upgrade cffi + + python3 -m venv dbt-env + source dbt-env/bin/activate + + pip install dbt-sqlserver==1.4.3 + pip install dbt-snowflake==1.4.3 + dbt --version + displayName: Install dbt + + - bash: | + source dbt-env/bin/activate + cd $(dbtProjectPath) + dbt deps + displayName: Install dbt dependencies + + - bash: | + source dbt-env/bin/activate + cd $(dbtProjectPath) + dbt build --profiles-dir $(Agent.BuildDirectory)/self/.pipelines --profile default -t sqlserver-ci \ + --vars '{"schema_sources": "$(DBT_SCHEMA_SOURCES)", "DBT_SQL_SERVER_SERVER": "$(DBT_SQL_SERVER_SERVER)", "DBT_SQL_SERVER_USER": "$(DBT_SQL_SERVER_USER)", "DBT_SQL_SERVER_PASSWORD": "$(DBT_SQL_SERVER_PASSWORD)", "DBT_SQL_SERVER_DATABASE": "$(DBT_SQL_SERVER_DATABASE)", "DBT_SCHEMA": "$(DBT_SCHEMA)"}' + displayName: Test (SQL Server) + + - bash: | + source dbt-env/bin/activate + cd $(dbtProjectPath) + dbt build --profiles-dir $(Agent.BuildDirectory)/self/.pipelines --profile default -t snowflake-ci \ + --vars '{"schema_sources": "$(DBT_SCHEMA_SOURCES)", "DBT_SNOWFLAKE_ACCOUNT": "$(DBT_SNOWFLAKE_ACCOUNT)", "DBT_SNOWFLAKE_USER": "$(DBT_SNOWFLAKE_USER)", "DBT_SNOWFLAKE_PASSWORD": "$(DBT_SNOWFLAKE_PASSWORD)", "DBT_SNOWFLAKE_ROLE": "$(DBT_SNOWFLAKE_ROLE)", "DBT_SNOWFLAKE_DATABASE": "$(DBT_SNOWFLAKE_DATABASE)", "DBT_SNOWFLAKE_WAREHOUSE": "$(DBT_SNOWFLAKE_WAREHOUSE)", "DBT_SCHEMA": "$(DBT_SCHEMA)"}' + displayName: Test (Snowflake) diff --git a/.pipelines/profiles.yml b/.pipelines/profiles.yml new file mode 100644 index 0000000..e17bab9 --- /dev/null +++ b/.pipelines/profiles.yml @@ -0,0 +1,27 @@ +default: + outputs: + + sqlserver-ci: + type: sqlserver + driver: 'ODBC Driver 17 for SQL Server' + port: 1433 + server: "{{ var('DBT_SQL_SERVER_SERVER') }}" + user: "{{ var('DBT_SQL_SERVER_USER') }}" + password: "{{ var('DBT_SQL_SERVER_PASSWORD') }}" + database: "{{ var('DBT_SQL_SERVER_DATABASE') }}" + schema: "{{ var('DBT_SCHEMA') }}" + encrypt: true + trust_cert: true + + snowflake-ci: + type: snowflake + account: "{{ var('DBT_SNOWFLAKE_ACCOUNT') }}" + user: "{{ var('DBT_SNOWFLAKE_USER') }}" + password: "{{ var('DBT_SNOWFLAKE_PASSWORD') }}" + role: "{{ var('DBT_SNOWFLAKE_ROLE') }}" + database: "{{ var('DBT_SNOWFLAKE_DATABASE') }}" + warehouse: "{{ var('DBT_SNOWFLAKE_WAREHOUSE') }}" + schema: "{{ var('DBT_SCHEMA') }}" + threads: 10 + + target: snowflake-ci From ca185a35bea78f0b814b74846260468e3b9e6ecc Mon Sep 17 00:00:00 2001 From: Celine Verhoef Date: Tue, 10 Oct 2023 13:48:11 +0200 Subject: [PATCH 3/4] ci: implement using variable names --- .pipelines/azure-pipelines-integration-tests.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.pipelines/azure-pipelines-integration-tests.yml b/.pipelines/azure-pipelines-integration-tests.yml index 098eec8..669ca08 100644 --- a/.pipelines/azure-pipelines-integration-tests.yml +++ b/.pipelines/azure-pipelines-integration-tests.yml @@ -32,8 +32,8 @@ jobs: steps: - template: ./.pipelines/templates/get_keyvault_secrets.job.yml@testTransformations parameters: - serviceConnection: AzureDevTestEASubscription-ProcessMining-apps-team-development - keyVaultName: pmc-dev-config-kv + serviceConnection: $(serviceConnection) + keyVaultName: $(keyVaultName) - job: integration_tests displayName: Integration tests From 0d5eb82aa1751ea7ace457c5b36f30387a4f9215 Mon Sep 17 00:00:00 2001 From: Celine Verhoef Date: Wed, 11 Oct 2023 13:49:38 +0200 Subject: [PATCH 4/4] ci: remove one secret which is not used --- .pipelines/azure-pipelines-integration-tests.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.pipelines/azure-pipelines-integration-tests.yml b/.pipelines/azure-pipelines-integration-tests.yml index 669ca08..9b12222 100644 --- a/.pipelines/azure-pipelines-integration-tests.yml +++ b/.pipelines/azure-pipelines-integration-tests.yml @@ -40,7 +40,6 @@ jobs: dependsOn: get_keyvault_secrets condition: eq(dependencies.get_keyvault_secrets.result, 'Succeeded') variables: - SNOWSQL_PWD_input: $[ dependencies.get_keyvault_secrets.outputs['set_variables.SNOWSQL_PWD_input'] ] DBT_SQL_SERVER_SERVER: $[ dependencies.get_keyvault_secrets.outputs['set_variables.DBT_SQL_SERVER_SERVER'] ] DBT_SQL_SERVER_USER: $[ dependencies.get_keyvault_secrets.outputs['set_variables.DBT_SQL_SERVER_USER'] ] DBT_SQL_SERVER_PASSWORD: $[ dependencies.get_keyvault_secrets.outputs['set_variables.DBT_SQL_SERVER_PASSWORD'] ]