From d0674ba9724c9ceea65841315a0e08c6214232a4 Mon Sep 17 00:00:00 2001 From: Celine Verhoef Date: Fri, 6 Oct 2023 16:06:20 +0200 Subject: [PATCH] test: implement azure pipeline --- .../azure-pipelines-integration-tests.yml | 91 +++++++++++++++++++ .pipelines/profiles.yml | 27 ++++++ 2 files changed, 118 insertions(+) create mode 100644 .pipelines/azure-pipelines-integration-tests.yml create mode 100644 .pipelines/profiles.yml diff --git a/.pipelines/azure-pipelines-integration-tests.yml b/.pipelines/azure-pipelines-integration-tests.yml new file mode 100644 index 0000000..f3c1a8e --- /dev/null +++ b/.pipelines/azure-pipelines-integration-tests.yml @@ -0,0 +1,91 @@ +name: Integration tests + +resources: + repositories: + - repository: testTransformations + endpoint: UiPath + type: github + name: UiPath/ProcessMining-framework-resources + ref: refs/tags/test_transformations.1.1.1 + +# Trigger the pipeline for all PRs we open, but do not start as long as there is no PR. +trigger: + none + +pr: + branches: + include: + - '*' + +pool: + vmImage: ubuntu-latest + +jobs: + - job: get_keyvault_secrets + displayName: Get keyvault secrets + steps: + - template: ./.pipelines/templates/get_keyvault_secrets.job.yml@testTransformations + parameters: + serviceConnection: AzureDevTestEASubscription-ProcessMining-apps-team-development + keyVaultName: pmc-dev-config-kv + + - job: integration_tests + displayName: Integration tests + dependsOn: get_keyvault_secrets + condition: eq(dependencies.get_keyvault_secrets.result, 'Succeeded') + variables: + SNOWSQL_PWD_input: $[ dependencies.get_keyvault_secrets.outputs['set_variables.SNOWSQL_PWD_input'] ] + DBT_SQL_SERVER_SERVER: $[ dependencies.get_keyvault_secrets.outputs['set_variables.DBT_SQL_SERVER_SERVER'] ] + DBT_SQL_SERVER_USER: $[ dependencies.get_keyvault_secrets.outputs['set_variables.DBT_SQL_SERVER_USER'] ] + DBT_SQL_SERVER_PASSWORD: $[ dependencies.get_keyvault_secrets.outputs['set_variables.DBT_SQL_SERVER_PASSWORD'] ] + DBT_SQL_SERVER_DATABASE: $[ dependencies.get_keyvault_secrets.outputs['set_variables.DBT_SQL_SERVER_DATABASE'] ] + DBT_SNOWFLAKE_ACCOUNT: $[ dependencies.get_keyvault_secrets.outputs['set_variables.DBT_SNOWFLAKE_ACCOUNT'] ] + DBT_SNOWFLAKE_USER: $[ dependencies.get_keyvault_secrets.outputs['set_variables.DBT_SNOWFLAKE_USER'] ] + DBT_SNOWFLAKE_PASSWORD: $[ dependencies.get_keyvault_secrets.outputs['set_variables.DBT_SNOWFLAKE_PASSWORD'] ] + DBT_SNOWFLAKE_ROLE: $[ dependencies.get_keyvault_secrets.outputs['set_variables.DBT_SNOWFLAKE_ROLE'] ] + DBT_SNOWFLAKE_DATABASE: $[ dependencies.get_keyvault_secrets.outputs['set_variables.DBT_SNOWFLAKE_DATABASE'] ] + DBT_SNOWFLAKE_WAREHOUSE: $[ dependencies.get_keyvault_secrets.outputs['set_variables.DBT_SNOWFLAKE_WAREHOUSE'] ] + steps: + - checkout: self + path: self + displayName: Checkout current repository + + - bash: | + echo "##vso[task.setvariable variable=DBT_SCHEMA]pm_utils_integration_tests" + echo "##vso[task.setvariable variable=dbtProjectPath]$(Agent.BuildDirectory)/self/integration_tests" + displayName: Set variables + + - bash: | + python --version + + sudo apt-get install git libpq-dev python-dev python3-pip + sudo apt-get remove python-cffi + sudo pip install --upgrade cffi + + python3 -m venv dbt-env + source dbt-env/bin/activate + + pip install dbt-sqlserver==1.4.3 + pip install dbt-snowflake==1.4.3 + dbt --version + displayName: Install dbt + + - bash: | + source dbt-env/bin/activate + cd $(dbtProjectPath) + dbt deps + displayName: Install dbt dependencies + + - bash: | + source dbt-env/bin/activate + cd $(dbtProjectPath) + dbt build --profiles-dir $(Agent.BuildDirectory)/self/.pipelines --profile default -t sqlserver-ci \ + --vars '{"schema_sources": "$(DBT_SCHEMA_SOURCES)", "DBT_SQL_SERVER_SERVER": "$(DBT_SQL_SERVER_SERVER)", "DBT_SQL_SERVER_USER": "$(DBT_SQL_SERVER_USER)", "DBT_SQL_SERVER_PASSWORD": "$(DBT_SQL_SERVER_PASSWORD)", "DBT_SQL_SERVER_DATABASE": "$(DBT_SQL_SERVER_DATABASE)", "DBT_SCHEMA": "$(DBT_SCHEMA)"}' + displayName: Test (SQL Server) + + - bash: | + source dbt-env/bin/activate + cd $(dbtProjectPath) + dbt build --profiles-dir $(Agent.BuildDirectory)/self/.pipelines --profile default -t snowflake-ci \ + --vars '{"schema_sources": "$(DBT_SCHEMA_SOURCES)", "DBT_SNOWFLAKE_ACCOUNT": "$(DBT_SNOWFLAKE_ACCOUNT)", "DBT_SNOWFLAKE_USER": "$(DBT_SNOWFLAKE_USER)", "DBT_SNOWFLAKE_PASSWORD": "$(DBT_SNOWFLAKE_PASSWORD)", "DBT_SNOWFLAKE_ROLE": "$(DBT_SNOWFLAKE_ROLE)", "DBT_SNOWFLAKE_DATABASE": "$(DBT_SNOWFLAKE_DATABASE)", "DBT_SNOWFLAKE_WAREHOUSE": "$(DBT_SNOWFLAKE_WAREHOUSE)", "DBT_SCHEMA": "$(DBT_SCHEMA)"}' + displayName: Test (Snowflake) diff --git a/.pipelines/profiles.yml b/.pipelines/profiles.yml new file mode 100644 index 0000000..e17bab9 --- /dev/null +++ b/.pipelines/profiles.yml @@ -0,0 +1,27 @@ +default: + outputs: + + sqlserver-ci: + type: sqlserver + driver: 'ODBC Driver 17 for SQL Server' + port: 1433 + server: "{{ var('DBT_SQL_SERVER_SERVER') }}" + user: "{{ var('DBT_SQL_SERVER_USER') }}" + password: "{{ var('DBT_SQL_SERVER_PASSWORD') }}" + database: "{{ var('DBT_SQL_SERVER_DATABASE') }}" + schema: "{{ var('DBT_SCHEMA') }}" + encrypt: true + trust_cert: true + + snowflake-ci: + type: snowflake + account: "{{ var('DBT_SNOWFLAKE_ACCOUNT') }}" + user: "{{ var('DBT_SNOWFLAKE_USER') }}" + password: "{{ var('DBT_SNOWFLAKE_PASSWORD') }}" + role: "{{ var('DBT_SNOWFLAKE_ROLE') }}" + database: "{{ var('DBT_SNOWFLAKE_DATABASE') }}" + warehouse: "{{ var('DBT_SNOWFLAKE_WAREHOUSE') }}" + schema: "{{ var('DBT_SCHEMA') }}" + threads: 10 + + target: snowflake-ci