test: implement azure pipeline

UiPath · Oct 6, 2023 · d0674ba · d0674ba
1 parent f269352
commit d0674ba
Show file tree

Hide file tree

Showing 2 changed files with 118 additions and 0 deletions.
diff --git a/.pipelines/azure-pipelines-integration-tests.yml b/.pipelines/azure-pipelines-integration-tests.yml
@@ -0,0 +1,91 @@
+name: Integration tests
+
+resources:
+  repositories:
+    - repository: testTransformations
+      endpoint: UiPath
+      type: github
+      name: UiPath/ProcessMining-framework-resources
+      ref: refs/tags/test_transformations.1.1.1
+
+# Trigger the pipeline for all PRs we open, but do not start as long as there is no PR.
+trigger:
+  none
+
+pr:
+  branches:
+    include:
+    - '*'
+
+pool:
+  vmImage: ubuntu-latest
+
+jobs:
+  - job: get_keyvault_secrets
+    displayName: Get keyvault secrets
+    steps:
+      - template: ./.pipelines/templates/get_keyvault_secrets.job.yml@testTransformations
+        parameters:
+          serviceConnection: AzureDevTestEASubscription-ProcessMining-apps-team-development
+          keyVaultName: pmc-dev-config-kv
+
+  - job: integration_tests
+    displayName: Integration tests
+    dependsOn: get_keyvault_secrets
+    condition: eq(dependencies.get_keyvault_secrets.result, 'Succeeded')
+    variables:
+        SNOWSQL_PWD_input: $[ dependencies.get_keyvault_secrets.outputs['set_variables.SNOWSQL_PWD_input'] ]
+        DBT_SQL_SERVER_SERVER: $[ dependencies.get_keyvault_secrets.outputs['set_variables.DBT_SQL_SERVER_SERVER'] ]
+        DBT_SQL_SERVER_USER: $[ dependencies.get_keyvault_secrets.outputs['set_variables.DBT_SQL_SERVER_USER'] ]
+        DBT_SQL_SERVER_PASSWORD: $[ dependencies.get_keyvault_secrets.outputs['set_variables.DBT_SQL_SERVER_PASSWORD'] ]
+        DBT_SQL_SERVER_DATABASE: $[ dependencies.get_keyvault_secrets.outputs['set_variables.DBT_SQL_SERVER_DATABASE'] ]
+        DBT_SNOWFLAKE_ACCOUNT: $[ dependencies.get_keyvault_secrets.outputs['set_variables.DBT_SNOWFLAKE_ACCOUNT'] ]
+        DBT_SNOWFLAKE_USER: $[ dependencies.get_keyvault_secrets.outputs['set_variables.DBT_SNOWFLAKE_USER'] ]
+        DBT_SNOWFLAKE_PASSWORD: $[ dependencies.get_keyvault_secrets.outputs['set_variables.DBT_SNOWFLAKE_PASSWORD'] ]
+        DBT_SNOWFLAKE_ROLE: $[ dependencies.get_keyvault_secrets.outputs['set_variables.DBT_SNOWFLAKE_ROLE'] ]
+        DBT_SNOWFLAKE_DATABASE: $[ dependencies.get_keyvault_secrets.outputs['set_variables.DBT_SNOWFLAKE_DATABASE'] ]
+        DBT_SNOWFLAKE_WAREHOUSE: $[ dependencies.get_keyvault_secrets.outputs['set_variables.DBT_SNOWFLAKE_WAREHOUSE'] ]
+    steps:
+      - checkout: self
+        path: self
+        displayName: Checkout current repository
+
+      - bash: |
+          echo "##vso[task.setvariable variable=DBT_SCHEMA]pm_utils_integration_tests"
+          echo "##vso[task.setvariable variable=dbtProjectPath]$(Agent.BuildDirectory)/self/integration_tests"
+        displayName: Set variables
+
+      - bash: |
+          python --version
+
+          sudo apt-get install git libpq-dev python-dev python3-pip
+          sudo apt-get remove python-cffi
+          sudo pip install --upgrade cffi
+
+          python3 -m venv dbt-env
+          source dbt-env/bin/activate
+
+          pip install dbt-sqlserver==1.4.3
+          pip install dbt-snowflake==1.4.3
+          dbt --version
+        displayName: Install dbt
+
+      - bash: |
+          source dbt-env/bin/activate
+          cd $(dbtProjectPath)
+          dbt deps
+        displayName: Install dbt dependencies
+
+      - bash: |
+          source dbt-env/bin/activate
+          cd $(dbtProjectPath)
+          dbt build --profiles-dir $(Agent.BuildDirectory)/self/.pipelines --profile default -t sqlserver-ci \
+            --vars '{"schema_sources": "$(DBT_SCHEMA_SOURCES)", "DBT_SQL_SERVER_SERVER": "$(DBT_SQL_SERVER_SERVER)", "DBT_SQL_SERVER_USER": "$(DBT_SQL_SERVER_USER)", "DBT_SQL_SERVER_PASSWORD": "$(DBT_SQL_SERVER_PASSWORD)", "DBT_SQL_SERVER_DATABASE": "$(DBT_SQL_SERVER_DATABASE)", "DBT_SCHEMA": "$(DBT_SCHEMA)"}'
+        displayName: Test (SQL Server)
+
+      - bash: |
+          source dbt-env/bin/activate
+          cd $(dbtProjectPath)
+          dbt build --profiles-dir $(Agent.BuildDirectory)/self/.pipelines --profile default -t snowflake-ci \
+            --vars '{"schema_sources": "$(DBT_SCHEMA_SOURCES)", "DBT_SNOWFLAKE_ACCOUNT": "$(DBT_SNOWFLAKE_ACCOUNT)", "DBT_SNOWFLAKE_USER": "$(DBT_SNOWFLAKE_USER)", "DBT_SNOWFLAKE_PASSWORD": "$(DBT_SNOWFLAKE_PASSWORD)", "DBT_SNOWFLAKE_ROLE": "$(DBT_SNOWFLAKE_ROLE)", "DBT_SNOWFLAKE_DATABASE": "$(DBT_SNOWFLAKE_DATABASE)", "DBT_SNOWFLAKE_WAREHOUSE": "$(DBT_SNOWFLAKE_WAREHOUSE)", "DBT_SCHEMA": "$(DBT_SCHEMA)"}'
+        displayName: Test (Snowflake)
diff --git a/.pipelines/profiles.yml b/.pipelines/profiles.yml
@@ -0,0 +1,27 @@
+default:
+  outputs:
+
+    sqlserver-ci:
+      type: sqlserver
+      driver: 'ODBC Driver 17 for SQL Server'
+      port: 1433
+      server: "{{ var('DBT_SQL_SERVER_SERVER') }}"
+      user: "{{ var('DBT_SQL_SERVER_USER') }}"
+      password: "{{ var('DBT_SQL_SERVER_PASSWORD') }}"
+      database: "{{ var('DBT_SQL_SERVER_DATABASE') }}"
+      schema: "{{ var('DBT_SCHEMA') }}"
+      encrypt: true
+      trust_cert: true
+
+    snowflake-ci:
+      type: snowflake
+      account: "{{ var('DBT_SNOWFLAKE_ACCOUNT') }}"
+      user: "{{ var('DBT_SNOWFLAKE_USER') }}"
+      password: "{{ var('DBT_SNOWFLAKE_PASSWORD') }}"
+      role: "{{ var('DBT_SNOWFLAKE_ROLE') }}"
+      database: "{{ var('DBT_SNOWFLAKE_DATABASE') }}"
+      warehouse: "{{ var('DBT_SNOWFLAKE_WAREHOUSE') }}"
+      schema: "{{ var('DBT_SCHEMA') }}"
+      threads: 10
+
+  target: snowflake-ci