setup terraform for snowflake ci instance

.github/workflows/terraform-apply.yml

@@ -0,0 +1,31 @@
+name: "Terraform apply"
+
+on:
+  workflow_dispatch:
+
+defaults:
+  run:
+    shell: bash
+    working-directory: ./.terraform
+
+jobs:
+  apply:
+    name: "Apply"
+    runs-on: ubuntu-latest
+    environment: "Snowflake: KTB38830"
+    steps:
+      - name: "Checkout"
+        uses: actions/checkout@v4
+        with:
+          ref: main
+
+      - name: "Setup Terraform"
+        uses: hashicorp/setup-terraform@v3
+        with:
+          terraform_version: ${{ vars.TERRAFORM_VERSION }}
+
+      - name: "Terraform init"
+        run: terraform init
+
+      - name: "Terraform apply"
+        run: terraform apply -auto-approve

.github/workflows/terraform-code-quality.yml

@@ -0,0 +1,34 @@
+name: "Terraform code quality"
+
+on:
+  pull_request:
+    branches:
+      - main
+
+defaults:
+  run:
+    shell: bash
+    working-directory: ./.terraform
+
+jobs:
+  code-quality:
+    name: "Code quality"
+    runs-on: ubuntu-latest
+    environment: "Snowflake: KTB38830"
+    steps:
+      - name: "Checkout"
+        uses: actions/checkout@v4
+
+      - name: "Setup Terraform"
+        uses: hashicorp/setup-terraform@v3
+        with:
+          terraform_version: ${{ vars.TERRAFORM_VERSION }}
+
+      - name: "Terraform init"
+        run: terraform init
+
+      - name: "Terraform format"
+        run: terraform fmt -check
+
+      - name: "Terraform validate"
+        run: terraform validate

.github/workflows/terraform-plan.yml

@@ -0,0 +1,36 @@
+name: "Terraform plan"
+
+on:
+  workflow_dispatch:
+    inputs:
+      branch:
+        description: "Branch to run plan on"
+        type: string
+        default: main
+
+defaults:
+  run:
+    shell: bash
+    working-directory: ./.terraform
+
+jobs:
+  plan:
+    name: "Plan"
+    runs-on: ubuntu-latest
+    environment: "Snowflake: KTB38830"
+    steps:
+      - name: "Checkout `${{ inputs.branch }}`"
+        uses: actions/checkout@v4
+        with:
+          ref: ${{ inputs.branch }}
+
+      - name: "Setup Terraform"
+        uses: hashicorp/setup-terraform@v3
+        with:
+          terraform_version: ${{ vars.TERRAFORM_VERSION }}
+
+      - name: "Terraform init"
+        run: terraform init
+
+      - name: "Terraform plan"
+        run: terraform plan

.gitignore

@@ -94,3 +94,27 @@ venv/
 
 # vscode
 .vscode/
+
+# Local .terraform directories
+**/.terraform/*
+
+# .tfstate files
+*.tfstate
+*.tfstate.*
+
+# Exclude all .tfvars files, which are likely to contain sensitive data, such as
+# password, private keys, and other secrets. These should not be part of version
+# control as they are data points which are potentially sensitive and subject
+# to change depending on the environment.
+*.tfvars
+*.tfvars.json
+
+# Ignore override files as they are usually used to override resources locally and so
+# are not checked in
+override.tf
+override.tf.json
+*_override.tf
+*_override.tf.json
+
+# Terraform lock files
+*.terraform.lock.hcl

.pre-commit-config.yaml

@@ -55,3 +55,11 @@ repos:
         additional_dependencies:
         -   types-pytz
         -   types-requests
+
+-   repo: local
+    hooks:
+    -   id: terraform-lint
+        name: terraform-lint
+        entry: bash -c 'cd infra && terraform fmt && terraform validate'
+        language: system
+        pass_filenames: false

infra/main.tf

@@ -0,0 +1,10 @@
+terraform {
+  required_version = "1.8.3"
+
+  required_providers {
+    snowflake = {
+      source  = "Snowflake-Labs/snowflake"
+      version = "0.91.0"
+    }
+  }
+}

infra/snowflake.tf

@@ -0,0 +1,103 @@
+provider "snowflake" {
+  alias = "security_admin"
+  role  = "SECURITYADMIN"
+  # SNOWFLAKE_ACCOUNT
+  # SNOWFLAKE_USER
+  # SNOWFLAKE_AUTHENTICATOR
+  # SNOWFLAKE_PRIVATE_KEY
+}
+
+# Resources needed to run dbt-snowflake
+
+resource "snowflake_database" "database" {
+  name                        = "DBT_SNOWFLAKE_DB"
+  data_retention_time_in_days = 0
+  comment                     = "Used by `dbt-snowflake` for CI"
+}
+
+resource "snowflake_warehouse" "warehouse" {
+  name           = "DBT_SNOWFLAKE_WH"
+  warehouse_size = "XSMALL"
+  auto_suspend   = 60
+  comment        = "Used by `dbt-snowflake` for CI"
+}
+
+resource "snowflake_role" "role" {
+  provider = snowflake.security_admin
+  name     = "DBT_SNOWFLAKE_ROLE"
+  comment  = "Application role for `dbt_snowflake`"
+}
+
+resource "snowflake_grant_privileges_to_account_role" "database_grant" {
+  provider          = snowflake.security_admin
+  privileges        = ["USAGE", "MODIFY", "CREATE SCHEMA"]
+  account_role_name = snowflake_role.role.name
+
+  on_account_object {
+    object_type = "DATABASE"
+    object_name = snowflake_database.database.name
+  }
+}
+
+resource "snowflake_grant_privileges_to_account_role" "warehouse_grant" {
+  provider          = snowflake.security_admin
+  privileges        = ["USAGE"]
+  account_role_name = snowflake_role.role.name
+
+  on_account_object {
+    object_type = "WAREHOUSE"
+    object_name = snowflake_warehouse.warehouse.name
+  }
+}
+
+resource "tls_private_key" "user" {
+  algorithm = "RSA"
+  rsa_bits  = 2048
+}
+
+resource "snowflake_user" "user" {
+  provider          = snowflake.security_admin
+  name              = "DBT_SNOWFLAKE"
+  display_name      = "dbt-snowflake"
+  rsa_public_key    = substr(tls_private_key.user.public_key_pem, 27, 398)
+  default_warehouse = snowflake_warehouse.warehouse.name
+  default_role      = snowflake_role.role.name
+  default_namespace = snowflake_database.database.name
+  comment           = "Application user for `dbt_snowflake`"
+}
+
+resource "snowflake_grant_account_role" "role_grant" {
+  provider  = snowflake.security_admin
+  role_name = snowflake_role.role.name
+  user_name = snowflake_user.user.name
+}
+
+output "dbt_snowflake_user_public_key" {
+  value = tls_private_key.user.public_key_pem
+}
+
+output "dbt_snowflake_user_private_key" {
+  value     = tls_private_key.user.private_key_pem
+  sensitive = true
+}
+
+# Additional resources required for integration tests
+
+resource "snowflake_database" "database_quoted" {
+  name                        = "DBT_SNOWFLAKE_DB_QUOTED"
+  data_retention_time_in_days = 0
+  comment                     = "Used by `dbt-snowflake` for CI"
+}
+
+resource "snowflake_database" "database_alt" {
+  name                        = "DBT_SNOWFLAKE_DB_ALT"
+  data_retention_time_in_days = 0
+  comment                     = "Used by `dbt-snowflake` for CI"
+}
+
+resource "snowflake_warehouse" "warehouse_alt" {
+  name           = "DBT_SNOWFLAKE_WH_ALT"
+  warehouse_size = "XSMALL"
+  auto_suspend   = 60
+  comment        = "Used by `dbt-snowflake` for CI"
+}

test.env.example

@@ -6,29 +6,33 @@
 # These will all be gathered from account information or created by you.
 
 # SNOWFLAKE_TEST_ACCOUNT: The name that uniquely identifies your Snowflake account.
-# SNOWFLAKE_TEST_ALT_DATABASE: Name of a secondary or alternate database to use for testing. You will need to create this database.
-# SNOWFLAKE_TEST_ALT_WAREHOUSE: Name of the secondary warehouse to use for testing.
 # SNOWFLAKE_TEST_DATABASE: Name of the primary database to use for testing.
+# SNOWFLAKE_TEST_WAREHOUSE: Warehouse name to be used as primary.
+# SNOWFLAKE_TEST_USER: Username of database user
+# SNOWFLAKE_TEST_PASSWORD:Password used for your database user.
+
 # SNOWFLAKE_TEST_OAUTH_CLIENT_ID: Client ID of the OAuth client integration. (only for oauth authentication)
 # SNOWFLAKE_TEST_OAUTH_CLIENT_SECRET: Client secret of your OAuth client id. (only for oauth authentication)
 # SNOWFLAKE_TEST_OAUTH_REFRESH_TOKEN: Boolean value defaulted to True keep connection alive. (only for oauth authentication)
-# SNOWFLAKE_TEST_PASSWORD:Password used for your database user.
+
 # SNOWFLAKE_TEST_QUOTED_DATABASE: Name of database to be used from warehouse.
-# SNOWFLAKE_TEST_USER: Username of database user
-# SNOWFLAKE_TEST_WAREHOUSE: Warehouse name to be used as primary.
+# SNOWFLAKE_TEST_ALT_DATABASE: Name of a secondary or alternate database to use for testing. You will need to create this database.
+# SNOWFLAKE_TEST_ALT_WAREHOUSE: Name of the secondary warehouse to use for testing.
 
 # Copy the following to a test.env, and replace example values with your information.
-SNOWFLAKE_TEST_ACCOUNT=my_account_id
-SNOWFLAKE_TEST_ALT_DATABASE=my_alt_database_name
-SNOWFLAKE_TEST_ALT_WAREHOUSE=my_alt_warehouse_name
-SNOWFLAKE_TEST_DATABASE=my_database_name
+SNOWFLAKE_TEST_ACCOUNT=my_account
+SNOWFLAKE_TEST_DATABASE=DBT_SNOWFLAKE_DB
+SNOWFLAKE_TEST_WAREHOUSE=DBT_SNOWFLAKE_WH
+SNOWFLAKE_TEST_USER=DBT_SNOWFLAKE
+SNOWFLAKE_TEST_PASSWORD=my_password
+
 SNOWFLAKE_TEST_OAUTH_CLIENT_ID=my_oauth_id
 SNOWFLAKE_TEST_OAUTH_CLIENT_SECRET=my_oauth_secret
 SNOWFLAKE_TEST_OAUTH_REFRESH_TOKEN=TRUE
-SNOWFLAKE_TEST_PASSWORD=my_password
-SNOWFLAKE_TEST_QUOTED_DATABASE=my_quoted_database_name
-SNOWFLAKE_TEST_USER=my_username
-SNOWFLAKE_TEST_WAREHOUSE=my_warehouse_name
+
+SNOWFLAKE_TEST_QUOTED_DATABASE=DBT_SNOWFLAKE_DB_QUOTED
+SNOWFLAKE_TEST_ALT_DATABASE=DBT_SNOWFLAKE_DB_ALT
+SNOWFLAKE_TEST_ALT_WAREHOUSE=DBT_SNOWFLAKE_WH_ALT
 
 DBT_TEST_USER_1=dbt_test_role_1
 DBT_TEST_USER_2=dbt_test_role_2