Merge pull request #68 from star3am/feature/dbt-airflow-updates

Feature/dbt airflow updates

dbt/dbt-global.sh

@@ -10,7 +10,7 @@ source /vagrant/dbt/common.sh
 install-dbt-mssql () {
 
   echo "Installing dbt core.  Version: ${dbt_sqlserver}"
-  python -m pip install --no-cache "git+https://github.com/dbt-labs/dbt-core@v${dbt_sqlserver}#egg=dbt-postgres&subdirectory=plugins/postgres"
+  python -m pip install --break-system-packages --no-cache "git+https://github.com/dbt-labs/dbt-core@v${dbt_sqlserver}#egg=dbt-postgres&subdirectory=plugins/postgres"
 
   # Install ODBC headers for MSSQL support
   if ! [[ "18.04 20.04 22.04" == *"$(lsb_release -rs)"* ]];
@@ -34,8 +34,8 @@ install-dbt-mssql () {
   # optional: for unixODBC development headers
   sudo apt-get install -y unixodbc-dev postgresql-client
 
-  pip install -U dbt-sqlserver==$dbt_sqlserver
-  pip install -U dbt-synapse==$dbt_synapse
+  pip install -U dbt-sqlserver==$dbt_sqlserver --break-system-packages
+  pip install -U dbt-synapse==$dbt_synapse --break-system-packages
 
 }
 
@@ -45,37 +45,37 @@ install-dbt-mssql () {
 function install-dbt () {
 
   echo -e '\e[38;5;198m'"++++ installing postgres adapter"
-  python -m pip install --no-cache "git+https://github.com/dbt-labs/${dbt_postgres_ref}#egg=dbt-postgres&subdirectory=plugins/postgres"
+  python -m pip install --break-system-packages --no-cache dbt-postgres
 }
 
 #####################################
 function install-dbt-redshift () {
   echo -e '\e[38;5;198m'"++++ installing redshift adapater"
-  python -m pip install --no-cache "git+https://github.com/dbt-labs/${dbt_redshift_ref}#egg=dbt-redshift"
+  python -m pip install --break-system-packages --no-cache dbt-redshift
 }
 
 #####################################
 function install-dbt-bigquery () {
   echo -e '\e[38;5;198m'"++++ installing bigquery adapater"
-  python -m pip install --no-cache "git+https://github.com/dbt-labs/${dbt_bigquery_ref}#egg=dbt-bigquery"
+  python -m pip install --break-system-packages --no-cache dbt-bigquery
 }
 
 #####################################
 function install-dbt-snowflake () {
   echo -e '\e[38;5;198m'"++++ installing snowflake adapater"
-  python -m pip install --no-cache "git+https://github.com/dbt-labs/${dbt_snowflake_ref}#egg=dbt-snowflake"
+  python -m pip install --break-system-packages --no-cache dbt-snowflake
 }
 
 #####################################
 function install-dbt-spark () {
   echo -e '\e[38;5;198m'"++++ installing spark adapter"
-  python -m pip install --no-cache "git+https://github.com/dbt-labs/${dbt_spark_ref}#egg=dbt-spark[${dbt_spark_version}]"
+  python -m pip install --break-system-packages --no-cache dbt-spark
 }
 
 #####################################
 function install-dbt-databricks () {
   echo -e '\e[38;5;198m'"++++ installing databricks adapter"
-  python -m pip install --no-cache "dbt_databricks==${dbt_databricks}"
+  python -m pip install --break-system-packages --no-cache dbt-databricks
 }
 
 ############################
@@ -88,14 +88,9 @@ else
 fi
 
 ############################
-# Python stuff
-python --version
-# Ensure pip is upgraded and Print pip version 
-python -m pip install --upgrade pip
-pip --version
 
 # Cleanup any existing dbt packages.
-[ $(pip list | grep dbt | wc -l) -gt 0 ] && pip list | grep dbt | xargs pip uninstall -y
+[ $(pip list | grep dbt | wc -l) -gt 0 ] && pip list | grep dbt | xargs pip uninstall -y --break-system-packages
 
 echo $DBT_WITH
 DBT_WITH="${DBT_WITH:=postgres}"; echo $DBT_WITH
@@ -174,8 +169,8 @@ fi
 echo -e '\e[38;5;198m'"++++ "
 echo -e '\e[38;5;198m'"++++ Ensure postgresql-client is installed"
 echo -e '\e[38;5;198m'"++++ "
-sudo apt-get install -y postgresql-client libpq-dev python3.10-dev
-python3 -m pip install --force-reinstall psycopg2==2.9.4
+sudo apt-get install -y postgresql-client libpq-dev
+python3 -m pip install --break-system-packages --force-reinstall psycopg2==2.9.4
 
 if pgrep -x "postgres" >/dev/null
 then

localstack/README.md

@@ -10,7 +10,106 @@ LocalStack provides an easy-to-use test/mocking framework for developing Cloud a
 
 To get Localstack installed and running in Hashiqube, you can use the following command
 
-`vagrant up --provision-with basetools,localstack`
+Github Codespace: `bash localstack/localstack.sh` <br />
+Vagrant: `vagrant up --provision-with basetools,localstack`
+
+## Terraform plan
+
+To do a terraform plan, you can simply do the following:
+
+Change Directory into /vagrant/localstack
+`cd /vagrant/localstack`
+
+And then to a `terraform init` and `terraform plan`
+
+## Terraform apply
+
+To do a terraform apply, you can simply do the following:
+
+Change Directory into /vagrant/localstack
+`cd /vagrant/localstack`
+
+And then to a `terraform init` and `terraform plan` and lastly `terraform apply`
+
+## Terraform and Vault
+
+Terraform has many providers, and you can use Terraform to create many resources of many providers, such as Github, AWS, Azure, Cloudfalre and many others. Of course you can also manage HashiCorp Vault with Terraform. 
+
+To do that, you can follow these simple staps.
+
+1. Get Terraform and Localstack running <br />
+Github Codespace: `bash localstack/localstack.sh` <br />
+Vagrant: `vagrant up --provision-with basetools,localstack`
+
+2. Bring Vault up <br />
+Github Codespace: `bash vault/vault.sh` <br />
+Vagrant: `vagrant up --provision-with basetools,vault`
+
+3. Run Terraform plan with the vault module enabled <br />
+Github Codespace: `VAULT_TOKEN="YOUR_VAULT_TOKEN" TF_VAR_vault_enabled=true terraform plan` <br />
+Vagrant: To be determined
+
+```
+  # module.hashicorp-vault[0].vault_kv_secret_v2.example will be created
+  + resource "vault_kv_secret_v2" "example" {
+      + cas                 = 1
+      + data                = (sensitive value)
+      + data_json           = (sensitive value)
+      + delete_all_versions = true
+      + disable_read        = false
+      + id                  = (known after apply)
+      + metadata            = (known after apply)
+      + mount               = "kvv2"
+      + name                = "secret"
+      + path                = (known after apply)
+
+      + custom_metadata {
+          + data         = {
+              + "bar" = "12345"
+              + "foo" = "[email protected]"
+            }
+          + max_versions = 5
+        }
+    }
+
+  # module.hashicorp-vault[0].vault_mount.kvv2 will be created
+  + resource "vault_mount" "kvv2" {
+      + accessor                     = (known after apply)
+      + audit_non_hmac_request_keys  = (known after apply)
+      + audit_non_hmac_response_keys = (known after apply)
+      + default_lease_ttl_seconds    = (known after apply)
+      + description                  = "KV Version 2 secret engine mount"
+      + external_entropy_access      = false
+      + id                           = (known after apply)
+      + max_lease_ttl_seconds        = (known after apply)
+      + options                      = {
+          + "version" = "2"
+        }
+      + path                         = "kvv2"
+      + seal_wrap                    = (known after apply)
+      + type                         = "kv"
+    }
+```
+
+4. Run Terraform apply with the vault module enabled <br />
+Github Codespace: `VAULT_TOKEN="YOUR_VAULT_TOKEN" TF_VAR_vault_enabled=true terraform apply` <br />
+Vagrant: To be determined
+
+```
+module.hashicorp-vault[0].vault_mount.kvv2: Creation complete after 1s [id=kvv2]
+module.hashicorp-vault[0].vault_kv_secret_v2.example: Creating...
+module.hashicorp-vault[0].vault_kv_secret_v2.example: Creation complete after 0s [id=kvv2/data/secret]
+```
+
+5. Aaccess Vault to see the Secret engine enabled <br />
+
+![Vault Secrets Egine KV2](images/localstack-terraform-vault-secret-engine-kv2.png?raw=true "Vault Secrets Egine KV2")
+
+![Vault Secrets Egine KV2 Secrets](images/localstack-terraform-vault-secret-engine-kv2-secrets.png?raw=true "Vault Secrets Egine KV2 Secrets")
+
+For further details, look at the code in `/vagrant/localstack`
+
+[filename](modules.tf ':include :type=code hcl')
 
 ## Localstack Web Interface
 

terraform/terraform.sh

@@ -20,7 +20,7 @@ function terraform-install() {
     (cd /usr/local/bin && unzip /tmp/terraform.zip)
     echo -e '\e[38;5;198m'"++++ Installed: `/usr/local/bin/terraform version`"
   fi
-  pip3 install --upgrade awscli-local
+  pip3 install --upgrade awscli-local --break-system-packages --quiet
   export PATH=$HOME/.local/bin:$PATH
   sudo rm -rf awscliv2.zip
   # https://aws.amazon.com/blogs/developer/aws-cli-v2-now-available-for-linux-arm/ aarch64