Dev Environment Destroy working-ish? (#1333)

* Adding 1 pass integration * service accoutn token * Removing quicksight * ignore dep errors * Mock outputs on destroy * mock on destroy * mock on destroy * adding missing mocks * manually destroy spot * manually destroy spot * manually destroy spot * wtf * retry * cleanup * Debug * Debug * perftest * oops * Force destroy athena * Removing continue on error for common --------- Co-authored-by: Mike Pond <[email protected]>
cds-snc · May 15, 2024 · a1856c8 · a1856c8
1 parent dd3a9ee
commit a1856c8
Show file tree

Hide file tree

Showing 25 changed files with 231 additions and 59 deletions.
diff --git a/.github/workflows/terragrunt_destroy_environment.yml b/.github/workflows/terragrunt_destroy_environment.yml
@@ -9,6 +9,7 @@ defaults:
 
 env:
   AWS_REGION: ca-central-1
+  OP_SERVICE_ACCOUNT_TOKEN: ${{ secrets.OP_SERVICE_ACCOUNT_TOKEN }}
 
 
 permissions:
@@ -31,10 +32,14 @@ jobs:
       - name: Checkout
         uses: actions/checkout@f43a0e5ff2bd294095638e18286ca9a3d1956744 # v3.6.0
 
-      - name: Set environment variables
-        uses: ./.github/actions/setvars
-        with:
-          envVarFile: ./.env
+      - name: Install 1Pass CLI
+        run: |
+          curl -o 1pass.deb https://downloads.1password.com/linux/debian/amd64/stable/1password-cli-amd64-latest.deb
+          sudo dpkg -i 1pass.deb
+
+      - name: One Password Fetch
+        run: |
+          op read op://4eyyuwddp6w4vxlabrr2i2duxm/"TFVars - Dev"/notesPlain > /var/tmp/dev.tfvars
 
       - name: Setup Terraform tools
         uses: cds-snc/terraform-tools-setup@v1
@@ -50,96 +55,142 @@ jobs:
           git config --global url."https://x-access-token:${{ secrets.GITHUB_TOKEN }}@github.com/".insteadOf "https://github.com/"
 
       - name: Destroy aws/system_status_static_site
+        continue-on-error: true
         run: |
           cd env/dev/system_status_static_site
-          terragrunt destroy --terragrunt-non-interactive -auto-approve
+          terragrunt destroy --terragrunt-non-interactive -auto-approve -var-file /var/tmp/dev.tfvars
 
       - name: Destroy aws/system_status
+        continue-on-error: true
         run: |
           cd env/dev/system_status
-          terragrunt destroy --terragrunt-non-interactive -auto-approve
+          terragrunt destroy --terragrunt-non-interactive -auto-approve -var-file /var/tmp/dev.tfvars
 
       - name: Destroy aws/lambda-google-cidr
+        continue-on-error: true
         run: |
           cd env/dev/lambda-google-cidr
-          terragrunt destroy --terragrunt-non-interactive -auto-approve
+          terragrunt destroy --terragrunt-non-interactive -auto-approve -var-file /var/tmp/dev.tfvars
 
-      - name: Destroy aws/quicksight
+      # This must be run before quicksight destroy
+      - name: Destroy quicksight VPC Connection
+        continue-on-error: true
         run: |
-          cd env/dev/quicksight
-          terragrunt destroy --terragrunt-non-interactive -auto-approve
+          aws quicksight delete-vpc-connection --aws-account-id 800095993820 --vpc-connection-id $(aws quicksight list-vpc-connections --aws-account-id 800095993820 --query 'VPCConnectionSummaries[].VPCConnectionId' --output text)
 
+      # - name: Destroy aws/quicksight
+      #   run: |
+      #     cd env/dev/quicksight
+      #     terragrunt destroy --terragrunt-non-interactive -auto-approve -var-file /var/tmp/dev.tfvars
+
       - name: Destroy aws/database-tools
+        continue-on-error: true
         run: |
           cd env/dev/database-tools
-          terragrunt destroy --terragrunt-non-interactive -auto-approve
+          terragrunt destroy --terragrunt-non-interactive -auto-approve -var-file /var/tmp/dev.tfvars
 
       - name: Destroy aws/heartbeat
+        continue-on-error: true
         run: |
           cd env/dev/heartbeat
-          terragrunt destroy --terragrunt-non-interactive -auto-approve
+          terragrunt destroy --terragrunt-non-interactive -auto-approve -var-file /var/tmp/dev.tfvars
 
       - name: Destroy aws/lambda-api
+        continue-on-error: true
         run: |
           cd env/dev/lambda-api
-          terragrunt destroy --terragrunt-non-interactive -auto-approve
+          terragrunt destroy --terragrunt-non-interactive -auto-approve -var-file /var/tmp/dev.tfvars
 
       - name: Destroy aws/rds
+        continue-on-error: true
         run: |
           cd env/dev/rds
-          terragrunt destroy --terragrunt-non-interactive -auto-approve
+          terragrunt destroy --terragrunt-non-interactive -auto-approve -var-file /var/tmp/dev.tfvars
 
       - name: Destroy aws/elasticache
+        continue-on-error: true
         run: |
           cd env/dev/elasticache
-          terragrunt destroy --terragrunt-non-interactive -auto-approve
+          terragrunt destroy --terragrunt-non-interactive -auto-approve -var-file /var/tmp/dev.tfvars
+
+      - name: Destroy aws/preformance-test
+        continue-on-error: true
+        run: |
+          cd env/dev/performance-test
+          terragrunt destroy --terragrunt-non-interactive -auto-approve -var-file /var/tmp/dev.tfvars
+
+      - name: Destroy aws/eks
+        continue-on-error: true
+        run: |
+          cd env/dev/eks
+          terragrunt destroy --terragrunt-non-interactive -auto-approve -var-file /var/tmp/dev.tfvars
+
+      - name: Clean Up Lingering EKS 
+        continue-on-error: true
+        run: |
+          ./scripts/eksDestroyCleanup.sh
 
       - name: Destroy aws/eks
+        continue-on-error: true
         run: |
           cd env/dev/eks
-          terragrunt destroy --terragrunt-non-interactive -auto-approve
+          terragrunt destroy --terragrunt-non-interactive -auto-approve -var-file /var/tmp/dev.tfvars
 
       - name: Destroy aws/cloudfront
+        continue-on-error: true
         run: |
           cd env/dev/cloudfront
-          terragrunt destroy --terragrunt-non-interactive -auto-approve
+          terragrunt destroy --terragrunt-non-interactive -auto-approve -var-file /var/tmp/dev.tfvars
 
       - name: Destroy aws/ses_validation_dns_entries
+        continue-on-error: true
         run: |
           cd env/dev/ses_validation_dns_entries
-          terragrunt destroy --terragrunt-non-interactive -auto-approve
+          terragrunt destroy --terragrunt-non-interactive -auto-approve -var-file /var/tmp/dev.tfvars
 
       - name: Destroy aws/dns
+        continue-on-error: true
         run: |
           cd env/dev/dns
-          terragrunt destroy --terragrunt-non-interactive -auto-approve
+          terragrunt destroy --terragrunt-non-interactive -auto-approve -var-file /var/tmp/dev.tfvars
 
       - name: Destroy aws/pinpoint_to_sqs_sms_callbacks
+        continue-on-error: true
         run: |
           cd env/dev/pinpoint_to_sqs_sms_callbacks
-          terragrunt destroy --terragrunt-non-interactive -auto-approve
+          terragrunt destroy --terragrunt-non-interactive -auto-approve -var-file /var/tmp/dev.tfvars
 
       - name: Destroy aws/sns_to_sqs_sms_callbacks
+        continue-on-error: true
         run: |
           cd env/dev/sns_to_sqs_sms_callbacks
-          terragrunt destroy --terragrunt-non-interactive -auto-approve
+          terragrunt destroy --terragrunt-non-interactive -auto-approve -var-file /var/tmp/dev.tfvars
 
       - name: Destroy aws/ses_to_sqs_email_callbacks
+        continue-on-error: true
         run: |
           cd env/dev/ses_to_sqs_email_callbacks
-          terragrunt destroy --terragrunt-non-interactive -auto-approve
+          terragrunt destroy --terragrunt-non-interactive -auto-approve -var-file /var/tmp/dev.tfvars
 
       - name: Destroy aws/ses_receiving_emails
+        continue-on-error: true
         run: |
           cd env/dev/ses_receiving_emails
-          terragrunt destroy --terragrunt-non-interactive -auto-approve
+          terragrunt destroy --terragrunt-non-interactive -auto-approve -var-file /var/tmp/dev.tfvars
 
       - name: Destroy aws/ecr
+        continue-on-error: true
         run: |
           cd env/dev/ecr
-          terragrunt destroy --terragrunt-non-interactive -auto-approve
+          terragrunt destroy --terragrunt-non-interactive -auto-approve -var-file /var/tmp/dev.tfvars
+
+      - name: Cleanup Internal Load Balancers
+        continue-on-error: true
+        run: |
+          ./scripts/cleanupELB.sh
 
+      # Do not continue on error here so that we can track whether or not this actually worked
       - name: Destroy aws/common
         run: |
           cd env/dev/common
-          terragrunt destroy --terragrunt-non-interactive -auto-approve
+          terragrunt destroy --terragrunt-non-interactive -auto-approve -var-file /var/tmp/dev.tfvars
diff --git a/aws/common/athena.tf b/aws/common/athena.tf
@@ -3,8 +3,9 @@
 ###
 
 resource "aws_athena_database" "notification_athena" {
-  name   = "notification_athena"
-  bucket = aws_s3_bucket.athena_bucket.bucket
+  name          = "notification_athena"
+  bucket        = aws_s3_bucket.athena_bucket.bucket
+  force_destroy = var.force_destroy_athena
 
   encryption_configuration {
     encryption_option = "SSE_S3"

diff --git a/aws/common/variables.tf b/aws/common/variables.tf
@@ -245,6 +245,12 @@ variable "force_destroy_s3" {
   default     = false
 }
 
+variable "force_destroy_athena" {
+  description = "Destroy Athena workgroups even if there are tables in them when running terraform destroy"
+  type        = bool
+  default     = false
+}
+
 variable "athena_workgroup_name" {
   description = "Set the name for the athena workgroup"
   type        = string

diff --git a/env/dev/cloudfront/terragrunt.hcl b/env/dev/cloudfront/terragrunt.hcl
@@ -7,7 +7,7 @@ dependency "common" {
 
   # Configure mock outputs for the `validate` command that are returned when there are no outputs available (e.g the
   # module hasn't been applied yet.
-  mock_outputs_allowed_terraform_commands = ["validate"]
+  mock_outputs_allowed_terraform_commands = ["validate", "destroy"]
   mock_outputs = {
     asset_bucket_regional_domain_name = ""
   }

diff --git a/env/dev/database-tools/.terraform.lock.hcl b/env/dev/database-tools/.terraform.lock.hcl
diff --git a/env/dev/database-tools/terragrunt.hcl b/env/dev/database-tools/terragrunt.hcl
@@ -7,7 +7,7 @@ dependency "common" {
 
   # Configure mock outputs for the `validate` command that are returned when there are no outputs available (e.g the
   # module hasn't been applied yet.
-  mock_outputs_allowed_terraform_commands = ["init", "fmt", "validate", "plan", "show"]
+  mock_outputs_allowed_terraform_commands = ["init", "fmt", "validate", "plan", "show", "destroy"]
   mock_outputs = {
     vpc_id = ""
     vpc_private_subnets = [
@@ -24,7 +24,7 @@ dependency "eks" {
 
   # Configure mock outputs for the `validate` command that are returned when there are no outputs available (e.g the
   # module hasn't been applied yet.
-  mock_outputs_allowed_terraform_commands = ["validate", "plan", "init", "fmt", "show"]
+  mock_outputs_allowed_terraform_commands = ["validate", "plan", "init", "fmt", "show", "destroy"]
   mock_outputs_merge_with_state           = true
   mock_outputs = {
     database-tools-securitygroup    = ""
@@ -34,6 +34,10 @@ dependency "eks" {
 
 dependency "rds" {
   config_path = "../rds"
+  mock_outputs_allowed_terraform_commands = ["validate", "plan", "init", "fmt", "show", "destroy"]
+  mock_outputs = {
+    database_read_only_proxy_endpoint = "adsfs"
+  }
 }
 
 include {

diff --git a/env/dev/dns/terragrunt.hcl b/env/dev/dns/terragrunt.hcl
@@ -7,7 +7,7 @@ dependency "common" {
 
   # Configure mock outputs for the `validate` command that are returned when there are no outputs available (e.g the
   # module hasn't been applied yet.
-  mock_outputs_allowed_terraform_commands = ["validate"]
+  mock_outputs_allowed_terraform_commands = ["validate", "destroy"]
   mock_outputs = {
     notification_canada_ca_ses_callback_arn = ""
   }
@@ -18,7 +18,7 @@ dependency "ses_receiving_emails" {
 
   # Configure mock outputs for the `validate` command that are returned when there are no outputs available (e.g the
   # module hasn't been applied yet.
-  mock_outputs_allowed_terraform_commands = ["init", "fmt", "validate", "plan", "show"]
+  mock_outputs_allowed_terraform_commands = ["init", "fmt", "validate", "plan", "show", "destroy"]
   mock_outputs_merge_with_state           = true
   mock_outputs = {
     lambda_ses_receiving_emails_image_arn = ""

diff --git a/env/dev/eks/terragrunt.hcl b/env/dev/eks/terragrunt.hcl
@@ -7,7 +7,7 @@ dependency "common" {
 
   # Configure mock outputs for the `validate` command that are returned when there are no outputs available (e.g the
   # module hasn't been applied yet.
-  mock_outputs_allowed_terraform_commands = ["init", "fmt", "validate", "plan", "show"]
+  mock_outputs_allowed_terraform_commands = ["init", "fmt", "validate", "plan", "show", "destroy"]
   mock_outputs_merge_with_state           = true
   mock_outputs = {
     vpc_private_subnets = [
@@ -59,7 +59,7 @@ dependency "dns" {
 
   # Configure mock outputs for the `validate` command that are returned when there are no outputs available (e.g the
   # module hasn't been applied yet.
-  mock_outputs_allowed_terraform_commands = ["validate"]
+  mock_outputs_allowed_terraform_commands = ["validate", "destroy"]
   mock_outputs = {
     internal_dns_certificate_arn = ""
     internal_dns_zone_id = ""
@@ -72,7 +72,7 @@ dependency "cloudfront" {
 
   # Configure mock outputs for the `validate` command that are returned when there are no outputs available (e.g the
   # module hasn't been applied yet.
-  mock_outputs_allowed_terraform_commands = ["validate"]
+  mock_outputs_allowed_terraform_commands = ["validate", "destroy"]
   mock_outputs = {
     cloudfront_assets_arn = ""
     internal_dns_zone_id  = "aoeui"

diff --git a/env/dev/elasticache/terragrunt.hcl b/env/dev/elasticache/terragrunt.hcl
@@ -7,14 +7,17 @@ dependency "common" {
 
   # Configure mock outputs for the `validate` command that are returned when there are no outputs available (e.g the
   # module hasn't been applied yet.
-  mock_outputs_allowed_terraform_commands = ["validate"]
+  mock_outputs_allowed_terraform_commands = ["validate", "destroy"]
   mock_outputs = {
     vpc_id = ""
     vpc_private_subnets = [
       "subnet-001e585d12cce4d1e",
       "subnet-08de34a9e1a7458dc",
       "subnet-0af8b8402f1d605ff",
     ]
+    sns_alert_warning_arn = ""
+    sns_alert_critical_arn = ""
+    kms_arn = ""
   }
 }
 
@@ -23,7 +26,7 @@ dependency "eks" {
 
   # Configure mock outputs for the `validate` command that are returned when there are no outputs available (e.g the
   # module hasn't been applied yet.
-  mock_outputs_allowed_terraform_commands = ["validate"]
+  mock_outputs_allowed_terraform_commands = ["validate", "destroy"]
   mock_outputs = {
     eks-cluster-securitygroup = "sg-0e2c3ef6c5c75b74c"
   }

diff --git a/env/dev/heartbeat/terragrunt.hcl b/env/dev/heartbeat/terragrunt.hcl
@@ -7,6 +7,7 @@ dependency "common" {
 
   # Configure mock outputs for the `validate` command that are returned when there are no outputs available (e.g the
   # module hasn't been applied yet.
+  mock_outputs_allowed_terraform_commands = ["validate", "plan", "init", "fmt", "show", "destroy"]
   mock_outputs = {
     sns_alert_warning_arn  = ""
     sns_alert_critical_arn = ""
@@ -15,8 +16,14 @@ dependency "common" {
 
 dependency "ecr" {
   config_path = "../ecr"
+  mock_outputs_allowed_terraform_commands = ["validate", "plan", "init", "fmt", "show", "destroy"]
+  mock_outputs = {
+    heartbeat_ecr_repository_url = ""
+    heartbeat_ecr_arn            = ""
+  }
 }
 
+
 include {
   path = find_in_parent_folders()
 }