Delete dev env wf (#1577)

* Fixing aws-nuke url * change directory * debug * debug * debug * debug * debug * lockfiles * delete cbs * remove s3 * delete s3 buckets manually * recursive delete on versioned buckets * bucket delete * [review] * removing comment * us-east-1 * workflow dependencies * typo in hcl * more typos * moving s3 destroy to the right place * Install boto3 * missing import * dev eks fix * k8s fix * dev fix * get rid of NR * adding other auth roles * mock outputs * Workflow updates
cds-snc · Oct 15, 2024 · a1e7687 · a1e7687
1 parent 43fd17d
commit a1e7687
Show file tree

Hide file tree

Showing 32 changed files with 804 additions and 133 deletions.
diff --git a/.github/workflows/merge_to_main_production.yml b/.github/workflows/merge_to_main_production.yml
@@ -70,6 +70,40 @@ jobs:
           role-session-name: NotifyApiGitHubActions
           aws-region: "ca-central-1"        
 
+      - name: Login to Amazon ECR
+        id: login-ecr
+        uses: aws-actions/amazon-ecr-login@v2
+
+      - name: setup-terraform
+        uses: ./.github/actions/setup-terraform
+        with:
+          role_to_assume: arn:aws:iam::${{env.ACCOUNT_ID}}:role/notification-terraform-apply
+          role_session_name: NotifyTerraformDevApply        
+
+      - name: Install 1Pass CLI
+        run: |
+          curl -o 1pass.deb https://downloads.1password.com/linux/debian/amd64/stable/1password-cli-amd64-latest.deb
+          sudo dpkg -i 1pass.deb
+          sudo mkdir -p aws
+          cd aws
+          op read op://ppnxsriom3alsxj4ogikyjxlzi/"TFVars - ${{env.ENVIRONMENT}}"/notesPlain > ${{env.ENVIRONMENT}}.tfvars   
+
+      - name: terragrunt apply ECR
+        run: |
+          cd env/${{env.ENVIRONMENT}}/ecr
+          terragrunt apply --terragrunt-non-interactive -auto-approve
+
+  terragrunt-apply-ecr-us-east:
+    if: |
+      always() &&
+      !contains(needs.*.result, 'failure') &&
+      !contains(needs.*.result, 'cancelled')
+    runs-on: ubuntu-latest  
+
+    steps:
+      - name: Checkout
+        uses: actions/checkout@f43a0e5ff2bd294095638e18286ca9a3d1956744 # v3.6.0
+
       - name: Configure credentials to Notify Private ECR using OIDC
         uses: aws-actions/configure-aws-credentials@master
         with:
@@ -97,7 +131,7 @@ jobs:
 
       - name: terragrunt apply ECR
         run: |
-          cd env/${{env.ENVIRONMENT}}/ecr
+          cd env/${{env.ENVIRONMENT}}/ecr-us-east
           terragrunt apply --terragrunt-non-interactive -auto-approve
 
   terragrunt-apply-ses_receiving_emails:
@@ -106,7 +140,7 @@ jobs:
       !contains(needs.*.result, 'failure') &&
       !contains(needs.*.result, 'cancelled')
     runs-on: ubuntu-latest  
-    needs: [terragrunt-apply-common,terragrunt-apply-ecr]
+    needs: [terragrunt-apply-common,terragrunt-apply-ecr, terragrunt-apply-ecr-us-east]
 
     steps:
       - name: Checkout

diff --git a/.github/workflows/merge_to_main_staging.yml b/.github/workflows/merge_to_main_staging.yml
@@ -73,6 +73,40 @@ jobs:
           role-session-name: NotifyApiGitHubActions
           aws-region: "ca-central-1"        
 
+      - name: Login to Amazon ECR
+        id: login-ecr
+        uses: aws-actions/amazon-ecr-login@v2
+
+      - name: setup-terraform
+        uses: ./.github/actions/setup-terraform
+        with:
+          role_to_assume: arn:aws:iam::${{env.ACCOUNT_ID}}:role/notification-terraform-apply
+          role_session_name: NotifyTerraformDevApply        
+
+      - name: Install 1Pass CLI
+        run: |
+          curl -o 1pass.deb https://downloads.1password.com/linux/debian/amd64/stable/1password-cli-amd64-latest.deb
+          sudo dpkg -i 1pass.deb
+          sudo mkdir -p aws
+          cd aws
+          op read op://4eyyuwddp6w4vxlabrr2i2duxm/"TFVars - ${{env.ENVIRONMENT}}"/notesPlain > ${{env.ENVIRONMENT}}.tfvars   
+
+      - name: terragrunt apply ECR
+        run: |
+          cd env/${{env.ENVIRONMENT}}/ecr
+          terragrunt apply --terragrunt-non-interactive -auto-approve
+
+  terragrunt-apply-ecr-us-east:
+    if: |
+      always() &&
+      !contains(needs.*.result, 'failure') &&
+      !contains(needs.*.result, 'cancelled')
+    runs-on: ubuntu-latest  
+
+    steps:
+      - name: Checkout
+        uses: actions/checkout@f43a0e5ff2bd294095638e18286ca9a3d1956744 # v3.6.0 
+
       - name: Configure credentials to Notify Private ECR using OIDC
         uses: aws-actions/configure-aws-credentials@master
         with:
@@ -100,16 +134,17 @@ jobs:
 
       - name: terragrunt apply ECR
         run: |
-          cd env/${{env.ENVIRONMENT}}/ecr
+          cd env/${{env.ENVIRONMENT}}/ecr-us-east
           terragrunt apply --terragrunt-non-interactive -auto-approve
 
+
   terragrunt-apply-ses_receiving_emails:
     if: |
       always() &&
       !contains(needs.*.result, 'failure') &&
       !contains(needs.*.result, 'cancelled')
     runs-on: ubuntu-latest  
-    needs: [terragrunt-apply-common,terragrunt-apply-ecr]
+    needs: [terragrunt-apply-common,terragrunt-apply-ecr,terragrunt-apply-ecr-us-east]
 
     steps:
       - name: Checkout

diff --git a/.github/workflows/terragrunt_create_dev_environment.yml b/.github/workflows/terragrunt_create_dev_environment.yml
@@ -43,9 +43,14 @@ jobs:
           op read op://4eyyuwddp6w4vxlabrr2i2duxm/"TFVars - ${{env.ENVIRONMENT}}"/notesPlain > ${{env.ENVIRONMENT}}.tfvars   
 
       - name: terragrunt apply COMMON
-        run: |
-          cd env/${{env.ENVIRONMENT}}/common
-          terragrunt apply --terragrunt-non-interactive -auto-approve
+        uses: nick-fields/retry@v3
+        with:
+          timeout_seconds: 600
+          max_attempts: 3
+          retry_on: error
+          command: |
+            cd env/${{env.ENVIRONMENT}}/common
+            terragrunt apply --terragrunt-non-interactive -auto-approve
 
   terragrunt-apply-ecr:
     if: |
@@ -65,6 +70,40 @@ jobs:
           role-session-name: NotifyApiGitHubActions
           aws-region: "ca-central-1"        
 
+      - name: Login to Amazon ECR
+        id: login-ecr
+        uses: aws-actions/amazon-ecr-login@v2
+
+      - name: setup-terraform
+        uses: ./.github/actions/setup-terraform
+        with:
+          role_to_assume: arn:aws:iam::${{env.ACCOUNT_ID}}:role/notification-terraform-apply
+          role_session_name: NotifyTerraformDevApply        
+
+      - name: Install 1Pass CLI
+        run: |
+          curl -o 1pass.deb https://downloads.1password.com/linux/debian/amd64/stable/1password-cli-amd64-latest.deb
+          sudo dpkg -i 1pass.deb
+          sudo mkdir -p aws
+          cd aws
+          op read op://4eyyuwddp6w4vxlabrr2i2duxm/"TFVars - ${{env.ENVIRONMENT}}"/notesPlain > ${{env.ENVIRONMENT}}.tfvars   
+
+      - name: terragrunt apply ECR
+        run: |
+          cd env/${{env.ENVIRONMENT}}/ecr
+          terragrunt apply --terragrunt-non-interactive -auto-approve
+
+  terragrunt-apply-ecr-us-east:
+    if: |
+      always() &&
+      !contains(needs.*.result, 'failure') &&
+      !contains(needs.*.result, 'cancelled')
+    runs-on: ubuntu-latest  
+
+    steps:
+      - name: Checkout
+        uses: actions/checkout@f43a0e5ff2bd294095638e18286ca9a3d1956744 # v3.6.0
+
       - name: Configure credentials to Notify Private ECR using OIDC
         uses: aws-actions/configure-aws-credentials@master
         with:
@@ -90,9 +129,9 @@ jobs:
           cd aws
           op read op://4eyyuwddp6w4vxlabrr2i2duxm/"TFVars - ${{env.ENVIRONMENT}}"/notesPlain > ${{env.ENVIRONMENT}}.tfvars   
 
-      - name: terragrunt apply ECR
+      - name: terragrunt apply ECR US East
         run: |
-          cd env/${{env.ENVIRONMENT}}/ecr
+          cd env/${{env.ENVIRONMENT}}/ecr-us-east
           terragrunt apply --terragrunt-non-interactive -auto-approve
 
   terragrunt-apply-ses_receiving_emails:
@@ -101,7 +140,7 @@ jobs:
       !contains(needs.*.result, 'failure') &&
       !contains(needs.*.result, 'cancelled')
     runs-on: ubuntu-latest  
-    needs: [terragrunt-apply-common,terragrunt-apply-ecr]
+    needs: [terragrunt-apply-common,terragrunt-apply-ecr,terragrunt-apply-ecr-us-east]
 
     steps:
       - name: Checkout
@@ -350,7 +389,7 @@ jobs:
       !contains(needs.*.result, 'failure') &&
       !contains(needs.*.result, 'cancelled')
     runs-on: ubuntu-latest  
-    needs: [terragrunt-apply-common,terragrunt-apply-elasticache,terragrunt-apply-ecr]
+    needs: [terragrunt-apply-common,terragrunt-apply-elasticache,terragrunt-apply-ecr,terragrunt-apply-ecr-us-east]
 
     steps:
       - name: Checkout
@@ -381,7 +420,7 @@ jobs:
       !contains(needs.*.result, 'failure') &&
       !contains(needs.*.result, 'cancelled')
     runs-on: ubuntu-latest  
-    needs: [terragrunt-apply-common,terragrunt-apply-eks,terragrunt-apply-ecr]
+    needs: [terragrunt-apply-common,terragrunt-apply-eks,terragrunt-apply-ecr-us-east]
 
     steps:
       - name: Checkout
@@ -412,7 +451,7 @@ jobs:
       !contains(needs.*.result, 'failure') &&
       !contains(needs.*.result, 'cancelled')
     runs-on: ubuntu-latest  
-    needs: [terragrunt-apply-common,terragrunt-apply-ecr]
+    needs: [terragrunt-apply-common,terragrunt-apply-ecr-us-east]
 
     steps:
       - name: Checkout
@@ -516,7 +555,7 @@ jobs:
       !contains(needs.*.result, 'failure') &&
       !contains(needs.*.result, 'cancelled')
     runs-on: ubuntu-latest  
-    needs: [terragrunt-apply-common,terragrunt-apply-eks,terragrunt-apply-ecr]
+    needs: [terragrunt-apply-common,terragrunt-apply-eks,terragrunt-apply-ecr-us-east]
 
     steps:
       - name: Checkout
@@ -547,7 +586,7 @@ jobs:
       !contains(needs.*.result, 'failure') &&
       !contains(needs.*.result, 'cancelled')
     runs-on: ubuntu-latest  
-    needs: [terragrunt-apply-common,terragrunt-apply-ecr]
+    needs: [terragrunt-apply-common,terragrunt-apply-ecr-us-east]
 
     steps:
       - name: Checkout
@@ -578,7 +617,7 @@ jobs:
       !contains(needs.*.result, 'failure') &&
       !contains(needs.*.result, 'cancelled')
     runs-on: ubuntu-latest  
-    needs: [terragrunt-apply-common,terragrunt-apply-ecr]
+    needs: [terragrunt-apply-common,terragrunt-apply-ecr-us-east]
 
     steps:
       - name: Checkout
@@ -609,7 +648,7 @@ jobs:
       !contains(needs.*.result, 'failure') &&
       !contains(needs.*.result, 'cancelled')
     runs-on: ubuntu-latest  
-    needs: [terragrunt-apply-common,terragrunt-apply-ecr]
+    needs: [terragrunt-apply-common,terragrunt-apply-ecr-us-east]
 
     steps:
       - name: Checkout
@@ -640,7 +679,7 @@ jobs:
       !contains(needs.*.result, 'failure') &&
       !contains(needs.*.result, 'cancelled')
     runs-on: ubuntu-latest  
-    needs: [terragrunt-apply-common,terragrunt-apply-ecr,terragrunt-apply-rds,terragrunt-apply-eks]
+    needs: [terragrunt-apply-common,terragrunt-apply-ecr-us-east,terragrunt-apply-rds,terragrunt-apply-eks]
 
     steps:
       - name: Checkout
@@ -696,36 +735,36 @@ jobs:
           cd env/${{env.ENVIRONMENT}}/system_status_static_site
           terragrunt apply --terragrunt-non-interactive -auto-approve
 
-  terragrunt-apply-newrelic:
-    if: |
-      always() &&
-      !contains(needs.*.result, 'failure') &&
-      !contains(needs.*.result, 'cancelled')
-    needs: [terragrunt-apply-common]
-    runs-on: ubuntu-latest  
+  # terragrunt-apply-newrelic:
+  #   if: |
+  #     always() &&
+  #     !contains(needs.*.result, 'failure') &&
+  #     !contains(needs.*.result, 'cancelled')
+  #   needs: [terragrunt-apply-common]
+  #   runs-on: ubuntu-latest  
 
-    steps:
-      - name: Checkout
-        uses: actions/checkout@f43a0e5ff2bd294095638e18286ca9a3d1956744 # v3.6.0
+  #   steps:
+  #     - name: Checkout
+  #       uses: actions/checkout@f43a0e5ff2bd294095638e18286ca9a3d1956744 # v3.6.0
 
-      - name: setup-terraform
-        uses: ./.github/actions/setup-terraform
-        with:
-          role_to_assume: arn:aws:iam::${{env.ACCOUNT_ID}}:role/notification-terraform-apply
-          role_session_name: NotifyTerraformApply        
+  #     - name: setup-terraform
+  #       uses: ./.github/actions/setup-terraform
+  #       with:
+  #         role_to_assume: arn:aws:iam::${{env.ACCOUNT_ID}}:role/notification-terraform-apply
+  #         role_session_name: NotifyTerraformApply        
 
-      - name: Install 1Pass CLI
-        run: |
-          curl -o 1pass.deb https://downloads.1password.com/linux/debian/amd64/stable/1password-cli-amd64-latest.deb
-          sudo dpkg -i 1pass.deb
-          sudo mkdir -p aws
-          cd aws
-          op read op://4eyyuwddp6w4vxlabrr2i2duxm/"TFVars - ${{env.ENVIRONMENT}}"/notesPlain > ${{env.ENVIRONMENT}}.tfvars   
+  #     - name: Install 1Pass CLI
+  #       run: |
+  #         curl -o 1pass.deb https://downloads.1password.com/linux/debian/amd64/stable/1password-cli-amd64-latest.deb
+  #         sudo dpkg -i 1pass.deb
+  #         sudo mkdir -p aws
+  #         cd aws
+  #         op read op://4eyyuwddp6w4vxlabrr2i2duxm/"TFVars - ${{env.ENVIRONMENT}}"/notesPlain > ${{env.ENVIRONMENT}}.tfvars   
 
-      - name: terragrunt apply newrelic
-        run: |
-          cd env/${{env.ENVIRONMENT}}/newrelic
-          terragrunt apply --terragrunt-non-interactive -auto-approve
+  #     - name: terragrunt apply newrelic
+  #       run: |
+  #         cd env/${{env.ENVIRONMENT}}/newrelic
+  #         terragrunt apply --terragrunt-non-interactive -auto-approve
 
   deploy-application:
     if: |
@@ -797,7 +836,6 @@ jobs:
           kubectl config rename-context arn:aws:eks:ca-central-1:${{env.ACCOUNT_ID}}:cluster/notification-canada-ca-dev-eks-cluster dev
 
       - name: terragrunt apply k8s-fix
-        continue-on-error: true 
         run: |
           curl -o 1pass.deb https://downloads.1password.com/linux/debian/amd64/stable/1password-cli-amd64-latest.deb
           sudo dpkg -i 1pass.deb

diff --git a/.github/workflows/terragrunt_destroy_environment.yml b/.github/workflows/terragrunt_destroy_environment.yml
@@ -21,7 +21,7 @@ permissions:
 
 jobs:
 
-  terragrunt-destroy-dns:
+  delete-dev-environment:
     if: |
       always() &&
 
@@ -41,7 +41,7 @@ jobs:
       - name: Install AWS nuke
         run: |
           mkdir bin
-          curl -Lo aws-nuke.tar.gz https://github.com/rebuy-de/aws-nuke/releases/download/v${{ env.AWS_NUKE_VERSION }}/aws-nuke-v${{ env.AWS_NUKE_VERSION }}-linux-amd64.tar.gz
+          curl -Lo aws-nuke.tar.gz https://github.com/ekristen/aws-nuke/releases/download/v${{ env.AWS_NUKE_VERSION }}/aws-nuke-v${{ env.AWS_NUKE_VERSION }}-linux-amd64.tar.gz
           tar -xzf aws-nuke.tar.gz -C bin
           chmod +x bin/*
           echo "$GITHUB_WORKSPACE/bin" >> $GITHUB_PATH
@@ -51,7 +51,11 @@ jobs:
           curl -o 1pass.deb https://downloads.1password.com/linux/debian/amd64/stable/1password-cli-amd64-latest.deb
           sudo dpkg -i 1pass.deb          
 
+      - name: Fetch secrets from 1Password
+        run: |
+          op read op://4eyyuwddp6w4vxlabrr2i2duxm/"TFVars - Dev"/notesPlain > aws/dev.tfvars
+
       - name: Destroy notify dev environment
         run: |
-          op read op://4eyyuwddp6w4vxlabrr2i2duxm/"TFVars - Dev"/notesPlain > /var/tmp/dev.tfvars
-          ./scripts/deleteEnvironment.sh dev 800095993820
+          cd scripts
+          ./deleteEnvironment.sh dev 800095993820