From 02261ba4d992ed72365b3fbacb2cabdef206b846 Mon Sep 17 00:00:00 2001
From: vacabor <166112501+vacabor@users.noreply.github.com>
Date: Thu, 20 Jun 2024 08:48:44 +0100
Subject: [PATCH 01/18] Production deployment workflow

---
 .github/workflows/build_and_deploy.yml        | 46 ++++++++++++++++++-
 terraform/application/application.tf          |  4 ++
 .../application/config/production.tfvars.json | 11 +++++
 .../application/config/production_Terrafile   |  3 ++
 .../application/config/production_app_env.yml |  1 +
 .../application/config/review.tfvars.json     |  2 +-
 terraform/application/variables.tf            | 12 +++++
 7 files changed, 77 insertions(+), 2 deletions(-)
 create mode 100644 terraform/application/config/production.tfvars.json
 create mode 100644 terraform/application/config/production_Terrafile
 create mode 100644 terraform/application/config/production_app_env.yml

diff --git a/.github/workflows/build_and_deploy.yml b/.github/workflows/build_and_deploy.yml
index 5db9b8b0c0..7310d690d5 100644
--- a/.github/workflows/build_and_deploy.yml
+++ b/.github/workflows/build_and_deploy.yml
@@ -1,6 +1,8 @@
 name: Build and deploy to AKS cluster
 
 on:
+  push:
+    branches: [master]
   pull_request:
     types: [labeled, opened, reopened, synchronize]
 
@@ -13,7 +15,6 @@ permissions:
 jobs:
   build:
     runs-on: ubuntu-latest
-    if: ${{ contains(github.event.pull_request.labels.*.name, 'deploy') }}
     outputs:
       docker-image-tag: ${{ steps.build-image.outputs.tag }}
 
@@ -33,6 +34,7 @@ jobs:
     name: Deploy to review environment
     concurrency: deploy_review_${{ github.event.pull_request.number }}
     runs-on: ubuntu-latest
+    if: ${{ contains(github.event.pull_request.labels.*.name, 'deploy') }}
     needs: [build]
     environment:
       name: review-aks
@@ -78,3 +80,45 @@ jobs:
             | Additional Payments | <${{ env.APP_URL }}/additional-payments/claim> |
             | Student Loans       | <${{ env.APP_URL }}/student-loans/claim>       |
             | Admin               | <${{ env.APP_URL }}/admin>                     |
+
+  deploy_production:
+    name: Deploy to production environment
+    concurrency: deploy_production
+    runs-on: ubuntu-latest
+    if: github.ref == 'refs/heads/master' && github.event_name == 'push'
+    needs: [build]
+    environment:
+      name: production-aks
+      url: ${{ steps.deploy.outputs.environment_url }}
+    outputs:
+      environment_url: ${{ steps.deploy.outputs.environment_url }}
+
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v4
+
+      - uses: azure/login@v2
+        with:
+          creds: ${{ secrets.AZURE_CREDENTIALS }}
+
+      - uses: ./.github/actions/deploy-environment
+        id: deploy
+        with:
+          environment: production-aks
+          docker-image: ${{ needs.build.outputs.docker-image-tag }}
+          azure-credentials: ${{ secrets.AZURE_CREDENTIALS }}
+
+      - name: Run migrations
+        shell: bash
+        run: |
+          make ci production-aks get-cluster-credentials
+          kubectl exec -n srtl-test deployment/claim-additional-payments-for-teaching-production-worker -- sh -c "DISABLE_DATABASE_ENVIRONMENT_CHECK=1 bin/prepare-database"
+
+      - name: Slack notification
+        if: failure()
+        uses: rtCamp/action-slack-notify@master
+        env:
+          SLACK_COLOR: failure
+          SLACK_TITLE: Failure deploying release to production
+          SLACK_MESSAGE: Failure deploying release to production - Docker tag ${{ needs.build.outputs.docker-image-tag }}
+          SLACK_WEBHOOK: ${{ secrets.SLACK_WEBHOOK_URL }}
diff --git a/terraform/application/application.tf b/terraform/application/application.tf
index 86e04a9e25..67ee359e52 100644
--- a/terraform/application/application.tf
+++ b/terraform/application/application.tf
@@ -38,6 +38,8 @@ module "web_application" {
 
   docker_image = var.docker_image
   command      = var.startup_command
+
+  replicas     = var.web_replicas
 }
 
 module "worker_application" {
@@ -56,4 +58,6 @@ module "worker_application" {
 
   docker_image = var.docker_image
   command      = var.worker_command
+
+  replicas     = var.worker_replicas
 }
diff --git a/terraform/application/config/production.tfvars.json b/terraform/application/config/production.tfvars.json
new file mode 100644
index 0000000000..58f0591935
--- /dev/null
+++ b/terraform/application/config/production.tfvars.json
@@ -0,0 +1,11 @@
+{
+  "cluster": "production",
+  "namespace": "srtl-production",
+  "config": "production",
+  "environment": "production",
+  "canonical_hostname": "claim-additional-teaching-payment.service.gov.uk",
+  "web_replicas": 2,
+  "worker_replicas": 2,
+  "startup_command": ["/bin/sh", "-c", "bin/rails server -b 0.0.0.0"],
+  "worker_command": ["/bin/sh", "-c", "bin/bundle exec bin/delayed_job run -n 1"]
+}
\ No newline at end of file
diff --git a/terraform/application/config/production_Terrafile b/terraform/application/config/production_Terrafile
new file mode 100644
index 0000000000..5b2b118f00
--- /dev/null
+++ b/terraform/application/config/production_Terrafile
@@ -0,0 +1,3 @@
+aks:
+    source:  "https://github.com/DFE-Digital/terraform-modules"
+    version: "stable"
diff --git a/terraform/application/config/production_app_env.yml b/terraform/application/config/production_app_env.yml
new file mode 100644
index 0000000000..ed97d539c0
--- /dev/null
+++ b/terraform/application/config/production_app_env.yml
@@ -0,0 +1 @@
+---
diff --git a/terraform/application/config/review.tfvars.json b/terraform/application/config/review.tfvars.json
index 39f0b0d215..5d80a8ed97 100644
--- a/terraform/application/config/review.tfvars.json
+++ b/terraform/application/config/review.tfvars.json
@@ -5,5 +5,5 @@
   "deploy_azure_backing_services": false,
   "enable_postgres_ssl": false,
   "startup_command": ["/bin/sh", "-c", "bin/rails server -b 0.0.0.0"],
-  "worker_command": ["/bin/sh", "-c", "bin/bundle exec bin/delayed_job run -n 4"]
+  "worker_command": ["/bin/sh", "-c", "bin/bundle exec bin/delayed_job run -n 1"]
 }
diff --git a/terraform/application/variables.tf b/terraform/application/variables.tf
index 323f7aeab2..7765c1cd2f 100644
--- a/terraform/application/variables.tf
+++ b/terraform/application/variables.tf
@@ -61,6 +61,18 @@ variable "canonical_hostname" {
   default     = null
 }
 
+variable "web_replicas" {
+  type        = integer
+  description = "Number of instances of the web container to run"
+  default     = 1
+}
+
+variable "worker_replicas" {
+  type        = integer
+  description = "Number of instances of the worker container to run"
+  default     = 1
+}
+
 locals {
   postgres_ssl_mode = var.enable_postgres_ssl ? "require" : "disable"
   canonical_hostname = var.canonical_hostname != null ? var.canonical_hostname : "${var.service_name}-${var.environment}-web.test.teacherservices.cloud"

From 85e9d48e66722a2f599801c7119b5aaff630bc4e Mon Sep 17 00:00:00 2001
From: vacabor <166112501+vacabor@users.noreply.github.com>
Date: Thu, 20 Jun 2024 08:54:25 +0100
Subject: [PATCH 02/18] Trailing newline

---
 terraform/application/config/production.tfvars.json | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/terraform/application/config/production.tfvars.json b/terraform/application/config/production.tfvars.json
index 58f0591935..f9321e2b61 100644
--- a/terraform/application/config/production.tfvars.json
+++ b/terraform/application/config/production.tfvars.json
@@ -8,4 +8,4 @@
   "worker_replicas": 2,
   "startup_command": ["/bin/sh", "-c", "bin/rails server -b 0.0.0.0"],
   "worker_command": ["/bin/sh", "-c", "bin/bundle exec bin/delayed_job run -n 1"]
-}
\ No newline at end of file
+}

From a75e7d9ad6bc16f6ef435dba709ad921a1d57920 Mon Sep 17 00:00:00 2001
From: vacabor <166112501+vacabor@users.noreply.github.com>
Date: Thu, 20 Jun 2024 12:12:24 +0100
Subject: [PATCH 03/18] Use a datatype that exists

---
 terraform/application/variables.tf | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/terraform/application/variables.tf b/terraform/application/variables.tf
index 7765c1cd2f..5f5f11e9b0 100644
--- a/terraform/application/variables.tf
+++ b/terraform/application/variables.tf
@@ -62,13 +62,13 @@ variable "canonical_hostname" {
 }
 
 variable "web_replicas" {
-  type        = integer
+  type        = number
   description = "Number of instances of the web container to run"
   default     = 1
 }
 
 variable "worker_replicas" {
-  type        = integer
+  type        = number
   description = "Number of instances of the worker container to run"
   default     = 1
 }

From 788c5e519fac3e08184d43f6ad7dd1ba908a79b9 Mon Sep 17 00:00:00 2001
From: vacabor <166112501+vacabor@users.noreply.github.com>
Date: Mon, 24 Jun 2024 09:12:19 +0100
Subject: [PATCH 04/18] Wait for test deployment before deploying to production

---
 .github/workflows/build_and_deploy.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/build_and_deploy.yml b/.github/workflows/build_and_deploy.yml
index 91320237cd..df3670839b 100644
--- a/.github/workflows/build_and_deploy.yml
+++ b/.github/workflows/build_and_deploy.yml
@@ -142,7 +142,7 @@ jobs:
     concurrency: deploy_production
     runs-on: ubuntu-latest
     if: github.ref == 'refs/heads/master' && github.event_name == 'push'
-    needs: [build]
+    needs: [deploy_test]
     environment:
       name: production-aks
       url: ${{ steps.deploy.outputs.environment_url }}

From 5c7c3342374c08bcd66ddb6b924faa952446b228 Mon Sep 17 00:00:00 2001
From: vacabor <166112501+vacabor@users.noreply.github.com>
Date: Mon, 24 Jun 2024 16:17:14 +0100
Subject: [PATCH 05/18] Add HA and PG backups for production

---
 terraform/application/application.tf          | 10 +++----
 .../application/config/production.tfvars.json | 10 ++++++-
 terraform/application/database.tf             | 27 ++++++++++---------
 terraform/application/variables.tf            | 20 +++++++++++---
 4 files changed, 45 insertions(+), 22 deletions(-)

diff --git a/terraform/application/application.tf b/terraform/application/application.tf
index 67ee359e52..a9436e58bf 100644
--- a/terraform/application/application.tf
+++ b/terraform/application/application.tf
@@ -13,10 +13,10 @@ module "application_configuration" {
   config_variables = merge(
     local.app_env_values,
     {
-      ENVIRONMENT_NAME = var.environment
-      PGSSLMODE        = local.postgres_ssl_mode
+      ENVIRONMENT_NAME   = var.environment
+      PGSSLMODE          = local.postgres_ssl_mode
       CANONICAL_HOSTNAME = local.canonical_hostname
-    })
+  })
   secret_variables = {
     DATABASE_URL = module.postgres.url
   }
@@ -39,7 +39,7 @@ module "web_application" {
   docker_image = var.docker_image
   command      = var.startup_command
 
-  replicas     = var.web_replicas
+  replicas = var.web_replicas
 }
 
 module "worker_application" {
@@ -59,5 +59,5 @@ module "worker_application" {
   docker_image = var.docker_image
   command      = var.worker_command
 
-  replicas     = var.worker_replicas
+  replicas = var.worker_replicas
 }
diff --git a/terraform/application/config/production.tfvars.json b/terraform/application/config/production.tfvars.json
index f9321e2b61..c557d51f54 100644
--- a/terraform/application/config/production.tfvars.json
+++ b/terraform/application/config/production.tfvars.json
@@ -7,5 +7,13 @@
   "web_replicas": 2,
   "worker_replicas": 2,
   "startup_command": ["/bin/sh", "-c", "bin/rails server -b 0.0.0.0"],
-  "worker_command": ["/bin/sh", "-c", "bin/bundle exec bin/delayed_job run -n 1"]
+  "worker_command": ["/bin/sh", "-c", "bin/bundle exec bin/delayed_job run -n 1"],
+  "postgres_flexible_server_sku": "GP_Standard_D2ds_v4",
+  "postgres_enable_high_availability": true,
+  "enable_postgres_backup_storage": true,
+  "azure_maintenance_window": {
+    "day_of_week": 0,
+    "start_hour": 3,
+    "start_minute": 0
+  }
 }
diff --git a/terraform/application/database.tf b/terraform/application/database.tf
index 957b6c2b82..1d363336c1 100644
--- a/terraform/application/database.tf
+++ b/terraform/application/database.tf
@@ -1,16 +1,19 @@
 module "postgres" {
   source = "./vendor/modules/aks//aks/postgres"
 
-  namespace                   = var.namespace
-  environment                 = var.environment
-  azure_resource_prefix       = var.azure_resource_prefix
-  service_name                = var.service_name
-  service_short               = var.service_short
-  config_short                = var.config_short
-  cluster_configuration_map   = module.cluster_data.configuration_map
-  use_azure                   = var.deploy_azure_backing_services
-  azure_enable_monitoring     = var.enable_monitoring
-  azure_enable_backup_storage = var.enable_postgres_backup_storage
-  azure_extensions            = ["pg_trgm", "pgcrypto", "plpgsql"]
-  server_version              = "16"
+  namespace                      = var.namespace
+  environment                    = var.environment
+  azure_resource_prefix          = var.azure_resource_prefix
+  service_name                   = var.service_name
+  service_short                  = var.service_short
+  config_short                   = var.config_short
+  cluster_configuration_map      = module.cluster_data.configuration_map
+  use_azure                      = var.deploy_azure_backing_services
+  azure_enable_monitoring        = var.enable_monitoring
+  azure_enable_backup_storage    = var.enable_postgres_backup_storage
+  azure_extensions               = ["pg_trgm", "pgcrypto", "plpgsql"]
+  server_version                 = "16"
+  azure_sku_name                 = var.postgres_flexible_server_sku
+  azure_enable_high_availability = var.postgres_enable_high_availability
+  azure_maintenance_window       = var.azure_maintenance_window
 }
diff --git a/terraform/application/variables.tf b/terraform/application/variables.tf
index 444a10a983..032cd11e6d 100644
--- a/terraform/application/variables.tf
+++ b/terraform/application/variables.tf
@@ -62,7 +62,7 @@ variable "canonical_hostname" {
 }
 variable "web_replicas" {
   description = "Number of replicas of the web app"
-  default = 1
+  default     = 1
 }
 
 variable "worker_replicas" {
@@ -70,9 +70,21 @@ variable "worker_replicas" {
   default     = 1
 }
 
+variable "azure_maintenance_window" {
+  default = null
+}
+
+variable "postgres_flexible_server_sku" {
+  default = "B_Standard_B1ms"
+}
+
+variable "postgres_enable_high_availability" {
+  default = false
+}
+
 locals {
-  postgres_ssl_mode = var.enable_postgres_ssl ? "require" : "disable"
-  canonical_hostname = var.canonical_hostname != null ? var.canonical_hostname : "${var.service_name}-${var.environment}-web.test.teacherservices.cloud"
+  postgres_ssl_mode       = var.enable_postgres_ssl ? "require" : "disable"
+  canonical_hostname      = var.canonical_hostname != null ? var.canonical_hostname : "${var.service_name}-${var.environment}-web.test.teacherservices.cloud"
   app_env_values_from_yml = yamldecode(file("${path.module}/config/${var.config}_app_env.yml"))
-  app_env_values = merge(local.app_env_values_from_yml)
+  app_env_values          = merge(local.app_env_values_from_yml)
 }

From 48c7070f2f9781acf0860887979b600725ff0c12 Mon Sep 17 00:00:00 2001
From: vacabor <166112501+vacabor@users.noreply.github.com>
Date: Tue, 25 Jun 2024 10:18:17 +0100
Subject: [PATCH 06/18] Refactor running migrations into deployment action

---
 .github/actions/deploy-environment/action.yml | 20 +++++++++++++++++++
 .github/workflows/build_and_deploy.yml        | 19 +++++++-----------
 2 files changed, 27 insertions(+), 12 deletions(-)

diff --git a/.github/actions/deploy-environment/action.yml b/.github/actions/deploy-environment/action.yml
index b1d052fd88..b373670342 100644
--- a/.github/actions/deploy-environment/action.yml
+++ b/.github/actions/deploy-environment/action.yml
@@ -14,6 +14,16 @@ inputs:
   pull-request-number:
     description: The pull request number which triggered this deploy.
     required: false
+  prepare-database:
+    description: Whether to prepare the database by running migrations or seeding if needed
+    required: false
+    default: true
+  aks-namespace:
+    description: The namespace under which kubectl will run
+    required: true
+  aks-deployment:
+    description: The deployment which kubectl will target for running migrations
+    required: true
 
 runs:
   using: composite
@@ -42,3 +52,13 @@ runs:
       run: |
         output=$(terraform -chdir=terraform/application output -raw url)
         echo "APP_URL=$output" >> $GITHUB_ENV
+
+    - name: Set kubectl
+      uses: DFE-Digital/github-actions/set-kubectl@master
+
+    - name: Prepare database
+      if: ${{ inputs.prepare-database }}
+      shell: bash
+      run: |
+        make ci ${{ inputs.environment }} get-cluster-credentials
+        kubectl exec -n ${{ inputs.aks-namespace }} deployment/${{ inputs.aks-deployment }}-worker -- sh -c "DISABLE_DATABASE_ENVIRONMENT_CHECK=1 bin/prepare-database"
diff --git a/.github/workflows/build_and_deploy.yml b/.github/workflows/build_and_deploy.yml
index 58f3d7ed8a..24dad30ebb 100644
--- a/.github/workflows/build_and_deploy.yml
+++ b/.github/workflows/build_and_deploy.yml
@@ -50,6 +50,9 @@ jobs:
           docker-image: ${{ needs.build.outputs.docker-image-tag }}
           azure-credentials: ${{ secrets.AZURE_CREDENTIALS }}
           pull-request-number: ${{ github.event.pull_request.number }}
+          aks-namespace: srtl-development
+          aks-deployment: claim-additional-payments-for-teaching-review-${{ github.event.pull_request.number }}
+          prepare-database: ${{ github.event.pull_request.number != '' }}
 
       - uses: azure/login@v2
         with:
@@ -108,12 +111,8 @@ jobs:
           environment: test-aks
           docker-image: ${{ needs.build.outputs.docker-image-tag }}
           azure-credentials: ${{ secrets.AZURE_CREDENTIALS }}
-
-      - name: Run migrations
-        shell: bash
-        run: |
-          make ci test-aks get-cluster-credentials
-          kubectl exec -n srtl-test deployment/claim-additional-payments-for-teaching-test-worker -- sh -c "DISABLE_DATABASE_ENVIRONMENT_CHECK=1 bin/prepare-database"
+          aks-namespace: srtl-test
+          aks-deployment: claim-additional-payments-for-teaching-test
 
       - name: Install Ruby
         uses: ruby/setup-ruby@v1
@@ -164,12 +163,8 @@ jobs:
           environment: production-aks
           docker-image: ${{ needs.build.outputs.docker-image-tag }}
           azure-credentials: ${{ secrets.AZURE_CREDENTIALS }}
-
-      - name: Run migrations
-        shell: bash
-        run: |
-          make ci production-aks get-cluster-credentials
-          kubectl exec -n srtl-production deployment/claim-additional-payments-for-teaching-production-worker -- sh -c "DISABLE_DATABASE_ENVIRONMENT_CHECK=1 bin/prepare-database"
+          aks-namespace: srtl-production
+          aks-deployment: claim-additional-payments-for-teaching-production
 
       - name: Install Ruby
         uses: ruby/setup-ruby@v1

From 21aaab548d526956d6ef93995187d6b4b5410a12 Mon Sep 17 00:00:00 2001
From: vacabor <166112501+vacabor@users.noreply.github.com>
Date: Tue, 25 Jun 2024 10:19:49 +0100
Subject: [PATCH 07/18] WIP: Run the action on this branch for testing

---
 .github/workflows/build_and_deploy.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/build_and_deploy.yml b/.github/workflows/build_and_deploy.yml
index 24dad30ebb..fae68e6174 100644
--- a/.github/workflows/build_and_deploy.yml
+++ b/.github/workflows/build_and_deploy.yml
@@ -2,7 +2,7 @@ name: Build and deploy to AKS cluster
 
 on:
   push:
-    branches: [master]
+    branches: [CAPT-1703-prod-aks-migration]
   pull_request:
     types: [labeled, opened, reopened, synchronize]
 

From a4dfafadf0b233bcf8887fb78cea794e44511373 Mon Sep 17 00:00:00 2001
From: vacabor <166112501+vacabor@users.noreply.github.com>
Date: Tue, 25 Jun 2024 11:34:29 +0100
Subject: [PATCH 08/18] Provide PR_NUMBER when running migrations

---
 .github/actions/deploy-environment/action.yml | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/.github/actions/deploy-environment/action.yml b/.github/actions/deploy-environment/action.yml
index b373670342..e102a453fe 100644
--- a/.github/actions/deploy-environment/action.yml
+++ b/.github/actions/deploy-environment/action.yml
@@ -62,3 +62,5 @@ runs:
       run: |
         make ci ${{ inputs.environment }} get-cluster-credentials
         kubectl exec -n ${{ inputs.aks-namespace }} deployment/${{ inputs.aks-deployment }}-worker -- sh -c "DISABLE_DATABASE_ENVIRONMENT_CHECK=1 bin/prepare-database"
+      env:
+        PR_NUMBER: ${{ inputs.pull-request-number }}

From 97ad20942d8af52c98fbe0e0a2e1c2f726402fe7 Mon Sep 17 00:00:00 2001
From: vacabor <166112501+vacabor@users.noreply.github.com>
Date: Tue, 25 Jun 2024 15:56:48 +0100
Subject: [PATCH 09/18] Perform azure/login ahead of migrations

---
 .github/actions/deploy-environment/action.yml |  4 ++++
 .github/workflows/build_and_deploy.yml        | 16 ----------------
 2 files changed, 4 insertions(+), 16 deletions(-)

diff --git a/.github/actions/deploy-environment/action.yml b/.github/actions/deploy-environment/action.yml
index e102a453fe..31ab84369e 100644
--- a/.github/actions/deploy-environment/action.yml
+++ b/.github/actions/deploy-environment/action.yml
@@ -53,6 +53,10 @@ runs:
         output=$(terraform -chdir=terraform/application output -raw url)
         echo "APP_URL=$output" >> $GITHUB_ENV
 
+    - uses: azure/login@v2
+      with:
+        creds: ${{ inputs.azure-credentials }}
+
     - name: Set kubectl
       uses: DFE-Digital/github-actions/set-kubectl@master
 
diff --git a/.github/workflows/build_and_deploy.yml b/.github/workflows/build_and_deploy.yml
index fae68e6174..3a8a82c89a 100644
--- a/.github/workflows/build_and_deploy.yml
+++ b/.github/workflows/build_and_deploy.yml
@@ -54,22 +54,6 @@ jobs:
           aks-deployment: claim-additional-payments-for-teaching-review-${{ github.event.pull_request.number }}
           prepare-database: ${{ github.event.pull_request.number != '' }}
 
-      - uses: azure/login@v2
-        with:
-          creds: ${{ secrets.AZURE_CREDENTIALS }}
-
-      - name: Set kubectl
-        uses: DFE-Digital/github-actions/set-kubectl@master
-
-      - name: Seed review app
-        shell: bash
-        if: github.event.number != ''
-        run: |
-          make ci review-aks get-cluster-credentials
-          kubectl exec -n srtl-development deployment/claim-additional-payments-for-teaching-review-${{ github.event.pull_request.number }}-worker -- sh -c "DISABLE_DATABASE_ENVIRONMENT_CHECK=1 bin/prepare-database"
-        env:
-          PR_NUMBER: ${{ github.event.pull_request.number }}
-
       - name: Post comment to Pull Request ${{ github.event.number }}
         if: ${{ github.event_name == 'pull_request' }}
         uses: marocchino/sticky-pull-request-comment@v2

From 3d551b3c1c65084e23491fc77eca3ad590e3be0b Mon Sep 17 00:00:00 2001
From: vacabor <166112501+vacabor@users.noreply.github.com>
Date: Tue, 25 Jun 2024 16:23:34 +0100
Subject: [PATCH 10/18] WIP: Run the test/prod deploys on this branch for
 testing

---
 .github/workflows/build_and_deploy.yml | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/.github/workflows/build_and_deploy.yml b/.github/workflows/build_and_deploy.yml
index 3a8a82c89a..b428be4bae 100644
--- a/.github/workflows/build_and_deploy.yml
+++ b/.github/workflows/build_and_deploy.yml
@@ -73,7 +73,7 @@ jobs:
     name: Deploy to test environment
     concurrency: deploy_test
     runs-on: ubuntu-latest
-    if: github.ref == 'refs/heads/master' && github.event_name == 'push'
+    # if: github.ref == 'refs/heads/master' && github.event_name == 'push'
     needs: [build]
     environment:
       name: test-aks
@@ -125,7 +125,7 @@ jobs:
     name: Deploy to production environment
     concurrency: deploy_production
     runs-on: ubuntu-latest
-    if: github.ref == 'refs/heads/master' && github.event_name == 'push'
+    # if: github.ref == 'refs/heads/master' && github.event_name == 'push'
     needs: [deploy_test]
     environment:
       name: production-aks

From 94014648f84663af467b280db4420a4b8f2ce175 Mon Sep 17 00:00:00 2001
From: vacabor <166112501+vacabor@users.noreply.github.com>
Date: Wed, 26 Jun 2024 10:31:22 +0100
Subject: [PATCH 11/18] Turn off HA initially to allow the SKU to change

---
 terraform/application/config/production.tfvars.json | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/terraform/application/config/production.tfvars.json b/terraform/application/config/production.tfvars.json
index c557d51f54..8ef86b971d 100644
--- a/terraform/application/config/production.tfvars.json
+++ b/terraform/application/config/production.tfvars.json
@@ -9,7 +9,7 @@
   "startup_command": ["/bin/sh", "-c", "bin/rails server -b 0.0.0.0"],
   "worker_command": ["/bin/sh", "-c", "bin/bundle exec bin/delayed_job run -n 1"],
   "postgres_flexible_server_sku": "GP_Standard_D2ds_v4",
-  "postgres_enable_high_availability": true,
+  "postgres_enable_high_availability": false,
   "enable_postgres_backup_storage": true,
   "azure_maintenance_window": {
     "day_of_week": 0,

From da478836c0270c8d1af73a786f72bbca57e835c2 Mon Sep 17 00:00:00 2001
From: vacabor <166112501+vacabor@users.noreply.github.com>
Date: Wed, 26 Jun 2024 10:55:15 +0100
Subject: [PATCH 12/18] Revert "Turn off HA initially to allow the SKU to
 change"

This reverts commit 94014648f84663af467b280db4420a4b8f2ce175.
---
 terraform/application/config/production.tfvars.json | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/terraform/application/config/production.tfvars.json b/terraform/application/config/production.tfvars.json
index 8ef86b971d..c557d51f54 100644
--- a/terraform/application/config/production.tfvars.json
+++ b/terraform/application/config/production.tfvars.json
@@ -9,7 +9,7 @@
   "startup_command": ["/bin/sh", "-c", "bin/rails server -b 0.0.0.0"],
   "worker_command": ["/bin/sh", "-c", "bin/bundle exec bin/delayed_job run -n 1"],
   "postgres_flexible_server_sku": "GP_Standard_D2ds_v4",
-  "postgres_enable_high_availability": false,
+  "postgres_enable_high_availability": true,
   "enable_postgres_backup_storage": true,
   "azure_maintenance_window": {
     "day_of_week": 0,

From e974aa684577ee18ae594c154463a3c107ac30db Mon Sep 17 00:00:00 2001
From: vacabor <166112501+vacabor@users.noreply.github.com>
Date: Thu, 27 Jun 2024 10:08:24 +0100
Subject: [PATCH 13/18] Require the build job for prod deploy per PR feedback

---
 .github/workflows/build_and_deploy.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/build_and_deploy.yml b/.github/workflows/build_and_deploy.yml
index b428be4bae..e0219e4d87 100644
--- a/.github/workflows/build_and_deploy.yml
+++ b/.github/workflows/build_and_deploy.yml
@@ -126,7 +126,7 @@ jobs:
     concurrency: deploy_production
     runs-on: ubuntu-latest
     # if: github.ref == 'refs/heads/master' && github.event_name == 'push'
-    needs: [deploy_test]
+    needs: [build, deploy_test]
     environment:
       name: production-aks
       url: ${{ steps.deploy.outputs.environment_url }}

From 7312f50925e3d0e8ea3ab4b43190c586c1fa4d6b Mon Sep 17 00:00:00 2001
From: vacabor <166112501+vacabor@users.noreply.github.com>
Date: Thu, 27 Jun 2024 17:40:14 +0100
Subject: [PATCH 14/18] Revert "WIP: Run the action on this branch for testing"

This reverts commit 21aaab548d526956d6ef93995187d6b4b5410a12.
---
 .github/workflows/build_and_deploy.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/build_and_deploy.yml b/.github/workflows/build_and_deploy.yml
index e0219e4d87..baa15108e6 100644
--- a/.github/workflows/build_and_deploy.yml
+++ b/.github/workflows/build_and_deploy.yml
@@ -2,7 +2,7 @@ name: Build and deploy to AKS cluster
 
 on:
   push:
-    branches: [CAPT-1703-prod-aks-migration]
+    branches: [master]
   pull_request:
     types: [labeled, opened, reopened, synchronize]
 

From a4ee510235f5f209f67bdbbff848219a5ad0c7bc Mon Sep 17 00:00:00 2001
From: vacabor <166112501+vacabor@users.noreply.github.com>
Date: Thu, 27 Jun 2024 17:43:22 +0100
Subject: [PATCH 15/18] Revert "WIP: Run the test/prod deploys on this branch
 for testing"

This reverts commit 3d551b3c1c65084e23491fc77eca3ad590e3be0b.
---
 .github/workflows/build_and_deploy.yml | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/.github/workflows/build_and_deploy.yml b/.github/workflows/build_and_deploy.yml
index baa15108e6..ea3b566ed5 100644
--- a/.github/workflows/build_and_deploy.yml
+++ b/.github/workflows/build_and_deploy.yml
@@ -73,7 +73,7 @@ jobs:
     name: Deploy to test environment
     concurrency: deploy_test
     runs-on: ubuntu-latest
-    # if: github.ref == 'refs/heads/master' && github.event_name == 'push'
+    if: github.ref == 'refs/heads/master' && github.event_name == 'push'
     needs: [build]
     environment:
       name: test-aks
@@ -125,7 +125,7 @@ jobs:
     name: Deploy to production environment
     concurrency: deploy_production
     runs-on: ubuntu-latest
-    # if: github.ref == 'refs/heads/master' && github.event_name == 'push'
+    if: github.ref == 'refs/heads/master' && github.event_name == 'push'
     needs: [build, deploy_test]
     environment:
       name: production-aks

From 24cd1fbb432ae55b05bb6a4791dfd3ee5313a7c3 Mon Sep 17 00:00:00 2001
From: vacabor <166112501+vacabor@users.noreply.github.com>
Date: Fri, 28 Jun 2024 10:14:36 +0100
Subject: [PATCH 16/18] WIP: Use the new production canonical_hostname for
 testing

---
 .github/workflows/build_and_deploy.yml              | 6 +++---
 terraform/application/config/production.tfvars.json | 2 +-
 2 files changed, 4 insertions(+), 4 deletions(-)

diff --git a/.github/workflows/build_and_deploy.yml b/.github/workflows/build_and_deploy.yml
index ea3b566ed5..e0219e4d87 100644
--- a/.github/workflows/build_and_deploy.yml
+++ b/.github/workflows/build_and_deploy.yml
@@ -2,7 +2,7 @@ name: Build and deploy to AKS cluster
 
 on:
   push:
-    branches: [master]
+    branches: [CAPT-1703-prod-aks-migration]
   pull_request:
     types: [labeled, opened, reopened, synchronize]
 
@@ -73,7 +73,7 @@ jobs:
     name: Deploy to test environment
     concurrency: deploy_test
     runs-on: ubuntu-latest
-    if: github.ref == 'refs/heads/master' && github.event_name == 'push'
+    # if: github.ref == 'refs/heads/master' && github.event_name == 'push'
     needs: [build]
     environment:
       name: test-aks
@@ -125,7 +125,7 @@ jobs:
     name: Deploy to production environment
     concurrency: deploy_production
     runs-on: ubuntu-latest
-    if: github.ref == 'refs/heads/master' && github.event_name == 'push'
+    # if: github.ref == 'refs/heads/master' && github.event_name == 'push'
     needs: [build, deploy_test]
     environment:
       name: production-aks
diff --git a/terraform/application/config/production.tfvars.json b/terraform/application/config/production.tfvars.json
index c557d51f54..3d3ad6831a 100644
--- a/terraform/application/config/production.tfvars.json
+++ b/terraform/application/config/production.tfvars.json
@@ -3,7 +3,7 @@
   "namespace": "srtl-production",
   "config": "production",
   "environment": "production",
-  "canonical_hostname": "claim-additional-teaching-payment.service.gov.uk",
+  "canonical_hostname": "claim-additional-payments-for-teaching-production-web.teacherservices.cloud",
   "web_replicas": 2,
   "worker_replicas": 2,
   "startup_command": ["/bin/sh", "-c", "bin/rails server -b 0.0.0.0"],

From 9c16a5caa4613d4e1f951504828f143f8ae0b7bc Mon Sep 17 00:00:00 2001
From: vacabor <166112501+vacabor@users.noreply.github.com>
Date: Fri, 5 Jul 2024 11:47:18 +0100
Subject: [PATCH 17/18] Clean up duplicate workflow steps with a matrix

---
 .github/workflows/build_and_deploy.yml | 74 +++++---------------------
 1 file changed, 13 insertions(+), 61 deletions(-)

diff --git a/.github/workflows/build_and_deploy.yml b/.github/workflows/build_and_deploy.yml
index e0219e4d87..a261cf1fee 100644
--- a/.github/workflows/build_and_deploy.yml
+++ b/.github/workflows/build_and_deploy.yml
@@ -69,17 +69,21 @@ jobs:
             | Further Education   | <${{ env.APP_URL }}/further-education-payments/landing-page> |
             | Admin               | <${{ env.APP_URL }}/admin>                                   |
 
-  deploy_test:
-    name: Deploy to test environment
-    concurrency: deploy_test
+  deploy:
+    name: Deploy to ${{ matrix.environment }}
     runs-on: ubuntu-latest
+    concurrency: deploy_${{ matrix.environment }}
     # if: github.ref == 'refs/heads/master' && github.event_name == 'push'
     needs: [build]
     environment:
-      name: test-aks
+      name: ${{ matrix.environment }}-aks
       url: ${{ steps.deploy.outputs.environment_url }}
     outputs:
       environment_url: ${{ steps.deploy.outputs.environment_url }}
+    strategy:
+      max-parallel: 1
+      matrix:
+        environment: [test, production]
 
     steps:
       - name: Checkout code
@@ -92,11 +96,11 @@ jobs:
       - uses: ./.github/actions/deploy-environment
         id: deploy
         with:
-          environment: test-aks
+          environment: ${{ matrix.environment }}-aks
           docker-image: ${{ needs.build.outputs.docker-image-tag }}
           azure-credentials: ${{ secrets.AZURE_CREDENTIALS }}
-          aks-namespace: srtl-test
-          aks-deployment: claim-additional-payments-for-teaching-test
+          aks-namespace: srtl-${{ matrix.environment }}
+          aks-deployment: claim-additional-payments-for-teaching-${{ matrix.environment }}
 
       - name: Install Ruby
         uses: ruby/setup-ruby@v1
@@ -117,58 +121,6 @@ jobs:
         uses: rtCamp/action-slack-notify@master
         env:
           SLACK_COLOR: failure
-          SLACK_TITLE: Failure deploying release to test
-          SLACK_MESSAGE: Failure deploying release to test - Docker tag ${{ needs.build.outputs.docker-image-tag }}
-          SLACK_WEBHOOK: ${{ secrets.SLACK_WEBHOOK_URL }}
-
-  deploy_production:
-    name: Deploy to production environment
-    concurrency: deploy_production
-    runs-on: ubuntu-latest
-    # if: github.ref == 'refs/heads/master' && github.event_name == 'push'
-    needs: [build, deploy_test]
-    environment:
-      name: production-aks
-      url: ${{ steps.deploy.outputs.environment_url }}
-    outputs:
-      environment_url: ${{ steps.deploy.outputs.environment_url }}
-
-    steps:
-      - name: Checkout code
-        uses: actions/checkout@v4
-
-      - uses: azure/login@v2
-        with:
-          creds: ${{ secrets.AZURE_CREDENTIALS }}
-
-      - uses: ./.github/actions/deploy-environment
-        id: deploy
-        with:
-          environment: production-aks
-          docker-image: ${{ needs.build.outputs.docker-image-tag }}
-          azure-credentials: ${{ secrets.AZURE_CREDENTIALS }}
-          aks-namespace: srtl-production
-          aks-deployment: claim-additional-payments-for-teaching-production
-
-      - name: Install Ruby
-        uses: ruby/setup-ruby@v1
-        with:
-          bundler-cache: true
-
-      - name: Run smoke tests
-        shell: bash
-        run: bundle exec rspec spec/smoke -t smoke:true -b
-        env:
-          RAILS_ENV: test
-          SMOKE_TEST_APP_HOST: ${{ env.APP_URL }}
-          BASIC_AUTH_USERNAME: ${{ secrets.BASIC_AUTH_USERNAME }}
-          BASIC_AUTH_PASSWORD: ${{ secrets.BASIC_AUTH_PASSWORD }}
-
-      - name: Notify on failure
-        if: failure()
-        uses: rtCamp/action-slack-notify@master
-        env:
-          SLACK_COLOR: failure
-          SLACK_TITLE: Failure deploying release to production
-          SLACK_MESSAGE: Failure deploying release to production - Docker tag ${{ needs.build.outputs.docker-image-tag }}
+          SLACK_TITLE: Failure deploying release to ${{ matrix.environment }}
+          SLACK_MESSAGE: Failure deploying release to ${{ matrix.environment }} - Docker tag ${{ needs.build.outputs.docker-image-tag }}
           SLACK_WEBHOOK: ${{ secrets.SLACK_WEBHOOK_URL }}

From e514f834bf0706e95ba489510363707a9da2b57f Mon Sep 17 00:00:00 2001
From: vacabor <166112501+vacabor@users.noreply.github.com>
Date: Wed, 10 Jul 2024 12:22:50 +0100
Subject: [PATCH 18/18] Switch back to targetting master

---
 .github/workflows/build_and_deploy.yml | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/.github/workflows/build_and_deploy.yml b/.github/workflows/build_and_deploy.yml
index 3d022a3454..fedfaff3d8 100644
--- a/.github/workflows/build_and_deploy.yml
+++ b/.github/workflows/build_and_deploy.yml
@@ -2,7 +2,7 @@ name: Build and deploy to AKS cluster
 
 on:
   push:
-    branches: [CAPT-1703-prod-aks-migration]
+    branches: [master]
   pull_request:
     types: [labeled, opened, reopened, synchronize]
 
@@ -74,7 +74,7 @@ jobs:
     name: Deploy to ${{ matrix.environment }}
     runs-on: ubuntu-latest
     concurrency: deploy_${{ matrix.environment }}
-    # if: github.ref == 'refs/heads/master' && github.event_name == 'push'
+    if: github.ref == 'refs/heads/master' && github.event_name == 'push'
     needs: [build]
     environment:
       name: ${{ matrix.environment }}-aks