JAPSER-101: Configure Vault and Secrets Manager (#52)

* - Updated sync secrets GHA to build the image and push to GCHR
- Added shells script that will pull secrets and push to AWS
- Added sync_secrets yaml for the CronJob template for Openshift

* Stringify the secret_string

* - Added aws_region as a secret
- Try adding back http proxy

* Added test s3 list command

* - Store the proxy-config as secrets
- Add code to sync aws secret keys/ids to OpenShift secrets

* Removed function to make is simpler

* Fixed sh error

* Change the way to parse json data

* Echo the keys

* Fixed missing ]

* Add missing whitespace

* Add debug mode and use =

* Removed debug mode and export new env vars

* Add oc cli to Dockerfile

* Add version checking

* Added oc cli

* Removed version checks

* Make oc executable

* Try using user=root

* add sudo

* - rolesanywhere and private ca commented code
- secret manager updates

* TF updates so secrets are consumed by ECS service

* Updates to install oc cli in docker container

* - Added update-aws-creds cronjob
- Simplify sync-secrets job settings
- Added IAM role policy for openshiftuser

* Fixed filename reference

* Fixed SRC path

* Change strategy matrix and add parsing to iterate on each item

* Added #!/bin/sh

* - Pass correct env variable
- Correct missing .dc.

* Added external/ prefix to secrets to be able to access from Openshift

* Ensure openshiftuser gets created if not exist

* Added kms permission to openshiftuser

* Fixed GHA for publishing openshift shell script images

---------

Co-authored-by: Ronaldo Macapobre <[email protected]>

.github/workflows/publish-openshift-images.yml

@@ -0,0 +1,71 @@
+name: Publish Openshift Images
+
+on:
+  workflow_dispatch:
+  push:
+    paths:
+      - "docker/openshift/**"
+      - "openshift/**"
+
+env:
+  GITHUB_IMAGE_REPO: ghcr.io/bcgov/jasper
+  SRC_PATH: ../../docker/openshift
+
+permissions:
+  id-token: write
+  packages: write
+
+jobs:
+  deploy:
+    runs-on: ubuntu-latest
+    name: Deploy Images
+    strategy:
+      matrix:
+        dockerfile-image:
+          - Dockerfile=./docker/openshift/Dockerfile.sync-secrets,image=sync-secrets
+          - Dockerfile=./docker/openshift/Dockerfile.update-aws-creds,image=update-aws-creds
+
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v4
+
+      - name: Log in to the GHCR
+        uses: docker/login-action@v3
+        with:
+          registry: ghcr.io
+          username: ${{ github.actor }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v3
+        with:
+          driver: docker
+
+      - name: Parse Dockerfile and Image Name
+        id: parse
+        run: |
+          echo "Dockerfile: ${{ matrix.dockerfile-image }}"
+          DOCKERFILE=$(echo "${{ matrix.dockerfile-image }}" | cut -d',' -f1 | cut -d'=' -f2)
+          IMAGE=$(echo "${{ matrix.dockerfile-image }}" | cut -d',' -f2 | cut -d'=' -f2)
+          echo "DOCKERFILE=$DOCKERFILE" >> $GITHUB_ENV
+          echo "IMAGE=$IMAGE" >> $GITHUB_ENV
+
+      - name: Setup Image Metadata
+        id: meta
+        uses: docker/metadata-action@v5
+        with:
+          images: |
+            ${{ env.GITHUB_IMAGE_REPO }}/${{ env.IMAGE }}
+          tags: |
+            type=raw,value=latest
+
+      - name: Build and Push Image to ghcr.io
+        uses: docker/build-push-action@v5
+        with:
+          push: true
+          context: .
+          tags: ${{ steps.meta.outputs.tags }}
+          labels: ${{ steps.meta.outputs.labels }}
+          file: ${{ env.DOCKERFILE }}
+          build-args: |
+            SRC=${{ env.SRC_PATH }}

docker/openshift/Dockerfile.sync-secrets

@@ -0,0 +1,20 @@
+FROM alpine:latest
+ARG VAULT_VERSION="1.17.6"
+ARG APP_ROOT=/usr/local/bin
+ARG SRC=./docker/openshift
+
+# Install dependencies
+RUN apk add --no-cache \
+    jq \
+    aws-cli
+
+WORKDIR ${APP_ROOT}
+
+# Copy the shell script to the container
+COPY ${SRC}/sync-secrets.sh ${APP_ROOT}/sync-secrets.sh
+
+# Ensure shell script has executable permissions
+RUN chmod +x ${APP_ROOT}/sync-secrets.sh
+
+# Command to run the script
+CMD [ "./sync-secrets.sh" ]

docker/openshift/Dockerfile.update-aws-creds

@@ -0,0 +1,32 @@
+FROM alpine:latest
+ARG VAULT_VERSION="1.17.6"
+ARG APP_ROOT=/usr/local/bin
+ARG SRC=./docker/openshift
+
+# Install dependencies
+RUN apk add --no-cache \
+    jq \
+    aws-cli \
+    curl \
+    tar \
+    bash \
+    libc6-compat
+
+# Download and install the OpenShift CLI (oc)
+RUN curl -L https://mirror.openshift.com/pub/openshift-v4/clients/ocp/stable/openshift-client-linux.tar.gz -o /tmp/openshift-client-linux.tar.gz && \
+    tar -zxvf /tmp/openshift-client-linux.tar.gz -C /usr/local/bin && \
+    rm /tmp/openshift-client-linux.tar.gz
+
+WORKDIR ${APP_ROOT}
+
+# Copy the shell script to the container
+COPY ${SRC}/update-aws-creds.sh ${APP_ROOT}/update-aws-creds.sh
+
+# Ensure that shell script and od has executable permissions
+RUN chmod +x ${APP_ROOT}/update-aws-creds.sh oc
+
+# Test if oc is installed correctly
+RUN oc version --client
+
+# Command to run the script
+CMD [ "./update-aws-creds.sh" ]

docker/openshift/sync-secrets.sh

@@ -0,0 +1,41 @@
+#!/bin/sh
+
+# Vault details
+VAULT_SECRET_ENV="${VAULT_SECRET_ENV}"
+LOCAL_SECRET_PATH="${LOCAL_SECRET_PATH}"
+
+aws_secret_format="external/jasper-X-secret-$VAULT_SECRET_ENV"
+secret_keys="\
+  aspnet_core \
+  auth \
+  database \
+  file_services_client \
+  keycloak \
+  location_services_client \
+  lookup_services_client \
+  misc \
+  request \
+  splunk \
+  user_services_client"
+
+echo "Syncing secrets..."
+
+# Iterate on each key to get the value from Vault and save to AWS secrets manager
+for key in $secret_keys; do
+  value=$(jq -r ".${VAULT_SECRET_ENV}_$key" "$LOCAL_SECRET_PATH")
+
+  sanitizedKey=$(echo "$key" | sed "s/_/-/g")
+  secret_name=$(echo "$aws_secret_format" | sed "s/X/$sanitizedKey/")
+  secret_string=$(echo "$value" | jq -c '.')
+
+  echo "Uploading $secret_name"
+  aws secretsmanager put-secret-value \
+    --secret-id $secret_name \
+    --secret-string "$secret_string"
+done
+
+if [ $? -eq 0 ]; then
+  echo "Secrets synced successfully from Vault to AWS Secrets Manager."
+else
+  echo "Failed to sync secrets."
+fi

docker/openshift/update-aws-creds.sh

@@ -0,0 +1,29 @@
+#!/bin/sh
+ENVIRONMENT="${ENVIRONMENT}"
+
+# AWS Access Keys/IDs has a scheduled rotation and needs to be kept up-to-date in OpenShift.
+# https://developer.gov.bc.ca/docs/default/component/public-cloud-techdocs/design-build-and-deploy-an-application/iam-user-service/#setup-automation-to-retrieve-and-use-keys
+echo "Checking if AWS keys needs to be updated..."
+param_value=$(aws ssm get-parameter --name "/iam_users/openshiftuser${ENVIRONMENT}_keys" --with-decryption | jq -r '.Parameter.Value')
+
+if [ $? -eq 0 ]; then
+  pendingAccessKeyId=$(echo "$param_value" | jq -r '.pending_deletion.AccessKeyID')
+  pendingSecretAccessKey=$(echo "$param_value" | jq -r '.pending_deletion.SecretAccessKey')
+  currentAccessKeyId=$(echo "$param_value" | jq -r '.current.AccessKeyID')
+  currentSecretAccessKey=$(echo "$param_value" | jq -r '.current.SecretAccessKey')
+
+  if [ "$AWS_ACCESS_KEY_ID" = "$pendingAccessKeyId" ] || [ "$AWS_SECRET_ACCESS_KEY" = "$pendingSecretAccessKey" ]; then
+    echo "Updating AWS_ACCESS_KEY_ID and AWS_SECRET_ACCESS_KEY..."
+
+    oc create secret generic aws-secret \
+      --from-literal=AWS_ACCESS_KEY_ID=$currentAccessKeyId \
+      --from-literal=AWS_SECRET_ACCESS_KEY=$currentSecretAccessKey \
+      --dry-run=client -o yaml | oc apply -f -
+
+    echo "Done."    
+  else
+    echo "Credentials are up-to-date."
+  fi
+else
+  echo "Failed to query credentials from AWS."
+fi

infrastructure/cloud/environments/dev/dev.tfvars

@@ -3,3 +3,5 @@ test_s3_bucket_name = "jasper-test-s3-bucket-dev"
 web_subnet_names    = ["Web_Dev_aza_net", "Web_Dev_azb_net"]
 app_subnet_names    = ["App_Dev_aza_net", "App_Dev_azb_net"]
 data_subnet_names   = ["Data_Dev_aza_net", "Data_Dev_azb_net"]
+openshift_iam_user  = "openshiftuserdev"
+iam_user_table_name = "BCGOV_IAM_USER_TABLE"

infrastructure/cloud/environments/dev/variables.tf

@@ -42,3 +42,13 @@ variable "data_subnet_names" {
   description = "List of Subnets for Data"
   type        = list(string)
 }
+
+variable "openshift_iam_user" {
+  description = "Openshift IAM Username"
+  type        = string
+}
+
+variable "iam_user_table_name" {
+  description = "The BCGOV DynamoDb IAM user table"
+  type        = string
+}

infrastructure/cloud/environments/dev/webapp.tf

@@ -6,6 +6,8 @@ module "security" {
   ecs_web_td_log_group_arn = module.monitoring.ecs_web_td_log_group_arn
   ecs_api_td_log_group_arn = module.monitoring.ecs_api_td_log_group_arn
   ecr_repository_arn       = module.container.ecr_repository_arn
+  openshift_iam_user       = var.openshift_iam_user
+  iam_user_table_name      = var.iam_user_table_name
 }
 
 module "storage" {
@@ -40,6 +42,10 @@ module "container" {
   ecs_web_td_log_group_name = module.monitoring.ecs_web_td_log_group_name
   ecs_api_td_log_group_name = module.monitoring.ecs_api_td_log_group_name
   kms_key_id                = module.security.kms_key_id
+  lb_dns_name               = module.networking.lb_dns_name
+  api_secrets               = module.security.api_secrets
+  web_secrets               = module.security.web_secrets
+  db_secrets                = module.security.db_secrets
   depends_on                = [module.monitoring]
 }
 

infrastructure/cloud/modules/container/ecs.tf

@@ -38,6 +38,12 @@ resource "aws_ecs_task_definition" "ecs_web_task_definition" {
           "awslogs-stream-prefix" = "ecs"
         }
       }
+      secrets = [
+        for secret in var.web_secrets : {
+          name      = secret[0]
+          valueFrom = secret[1]
+        }
+      ]
     }
   ])
 }
@@ -81,6 +87,18 @@ resource "aws_ecs_task_definition" "ecs_api_task_definition" {
           containerPort = 5000
         }
       ]
+      environment = [
+        {
+          name  = "CORS_DOMAIN"
+          value = var.lb_dns_name
+        }
+      ]
+      secrets = [
+        for secret in var.api_secrets : {
+          name      = secret[0]
+          valueFrom = secret[1]
+        }
+      ]
       logConfiguration = {
         logDriver = "awslogs"
         options = {

infrastructure/cloud/modules/container/variables.tf

@@ -48,3 +48,22 @@ variable "kms_key_id" {
   type        = string
 }
 
+variable "lb_dns_name" {
+  description = "Load Balancer DNS Name"
+  type        = string
+}
+
+variable "api_secrets" {
+  description = "List if env variable secrets used in API"
+  type        = list(list(string))
+}
+
+variable "web_secrets" {
+  description = "List if env variable secrets used in Web"
+  type        = list(list(string))
+}
+
+variable "db_secrets" {
+  description = "List if env variable secrets used in Database"
+  type        = list(list(string))
+}

infrastructure/cloud/modules/networking/outputs.tf

@@ -9,3 +9,7 @@ output "ecs_sg_id" {
 output "web_subnets_ids" {
   value = local.web_subnets
 }
+
+output "lb_dns_name" {
+  value = aws_lb.lb.dns_name
+}

infrastructure/cloud/modules/security/acmpca.tf

@@ -0,0 +1,46 @@
+# resource "aws_acmpca_certificate_authority" "acmpca_ca" {
+#   type                          = "ROOT"
+#   usage_mode                    = "GENERAL_PURPOSE"
+#   key_storage_security_standard = "FIPS_140_2_LEVEL_3_OR_HIGHER"
+#   certificate_authority_configuration {
+#     key_algorithm     = "RSA_2048"
+#     signing_algorithm = "SHA256WITHRSA"
+#     subject {
+#       country                      = "CA"
+#       organization                 = "bcgov"
+#       organizational_unit          = "bccourts"
+#       distinguished_name_qualifier = "${var.app_name}-ca-${var.environment}"
+#       common_name                  = "${var.app_name}-ca-${var.environment}"
+#       state                        = "BC"
+#       locality                     = "Vancouver"
+#     }
+#   }
+
+#   tags = {
+#     Name = "${var.app_name}-acmpca-${var.environment}"
+#   }
+# }
+
+# resource "aws_acmpca_permission" "acmpca_permission" {
+#   certificate_authority_arn = aws_acmpca_certificate_authority.acmpca_ca.arn
+#   actions                   = ["IssueCertificate", "GetCertificate", "ListPermissions"]
+#   principal                 = "acm.amazonaws.com"
+# }
+
+# resource "aws_acmpca_certificate" "acmpca_certificate" {
+#   certificate_authority_arn   = aws_acmpca_certificate_authority.acmpca_ca.arn
+#   certificate_signing_request = aws_acmpca_certificate_authority.acmpca_ca.certificate_signing_request
+#   signing_algorithm           = "SHA256WITHRSA"
+#   template_arn                = "arn:aws:acm-pca:::template/RootCACertificate/V1"
+#   validity {
+#     type  = "YEARS"
+#     value = 3
+#   }
+# }
+
+# resource "aws_acmpca_certificate_authority_certificate" "acmpca_cac" {
+#   certificate_authority_arn = aws_acmpca_certificate_authority.acmpca_ca.arn
+
+#   certificate       = aws_acmpca_certificate.acmpca_certificate.certificate
+#   certificate_chain = aws_acmpca_certificate.acmpca_certificate.certificate_chain
+# }