diff --git a/.github/workflows/merge_to_main_production.yml b/.github/workflows/merge_to_main_production.yml index dfdad9c81..2b177e033 100644 --- a/.github/workflows/merge_to_main_production.yml +++ b/.github/workflows/merge_to_main_production.yml @@ -693,6 +693,39 @@ jobs: cd env/${{env.ENVIRONMENT}}/newrelic terragrunt apply --terragrunt-non-interactive -auto-approve + terragrunt-apply-manifest_secrets: + if: | + always() && + !contains(needs.*.result, 'failure') && + !contains(needs.*.result, 'cancelled') + needs: [terragrunt-apply-rds, terragrunt-apply-elasticache, terragrunt-apply-eks, terragrunt-apply-ecr, terragrunt-apply-lambda-api, terragrunt-apply-heartbeat, terragrunt-apply-database-tools, terragrunt-apply-quicksight, terragrunt-apply-lambda-google-cidr, terragrunt-apply-ses_to_sqs_email_callbacks, terragrunt-apply-sns_to_sqs_sms_callbacks, terragrunt-apply-pinpoint_to_sqs_sms_callbacks, terragrunt-apply-system_status, terragrunt-apply-system_status_static_site, terragrunt-apply-newrelic] + runs-on: ubuntu-latest + + steps: + - name: Checkout + uses: actions/checkout@f43a0e5ff2bd294095638e18286ca9a3d1956744 # v3.6.0 + + - name: setup-terraform + uses: ./.github/actions/setup-terraform + with: + role_to_assume: arn:aws:iam::${{env.ACCOUNT_ID}}:role/notification-terraform-apply + role_session_name: NotifyTerraformApply + + - name: Install 1Pass CLI + run: | + curl -o 1pass.deb https://downloads.1password.com/linux/debian/amd64/stable/1password-cli-amd64-latest.deb + sudo dpkg -i 1pass.deb + sudo mkdir -p aws + cd aws + op read op://ppnxsriom3alsxj4ogikyjxlzi/"TERRAFORM_SECRETS_${{env.ENVIRONMENT}}"/notesPlain > ${{env.ENVIRONMENT}}.tfvars + + - name: terragrunt apply manifest_secrets + run: | + cd env/${{env.ENVIRONMENT}}/manifest_secrets + terragrunt apply --terragrunt-non-interactive -auto-approve + + + bump-version-and-push-tag: if: | always() && diff --git a/.github/workflows/merge_to_main_staging.yml b/.github/workflows/merge_to_main_staging.yml index 3dbb62b2d..c627e8a1b 100644 --- a/.github/workflows/merge_to_main_staging.yml +++ b/.github/workflows/merge_to_main_staging.yml @@ -765,7 +765,7 @@ jobs: always() && !contains(needs.*.result, 'failure') && !contains(needs.*.result, 'cancelled') - needs: [terragrunt-apply-rds] + needs: [terragrunt-apply-rds, terragrunt-apply-elasticache, terragrunt-apply-eks, terragrunt-apply-ecr, terragrunt-apply-lambda-api, terragrunt-apply-lambda-admin-pr, terragrunt-apply-performance-test, terragrunt-apply-heartbeat, terragrunt-apply-database-tools, terragrunt-apply-quicksight, terragrunt-apply-lambda-google-cidr, terragrunt-apply-ses_to_sqs_email_callbacks, terragrunt-apply-sns_to_sqs_sms_callbacks, terragrunt-apply-pinpoint_to_sqs_sms_callbacks, terragrunt-apply-system_status, terragrunt-apply-system_status_static_site, terragrunt-apply-newrelic] runs-on: ubuntu-latest steps: diff --git a/.github/workflows/terragrunt_create_dev_environment.yml b/.github/workflows/terragrunt_create_dev_environment.yml index 260c597c1..1ec9359e0 100644 --- a/.github/workflows/terragrunt_create_dev_environment.yml +++ b/.github/workflows/terragrunt_create_dev_environment.yml @@ -772,6 +772,37 @@ jobs: # cd env/${{env.ENVIRONMENT}}/newrelic # terragrunt apply --terragrunt-non-interactive -auto-approve + terragrunt-apply-manifest_secrets: + if: | + always() && + !contains(needs.*.result, 'failure') && + !contains(needs.*.result, 'cancelled') + needs: [terragrunt-apply-common, terragrunt-apply-eks, terragrunt-apply-rds, terragrunt-apply-elasticache, terragrunt-apply-ecr, terragrunt-apply-ecr-us-east, terragrunt-apply-lambda-admin-pr, terragrunt-apply-performance-test, terragrunt-apply-heartbeat, terragrunt-apply-database-tools, terragrunt-apply-lambda-google-cidr, terragrunt-apply-ses_to_sqs_email_callbacks, terragrunt-apply-sns_to_sqs_sms_callbacks, terragrunt-apply-pinpoint_to_sqs_sms_callbacks, terragrunt-apply-system_status, terragrunt-apply-system_status_static_site] + runs-on: ubuntu-latest + + steps: + - name: Checkout + uses: actions/checkout@f43a0e5ff2bd294095638e18286ca9a3d1956744 # v3.6.0 + + - name: setup-terraform + uses: ./.github/actions/setup-terraform + with: + role_to_assume: arn:aws:iam::${{env.ACCOUNT_ID}}:role/notification-terraform-apply + role_session_name: NotifyTerraformApply + + - name: Install 1Pass CLI + run: | + curl -o 1pass.deb https://downloads.1password.com/linux/debian/amd64/stable/1password-cli-amd64-latest.deb + sudo dpkg -i 1pass.deb + sudo mkdir -p aws + cd aws + op read op://4eyyuwddp6w4vxlabrr2i2duxm/"TERRAFORM_SECRETS_${{env.ENVIRONMENT}}"/notesPlain > ${{env.ENVIRONMENT}}.tfvars + + - name: terragrunt apply manifest_secrets + run: | + cd env/${{env.ENVIRONMENT}}/manifest_secrets + terragrunt apply --terragrunt-non-interactive -auto-approve + deploy-application: if: | always() && diff --git a/.github/workflows/terragrunt_plan_dev.yml b/.github/workflows/terragrunt_plan_dev.yml index 8dd6600c3..2f36b99da 100644 --- a/.github/workflows/terragrunt_plan_dev.yml +++ b/.github/workflows/terragrunt_plan_dev.yml @@ -38,7 +38,8 @@ jobs: system_status: ${{ steps.filter.outputs.system_status }} system_status_static_site: ${{ steps.filter.outputs.system_status_static_site }} newrelic: ${{ steps.filter.outputs.newrelic }} - + manifest_secrets: ${{ steps.filter.outputs.manifest_secrets }} + steps: - uses: dorny/paths-filter@4512585405083f25c027a35db413c2b3b9006d50 # v2.11.1 id: filter @@ -112,6 +113,9 @@ jobs: newrelic: - 'aws/newrelic/**' - 'env/${{env.ENVIRONMENT}}/newrelic/**' + manifest_secrets: + - 'aws/manifest_secrets/**' + - 'env/${{env.ENVIRONMENT}}/manifest_secrets/**' terragrunt-plan-common: runs-on: ubuntu-latest @@ -867,3 +871,37 @@ jobs: terragrunt: "true" terraform-init: | -upgrade + + + terragrunt-plan-manifest_secrets: + if: | + always() && + !contains(needs.*.result, 'failure') && + !contains(needs.*.result, 'cancelled') + needs: [terragrunt-plan-rds, terragrunt-plan-elasticache, terragrunt-plan-eks, terragrunt-plan-lambda-api, terragrunt-plan-lambda-admin-pr, terragrunt-plan-performance-test, terragrunt-plan-heartbeat, terragrunt-plan-database-tools, terragrunt-plan-quicksight, terragrunt-plan-lambda-google-cidr, terragrunt-plan-ses_to_sqs_email_callbacks, terragrunt-plan-sns_to_sqs_sms_callbacks, terragrunt-plan-pinpoint_to_sqs_sms_callbacks, terragrunt-plan-system_status, terragrunt-plan-system_status_static_site, terragrunt-plan-newrelic] + runs-on: ubuntu-latest + env: + COMPONENT: "manifest_secrets" + steps: + - name: Checkout + uses: actions/checkout@f43a0e5ff2bd294095638e18286ca9a3d1956744 # v3.6.0 + - uses: ./.github/actions/setup-terraform + with: + role_to_assume: arn:aws:iam::${{env.ACCOUNT_ID}}:role/notification-terraform-apply + role_session_name: NotifyTerraformApply + - name: Install 1Pass CLI and Download TFVars + run: | + curl -o 1pass.deb https://downloads.1password.com/linux/debian/amd64/stable/1password-cli-amd64-latest.deb + sudo dpkg -i 1pass.deb + sudo mkdir -p aws && cd aws + op read op://4eyyuwddp6w4vxlabrr2i2duxm/"TERRAFORM_SECRETS_${{env.ENVIRONMENT}}"/notesPlain > ${{env.ENVIRONMENT}}.tfvars + - name: Terragrunt plan ${{env.COMPONENT}} + uses: cds-snc/terraform-plan@7f4ce4a4bdffaba639d32a45272804e37a569408 # v3.0.6 + with: + directory: "env/${{env.ENVIRONMENT}}/${{env.COMPONENT}}" + comment-delete: "true" + comment-title: "${{env.ENVIRONMENT}}: ${{env.COMPONENT}}" + github-token: "${{ secrets.GITHUB_TOKEN }}" + terragrunt: "true" + terraform-init: | + -upgrade \ No newline at end of file diff --git a/.github/workflows/terragrunt_plan_production.yml b/.github/workflows/terragrunt_plan_production.yml index 2da901f55..10e69e860 100644 --- a/.github/workflows/terragrunt_plan_production.yml +++ b/.github/workflows/terragrunt_plan_production.yml @@ -706,3 +706,36 @@ jobs: terragrunt: "true" terraform-init: | -upgrade + + terragrunt-plan-manifest_secrets: + if: | + always() && + !contains(needs.*.result, 'failure') && + !contains(needs.*.result, 'cancelled') + needs: [terragrunt-plan-rds, terragrunt-plan-elasticache, terragrunt-plan-eks, terragrunt-plan-lambda-api, terragrunt-plan-heartbeat, terragrunt-plan-database-tools, terragrunt-plan-quicksight, terragrunt-plan-lambda-google-cidr, terragrunt-plan-ses_to_sqs_email_callbacks, terragrunt-plan-sns_to_sqs_sms_callbacks, terragrunt-plan-pinpoint_to_sqs_sms_callbacks, terragrunt-plan-system_status, terragrunt-plan-system_status_static_site, terragrunt-plan-newrelic] + runs-on: ubuntu-latest + env: + COMPONENT: "manifest_secrets" + steps: + - name: Checkout + uses: actions/checkout@f43a0e5ff2bd294095638e18286ca9a3d1956744 # v3.6.0 + - uses: ./.github/actions/setup-terraform + with: + role_to_assume: arn:aws:iam::${{env.ACCOUNT_ID}}:role/notification-terraform-plan + role_session_name: NotifyTerraformPlan + - name: Install 1Pass CLI and Download TFVars + run: | + curl -o 1pass.deb https://downloads.1password.com/linux/debian/amd64/stable/1password-cli-amd64-latest.deb + sudo dpkg -i 1pass.deb + sudo mkdir -p aws && cd aws + op read op://ppnxsriom3alsxj4ogikyjxlzi/"TERRAFORM_SECRETS_${{env.ENVIRONMENT}}"/notesPlain > ${{env.ENVIRONMENT}}.tfvars + - name: Terragrunt plan ${{env.COMPONENT}} + uses: cds-snc/terraform-plan@7f4ce4a4bdffaba639d32a45272804e37a569408 # v3.0.6 + with: + directory: "env/${{env.ENVIRONMENT}}/${{env.COMPONENT}}" + comment-delete: "true" + comment-title: "${{env.ENVIRONMENT}}: ${{env.COMPONENT}}" + github-token: "${{ secrets.GITHUB_TOKEN }}" + terragrunt: "true" + terraform-init: | + -upgrade \ No newline at end of file diff --git a/.github/workflows/terragrunt_plan_staging.yml b/.github/workflows/terragrunt_plan_staging.yml index 432d731eb..806275a07 100644 --- a/.github/workflows/terragrunt_plan_staging.yml +++ b/.github/workflows/terragrunt_plan_staging.yml @@ -886,7 +886,7 @@ jobs: always() && !contains(needs.*.result, 'failure') && !contains(needs.*.result, 'cancelled') - needs: [terragrunt-plan-rds] + needs: [terragrunt-plan-rds, terragrunt-plan-elasticache, terragrunt-plan-eks, terragrunt-plan-lambda-api, terragrunt-plan-lambda-admin-pr, terragrunt-plan-performance-test, terragrunt-plan-heartbeat, terragrunt-plan-database-tools, terragrunt-plan-quicksight, terragrunt-plan-lambda-google-cidr, terragrunt-plan-ses_to_sqs_email_callbacks, terragrunt-plan-sns_to_sqs_sms_callbacks, terragrunt-plan-pinpoint_to_sqs_sms_callbacks, terragrunt-plan-system_status, terragrunt-plan-system_status_static_site, terragrunt-plan-newrelic] runs-on: ubuntu-latest env: COMPONENT: "manifest_secrets" diff --git a/aws/manifest_secrets/secrets.tf b/aws/manifest_secrets/secrets.tf index d371ca6e2..98d98d150 100644 --- a/aws/manifest_secrets/secrets.tf +++ b/aws/manifest_secrets/secrets.tf @@ -329,17 +329,19 @@ resource "aws_secretsmanager_secret_version" "manifest_aws_pinpoint_default_pool } resource "aws_secretsmanager_secret" "manifest_sqlalachemy_database_uri" { - name = "SQLALCHEMY_DATABASE_URI" + name = "MANIFEST_SQLALCHEMY_DATABASE_URI" recovery_window_in_days = 0 } +# THESE BELOW ARE ARE DEPENDENT ON DYNAMICALLY GENERATED AWS INFORMATION + resource "aws_secretsmanager_secret_version" "manifest_sqlalachemy_database_uri" { secret_id = aws_secretsmanager_secret.manifest_sqlalachemy_database_uri.id secret_string = "postgresql://${var.app_db_user}:${var.app_db_user_password}@${var.database_read_write_proxy_endpoint}/${var.app_db_database_name}" } resource "aws_secretsmanager_secret" "manifest_sqlalachemy_database_reader_uri" { - name = "SQLALCHEMY_DATABASE_READER_URI" + name = "MANIFEST_SQLALCHEMY_DATABASE_READER_URI" recovery_window_in_days = 0 } @@ -355,23 +357,35 @@ resource "aws_secretsmanager_secret" "manifest_postgres_host" { resource "aws_secretsmanager_secret_version" "manifest_postgres_host_version" { secret_id = aws_secretsmanager_secret.manifest_postgres_host.id - secret_string = "notification-canada-ca-${var.env}-cluster.${var.postgres_rds_instance_id}.${var.region}.rds.amazonaws.com" + secret_string = var.postgres_cluster_endpoint +} + +resource "aws_secretsmanager_secret" "manifest_postgres_sql" { + name = "MANIFEST_POSTGRES_SQL" + recovery_window_in_days = 0 +} + +resource "aws_secretsmanager_secret_version" "manifest_postgres_sql_version" { + secret_id = aws_secretsmanager_secret.manifest_postgres_sql.id + secret_string = "postgresql://${var.app_db_user}:${var.app_db_user_password}@${var.database_read_write_proxy_endpoint}/${var.app_db_database_name}" } resource "aws_secretsmanager_secret" "manifest_redis_publish_url" { - name = "MANIFEST_REDIS_PUBLISH_URL" + name = "MANIFEST_REDIS_PUBLISH_URL" + recovery_window_in_days = 0 } resource "aws_secretsmanager_secret_version" "manifest_redis_publish_url" { secret_id = aws_secretsmanager_secret.manifest_redis_publish_url.id - secret_string = "redis://notify-${var.env}-cluster-cache-az.${var.redis_cluster_security_group_id}.ng.0001.cac1.cache.amazonaws.com:6379" + secret_string = "redis://${var.redis_primary_endpoint_address}" } resource "aws_secretsmanager_secret" "manifest_redis_url" { - name = "MANIFEST_REDIS_URL" + name = "MANIFEST_REDIS_URL" + recovery_window_in_days = 0 } resource "aws_secretsmanager_secret_version" "manifest_redis_url" { secret_id = aws_secretsmanager_secret.manifest_redis_url.id - secret_string = "redis://notify-${var.env}-cluster-cache-az.${var.redis_cluster_security_group_id}.ng.0001.cac1.cache.amazonaws.com:6379" + secret_string = "redis://${var.redis_primary_endpoint_address}" } diff --git a/aws/manifest_secrets/variables.tf b/aws/manifest_secrets/variables.tf index 740fd7f0b..a8ea0f64f 100644 --- a/aws/manifest_secrets/variables.tf +++ b/aws/manifest_secrets/variables.tf @@ -6,10 +6,10 @@ variable "database_read_only_proxy_endpoint" { type = string } -variable "postgres_rds_instance_id" { +variable "postgres_cluster_endpoint" { type = string } -variable "redis_cluster_security_group_id" { +variable "redis_primary_endpoint_address" { type = string -} \ No newline at end of file +} diff --git a/env/dev/manifest_secrets/terragrunt.hcl b/env/dev/manifest_secrets/terragrunt.hcl new file mode 100644 index 000000000..f9390d309 --- /dev/null +++ b/env/dev/manifest_secrets/terragrunt.hcl @@ -0,0 +1,36 @@ +terraform { + source = "${get_env("ENVIRONMENT") == "production" ? "git::https://github.com/cds-snc/notification-terraform//aws/manifest_secrets?ref=v${get_env("INFRASTRUCTURE_VERSION")}" : "../../../aws//manifest_secrets"}" +} + +dependencies { + paths = ["../rds", "../elasticache"] +} + +dependency "rds" { + config_path = "../rds" + mock_outputs_allowed_terraform_commands = ["validate", "plan"] + mock_outputs = { + database_read_only_proxy_endpoint = "thisisamockstring_database_read_only_proxy_endpoint" + database_read_write_proxy_endpoint = "thisisamockstring_database_read_write_proxy_endpoint" + postgres_cluster_endpoint = "thisisamockstring_postgres_cluster_endpoint" + } +} + +dependency "elasticache" { + config_path = "../elasticache" + mock_outputs_allowed_terraform_commands = ["validate", "plan"] + mock_outputs = { + redis_primary_endpoint_address = "thisisamockstring_redis_primary_endpoint_address" + } +} + +include { + path = find_in_parent_folders() +} + +inputs = { + database_read_only_proxy_endpoint = dependency.rds.outputs.database_read_only_proxy_endpoint + database_read_write_proxy_endpoint = dependency.rds.outputs.database_read_write_proxy_endpoint + postgres_cluster_endpoint = dependency.rds.outputs.postgres_cluster_endpoint + redis_primary_endpoint_address = dependency.elasticache.outputs.redis_primary_endpoint_address +} diff --git a/env/production/manifest_secrets/terragrunt.hcl b/env/production/manifest_secrets/terragrunt.hcl new file mode 100644 index 000000000..f9390d309 --- /dev/null +++ b/env/production/manifest_secrets/terragrunt.hcl @@ -0,0 +1,36 @@ +terraform { + source = "${get_env("ENVIRONMENT") == "production" ? "git::https://github.com/cds-snc/notification-terraform//aws/manifest_secrets?ref=v${get_env("INFRASTRUCTURE_VERSION")}" : "../../../aws//manifest_secrets"}" +} + +dependencies { + paths = ["../rds", "../elasticache"] +} + +dependency "rds" { + config_path = "../rds" + mock_outputs_allowed_terraform_commands = ["validate", "plan"] + mock_outputs = { + database_read_only_proxy_endpoint = "thisisamockstring_database_read_only_proxy_endpoint" + database_read_write_proxy_endpoint = "thisisamockstring_database_read_write_proxy_endpoint" + postgres_cluster_endpoint = "thisisamockstring_postgres_cluster_endpoint" + } +} + +dependency "elasticache" { + config_path = "../elasticache" + mock_outputs_allowed_terraform_commands = ["validate", "plan"] + mock_outputs = { + redis_primary_endpoint_address = "thisisamockstring_redis_primary_endpoint_address" + } +} + +include { + path = find_in_parent_folders() +} + +inputs = { + database_read_only_proxy_endpoint = dependency.rds.outputs.database_read_only_proxy_endpoint + database_read_write_proxy_endpoint = dependency.rds.outputs.database_read_write_proxy_endpoint + postgres_cluster_endpoint = dependency.rds.outputs.postgres_cluster_endpoint + redis_primary_endpoint_address = dependency.elasticache.outputs.redis_primary_endpoint_address +} diff --git a/env/sandbox/manifest_secrets/terragrunt.hcl b/env/sandbox/manifest_secrets/terragrunt.hcl new file mode 100644 index 000000000..f9390d309 --- /dev/null +++ b/env/sandbox/manifest_secrets/terragrunt.hcl @@ -0,0 +1,36 @@ +terraform { + source = "${get_env("ENVIRONMENT") == "production" ? "git::https://github.com/cds-snc/notification-terraform//aws/manifest_secrets?ref=v${get_env("INFRASTRUCTURE_VERSION")}" : "../../../aws//manifest_secrets"}" +} + +dependencies { + paths = ["../rds", "../elasticache"] +} + +dependency "rds" { + config_path = "../rds" + mock_outputs_allowed_terraform_commands = ["validate", "plan"] + mock_outputs = { + database_read_only_proxy_endpoint = "thisisamockstring_database_read_only_proxy_endpoint" + database_read_write_proxy_endpoint = "thisisamockstring_database_read_write_proxy_endpoint" + postgres_cluster_endpoint = "thisisamockstring_postgres_cluster_endpoint" + } +} + +dependency "elasticache" { + config_path = "../elasticache" + mock_outputs_allowed_terraform_commands = ["validate", "plan"] + mock_outputs = { + redis_primary_endpoint_address = "thisisamockstring_redis_primary_endpoint_address" + } +} + +include { + path = find_in_parent_folders() +} + +inputs = { + database_read_only_proxy_endpoint = dependency.rds.outputs.database_read_only_proxy_endpoint + database_read_write_proxy_endpoint = dependency.rds.outputs.database_read_write_proxy_endpoint + postgres_cluster_endpoint = dependency.rds.outputs.postgres_cluster_endpoint + redis_primary_endpoint_address = dependency.elasticache.outputs.redis_primary_endpoint_address +} diff --git a/env/staging/manifest_secrets/.terraform.lock.hcl b/env/staging/elasticache/.terraform.lock.hcl similarity index 100% rename from env/staging/manifest_secrets/.terraform.lock.hcl rename to env/staging/elasticache/.terraform.lock.hcl diff --git a/env/staging/manifest_secrets/terragrunt.hcl b/env/staging/manifest_secrets/terragrunt.hcl index f71609190..f9390d309 100644 --- a/env/staging/manifest_secrets/terragrunt.hcl +++ b/env/staging/manifest_secrets/terragrunt.hcl @@ -8,10 +8,20 @@ dependencies { dependency "rds" { config_path = "../rds" + mock_outputs_allowed_terraform_commands = ["validate", "plan"] + mock_outputs = { + database_read_only_proxy_endpoint = "thisisamockstring_database_read_only_proxy_endpoint" + database_read_write_proxy_endpoint = "thisisamockstring_database_read_write_proxy_endpoint" + postgres_cluster_endpoint = "thisisamockstring_postgres_cluster_endpoint" + } } dependency "elasticache" { config_path = "../elasticache" + mock_outputs_allowed_terraform_commands = ["validate", "plan"] + mock_outputs = { + redis_primary_endpoint_address = "thisisamockstring_redis_primary_endpoint_address" + } } include { @@ -21,6 +31,6 @@ include { inputs = { database_read_only_proxy_endpoint = dependency.rds.outputs.database_read_only_proxy_endpoint database_read_write_proxy_endpoint = dependency.rds.outputs.database_read_write_proxy_endpoint - postgres_rds_instance_id = dependency.rds.outputs.rds_instance_id - redis_cluster_security_group_id = dependency.elasticache.outputs.redis_cluster_security_group_id + postgres_cluster_endpoint = dependency.rds.outputs.postgres_cluster_endpoint + redis_primary_endpoint_address = dependency.elasticache.outputs.redis_primary_endpoint_address }