From 297ce130e5c29afd081cd218b7f4f8107490d537 Mon Sep 17 00:00:00 2001 From: Jeremy Frasier Date: Tue, 21 May 2024 10:08:34 -0400 Subject: [PATCH 01/25] Modify the ansible-core pin to ensure a good version is used We can do this because new versions of ansible-core (2.16.7 and 2.17.0) have been released that do not suffer from the bug discussed in ansible/ansible#82702. This bug broke any symlinked files in vars, tasks, etc. for any Ansible role installed via ansible-galaxy. All versions later than ansible-core 2.16.7 and 2.17.0 should function as expected. --- requirements.txt | 16 ++++++---------- 1 file changed, 6 insertions(+), 10 deletions(-) diff --git a/requirements.txt b/requirements.txt index ec3e967..bdf26b2 100644 --- a/requirements.txt +++ b/requirements.txt @@ -13,19 +13,15 @@ # often breaking changes across major versions. This is the reason # for the upper bound. ansible>=8,<10 -# TODO: Remove this pin when possible. See -# cisagov/skeleton-packer#312 for more details. -# -# ansible-core 2.16.3 and later suffer from the bug discussed in +# ansible-core 2.16.3 through 2.16.6 suffer from the bug discussed in # ansible/ansible#82702, which breaks any symlinked files in vars, # tasks, etc. for any Ansible role installed via ansible-galaxy. +# Hence we never want to install those versions. # -# See also cisagov/skeleton-ansible-role#178 and -# cisagov/skeleton-generic#180. Note from these PRs that any changes -# made to this dependency must also be made in requirements-test.txt -# in cisagov/skeleton-ansible-role and .pre-commit-config.yaml in -# cisagov/skeleton-generic. -ansible-core<2.16.3 +# Note that any changes made to this dependency must also be made in +# requirements-test.txt in cisagov/skeleton-ansible-role and +# .pre-commit-config.yaml in cisagov/skeleton-generic. +ansible-core>=2.16.7 boto3 docopt semver From 07e2b60f912ada6de4de0dcf0573bbecc0f2037c Mon Sep 17 00:00:00 2001 From: Jeremy Frasier Date: Sat, 4 May 2024 10:09:12 -0400 Subject: [PATCH 02/25] Pin ansible-core when running the ansible-lint linter New versions of ansible-core (2.16.7 and 2.17.0) have been released that do not suffer from the bug discussed in ansible/ansible#82702. This bug broke any symlinked files in vars, tasks, etc. for any Ansible role installed via ansible-galaxy. All versions later than ansible-core 2.16.7 and 2.17.0 should function as expected. Co-authored-by: Nick <50747025+mcdonnnj@users.noreply.github.com> --- .pre-commit-config.yaml | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 2c5b3c8..895384c 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -155,6 +155,17 @@ repos: rev: v24.2.0 hooks: - id: ansible-lint + additional_dependencies: + # ansible-core 2.16.3 through 2.16.6 suffer from the bug + # discussed in ansible/ansible#82702, which breaks any + # symlinked files in vars, tasks, etc. for any Ansible role + # installed via ansible-galaxy. Hence we never want to + # install those versions. + # + # Note that any changes made to this dependency must also be + # made in requirements.txt in cisagov/skeleton-packer and + # requirements-test.txt in cisagov/skeleton-ansible-role. + - ansible-core>=2.16.7 # files: molecule/default/playbook.yml # Terraform hooks From c74e5db75b7e3785a3f7196365b7ba99f9a004ea Mon Sep 17 00:00:00 2001 From: Jeremy Frasier Date: Sat, 4 May 2024 10:11:08 -0400 Subject: [PATCH 03/25] Remove unnecessary line The line is not only unnecessary, it was commented out to boot! --- .pre-commit-config.yaml | 1 - 1 file changed, 1 deletion(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 895384c..11772d9 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -166,7 +166,6 @@ repos: # made in requirements.txt in cisagov/skeleton-packer and # requirements-test.txt in cisagov/skeleton-ansible-role. - ansible-core>=2.16.7 - # files: molecule/default/playbook.yml # Terraform hooks - repo: https://github.com/antonbabenko/pre-commit-terraform From 2e53e0de98caa5d5a42320618d8e604c770da7d9 Mon Sep 17 00:00:00 2001 From: Jeremy Frasier Date: Tue, 7 May 2024 15:21:06 -0400 Subject: [PATCH 04/25] Explain why ansible may need to be added as a dependency for ansible-lint On its own ansible-lint does not pull in ansible, only ansible-core. Therefore, if an Ansible module lives in ansible instead of ansible-core, the linter will complain that the module is unknown. In these cases it is necessary to add the ansible package itself as an additional dependency, with the same pinning as is done in requirements-test.txt of cisagov/skeleton-ansible-role. --- .pre-commit-config.yaml | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 11772d9..a48e196 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -156,6 +156,14 @@ repos: hooks: - id: ansible-lint additional_dependencies: + # On its own ansible-lint does not pull in ansible, only + # ansible-core. Therefore, if an Ansible module lives in + # ansible instead of ansible-core, the linter will complain + # that the module is unknown. In these cases it is + # necessary to add the ansible package itself as an + # additional dependency, with the same pinning as is done in + # requirements-test.txt of cisagov/skeleton-ansible-role. + # - ansible>=8,<10 # ansible-core 2.16.3 through 2.16.6 suffer from the bug # discussed in ansible/ansible#82702, which breaks any # symlinked files in vars, tasks, etc. for any Ansible role From f51fe623bb34e68bb874cd30756fee2692d34b5a Mon Sep 17 00:00:00 2001 From: Nicholas McDonnell <50747025+mcdonnnj@users.noreply.github.com> Date: Sat, 11 May 2024 00:38:50 -0400 Subject: [PATCH 05/25] Update pre-commit hook versions This is done automatically with the `pre-commit autoupdate` command. The pre-commit/mirrors-prettier was manually held back because the latest tags are for alpha releases of the next major version. --- .pre-commit-config.yaml | 24 ++++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 2c5b3c8..d315345 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -5,7 +5,7 @@ default_language_version: repos: - repo: https://github.com/pre-commit/pre-commit-hooks - rev: v4.5.0 + rev: v4.6.0 hooks: - id: check-case-conflict - id: check-executables-have-shebangs @@ -31,7 +31,7 @@ repos: # Text file hooks - repo: https://github.com/igorshubovych/markdownlint-cli - rev: v0.39.0 + rev: v0.41.0 hooks: - id: markdownlint args: @@ -56,14 +56,14 @@ repos: # GitHub Actions hooks - repo: https://github.com/python-jsonschema/check-jsonschema - rev: 0.28.0 + rev: 0.28.4 hooks: - id: check-github-actions - id: check-github-workflows # pre-commit hooks - repo: https://github.com/pre-commit/pre-commit - rev: v3.6.2 + rev: v3.7.1 hooks: - id: validate_manifest @@ -98,7 +98,7 @@ repos: # Shell script hooks - repo: https://github.com/scop/pre-commit-shfmt - rev: v3.7.0-4 + rev: v3.8.0-1 hooks: - id: shfmt args: @@ -116,19 +116,19 @@ repos: # Redirect operators are followed by a space - --space-redirects - repo: https://github.com/shellcheck-py/shellcheck-py - rev: v0.9.0.6 + rev: v0.10.0.1 hooks: - id: shellcheck # Python hooks - repo: https://github.com/PyCQA/bandit - rev: 1.7.7 + rev: 1.7.8 hooks: - id: bandit args: - --config=.bandit.yml - repo: https://github.com/psf/black-pre-commit-mirror - rev: 24.2.0 + rev: 24.4.2 hooks: - id: black - repo: https://github.com/PyCQA/flake8 @@ -142,24 +142,24 @@ repos: hooks: - id: isort - repo: https://github.com/pre-commit/mirrors-mypy - rev: v1.8.0 + rev: v1.10.0 hooks: - id: mypy - repo: https://github.com/asottile/pyupgrade - rev: v3.15.1 + rev: v3.15.2 hooks: - id: pyupgrade # Ansible hooks - repo: https://github.com/ansible/ansible-lint - rev: v24.2.0 + rev: v24.6.0 hooks: - id: ansible-lint # files: molecule/default/playbook.yml # Terraform hooks - repo: https://github.com/antonbabenko/pre-commit-terraform - rev: v1.88.0 + rev: v1.90.0 hooks: - id: terraform_fmt - id: terraform_validate From 8e55b8e24e047008b1fa80bc87a76163888e4e5e Mon Sep 17 00:00:00 2001 From: Nicholas McDonnell <50747025+mcdonnnj@users.noreply.github.com> Date: Thu, 6 Jun 2024 13:58:38 -0400 Subject: [PATCH 06/25] Manually update the prettier hook Use the latest v3 release available from NPM. --- .pre-commit-config.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index d315345..2ece2c0 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -46,7 +46,7 @@ repos: # mirror does not pull tags for old major versions once a new major # version tag is published. additional_dependencies: - - prettier@3.2.5 + - prettier@3.3.1 - repo: https://github.com/adrienverge/yamllint rev: v1.35.1 hooks: From c617bb92af0bd01d0ef9bb7e51c007e34f91a915 Mon Sep 17 00:00:00 2001 From: Shane Frasier Date: Thu, 6 Jun 2024 16:42:24 -0400 Subject: [PATCH 07/25] Correct commented-out ansible pin The pin now agrees with what is in cisagov/skeleton-ansible-role. Co-authored-by: Nick <50747025+mcdonnnj@users.noreply.github.com> --- .pre-commit-config.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index a48e196..3071c44 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -163,7 +163,7 @@ repos: # necessary to add the ansible package itself as an # additional dependency, with the same pinning as is done in # requirements-test.txt of cisagov/skeleton-ansible-role. - # - ansible>=8,<10 + # - ansible>=9,<10 # ansible-core 2.16.3 through 2.16.6 suffer from the bug # discussed in ansible/ansible#82702, which breaks any # symlinked files in vars, tasks, etc. for any Ansible role From 6041172643f1290ba73e5056cef86c23fe742a6a Mon Sep 17 00:00:00 2001 From: Jeremy Frasier Date: Sun, 7 Jul 2024 11:00:21 -0400 Subject: [PATCH 08/25] Alphabetize filters by name --- terraform-post-packer/main.tf | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/terraform-post-packer/main.tf b/terraform-post-packer/main.tf index 7b3b4ff..59d5f44 100644 --- a/terraform-post-packer/main.tf +++ b/terraform-post-packer/main.tf @@ -26,6 +26,8 @@ locals { # The IDs of all cisagov/skeleton-packer AMIs data "aws_ami_ids" "historical_amis" { + owners = [data.aws_caller_identity.images.account_id] + filter { name = "name" values = [ @@ -33,17 +35,15 @@ data "aws_ami_ids" "historical_amis" { ] } - filter { - name = "virtualization-type" - values = ["hvm"] - } - filter { name = "root-device-type" values = ["ebs"] } - owners = [data.aws_caller_identity.images.account_id] + filter { + name = "virtualization-type" + values = ["hvm"] + } } # Assign launch permissions to the AMI From 89c7e1e523ddd401e266b653a01d8e230403b138 Mon Sep 17 00:00:00 2001 From: Jeremy Frasier Date: Sun, 7 Jul 2024 11:03:40 -0400 Subject: [PATCH 09/25] Add a filter for the AMI's architecture This is more flexible than filtering the architecture just based on the name, particularly since the name filter does not support full-fledged regexes. --- terraform-post-packer/main.tf | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/terraform-post-packer/main.tf b/terraform-post-packer/main.tf index 59d5f44..6cb2c42 100644 --- a/terraform-post-packer/main.tf +++ b/terraform-post-packer/main.tf @@ -28,6 +28,13 @@ locals { data "aws_ami_ids" "historical_amis" { owners = [data.aws_caller_identity.images.account_id] + filter { + name = "architecture" + values = [ + "x86_64", + ] + } + filter { name = "name" values = [ From d58b66089798a3523df1837c110d21d38fe1da6a Mon Sep 17 00:00:00 2001 From: Jeremy Frasier Date: Mon, 8 Jul 2024 17:03:45 -0400 Subject: [PATCH 10/25] Add an architecture filter in the Packer config This should make the search more efficient. --- src/packer.pkr.hcl | 1 + 1 file changed, 1 insertion(+) diff --git a/src/packer.pkr.hcl b/src/packer.pkr.hcl index 849c29d..47f2b81 100644 --- a/src/packer.pkr.hcl +++ b/src/packer.pkr.hcl @@ -65,6 +65,7 @@ variable "skip_create_ami" { data "amazon-ami" "debian_bookworm" { filters = { + architecture = "x86_64" name = "debian-12-amd64-*" root-device-type = "ebs" virtualization-type = "hvm" From aaeb8bde984f3674cecbaf7c1e160294fde081c2 Mon Sep 17 00:00:00 2001 From: Jeremy Frasier Date: Wed, 10 Jul 2024 00:29:29 -0400 Subject: [PATCH 11/25] Add an ARM64 source to the Packer configuration This will allow us to build both x86-64 and ARM64 versions of our AMI. --- src/packer.pkr.hcl | 70 ++++++++++++++++++++++++++++++++++++++++++---- 1 file changed, 65 insertions(+), 5 deletions(-) diff --git a/src/packer.pkr.hcl b/src/packer.pkr.hcl index 47f2b81..2172d3f 100644 --- a/src/packer.pkr.hcl +++ b/src/packer.pkr.hcl @@ -63,7 +63,7 @@ variable "skip_create_ami" { type = bool } -data "amazon-ami" "debian_bookworm" { +data "amazon-ami" "debian_bookworm_x86_64" { filters = { architecture = "x86_64" name = "debian-12-amd64-*" @@ -75,9 +75,65 @@ data "amazon-ami" "debian_bookworm" { region = var.build_region } +data "amazon-ami" "debian_bookworm_arm64" { + filters = { + architecture = "arm64" + name = "debian-12-arm64-*" + root-device-type = "ebs" + virtualization-type = "hvm" + } + most_recent = true + owners = ["136693071363"] + region = var.build_region +} + locals { timestamp = regex_replace(timestamp(), "[- TZ:]", "") } -source "amazon-ebs" "example" { +source "amazon-ebs" "arm64" { + ami_name = "example-hvm-${local.timestamp}-arm64-ebs" + ami_regions = var.ami_regions + associate_public_ip_address = true + encrypt_boot = true + instance_type = "t4g.small" + kms_key_id = var.build_region_kms + launch_block_device_mappings { + delete_on_termination = true + device_name = "/dev/xvda" + encrypted = true + volume_size = 8 + volume_type = "gp3" + } + region = var.build_region + region_kms_key_ids = var.region_kms_keys + skip_create_ami = var.skip_create_ami + source_ami = data.amazon-ami.debian_bookworm_arm64.id + ssh_username = "admin" + subnet_filter { + filters = { + "tag:Name" = "AMI Build" + } + } + tags = { + Application = "Example" + Architecture = "arm64" + Base_AMI_Name = data.amazon-ami.debian_bookworm_arm64.name + GitHub_Release_URL = var.release_url + OS_Version = "Debian Bookworm" + Pre_Release = var.is_prerelease + Release = var.release_tag + Team = "VM Fusion - Development" + } + # Many Linux distributions are now disallowing the use of RSA keys, + # so it makes sense to use an ED25519 key instead. + temporary_key_pair_type = "ed25519" + vpc_filter { + filters = { + "tag:Name" = "AMI Build" + } + } +} + +source "amazon-ebs" "x86_64" { ami_name = "example-hvm-${local.timestamp}-x86_64-ebs" ami_regions = var.ami_regions associate_public_ip_address = true @@ -94,7 +150,7 @@ source "amazon-ebs" "example" { region = var.build_region region_kms_key_ids = var.region_kms_keys skip_create_ami = var.skip_create_ami - source_ami = data.amazon-ami.debian_bookworm.id + source_ami = data.amazon-ami.debian_bookworm_x86_64.id ssh_username = "admin" subnet_filter { filters = { @@ -103,7 +159,8 @@ source "amazon-ebs" "example" { } tags = { Application = "Example" - Base_AMI_Name = data.amazon-ami.debian_bookworm.name + Architecture = "x86_64" + Base_AMI_Name = data.amazon-ami.debian_bookworm_x86_64.name GitHub_Release_URL = var.release_url OS_Version = "Debian Bookworm" Pre_Release = var.is_prerelease @@ -121,7 +178,10 @@ source "amazon-ebs" "example" { } build { - sources = ["source.amazon-ebs.example"] + sources = [ + "source.amazon-ebs.arm64", + "source.amazon-ebs.x86_64", + ] provisioner "ansible" { playbook_file = "src/upgrade.yml" From 2452ca585ab88e4ba80a34acbe0127492d7d1cc9 Mon Sep 17 00:00:00 2001 From: Jeremy Frasier Date: Wed, 10 Jul 2024 00:31:12 -0400 Subject: [PATCH 12/25] Build both the x86-64 and ARM64 versions of the AMI In order that the builds should complete as soon as possible, the different architectures are built in parallel. --- .github/workflows/build.yml | 9 ++++++++- .github/workflows/prerelease.yml | 9 ++++++++- .github/workflows/release.yml | 9 ++++++++- 3 files changed, 24 insertions(+), 3 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 2cc2474..3928aa6 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -238,6 +238,12 @@ jobs: - lint - test runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + architecture: + - arm64 + - x86_64 steps: - id: harden-runner name: Harden the runner @@ -315,7 +321,8 @@ jobs: # This runs through the AMI creation process but does not # actually create an AMI run: | - packer build -timestamp-ui \ + packer build -only amazon-ebs.${{ matrix.architecture }} \ + -timestamp-ui \ -var skip_create_ami=true \ src/packer.pkr.hcl - name: Remove /usr/bin/python3 symlink to the installed Python diff --git a/.github/workflows/prerelease.yml b/.github/workflows/prerelease.yml index 1f8dcfa..dcc3d6a 100644 --- a/.github/workflows/prerelease.yml +++ b/.github/workflows/prerelease.yml @@ -33,6 +33,12 @@ jobs: needs: - diagnostics runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + architecture: + - arm64 + - x86_64 steps: - id: harden-runner name: Harden the runner @@ -108,7 +114,8 @@ jobs: run: packer init src - name: Create machine image run: | - packer build -timestamp-ui \ + packer build -only amazon-ebs.${{ matrix.architecture }} \ + -timestamp-ui \ -var is_prerelease=${{ github.event.release.prerelease }} \ -var release_tag=${{ github.event.release.tag_name }} \ -var release_url=${{ github.event.release.html_url }} \ diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index b50f9b6..f055508 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -40,6 +40,12 @@ jobs: needs: - diagnostics runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + architecture: + - arm64 + - x86_64 steps: - id: harden-runner name: Harden the runner @@ -123,7 +129,8 @@ jobs: run: packer init src - name: Create machine image run: | - packer build -timestamp-ui \ + packer build -only amazon-ebs.${{ matrix.architecture }} \ + -timestamp-ui \ -var is_prerelease=${{ github.event.release.prerelease }} \ -var release_tag=${{ github.event.release.tag_name }} \ -var release_url=${{ github.event.release.html_url }} \ From 8dec77525fc89f41a28c070fc40ab2e6da28c5e0 Mon Sep 17 00:00:00 2001 From: Jeremy Frasier Date: Wed, 10 Jul 2024 00:32:55 -0400 Subject: [PATCH 13/25] Update the AMI filter to include both the x86-64 and ARM64 AMIs We want to be able to offer both wherever possible. --- terraform-post-packer/main.tf | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/terraform-post-packer/main.tf b/terraform-post-packer/main.tf index 6cb2c42..93a0c76 100644 --- a/terraform-post-packer/main.tf +++ b/terraform-post-packer/main.tf @@ -31,6 +31,7 @@ data "aws_ami_ids" "historical_amis" { filter { name = "architecture" values = [ + "arm64", "x86_64", ] } @@ -38,7 +39,7 @@ data "aws_ami_ids" "historical_amis" { filter { name = "name" values = [ - "example-hvm-*-x86_64-ebs", + "example-hvm-*-*-ebs", ] } From 793b46d9424b3ab0818f170f9c05356ae26e8049 Mon Sep 17 00:00:00 2001 From: Jeremy Frasier Date: Wed, 10 Jul 2024 00:35:09 -0400 Subject: [PATCH 14/25] Bump version from 1.0.1 to 2.0.0 --- src/version.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/version.txt b/src/version.txt index 5c4105c..8c0d5d5 100644 --- a/src/version.txt +++ b/src/version.txt @@ -1 +1 @@ -__version__ = "1.0.1" +__version__ = "2.0.0" From 95dc9a38a11dd0df0be7276e696ffb3bdbb15d77 Mon Sep 17 00:00:00 2001 From: Jeremy Frasier Date: Wed, 10 Jul 2024 00:35:19 -0400 Subject: [PATCH 15/25] Bump version from 2.0.0 to 2.0.0-rc.1 --- src/version.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/version.txt b/src/version.txt index 8c0d5d5..8cc9434 100644 --- a/src/version.txt +++ b/src/version.txt @@ -1 +1 @@ -__version__ = "2.0.0" +__version__ = "2.0.0-rc.1" From f6dbbb8ccad588525262b3885e75a1094f3679b4 Mon Sep 17 00:00:00 2001 From: Jeremy Frasier Date: Thu, 11 Jul 2024 13:01:26 -0400 Subject: [PATCH 16/25] Reorder data blocks to match order of source blocks Co-authored-by: Nick <50747025+mcdonnnj@users.noreply.github.com> --- src/packer.pkr.hcl | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/src/packer.pkr.hcl b/src/packer.pkr.hcl index 2172d3f..437adf0 100644 --- a/src/packer.pkr.hcl +++ b/src/packer.pkr.hcl @@ -63,10 +63,10 @@ variable "skip_create_ami" { type = bool } -data "amazon-ami" "debian_bookworm_x86_64" { +data "amazon-ami" "debian_bookworm_arm64" { filters = { - architecture = "x86_64" - name = "debian-12-amd64-*" + architecture = "arm64" + name = "debian-12-arm64-*" root-device-type = "ebs" virtualization-type = "hvm" } @@ -75,10 +75,10 @@ data "amazon-ami" "debian_bookworm_x86_64" { region = var.build_region } -data "amazon-ami" "debian_bookworm_arm64" { +data "amazon-ami" "debian_bookworm_x86_64" { filters = { - architecture = "arm64" - name = "debian-12-arm64-*" + architecture = "x86_64" + name = "debian-12-amd64-*" root-device-type = "ebs" virtualization-type = "hvm" } From 9332c3832ae3383ce595741f344f6fa693dae113 Mon Sep 17 00:00:00 2001 From: Jeremy Frasier Date: Thu, 11 Jul 2024 14:26:16 -0400 Subject: [PATCH 17/25] Create separate resources for ARM64 and x86-64 AMIs and launch permissions Co-authored-by: Nick <50747025+mcdonnnj@users.noreply.github.com> --- terraform-post-packer/main.tf | 67 +++++++++++++++++++++++------- terraform-post-packer/outputs.tf | 11 +++-- terraform-post-packer/variables.tf | 2 +- 3 files changed, 62 insertions(+), 18 deletions(-) diff --git a/terraform-post-packer/main.tf b/terraform-post-packer/main.tf index 93a0c76..f721ecd 100644 --- a/terraform-post-packer/main.tf +++ b/terraform-post-packer/main.tf @@ -24,23 +24,18 @@ locals { account_name_regex = format("^env[[:digit:]]+ \\(%s\\)$", local.images_account_type) } -# The IDs of all cisagov/skeleton-packer AMIs -data "aws_ami_ids" "historical_amis" { +# The IDs of all ARM64 cisagov/skeleton-packer AMIs +data "aws_ami_ids" "historical_amis_arm64" { owners = [data.aws_caller_identity.images.account_id] filter { - name = "architecture" - values = [ - "arm64", - "x86_64", - ] + name = "architecture" + values = ["arm64"] } filter { - name = "name" - values = [ - "example-hvm-*-*-ebs", - ] + name = "name" + values = ["example-hvm-*-arm64-ebs"] } filter { @@ -54,12 +49,56 @@ data "aws_ami_ids" "historical_amis" { } } -# Assign launch permissions to the AMI -module "ami_launch_permission" { +# Assign launch permissions to the ARM64 AMIs +module "ami_launch_permission_arm64" { # Really we only want the var.recent_ami_count most recent AMIs, but # we have to cover the case where there are fewer than that many # AMIs in existence. Hence the min()/length() tomfoolery. - for_each = toset(slice(data.aws_ami_ids.historical_amis.ids, 0, min(var.recent_ami_count, length(data.aws_ami_ids.historical_amis.ids)))) + for_each = toset(slice(data.aws_ami_ids.historical_amis_arm64.ids, 0, min(var.recent_ami_count, length(data.aws_ami_ids.historical_amis_arm64.ids)))) + + source = "github.com/cisagov/ami-launch-permission-tf-module" + + providers = { + aws = aws + aws.master = aws.master + } + + account_name_regex = local.account_name_regex + ami_id = each.value + extraorg_account_ids = var.extraorg_account_ids +} + +# The IDs of all x86-64 cisagov/skeleton-packer AMIs +data "aws_ami_ids" "historical_amis_x86_64" { + owners = [data.aws_caller_identity.images.account_id] + + filter { + name = "architecture" + values = ["x86_64"] + } + + filter { + name = "name" + values = ["example-hvm-*-x86_64-ebs"] + } + + filter { + name = "root-device-type" + values = ["ebs"] + } + + filter { + name = "virtualization-type" + values = ["hvm"] + } +} + +# Assign launch permissions to the x86-64 AMIs +module "ami_launch_permission_x86_64" { + # Really we only want the var.recent_ami_count most recent AMIs, but + # we have to cover the case where there are fewer than that many + # AMIs in existence. Hence the min()/length() tomfoolery. + for_each = toset(slice(data.aws_ami_ids.historical_amis_x86_64.ids, 0, min(var.recent_ami_count, length(data.aws_ami_ids.historical_amis_x86_64.ids)))) source = "github.com/cisagov/ami-launch-permission-tf-module" diff --git a/terraform-post-packer/outputs.tf b/terraform-post-packer/outputs.tf index ef5a669..2937e6e 100644 --- a/terraform-post-packer/outputs.tf +++ b/terraform-post-packer/outputs.tf @@ -1,4 +1,9 @@ -output "launch_permissions" { - value = module.ami_launch_permission - description = "The cisagov/ami-launch-permission-tf-module for each AMI to which launch permission is being granted." +output "launch_permissions_arm64" { + value = module.ami_launch_permission_arm64 + description = "The cisagov/ami-launch-permission-tf-module for each ARM64 AMI to which launch permission is being granted." +} + +output "launch_permissions_x86_64" { + value = module.ami_launch_permission_x86_64 + description = "The cisagov/ami-launch-permission-tf-module for each x86_64 AMI to which launch permission is being granted." } diff --git a/terraform-post-packer/variables.tf b/terraform-post-packer/variables.tf index 22b2a33..1b28b61 100644 --- a/terraform-post-packer/variables.tf +++ b/terraform-post-packer/variables.tf @@ -12,6 +12,6 @@ variable "extraorg_account_ids" { variable "recent_ami_count" { default = 12 - description = "The number of most-recent AMIs for which to grant launch permission (e.g. \"3\"). If this variable is set to three, for example, then accounts will be granted permission to launch the three most recent AMIs (or all most recent AMIs, if there are only one or two of them in existence)." + description = "The number of most-recent AMIs (per architecture) for which to grant launch permission (e.g. \"3\"). If this variable is set to three, for example, then accounts will be granted permission to launch the three most recent AMIs (or all most recent AMIs, if there are only one or two of them in existence)." type = number } From 18765c6b723da395fe311415f4baee1c33f5e3aa Mon Sep 17 00:00:00 2001 From: Jeremy Frasier Date: Thu, 11 Jul 2024 14:35:34 -0400 Subject: [PATCH 18/25] Add moved blocks for renamed resources --- terraform-post-packer/main.tf | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/terraform-post-packer/main.tf b/terraform-post-packer/main.tf index f721ecd..7e146c5 100644 --- a/terraform-post-packer/main.tf +++ b/terraform-post-packer/main.tf @@ -93,6 +93,11 @@ data "aws_ami_ids" "historical_amis_x86_64" { } } +moved { + from = aws_ami_ids.historical_amis + to = aws_ami_ids.historical_amis_x86_64 +} + # Assign launch permissions to the x86-64 AMIs module "ami_launch_permission_x86_64" { # Really we only want the var.recent_ami_count most recent AMIs, but @@ -111,3 +116,8 @@ module "ami_launch_permission_x86_64" { ami_id = each.value extraorg_account_ids = var.extraorg_account_ids } + +moved { + from = module.ami_launch_permission + to = module.ami_launch_permission_x86_64 +} From 51944d044cfbd7291736ff49e9c90d3589819a76 Mon Sep 17 00:00:00 2001 From: Jeremy Frasier Date: Thu, 8 Aug 2024 15:21:22 -0400 Subject: [PATCH 19/25] Add comments to moved blocks to explain the intent Also reference issue cisagov/skeleton-packer#369 to consider removing the moved blocks when they are no longer necessary. --- terraform-post-packer/main.tf | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/terraform-post-packer/main.tf b/terraform-post-packer/main.tf index 7e146c5..edb7239 100644 --- a/terraform-post-packer/main.tf +++ b/terraform-post-packer/main.tf @@ -93,6 +93,12 @@ data "aws_ami_ids" "historical_amis_x86_64" { } } +# This moved block allows us to rename the resources at +# aws_ami_ids.historical_amis to aws_ami_ids.historical_amis_x86_64 +# instead of destroying and recreating them with a new name. +# +# TODO: Consider removing this moved block when it is no longer +# needed. Seecisagov/skeleton-packer#369 for more details. moved { from = aws_ami_ids.historical_amis to = aws_ami_ids.historical_amis_x86_64 @@ -117,6 +123,12 @@ module "ami_launch_permission_x86_64" { extraorg_account_ids = var.extraorg_account_ids } +# This moved block allows us to rename the resources at +# module.ami_launch_permission to module.ami_launch_permission_x86_64 +# instead of destroying and recreating them with a new name. +# +# TODO: Consider removing this moved block when it is no longer +# needed. Seecisagov/skeleton-packer#369 for more details. moved { from = module.ami_launch_permission to = module.ami_launch_permission_x86_64 From 88528fe50eedc5828e099f558b442ef81eeb63d2 Mon Sep 17 00:00:00 2001 From: Jeremy Frasier Date: Fri, 9 Aug 2024 10:03:18 -0400 Subject: [PATCH 20/25] Fix typos (missing spaces) Co-authored-by: David Redmin --- terraform-post-packer/main.tf | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/terraform-post-packer/main.tf b/terraform-post-packer/main.tf index edb7239..d30cfdf 100644 --- a/terraform-post-packer/main.tf +++ b/terraform-post-packer/main.tf @@ -98,7 +98,7 @@ data "aws_ami_ids" "historical_amis_x86_64" { # instead of destroying and recreating them with a new name. # # TODO: Consider removing this moved block when it is no longer -# needed. Seecisagov/skeleton-packer#369 for more details. +# needed. See cisagov/skeleton-packer#369 for more details. moved { from = aws_ami_ids.historical_amis to = aws_ami_ids.historical_amis_x86_64 @@ -128,7 +128,7 @@ module "ami_launch_permission_x86_64" { # instead of destroying and recreating them with a new name. # # TODO: Consider removing this moved block when it is no longer -# needed. Seecisagov/skeleton-packer#369 for more details. +# needed. See cisagov/skeleton-packer#369 for more details. moved { from = module.ami_launch_permission to = module.ami_launch_permission_x86_64 From f161416effe0aa371d12881c83212cf53875f32e Mon Sep 17 00:00:00 2001 From: Jeremy Frasier Date: Mon, 12 Aug 2024 10:21:18 -0400 Subject: [PATCH 21/25] Finalize version from 2.0.0-rc.1 to 2.0.0 --- src/version.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/version.txt b/src/version.txt index 8cc9434..8c0d5d5 100644 --- a/src/version.txt +++ b/src/version.txt @@ -1 +1 @@ -__version__ = "2.0.0-rc.1" +__version__ = "2.0.0" From c804038589ea0eb3f9cdc2a2e63e8977c3b52753 Mon Sep 17 00:00:00 2001 From: Jeremy Frasier Date: Fri, 16 Aug 2024 11:08:50 -0400 Subject: [PATCH 22/25] Remove support for ARM64 The Assessor Workbench Docker image only supports x86_64. --- .github/workflows/build.yml | 3 +- .github/workflows/prerelease.yml | 3 +- .github/workflows/release.yml | 3 +- src/packer.pkr.hcl | 113 ++++++++++++++++--------------- terraform-post-packer/main.tf | 82 +++++++++++----------- terraform-post-packer/outputs.tf | 9 +-- 6 files changed, 111 insertions(+), 102 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 92b8474..36ab166 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -242,7 +242,8 @@ jobs: fail-fast: false matrix: architecture: - - arm64 + # The Assessor Workbench Docker image only supports x86_64. + # - arm64 - x86_64 steps: - id: harden-runner diff --git a/.github/workflows/prerelease.yml b/.github/workflows/prerelease.yml index 5bcc8c8..d68d106 100644 --- a/.github/workflows/prerelease.yml +++ b/.github/workflows/prerelease.yml @@ -37,7 +37,8 @@ jobs: fail-fast: false matrix: architecture: - - arm64 + # The Assessor Workbench Docker image only supports x86_64. + # - arm64 - x86_64 steps: - id: harden-runner diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 995db05..dc4fc27 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -44,7 +44,8 @@ jobs: fail-fast: false matrix: architecture: - - arm64 + # The Assessor Workbench Docker image only supports x86_64. + # - arm64 - x86_64 steps: - id: harden-runner diff --git a/src/packer.pkr.hcl b/src/packer.pkr.hcl index a7e580a..e75aed4 100644 --- a/src/packer.pkr.hcl +++ b/src/packer.pkr.hcl @@ -73,17 +73,18 @@ variable "skip_create_ami" { type = bool } -data "amazon-ami" "debian_bookworm_arm64" { - filters = { - architecture = "arm64" - name = "debian-12-arm64-*" - root-device-type = "ebs" - virtualization-type = "hvm" - } - most_recent = true - owners = ["136693071363"] - region = var.build_region -} +# The Assessor Workbench Docker image only supports x86_64. +# data "amazon-ami" "debian_bookworm_arm64" { +# filters = { +# architecture = "arm64" +# name = "debian-12-arm64-*" +# root-device-type = "ebs" +# virtualization-type = "hvm" +# } +# most_recent = true +# owners = ["136693071363"] +# region = var.build_region +# } data "amazon-ami" "debian_bookworm_x86_64" { filters = { @@ -99,49 +100,50 @@ data "amazon-ami" "debian_bookworm_x86_64" { locals { timestamp = regex_replace(timestamp(), "[- TZ:]", "") } -source "amazon-ebs" "arm64" { - ami_name = "assessor-workbench-hvm-${local.timestamp}-arm64-ebs" - ami_regions = var.ami_regions - associate_public_ip_address = true - encrypt_boot = true - instance_type = "t4g.small" - kms_key_id = var.build_region_kms - launch_block_device_mappings { - delete_on_termination = true - device_name = "/dev/xvda" - encrypted = true - volume_size = 8 - volume_type = "gp3" - } - region = var.build_region - region_kms_key_ids = var.region_kms_keys - skip_create_ami = var.skip_create_ami - source_ami = data.amazon-ami.debian_bookworm_arm64.id - ssh_username = "admin" - subnet_filter { - filters = { - "tag:Name" = "AMI Build" - } - } - tags = { - Application = "Assessor Workbench" - Architecture = "arm64" - Base_AMI_Name = data.amazon-ami.debian_bookworm_arm64.name - GitHub_Release_URL = var.release_url - OS_Version = "Debian Bookworm" - Pre_Release = var.is_prerelease - Release = var.release_tag - Team = "VM Fusion - Development" - } - # Many Linux distributions are now disallowing the use of RSA keys, - # so it makes sense to use an ED25519 key instead. - temporary_key_pair_type = "ed25519" - vpc_filter { - filters = { - "tag:Name" = "AMI Build" - } - } -} +# The Assessor Workbench Docker image only supports x86_64. +# source "amazon-ebs" "arm64" { +# ami_name = "assessor-workbench-hvm-${local.timestamp}-arm64-ebs" +# ami_regions = var.ami_regions +# associate_public_ip_address = true +# encrypt_boot = true +# instance_type = "t4g.small" +# kms_key_id = var.build_region_kms +# launch_block_device_mappings { +# delete_on_termination = true +# device_name = "/dev/xvda" +# encrypted = true +# volume_size = 8 +# volume_type = "gp3" +# } +# region = var.build_region +# region_kms_key_ids = var.region_kms_keys +# skip_create_ami = var.skip_create_ami +# source_ami = data.amazon-ami.debian_bookworm_arm64.id +# ssh_username = "admin" +# subnet_filter { +# filters = { +# "tag:Name" = "AMI Build" +# } +# } +# tags = { +# Application = "Assessor Workbench" +# Architecture = "arm64" +# Base_AMI_Name = data.amazon-ami.debian_bookworm_arm64.name +# GitHub_Release_URL = var.release_url +# OS_Version = "Debian Bookworm" +# Pre_Release = var.is_prerelease +# Release = var.release_tag +# Team = "VM Fusion - Development" +# } +# # Many Linux distributions are now disallowing the use of RSA keys, +# # so it makes sense to use an ED25519 key instead. +# temporary_key_pair_type = "ed25519" +# vpc_filter { +# filters = { +# "tag:Name" = "AMI Build" +# } +# } +# } source "amazon-ebs" "x86_64" { ami_name = "assessor-workbench-hvm-${local.timestamp}-x86_64-ebs" @@ -189,7 +191,8 @@ source "amazon-ebs" "x86_64" { build { sources = [ - "source.amazon-ebs.arm64", + # The Assessor Workbench Docker image only supports x86_64. + # "source.amazon-ebs.arm64", "source.amazon-ebs.x86_64", ] diff --git a/terraform-post-packer/main.tf b/terraform-post-packer/main.tf index f135962..8fdf29e 100644 --- a/terraform-post-packer/main.tf +++ b/terraform-post-packer/main.tf @@ -24,49 +24,51 @@ locals { account_name_regex = format("^env[[:digit:]]+ \\(%s\\)$", local.images_account_type) } +# The Assessor Workbench Docker image only supports x86_64. # The IDs of all ARM64 cisagov/assessor-workbench-packer AMIs -data "aws_ami_ids" "historical_amis_arm64" { - owners = [data.aws_caller_identity.images.account_id] - - filter { - name = "architecture" - values = ["arm64"] - } - - filter { - name = "name" - values = ["assessor-workbench-hvm-*-arm64-ebs"] - } - - filter { - name = "root-device-type" - values = ["ebs"] - } - - filter { - name = "virtualization-type" - values = ["hvm"] - } -} +# data "aws_ami_ids" "historical_amis_arm64" { +# owners = [data.aws_caller_identity.images.account_id] +# +# filter { +# name = "architecture" +# values = ["arm64"] +# } +# +# filter { +# name = "name" +# values = ["assessor-workbench-hvm-*-arm64-ebs"] +# } +# +# filter { +# name = "root-device-type" +# values = ["ebs"] +# } +# +# filter { +# name = "virtualization-type" +# values = ["hvm"] +# } +# } +# The Assessor Workbench Docker image only supports x86_64. # Assign launch permissions to the ARM64 AMIs -module "ami_launch_permission_arm64" { - # Really we only want the var.recent_ami_count most recent AMIs, but - # we have to cover the case where there are fewer than that many - # AMIs in existence. Hence the min()/length() tomfoolery. - for_each = toset(slice(data.aws_ami_ids.historical_amis_arm64.ids, 0, min(var.recent_ami_count, length(data.aws_ami_ids.historical_amis_arm64.ids)))) - - source = "github.com/cisagov/ami-launch-permission-tf-module" - - providers = { - aws = aws - aws.master = aws.master - } - - account_name_regex = local.account_name_regex - ami_id = each.value - extraorg_account_ids = var.extraorg_account_ids -} +# module "ami_launch_permission_arm64" { +# # Really we only want the var.recent_ami_count most recent AMIs, but +# # we have to cover the case where there are fewer than that many +# # AMIs in existence. Hence the min()/length() tomfoolery. +# for_each = toset(slice(data.aws_ami_ids.historical_amis_arm64.ids, 0, min(var.recent_ami_count, length(data.aws_ami_ids.historical_amis_arm64.ids)))) +# +# source = "github.com/cisagov/ami-launch-permission-tf-module" +# +# providers = { +# aws = aws +# aws.master = aws.master +# } +# +# account_name_regex = local.account_name_regex +# ami_id = each.value +# extraorg_account_ids = var.extraorg_account_ids +# } # The IDs of all x86-64 cisagov/assessor-workbench-packer AMIs data "aws_ami_ids" "historical_amis_x86_64" { diff --git a/terraform-post-packer/outputs.tf b/terraform-post-packer/outputs.tf index 2937e6e..20a3517 100644 --- a/terraform-post-packer/outputs.tf +++ b/terraform-post-packer/outputs.tf @@ -1,7 +1,8 @@ -output "launch_permissions_arm64" { - value = module.ami_launch_permission_arm64 - description = "The cisagov/ami-launch-permission-tf-module for each ARM64 AMI to which launch permission is being granted." -} +# The Assessor Workbench Docker image only supports x86_64. +# output "launch_permissions_arm64" { +# value = module.ami_launch_permission_arm64 +# description = "The cisagov/ami-launch-permission-tf-module for each ARM64 AMI to which launch permission is being granted." +# } output "launch_permissions_x86_64" { value = module.ami_launch_permission_x86_64 From 6dd0836c4e08bf53040c37c3b2a33069212fc1e8 Mon Sep 17 00:00:00 2001 From: Jeremy Frasier Date: Fri, 16 Aug 2024 11:09:24 -0400 Subject: [PATCH 23/25] Bump version from 0.1.1 to 0.1.2 --- src/version.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/version.txt b/src/version.txt index 485f44a..b3f4756 100644 --- a/src/version.txt +++ b/src/version.txt @@ -1 +1 @@ -__version__ = "0.1.1" +__version__ = "0.1.2" From 266d48716eb118b04416541eb8c462013f423328 Mon Sep 17 00:00:00 2001 From: Jeremy Frasier Date: Fri, 16 Aug 2024 11:09:38 -0400 Subject: [PATCH 24/25] Bump version from 0.1.2 to 0.1.2-rc.1 --- src/version.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/version.txt b/src/version.txt index b3f4756..b3d524c 100644 --- a/src/version.txt +++ b/src/version.txt @@ -1 +1 @@ -__version__ = "0.1.2" +__version__ = "0.1.2-rc.1" From 75510a138bd06afc25fbd80c26eb594eac5a13c9 Mon Sep 17 00:00:00 2001 From: Jeremy Frasier Date: Fri, 16 Aug 2024 15:48:11 -0400 Subject: [PATCH 25/25] Finalize version from 0.1.2-rc.1 to 0.1.2 --- src/version.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/version.txt b/src/version.txt index b3d524c..b3f4756 100644 --- a/src/version.txt +++ b/src/version.txt @@ -1 +1 @@ -__version__ = "0.1.2-rc.1" +__version__ = "0.1.2"