From 33582a11f73d4e5c3dc588a040e59941c381d09a Mon Sep 17 00:00:00 2001 From: Michael Saki Date: Wed, 14 Feb 2024 12:59:30 -0600 Subject: [PATCH 01/75] Add checks for semantic python versions --- setup-env | 39 +++++++++++++++++++++------------------ 1 file changed, 21 insertions(+), 18 deletions(-) diff --git a/setup-env b/setup-env index 05b010b..44368b3 100755 --- a/setup-env +++ b/setup-env @@ -39,6 +39,14 @@ python_versions() { pyenv versions --bare --skip-aliases --skip-envs } +check_semantic_version() { + local version=$1 + local regex="^(0|[1-9]\\d*)\\.(0|[1-9]\\d*)\\.(0|[1-9]\\d*)(?:-((?:0|[1-9]\\d*|\\d*[a-zA-Z-][0-9a-zA-Z-]*)(?:\\.(?:0|[1-9]\\d*|\\d*[a-zA-Z-][0-9a-zA-Z-]*))*))?(?:\\+([0-9a-zA-Z-]+(?:\\.[0-9a-zA-Z-]+)*))?\$" + + # Use Perl for regex matching and output true or false + echo "$version" | perl -ne "exit(!/$regex/)" +} + # Flag to force deletion and creation of virtual environment FORCE=0 @@ -103,16 +111,18 @@ while true; do -p | --python-version) PYTHON_VERSION="$2" shift 2 - # Check the Python versions being passed in. - if [ -n "${PYTHON_VERSION+x}" ]; then - if python_versions | grep -E "^${PYTHON_VERSION}$" > /dev/null; then - echo Using Python version "$PYTHON_VERSION" - else - echo Error: Python version "$PYTHON_VERSION" is not installed. - echo Installed Python versions are: - python_versions - exit 1 - fi + # Validate the semantic version format + if ! check_semantic_version "$PYTHON_VERSION"; then + echo "Error: The specified Python version $PYTHON_VERSION does not follow the semantic versioning standard." + echo "Example of a valid version: 3.8.1, 3.8.1-alpha.1, or 3.8.1+20130313144700" + exit 1 + elif ! python_versions | grep -E "^${PYTHON_VERSION}$" > /dev/null; then + echo "Error: Python version $PYTHON_VERSION is not installed." + echo "Installed Python versions are:" + python_versions + exit 1 + else + echo "Using Python version $PYTHON_VERSION" fi ;; -v | --venv-name) @@ -181,14 +191,7 @@ if [ $LIST_VERSIONS -ne 0 ]; then python_versions read -r -p "Enter the desired Python version: " PYTHON_VERSION # Check the Python versions being passed in. - if [ -n "${PYTHON_VERSION+x}" ]; then - if python_versions | grep -E "^${PYTHON_VERSION}$" > /dev/null; then - echo Using Python version "$PYTHON_VERSION" - else - echo Error: Python version "$PYTHON_VERSION" is not installed. - exit 1 - fi - fi + check_semantic_version "$PYTHON_VERSION" fi # Remove any lingering local configuration. From 94381940a9d28f87da2b85c5e1647a5a80d4a18d Mon Sep 17 00:00:00 2001 From: Michael Saki Date: Wed, 14 Feb 2024 14:29:13 -0600 Subject: [PATCH 02/75] Refactor code for the semantic check This commit will make a few changes. The orginal version of the semantic checking function was a bit more difficult to read. It is now somewhat easier to follow how the regex is structured. Also the function has been renamed to check_python_version since it has 2 functions, making sure that the version is semantically correct and the second is to make sure that it is installed on the user's machine. This makes it easier to follow the logic for the flags, -p or --python-version and -l or --list-versions --- setup-env | 54 ++++++++++++++++++++++++++++++++++++------------------ 1 file changed, 36 insertions(+), 18 deletions(-) diff --git a/setup-env b/setup-env index 44368b3..11ec170 100755 --- a/setup-env +++ b/setup-env @@ -39,12 +39,41 @@ python_versions() { pyenv versions --bare --skip-aliases --skip-envs } -check_semantic_version() { +check_python_version() { local version=$1 - local regex="^(0|[1-9]\\d*)\\.(0|[1-9]\\d*)\\.(0|[1-9]\\d*)(?:-((?:0|[1-9]\\d*|\\d*[a-zA-Z-][0-9a-zA-Z-]*)(?:\\.(?:0|[1-9]\\d*|\\d*[a-zA-Z-][0-9a-zA-Z-]*))*))?(?:\\+([0-9a-zA-Z-]+(?:\\.[0-9a-zA-Z-]+)*))?\$" - # Use Perl for regex matching and output true or false - echo "$version" | perl -ne "exit(!/$regex/)" + # Break down the regex into readable parts major.minor.patch + local major="0|[1-9]\\d*" + local minor="0|[1-9]\\d*" + local patch="0|[1-9]\\d*" + + # Splitting the prerelease part for readability + # Start of prerelease + local prerelease="(?:-" + # Numeric or alphanumeric identifiers + local prerelease+="(?:0|[1-9]\\d*|\\d*[a-zA-Z-][0-9a-zA-Z-]*)" + # Additional dot-separated identifiers + local prerelease+="(?:\\.(?:0|[1-9]\\d*|\\d*[a-zA-Z-][0-9a-zA-Z-]*))*" + # End of prerelease, making it optional + local prerelease+=")?" + # Optional build metadata + local build="(?:\\+([0-9a-zA-Z-]+(?:\\.[0-9a-zA-Z-]+)*))?" + + # Final regex composed of parts + local regex="^($major)\\.($minor)\\.($patch)$prerelease$build\$" + + if ! echo "$version" | perl -ne "exit(!/$regex/)"; then + echo "Error: The specified Python version $version does not follow the semantic versioning standard." + echo "Example of a valid version: 3.8.1, 3.8.1-alpha.1, or 3.8.1+20130313144700" + exit 1 + elif ! python_versions | grep -E "^${version}$" > /dev/null; then + echo "Error: Python version $version is not installed." + echo "Installed Python versions are:" + python_versions + exit 1 + else + echo "Using Python version $version" + fi } # Flag to force deletion and creation of virtual environment @@ -111,19 +140,8 @@ while true; do -p | --python-version) PYTHON_VERSION="$2" shift 2 - # Validate the semantic version format - if ! check_semantic_version "$PYTHON_VERSION"; then - echo "Error: The specified Python version $PYTHON_VERSION does not follow the semantic versioning standard." - echo "Example of a valid version: 3.8.1, 3.8.1-alpha.1, or 3.8.1+20130313144700" - exit 1 - elif ! python_versions | grep -E "^${PYTHON_VERSION}$" > /dev/null; then - echo "Error: Python version $PYTHON_VERSION is not installed." - echo "Installed Python versions are:" - python_versions - exit 1 - else - echo "Using Python version $PYTHON_VERSION" - fi + # Check the Python version being passed in. + check_python_version "$PYTHON_VERSION" ;; -v | --venv-name) VENV_NAME="$2" @@ -191,7 +209,7 @@ if [ $LIST_VERSIONS -ne 0 ]; then python_versions read -r -p "Enter the desired Python version: " PYTHON_VERSION # Check the Python versions being passed in. - check_semantic_version "$PYTHON_VERSION" + check_python_version "$PYTHON_VERSION" fi # Remove any lingering local configuration. From cea8edc5bcdcec8a06b6b810514b0222fc03f42e Mon Sep 17 00:00:00 2001 From: Michael Saki Date: Wed, 14 Feb 2024 12:59:30 -0600 Subject: [PATCH 03/75] Add checks for semantic python versions --- setup-env | 39 +++++++++++++++++++++------------------ 1 file changed, 21 insertions(+), 18 deletions(-) diff --git a/setup-env b/setup-env index 3a22d43..5e537bc 100755 --- a/setup-env +++ b/setup-env @@ -39,6 +39,14 @@ python_versions() { pyenv versions --bare --skip-aliases --skip-envs } +check_semantic_version() { + local version=$1 + local regex="^(0|[1-9]\\d*)\\.(0|[1-9]\\d*)\\.(0|[1-9]\\d*)(?:-((?:0|[1-9]\\d*|\\d*[a-zA-Z-][0-9a-zA-Z-]*)(?:\\.(?:0|[1-9]\\d*|\\d*[a-zA-Z-][0-9a-zA-Z-]*))*))?(?:\\+([0-9a-zA-Z-]+(?:\\.[0-9a-zA-Z-]+)*))?\$" + + # Use Perl for regex matching and output true or false + echo "$version" | perl -ne "exit(!/$regex/)" +} + # Flag to force deletion and creation of virtual environment FORCE=0 @@ -144,16 +152,18 @@ while true; do -p | --python-version) PYTHON_VERSION="$2" shift 2 - # Check the Python versions being passed in. - if [ -n "${PYTHON_VERSION+x}" ]; then - if python_versions | grep -E "^${PYTHON_VERSION}$" > /dev/null; then - echo Using Python version "$PYTHON_VERSION" - else - echo Error: Python version "$PYTHON_VERSION" is not installed. - echo Installed Python versions are: - python_versions - exit 1 - fi + # Validate the semantic version format + if ! check_semantic_version "$PYTHON_VERSION"; then + echo "Error: The specified Python version $PYTHON_VERSION does not follow the semantic versioning standard." + echo "Example of a valid version: 3.8.1, 3.8.1-alpha.1, or 3.8.1+20130313144700" + exit 1 + elif ! python_versions | grep -E "^${PYTHON_VERSION}$" > /dev/null; then + echo "Error: Python version $PYTHON_VERSION is not installed." + echo "Installed Python versions are:" + python_versions + exit 1 + else + echo "Using Python version $PYTHON_VERSION" fi ;; -v | --venv-name) @@ -189,14 +199,7 @@ if [ $LIST_VERSIONS -ne 0 ]; then # -r: treat backslashes as literal, -p: display prompt before input. read -r -p "Enter the desired Python version: " PYTHON_VERSION # Check the Python versions being passed in. - if [ -n "${PYTHON_VERSION+x}" ]; then - if python_versions | grep -E "^${PYTHON_VERSION}$" > /dev/null; then - echo Using Python version "$PYTHON_VERSION" - else - echo Error: Python version "$PYTHON_VERSION" is not installed. - exit 1 - fi - fi + check_semantic_version "$PYTHON_VERSION" fi # Remove any lingering local configuration. From d5c7c4a566f88f7575f06ff2e0829f257a00cb08 Mon Sep 17 00:00:00 2001 From: Michael Saki Date: Wed, 14 Feb 2024 14:29:13 -0600 Subject: [PATCH 04/75] Refactor code for the semantic check This commit will make a few changes. The orginal version of the semantic checking function was a bit more difficult to read. It is now somewhat easier to follow how the regex is structured. Also the function has been renamed to check_python_version since it has 2 functions, making sure that the version is semantically correct and the second is to make sure that it is installed on the user's machine. This makes it easier to follow the logic for the flags, -p or --python-version and -l or --list-versions --- setup-env | 54 ++++++++++++++++++++++++++++++++++++------------------ 1 file changed, 36 insertions(+), 18 deletions(-) diff --git a/setup-env b/setup-env index 5e537bc..92540d1 100755 --- a/setup-env +++ b/setup-env @@ -39,12 +39,41 @@ python_versions() { pyenv versions --bare --skip-aliases --skip-envs } -check_semantic_version() { +check_python_version() { local version=$1 - local regex="^(0|[1-9]\\d*)\\.(0|[1-9]\\d*)\\.(0|[1-9]\\d*)(?:-((?:0|[1-9]\\d*|\\d*[a-zA-Z-][0-9a-zA-Z-]*)(?:\\.(?:0|[1-9]\\d*|\\d*[a-zA-Z-][0-9a-zA-Z-]*))*))?(?:\\+([0-9a-zA-Z-]+(?:\\.[0-9a-zA-Z-]+)*))?\$" - # Use Perl for regex matching and output true or false - echo "$version" | perl -ne "exit(!/$regex/)" + # Break down the regex into readable parts major.minor.patch + local major="0|[1-9]\\d*" + local minor="0|[1-9]\\d*" + local patch="0|[1-9]\\d*" + + # Splitting the prerelease part for readability + # Start of prerelease + local prerelease="(?:-" + # Numeric or alphanumeric identifiers + local prerelease+="(?:0|[1-9]\\d*|\\d*[a-zA-Z-][0-9a-zA-Z-]*)" + # Additional dot-separated identifiers + local prerelease+="(?:\\.(?:0|[1-9]\\d*|\\d*[a-zA-Z-][0-9a-zA-Z-]*))*" + # End of prerelease, making it optional + local prerelease+=")?" + # Optional build metadata + local build="(?:\\+([0-9a-zA-Z-]+(?:\\.[0-9a-zA-Z-]+)*))?" + + # Final regex composed of parts + local regex="^($major)\\.($minor)\\.($patch)$prerelease$build\$" + + if ! echo "$version" | perl -ne "exit(!/$regex/)"; then + echo "Error: The specified Python version $version does not follow the semantic versioning standard." + echo "Example of a valid version: 3.8.1, 3.8.1-alpha.1, or 3.8.1+20130313144700" + exit 1 + elif ! python_versions | grep -E "^${version}$" > /dev/null; then + echo "Error: Python version $version is not installed." + echo "Installed Python versions are:" + python_versions + exit 1 + else + echo "Using Python version $version" + fi } # Flag to force deletion and creation of virtual environment @@ -152,19 +181,8 @@ while true; do -p | --python-version) PYTHON_VERSION="$2" shift 2 - # Validate the semantic version format - if ! check_semantic_version "$PYTHON_VERSION"; then - echo "Error: The specified Python version $PYTHON_VERSION does not follow the semantic versioning standard." - echo "Example of a valid version: 3.8.1, 3.8.1-alpha.1, or 3.8.1+20130313144700" - exit 1 - elif ! python_versions | grep -E "^${PYTHON_VERSION}$" > /dev/null; then - echo "Error: Python version $PYTHON_VERSION is not installed." - echo "Installed Python versions are:" - python_versions - exit 1 - else - echo "Using Python version $PYTHON_VERSION" - fi + # Check the Python version being passed in. + check_python_version "$PYTHON_VERSION" ;; -v | --venv-name) VENV_NAME="$2" @@ -199,7 +217,7 @@ if [ $LIST_VERSIONS -ne 0 ]; then # -r: treat backslashes as literal, -p: display prompt before input. read -r -p "Enter the desired Python version: " PYTHON_VERSION # Check the Python versions being passed in. - check_semantic_version "$PYTHON_VERSION" + check_python_version "$PYTHON_VERSION" fi # Remove any lingering local configuration. From 327ab733aeaaad6a4916eb86b20d86618c9351e3 Mon Sep 17 00:00:00 2001 From: Michael Saki Date: Mon, 18 Mar 2024 12:36:02 -0500 Subject: [PATCH 05/75] Remove example of correct semantic version --- setup-env | 1 - 1 file changed, 1 deletion(-) diff --git a/setup-env b/setup-env index 92540d1..bacd2d5 100755 --- a/setup-env +++ b/setup-env @@ -64,7 +64,6 @@ check_python_version() { if ! echo "$version" | perl -ne "exit(!/$regex/)"; then echo "Error: The specified Python version $version does not follow the semantic versioning standard." - echo "Example of a valid version: 3.8.1, 3.8.1-alpha.1, or 3.8.1+20130313144700" exit 1 elif ! python_versions | grep -E "^${version}$" > /dev/null; then echo "Error: Python version $version is not installed." From 4dedf50886fd47c67895deb07367fca5c36ca33f Mon Sep 17 00:00:00 2001 From: Michael Saki Date: Wed, 20 Mar 2024 12:58:03 -0500 Subject: [PATCH 06/75] Refactor the error message for the user --- setup-env | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/setup-env b/setup-env index bacd2d5..d7824cb 100755 --- a/setup-env +++ b/setup-env @@ -63,7 +63,9 @@ check_python_version() { local regex="^($major)\\.($minor)\\.($patch)$prerelease$build\$" if ! echo "$version" | perl -ne "exit(!/$regex/)"; then - echo "Error: The specified Python version $version does not follow the semantic versioning standard." + echo "Invalid version of Python: Python follows semantic versioning, " \ + "so any version string that is not a valid semantic version is an " \ + "invalid version of Python." exit 1 elif ! python_versions | grep -E "^${version}$" > /dev/null; then echo "Error: Python version $version is not installed." From e84deea5181f27471f01343113c91dc2b13e159e Mon Sep 17 00:00:00 2001 From: Michael Saki Date: Wed, 20 Mar 2024 14:52:16 -0500 Subject: [PATCH 07/75] Improve the semantic error message --- setup-env | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/setup-env b/setup-env index d7824cb..bba5f9e 100755 --- a/setup-env +++ b/setup-env @@ -63,8 +63,8 @@ check_python_version() { local regex="^($major)\\.($minor)\\.($patch)$prerelease$build\$" if ! echo "$version" | perl -ne "exit(!/$regex/)"; then - echo "Invalid version of Python: Python follows semantic versioning, " \ - "so any version string that is not a valid semantic version is an " \ + echo "Invalid version of Python: Python follows semantic versioning," \ + "so any version string that is not a valid semantic version is an" \ "invalid version of Python." exit 1 elif ! python_versions | grep -E "^${version}$" > /dev/null; then From 5fdc7befc1d1d4811c4550ca1e4c65a711971c21 Mon Sep 17 00:00:00 2001 From: Michael Saki Date: Wed, 20 Mar 2024 15:39:07 -0500 Subject: [PATCH 08/75] Fix grammar Co-authored-by: dav3r --- setup-env | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup-env b/setup-env index bba5f9e..b93810c 100755 --- a/setup-env +++ b/setup-env @@ -217,7 +217,7 @@ if [ $LIST_VERSIONS -ne 0 ]; then # Read the user's desired Python version. # -r: treat backslashes as literal, -p: display prompt before input. read -r -p "Enter the desired Python version: " PYTHON_VERSION - # Check the Python versions being passed in. + # Check the Python version being passed in. check_python_version "$PYTHON_VERSION" fi From 42ef8c2d7b54cde82d4390a0050622cddfccf92a Mon Sep 17 00:00:00 2001 From: Michael Saki Date: Thu, 21 Mar 2024 09:19:42 -0500 Subject: [PATCH 09/75] Refactor regex, add link, and improve comments --- setup-env | 25 ++++++++++++++++--------- 1 file changed, 16 insertions(+), 9 deletions(-) diff --git a/setup-env b/setup-env index b93810c..2f30021 100755 --- a/setup-env +++ b/setup-env @@ -42,31 +42,38 @@ python_versions() { check_python_version() { local version=$1 + # This is a valid regex for semantically correct Python version strings. + # For more information see here: https://regex101.com/r/vkijKf/1/. # Break down the regex into readable parts major.minor.patch - local major="0|[1-9]\\d*" - local minor="0|[1-9]\\d*" - local patch="0|[1-9]\\d*" + local major="0|[1-9]\d*" + local minor="0|[1-9]\d*" + local patch="0|[1-9]\d*" # Splitting the prerelease part for readability - # Start of prerelease + # Start of the prerelease local prerelease="(?:-" # Numeric or alphanumeric identifiers - local prerelease+="(?:0|[1-9]\\d*|\\d*[a-zA-Z-][0-9a-zA-Z-]*)" + local prerelease+="(?:0|[1-9]\d*|\d*[a-zA-Z-][0-9a-zA-Z-]*)" # Additional dot-separated identifiers - local prerelease+="(?:\\.(?:0|[1-9]\\d*|\\d*[a-zA-Z-][0-9a-zA-Z-]*))*" - # End of prerelease, making it optional + local prerelease+="(?:\.(?:0|[1-9]\d*|\d*[a-zA-Z-][0-9a-zA-Z-]*))*" + # End of the prerelease, making it optional local prerelease+=")?" # Optional build metadata - local build="(?:\\+([0-9a-zA-Z-]+(?:\\.[0-9a-zA-Z-]+)*))?" + local build="(?:\+([0-9a-zA-Z-]+(?:\.[0-9a-zA-Z-]+)*))?" # Final regex composed of parts - local regex="^($major)\\.($minor)\\.($patch)$prerelease$build\$" + local regex="^($major)\.($minor)\.($patch)$prerelease$build$" + # This checks if the Python version does not match the regex pattern specified in $regex, + # using Perl for regex matching. If the pattern is not found, then prompt the user with + # the invalid version message. if ! echo "$version" | perl -ne "exit(!/$regex/)"; then echo "Invalid version of Python: Python follows semantic versioning," \ "so any version string that is not a valid semantic version is an" \ "invalid version of Python." exit 1 + # Else if the Python version isn't installed then notify the user. + # grep -E is used for searching through text lines that match the specific verison. elif ! python_versions | grep -E "^${version}$" > /dev/null; then echo "Error: Python version $version is not installed." echo "Installed Python versions are:" From a77e5e1c9a8752a2072a6a974d4164be116069e9 Mon Sep 17 00:00:00 2001 From: Michael Saki Date: Thu, 21 Mar 2024 10:13:11 -0500 Subject: [PATCH 10/75] Update link to use semver.org over regex101.com --- setup-env | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/setup-env b/setup-env index 2f30021..8d7b347 100755 --- a/setup-env +++ b/setup-env @@ -43,7 +43,8 @@ check_python_version() { local version=$1 # This is a valid regex for semantically correct Python version strings. - # For more information see here: https://regex101.com/r/vkijKf/1/. + # For more information see here: + # https://semver.org/#is-there-a-suggested-regular-expression-regex-to-check-a-semver-string. # Break down the regex into readable parts major.minor.patch local major="0|[1-9]\d*" local minor="0|[1-9]\d*" From 5fe14c7c6066d30381f6746eb313a56e4d447ac5 Mon Sep 17 00:00:00 2001 From: Michael Saki Date: Thu, 21 Mar 2024 10:29:58 -0500 Subject: [PATCH 11/75] Remove unnecessary period Co-authored-by: dav3r --- setup-env | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup-env b/setup-env index 8d7b347..059ccad 100755 --- a/setup-env +++ b/setup-env @@ -44,7 +44,7 @@ check_python_version() { # This is a valid regex for semantically correct Python version strings. # For more information see here: - # https://semver.org/#is-there-a-suggested-regular-expression-regex-to-check-a-semver-string. + # https://semver.org/#is-there-a-suggested-regular-expression-regex-to-check-a-semver-string # Break down the regex into readable parts major.minor.patch local major="0|[1-9]\d*" local minor="0|[1-9]\d*" From b7896a0a2790cc121842c6ac1602734bbd5dd726 Mon Sep 17 00:00:00 2001 From: Nicholas McDonnell <50747025+mcdonnnj@users.noreply.github.com> Date: Sat, 20 Apr 2024 04:11:57 -0400 Subject: [PATCH 12/75] Add a meta hook to the pre-commit configuration Add the `check-useless-excludes` meta hook to verify that any defined `exclude` directives apply to at least one file in the repository. --- .pre-commit-config.yaml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 2c5b3c8..de8c587 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -4,6 +4,11 @@ default_language_version: python: python3 repos: + # Check the pre-commit configuration + - repo: meta + hooks: + - id: check-useless-excludes + - repo: https://github.com/pre-commit/pre-commit-hooks rev: v4.5.0 hooks: From 260566f177520175530963c469e50d124e5bc0e4 Mon Sep 17 00:00:00 2001 From: Nicholas McDonnell <50747025+mcdonnnj@users.noreply.github.com> Date: Sat, 20 Apr 2024 04:15:52 -0400 Subject: [PATCH 13/75] Remove `exclude` directive that does not apply to any files --- .pre-commit-config.yaml | 1 - 1 file changed, 1 deletion(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index de8c587..5ec468e 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -24,7 +24,6 @@ repos: - --allow-missing-credentials - id: detect-private-key - id: end-of-file-fixer - exclude: files/(issue|motd) - id: mixed-line-ending args: - --fix=lf From a68994d17dcc11e9b90132c50fe52732d5fda07b Mon Sep 17 00:00:00 2001 From: Jeremy Frasier Date: Mon, 1 Jul 2024 16:19:46 -0400 Subject: [PATCH 14/75] Add a lower-bound pin for flake8-docstrings --- .pre-commit-config.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 386c83f..74c9c76 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -136,7 +136,7 @@ repos: hooks: - id: flake8 additional_dependencies: - - flake8-docstrings + - flake8-docstrings>=1.7.0 - repo: https://github.com/PyCQA/isort rev: 5.13.2 hooks: From 43b91c74754e912172c702e20f12ba9f767ac202 Mon Sep 17 00:00:00 2001 From: Nicholas McDonnell <50747025+mcdonnnj@users.noreply.github.com> Date: Mon, 12 Aug 2024 06:24:06 -0400 Subject: [PATCH 15/75] Use the hashicorp/setup-packer GitHub Action Instead of manually installing Packer we can instead leverage the hashicorp/setup-packer Action just as we do for Terraform. --- .github/workflows/build.yml | 20 +++----------------- 1 file changed, 3 insertions(+), 17 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 9bb221a..e12b842 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -20,7 +20,6 @@ defaults: shell: bash -Eueo pipefail -x {0} env: - CURL_CACHE_DIR: ~/.cache/curl PIP_CACHE_DIR: ~/.cache/pip PRE_COMMIT_CACHE_DIR: ~/.cache/pre-commit RUN_TMATE: ${{ secrets.RUN_TMATE }} @@ -97,25 +96,12 @@ jobs: path: | ${{ env.PIP_CACHE_DIR }} ${{ env.PRE_COMMIT_CACHE_DIR }} - ${{ env.CURL_CACHE_DIR }} ${{ steps.go-cache.outputs.dir }} restore-keys: | ${{ env.BASE_CACHE_KEY }} - - name: Setup curl cache - run: mkdir -p ${{ env.CURL_CACHE_DIR }} - - name: Install Packer - env: - PACKER_VERSION: ${{ steps.setup-env.outputs.packer-version }} - run: | - PACKER_ZIP="packer_${PACKER_VERSION}_linux_amd64.zip" - curl --output ${{ env.CURL_CACHE_DIR }}/"${PACKER_ZIP}" \ - --time-cond ${{ env.CURL_CACHE_DIR }}/"${PACKER_ZIP}" \ - --location \ - "https://releases.hashicorp.com/packer/${PACKER_VERSION}/${PACKER_ZIP}" - sudo unzip -d /opt/packer \ - ${{ env.CURL_CACHE_DIR }}/"${PACKER_ZIP}" - sudo mv /usr/local/bin/packer /usr/local/bin/packer-default - sudo ln -s /opt/packer/packer /usr/local/bin/packer + - uses: hashicorp/setup-packer@v3 + with: + version: ${{ steps.setup-env.outputs.packer-version }} - uses: hashicorp/setup-terraform@v3 with: terraform_version: ${{ steps.setup-env.outputs.terraform-version }} From 8ada75d419c3ea546843fc0772d9d0b678beeea4 Mon Sep 17 00:00:00 2001 From: Jeremy Frasier Date: Fri, 23 Aug 2024 00:54:54 -0400 Subject: [PATCH 16/75] Remove @jasonodoom as a codeowner He is no longer a member of @cisagov/vm-dev. --- .github/CODEOWNERS | 28 ++++++++++++++-------------- 1 file changed, 14 insertions(+), 14 deletions(-) diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index 229920c..3af99ba 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -3,22 +3,22 @@ # These owners will be the default owners for everything in the # repo. Unless a later match takes precedence, these owners will be # requested for review when someone opens a pull request. -* @dav3r @felddy @jasonodoom @jsf9k @mcdonnnj +* @dav3r @felddy @jsf9k @mcdonnnj # These folks own any files in the .github directory at the root of # the repository and any of its subdirectories. -/.github/ @dav3r @felddy @jasonodoom @jsf9k @mcdonnnj +/.github/ @dav3r @felddy @jsf9k @mcdonnnj # These folks own all linting configuration files. -/.ansible-lint @dav3r @felddy @jasonodoom @jsf9k @mcdonnnj -/.bandit.yml @dav3r @felddy @jasonodoom @jsf9k @mcdonnnj -/.flake8 @dav3r @felddy @jasonodoom @jsf9k @mcdonnnj -/.isort.cfg @dav3r @felddy @jasonodoom @jsf9k @mcdonnnj -/.mdl_config.yaml @dav3r @felddy @jasonodoom @jsf9k @mcdonnnj -/.pre-commit-config.yaml @dav3r @felddy @jasonodoom @jsf9k @mcdonnnj -/.prettierignore @dav3r @felddy @jasonodoom @jsf9k @mcdonnnj -/.yamllint @dav3r @felddy @jasonodoom @jsf9k @mcdonnnj -/requirements.txt @dav3r @felddy @jasonodoom @jsf9k @mcdonnnj -/requirements-dev.txt @dav3r @felddy @jasonodoom @jsf9k @mcdonnnj -/requirements-test.txt @dav3r @felddy @jasonodoom @jsf9k @mcdonnnj -/setup-env @dav3r @felddy @jasonodoom @jsf9k @mcdonnnj +/.ansible-lint @dav3r @felddy @jsf9k @mcdonnnj +/.bandit.yml @dav3r @felddy @jsf9k @mcdonnnj +/.flake8 @dav3r @felddy @jsf9k @mcdonnnj +/.isort.cfg @dav3r @felddy @jsf9k @mcdonnnj +/.mdl_config.yaml @dav3r @felddy @jsf9k @mcdonnnj +/.pre-commit-config.yaml @dav3r @felddy @jsf9k @mcdonnnj +/.prettierignore @dav3r @felddy @jsf9k @mcdonnnj +/.yamllint @dav3r @felddy @jsf9k @mcdonnnj +/requirements.txt @dav3r @felddy @jsf9k @mcdonnnj +/requirements-dev.txt @dav3r @felddy @jsf9k @mcdonnnj +/requirements-test.txt @dav3r @felddy @jsf9k @mcdonnnj +/setup-env @dav3r @felddy @jsf9k @mcdonnnj From 293020830fb6830a7324b5eacb8c3122979d9882 Mon Sep 17 00:00:00 2001 From: Shane Frasier Date: Mon, 26 Aug 2024 09:27:58 -0400 Subject: [PATCH 17/75] Pin to a specific version Previously we only provided a lower bound for the version, but pinning to a specific version aligns with what has been done with the prettier hook and how pre-commit hooks are pinned in general. The flake8-docstrings package is rarely updated, so there is no real downside to pinning to a specific version. Co-authored-by: Nick <50747025+mcdonnnj@users.noreply.github.com> --- .pre-commit-config.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 74c9c76..236eeda 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -136,7 +136,7 @@ repos: hooks: - id: flake8 additional_dependencies: - - flake8-docstrings>=1.7.0 + - flake8-docstrings==1.7.0 - repo: https://github.com/PyCQA/isort rev: 5.13.2 hooks: From 46e055367c1e34711ed0980b2934b9df54bf33fe Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 24 Sep 2024 18:23:01 +0000 Subject: [PATCH 18/75] Bump actions/cache from 3 to 4 Bumps [actions/cache](https://github.com/actions/cache) from 3 to 4. - [Release notes](https://github.com/actions/cache/releases) - [Changelog](https://github.com/actions/cache/blob/main/RELEASES.md) - [Commits](https://github.com/actions/cache/compare/v3...v4) --- updated-dependencies: - dependency-name: actions/cache dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] --- .github/workflows/build.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 9bb221a..a403ea9 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -76,7 +76,7 @@ jobs: name: Lookup Go cache directory run: | echo "dir=$(go env GOCACHE)" >> $GITHUB_OUTPUT - - uses: actions/cache@v3 + - uses: actions/cache@v4 env: BASE_CACHE_KEY: "${{ github.job }}-${{ runner.os }}-\ py${{ steps.setup-python.outputs.python-version }}-\ From 3167421109abf3fe94dc801203587e1bf3ce33a3 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 24 Sep 2024 18:23:14 +0000 Subject: [PATCH 19/75] Bump crazy-max/ghaction-github-status from 3 to 4 Bumps [crazy-max/ghaction-github-status](https://github.com/crazy-max/ghaction-github-status) from 3 to 4. - [Release notes](https://github.com/crazy-max/ghaction-github-status/releases) - [Commits](https://github.com/crazy-max/ghaction-github-status/compare/v3...v4) --- updated-dependencies: - dependency-name: crazy-max/ghaction-github-status dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] --- .github/workflows/sync-labels.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/sync-labels.yml b/.github/workflows/sync-labels.yml index 5a20438..e83bd41 100644 --- a/.github/workflows/sync-labels.yml +++ b/.github/workflows/sync-labels.yml @@ -24,7 +24,7 @@ jobs: egress-policy: audit - id: github-status name: Check GitHub status - uses: crazy-max/ghaction-github-status@v3 + uses: crazy-max/ghaction-github-status@v4 - id: dump-context name: Dump context uses: crazy-max/ghaction-dump-context@v2 From 6a58c2c24ef1eb15c7a69a44f16c63964f1c7f82 Mon Sep 17 00:00:00 2001 From: Nicholas McDonnell <50747025+mcdonnnj@users.noreply.github.com> Date: Mon, 12 Aug 2024 16:23:58 -0400 Subject: [PATCH 20/75] Update pre-commit hook versions This is done automatically with the `pre-commit autoupdate` command. The pre-commit/mirrors-prettier hook was manually held back because the latest tags are for alpha releases of the next major version. --- .pre-commit-config.yaml | 24 ++++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 386c83f..81f3276 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -31,7 +31,7 @@ repos: # Text file hooks - repo: https://github.com/igorshubovych/markdownlint-cli - rev: v0.41.0 + rev: v0.42.0 hooks: - id: markdownlint args: @@ -56,14 +56,14 @@ repos: # GitHub Actions hooks - repo: https://github.com/python-jsonschema/check-jsonschema - rev: 0.28.4 + rev: 0.29.2 hooks: - id: check-github-actions - id: check-github-workflows # pre-commit hooks - repo: https://github.com/pre-commit/pre-commit - rev: v3.7.1 + rev: v3.8.0 hooks: - id: validate_manifest @@ -98,7 +98,7 @@ repos: # Shell script hooks - repo: https://github.com/scop/pre-commit-shfmt - rev: v3.8.0-1 + rev: v3.9.0-1 hooks: - id: shfmt args: @@ -122,17 +122,17 @@ repos: # Python hooks - repo: https://github.com/PyCQA/bandit - rev: 1.7.8 + rev: 1.7.10 hooks: - id: bandit args: - --config=.bandit.yml - repo: https://github.com/psf/black-pre-commit-mirror - rev: 24.4.2 + rev: 24.8.0 hooks: - id: black - repo: https://github.com/PyCQA/flake8 - rev: 7.0.0 + rev: 7.1.1 hooks: - id: flake8 additional_dependencies: @@ -142,17 +142,17 @@ repos: hooks: - id: isort - repo: https://github.com/pre-commit/mirrors-mypy - rev: v1.10.0 + rev: v1.11.2 hooks: - id: mypy - repo: https://github.com/asottile/pyupgrade - rev: v3.15.2 + rev: v3.17.0 hooks: - id: pyupgrade # Ansible hooks - repo: https://github.com/ansible/ansible-lint - rev: v24.6.0 + rev: v24.9.2 hooks: - id: ansible-lint additional_dependencies: @@ -177,7 +177,7 @@ repos: # Terraform hooks - repo: https://github.com/antonbabenko/pre-commit-terraform - rev: v1.90.0 + rev: v1.96.1 hooks: - id: terraform_fmt - id: terraform_validate @@ -190,7 +190,7 @@ repos: # Packer hooks - repo: https://github.com/cisagov/pre-commit-packer - rev: v0.0.2 + rev: v0.1.0 hooks: - id: packer_validate - id: packer_fmt From 553efcb0d4e755ebd47abb49c865367ed6d0a236 Mon Sep 17 00:00:00 2001 From: Nicholas McDonnell <50747025+mcdonnnj@users.noreply.github.com> Date: Mon, 12 Aug 2024 16:30:49 -0400 Subject: [PATCH 21/75] Manually update the prettier hook Use the latest v3 release available from NPM. --- .pre-commit-config.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 81f3276..2104775 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -46,7 +46,7 @@ repos: # mirror does not pull tags for old major versions once a new major # version tag is published. additional_dependencies: - - prettier@3.3.1 + - prettier@3.3.3 - repo: https://github.com/adrienverge/yamllint rev: v1.35.1 hooks: From 045a998dcf14dc7e3de9301ba7ee2103272b0ac4 Mon Sep 17 00:00:00 2001 From: Nicholas McDonnell <50747025+mcdonnnj@users.noreply.github.com> Date: Thu, 18 Jan 2024 16:11:15 -0500 Subject: [PATCH 22/75] Add a pre-commit hook to run pip-audit The pip-audit tool will audit any supplied pip requirements files for vulnerable packages. --- .pre-commit-config.yaml | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 2c5b3c8..78140ff 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -145,6 +145,18 @@ repos: rev: v1.8.0 hooks: - id: mypy + - repo: https://github.com/pypa/pip-audit + rev: v2.7.3 + hooks: + - id: pip-audit + args: + # Add any pip requirements files to scan + - --requirement + - requirements-dev.txt + - --requirement + - requirements-test.txt + - --requirement + - requirements.txt - repo: https://github.com/asottile/pyupgrade rev: v3.15.1 hooks: From c502f1ab7cca8bd383a34360ce456b50fd6e8b21 Mon Sep 17 00:00:00 2001 From: Nicholas McDonnell <50747025+mcdonnnj@users.noreply.github.com> Date: Mon, 12 Aug 2024 16:32:02 -0400 Subject: [PATCH 23/75] Use the rbubley/mirrors-prettier hook for prettier This replaces the now archived pre-commit/mirrors-prettier hook. --- .pre-commit-config.yaml | 11 ++--------- 1 file changed, 2 insertions(+), 9 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index ca59d6f..3cb1f85 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -40,17 +40,10 @@ repos: - id: markdownlint args: - --config=.mdl_config.yaml - - repo: https://github.com/pre-commit/mirrors-prettier - # This is the last version of v3 available from the mirror. We should hold - # here until v4, which is currently in alpha, is more stable. - rev: v3.1.0 + - repo: https://github.com/rbubley/mirrors-prettier + rev: v3.3.3 hooks: - id: prettier - # This is the latest version of v3 available from NPM. The pre-commit - # mirror does not pull tags for old major versions once a new major - # version tag is published. - additional_dependencies: - - prettier@3.3.3 - repo: https://github.com/adrienverge/yamllint rev: v1.35.1 hooks: From 942c0dc98f605282fdf3c0ac6b9a549647f89f41 Mon Sep 17 00:00:00 2001 From: Nicholas McDonnell <50747025+mcdonnnj@users.noreply.github.com> Date: Tue, 13 Aug 2024 06:17:33 -0400 Subject: [PATCH 24/75] Add a new trigger for the sync-labels GitHub Actions workflow Add a `workflow_dispatch` trigger so we can manually run the workflow if needed. --- .github/workflows/sync-labels.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/sync-labels.yml b/.github/workflows/sync-labels.yml index e83bd41..59aefe4 100644 --- a/.github/workflows/sync-labels.yml +++ b/.github/workflows/sync-labels.yml @@ -6,6 +6,7 @@ on: paths: - '.github/labels.yml' - '.github/workflows/sync-labels.yml' + workflow_dispatch: permissions: contents: read From a267662455c30986086d4ca14173cc20af7161d4 Mon Sep 17 00:00:00 2001 From: Nicholas McDonnell <50747025+mcdonnnj@users.noreply.github.com> Date: Tue, 13 Aug 2024 06:19:38 -0400 Subject: [PATCH 25/75] Remove unnecessary quotes in the sync-labels workflow --- .github/workflows/sync-labels.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/sync-labels.yml b/.github/workflows/sync-labels.yml index 59aefe4..5d5ab41 100644 --- a/.github/workflows/sync-labels.yml +++ b/.github/workflows/sync-labels.yml @@ -4,8 +4,8 @@ name: sync-labels on: push: paths: - - '.github/labels.yml' - - '.github/workflows/sync-labels.yml' + - .github/labels.yml + - .github/workflows/sync-labels.yml workflow_dispatch: permissions: From dc7f09e29b8466af0fa2f788761e22dd2fcbd0ce Mon Sep 17 00:00:00 2001 From: Nicholas McDonnell <50747025+mcdonnnj@users.noreply.github.com> Date: Sat, 14 Sep 2024 18:44:01 -0400 Subject: [PATCH 26/75] Add four new hooks from pre-commit/pre-commit-hooks --- .pre-commit-config.yaml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 3cb1f85..c98ded8 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -16,9 +16,13 @@ repos: - id: check-executables-have-shebangs - id: check-json - id: check-merge-conflict + - id: check-shebang-scripts-are-executable + - id: check-symlinks - id: check-toml + - id: check-vcs-permalinks - id: check-xml - id: debug-statements + - id: destroyed-symlinks - id: detect-aws-credentials args: - --allow-missing-credentials From 343d2ccbd1cd983374235e5d3bfcecd3187c00d5 Mon Sep 17 00:00:00 2001 From: Nicholas McDonnell <50747025+mcdonnnj@users.noreply.github.com> Date: Mon, 28 Oct 2024 17:47:53 -0400 Subject: [PATCH 27/75] Add the GitHubSecurityLab/actions-permissions/monitor Action This Action will provide information about the usage of GITHUB_TOKEN in the workflow. It should be added to _every_ job in _any_ workflow to provide information for analysis. --- .github/dependabot.yml | 1 + .github/workflows/build.yml | 10 ++++++++++ .github/workflows/sync-labels.yml | 10 ++++++++++ 3 files changed, 21 insertions(+) diff --git a/.github/dependabot.yml b/.github/dependabot.yml index 17220c6..4a6667f 100644 --- a/.github/dependabot.yml +++ b/.github/dependabot.yml @@ -16,6 +16,7 @@ updates: # - dependency-name: crazy-max/ghaction-dump-context # - dependency-name: crazy-max/ghaction-github-labeler # - dependency-name: crazy-max/ghaction-github-status + # - dependency-name: GitHubSecurityLab/actions-permissions # - dependency-name: hashicorp/setup-terraform # - dependency-name: mxschmitt/action-tmate # - dependency-name: step-security/harden-runner diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index e7a60b2..2cdd921 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -34,6 +34,12 @@ jobs: steps: # Note that a duplicate of this step must be added at the top of # each job. + - uses: GitHubSecurityLab/actions-permissions/monitor@v1 + with: + # Uses the organization variable unless overridden + config: ${{ vars.ACTIONS_PERMISSIONS_CONFIG }} + # Note that a duplicate of this step must be added at the top of + # each job. - id: harden-runner name: Harden the runner uses: step-security/harden-runner@v2 @@ -50,6 +56,10 @@ jobs: - diagnostics runs-on: ubuntu-latest steps: + - uses: GitHubSecurityLab/actions-permissions/monitor@v1 + with: + # Uses the organization variable unless overridden + config: ${{ vars.ACTIONS_PERMISSIONS_CONFIG }} - id: harden-runner name: Harden the runner uses: step-security/harden-runner@v2 diff --git a/.github/workflows/sync-labels.yml b/.github/workflows/sync-labels.yml index e83bd41..d2458d1 100644 --- a/.github/workflows/sync-labels.yml +++ b/.github/workflows/sync-labels.yml @@ -17,6 +17,12 @@ jobs: steps: # Note that a duplicate of this step must be added at the top of # each job. + - uses: GitHubSecurityLab/actions-permissions/monitor@v1 + with: + # Uses the organization variable unless overridden + config: ${{ vars.ACTIONS_PERMISSIONS_CONFIG }} + # Note that a duplicate of this step must be added at the top of + # each job. - id: harden-runner name: Harden the runner uses: step-security/harden-runner@v2 @@ -38,6 +44,10 @@ jobs: issues: write runs-on: ubuntu-latest steps: + - uses: GitHubSecurityLab/actions-permissions/monitor@v1 + with: + # Uses the organization variable unless overridden + config: ${{ vars.ACTIONS_PERMISSIONS_CONFIG }} - id: harden-runner name: Harden the runner uses: step-security/harden-runner@v2 From 8a77a8b77a7d5e5247e8ff563d93a14510e09b9a Mon Sep 17 00:00:00 2001 From: Nicholas McDonnell <50747025+mcdonnnj@users.noreply.github.com> Date: Mon, 28 Oct 2024 17:09:15 -0400 Subject: [PATCH 28/75] Restrict permissions of GITHUB_TOKEN This changes the default permissions for the GITHUB_TOKEN used in our GitHub Actions configuration to the minimum required to successfully run. --- .github/workflows/build.yml | 5 +++++ .github/workflows/sync-labels.yml | 2 ++ 2 files changed, 7 insertions(+) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index e7a60b2..d4340af 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -30,6 +30,8 @@ env: jobs: diagnostics: name: Run diagnostics + # This job does not need any permissions + permissions: {} runs-on: ubuntu-latest steps: # Note that a duplicate of this step must be added at the top of @@ -48,6 +50,9 @@ jobs: lint: needs: - diagnostics + permissions: + # actions/checkout needs this to fetch code + contents: read runs-on: ubuntu-latest steps: - id: harden-runner diff --git a/.github/workflows/sync-labels.yml b/.github/workflows/sync-labels.yml index e83bd41..39e7379 100644 --- a/.github/workflows/sync-labels.yml +++ b/.github/workflows/sync-labels.yml @@ -13,6 +13,8 @@ permissions: jobs: diagnostics: name: Run diagnostics + # This job does not need any permissions + permissions: {} runs-on: ubuntu-latest steps: # Note that a duplicate of this step must be added at the top of From 3b1d4ef0fae08e6444e9b414ce1315841e681322 Mon Sep 17 00:00:00 2001 From: Nicholas McDonnell <50747025+mcdonnnj@users.noreply.github.com> Date: Wed, 16 Oct 2024 12:53:42 -0400 Subject: [PATCH 29/75] Update pre-commit hook versions This is done automatically with the `pre-commit autoupdate` command. --- .pre-commit-config.yaml | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 3cb1f85..26b399d 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -10,7 +10,7 @@ repos: - id: check-useless-excludes - repo: https://github.com/pre-commit/pre-commit-hooks - rev: v4.6.0 + rev: v5.0.0 hooks: - id: check-case-conflict - id: check-executables-have-shebangs @@ -53,14 +53,14 @@ repos: # GitHub Actions hooks - repo: https://github.com/python-jsonschema/check-jsonschema - rev: 0.29.2 + rev: 0.29.4 hooks: - id: check-github-actions - id: check-github-workflows # pre-commit hooks - repo: https://github.com/pre-commit/pre-commit - rev: v3.8.0 + rev: v4.0.1 hooks: - id: validate_manifest @@ -95,7 +95,7 @@ repos: # Shell script hooks - repo: https://github.com/scop/pre-commit-shfmt - rev: v3.9.0-1 + rev: v3.10.0-1 hooks: - id: shfmt args: @@ -125,7 +125,7 @@ repos: args: - --config=.bandit.yml - repo: https://github.com/psf/black-pre-commit-mirror - rev: 24.8.0 + rev: 24.10.0 hooks: - id: black - repo: https://github.com/PyCQA/flake8 @@ -139,7 +139,7 @@ repos: hooks: - id: isort - repo: https://github.com/pre-commit/mirrors-mypy - rev: v1.11.2 + rev: v1.13.0 hooks: - id: mypy - repo: https://github.com/pypa/pip-audit @@ -155,7 +155,7 @@ repos: - --requirement - requirements.txt - repo: https://github.com/asottile/pyupgrade - rev: v3.17.0 + rev: v3.19.0 hooks: - id: pyupgrade @@ -199,7 +199,7 @@ repos: # Packer hooks - repo: https://github.com/cisagov/pre-commit-packer - rev: v0.1.0 + rev: v0.3.0 hooks: - id: packer_validate - id: packer_fmt From 1d285f2d851926effdbfbdcf58853ce70d1bf016 Mon Sep 17 00:00:00 2001 From: Nicholas McDonnell <50747025+mcdonnnj@users.noreply.github.com> Date: Tue, 29 Oct 2024 16:36:27 -0400 Subject: [PATCH 30/75] Sort hook ids in each pre-commit hook entry Ensure that all hook ids are sorted alphabetically in each hook entry in our pre-commit configuration. --- .pre-commit-config.yaml | 24 ++++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 3cb1f85..0fd3234 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -68,25 +68,25 @@ repos: - repo: https://github.com/TekWizely/pre-commit-golang rev: v1.0.0-rc.1 hooks: - # Style Checkers - - id: go-critic - # StaticCheck - - id: go-staticcheck-repo-mod # Go Build - id: go-build-repo-mod + # Style Checkers + - id: go-critic + # goimports + - id: go-imports-repo + args: + # Write changes to files + - -w # Go Mod Tidy - id: go-mod-tidy-repo + # GoSec + - id: go-sec-repo-mod + # StaticCheck + - id: go-staticcheck-repo-mod # Go Test - id: go-test-repo-mod # Go Vet - id: go-vet-repo-mod - # GoSec - - id: go-sec-repo-mod - # goimports - - id: go-imports-repo - args: - # Write changes to files - - -w # Nix hooks - repo: https://github.com/nix-community/nixpkgs-fmt rev: v1.3.0 @@ -201,5 +201,5 @@ repos: - repo: https://github.com/cisagov/pre-commit-packer rev: v0.1.0 hooks: - - id: packer_validate - id: packer_fmt + - id: packer_validate From 3843f1799f4ef95f62ef9c280fb4351005029048 Mon Sep 17 00:00:00 2001 From: Jeremy Frasier Date: Wed, 30 Oct 2024 13:17:04 -0400 Subject: [PATCH 31/75] Uncomment new Dependabot ignore directive from upstream --- .github/dependabot.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/dependabot.yml b/.github/dependabot.yml index a11003a..27bad88 100644 --- a/.github/dependabot.yml +++ b/.github/dependabot.yml @@ -16,7 +16,7 @@ updates: - dependency-name: crazy-max/ghaction-dump-context - dependency-name: crazy-max/ghaction-github-labeler - dependency-name: crazy-max/ghaction-github-status - # - dependency-name: GitHubSecurityLab/actions-permissions + - dependency-name: GitHubSecurityLab/actions-permissions - dependency-name: hashicorp/setup-terraform - dependency-name: mxschmitt/action-tmate - dependency-name: step-security/harden-runner From 8824475dfadd1a9cbc9ce9bd1c9f31e4a688994b Mon Sep 17 00:00:00 2001 From: Nicholas McDonnell <50747025+mcdonnnj@users.noreply.github.com> Date: Fri, 1 Nov 2024 12:25:35 -0400 Subject: [PATCH 32/75] Update the commented out dependabot ignore directives Add a directive for hashicorp/setup-packer that was missed when it was added to the `build` workflow. Add a directive for cisagov/setup-env-github-action that is not strictly necessary since we currently just pull from the `develop` branch, but is good to have in case we were to change that in the future. --- .github/dependabot.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/dependabot.yml b/.github/dependabot.yml index 4a6667f..81cd6bd 100644 --- a/.github/dependabot.yml +++ b/.github/dependabot.yml @@ -13,10 +13,12 @@ updates: # - dependency-name: actions/checkout # - dependency-name: actions/setup-go # - dependency-name: actions/setup-python + # - dependency-name: cisagov/setup-env-github-action # - dependency-name: crazy-max/ghaction-dump-context # - dependency-name: crazy-max/ghaction-github-labeler # - dependency-name: crazy-max/ghaction-github-status # - dependency-name: GitHubSecurityLab/actions-permissions + # - dependency-name: hashicorp/setup-packer # - dependency-name: hashicorp/setup-terraform # - dependency-name: mxschmitt/action-tmate # - dependency-name: step-security/harden-runner From 5a6801b6de42f7126233ea0ec0582e6208d2078c Mon Sep 17 00:00:00 2001 From: Jeremy Frasier Date: Wed, 30 Oct 2024 13:22:08 -0400 Subject: [PATCH 33/75] Install Packer via hashicorp/setup-packer everywhere --- .github/workflows/build.yml | 39 +++++--------------------------- .github/workflows/prerelease.yml | 20 +++------------- .github/workflows/release.yml | 20 +++------------- 3 files changed, 12 insertions(+), 67 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index e160f4f..aba9570 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -21,7 +21,6 @@ defaults: env: AWS_DEFAULT_REGION: us-east-1 - CURL_CACHE_DIR: ~/.cache/curl PIP_CACHE_DIR: ~/.cache/pip PRE_COMMIT_CACHE_DIR: ~/.cache/pre-commit RUN_TMATE: ${{ secrets.RUN_TMATE }} @@ -201,27 +200,14 @@ jobs: with: path: | ${{ env.PIP_CACHE_DIR }} - ${{ env.CURL_CACHE_DIR }} key: "${{ env.BASE_CACHE_KEY }}\ ${{ hashFiles('**/requirements-test.txt') }}-\ ${{ hashFiles('**/requirements.txt') }}" restore-keys: | ${{ env.BASE_CACHE_KEY }} - - name: Setup curl cache - run: mkdir -p ${{ env.CURL_CACHE_DIR }} - - name: Install Packer - env: - PACKER_VERSION: ${{ steps.setup-env.outputs.packer-version }} - run: | - PACKER_ZIP="packer_${PACKER_VERSION}_linux_amd64.zip" - curl --output ${{ env.CURL_CACHE_DIR }}/"${PACKER_ZIP}" \ - --time-cond ${{ env.CURL_CACHE_DIR }}/"${PACKER_ZIP}" \ - --location \ - "https://releases.hashicorp.com/packer/${PACKER_VERSION}/${PACKER_ZIP}" - sudo unzip -d /opt/packer \ - ${{ env.CURL_CACHE_DIR }}/"${PACKER_ZIP}" - sudo mv /usr/local/bin/packer /usr/local/bin/packer-default - sudo ln -s /opt/packer/packer /usr/local/bin/packer + - uses: hashicorp/setup-packer@v3 + with: + version: ${{ steps.setup-env.outputs.packer-version }} - name: Install dependencies run: | python -m pip install --upgrade pip @@ -268,26 +254,13 @@ jobs: with: path: | ${{ env.PIP_CACHE_DIR }} - ${{ env.CURL_CACHE_DIR }} key: "${{ env.BASE_CACHE_KEY }}\ ${{ hashFiles('**/requirements.txt') }}" restore-keys: | ${{ env.BASE_CACHE_KEY }} - - name: Setup curl cache - run: mkdir -p ${{ env.CURL_CACHE_DIR }} - - name: Install Packer - env: - PACKER_VERSION: ${{ steps.setup-env.outputs.packer-version }} - run: | - PACKER_ZIP="packer_${PACKER_VERSION}_linux_amd64.zip" - curl --output ${{ env.CURL_CACHE_DIR }}/"${PACKER_ZIP}" \ - --time-cond ${{ env.CURL_CACHE_DIR }}/"${PACKER_ZIP}" \ - --location \ - "https://releases.hashicorp.com/packer/${PACKER_VERSION}/${PACKER_ZIP}" - sudo unzip -d /opt/packer \ - ${{ env.CURL_CACHE_DIR }}/"${PACKER_ZIP}" - sudo mv /usr/local/bin/packer /usr/local/bin/packer-default - sudo ln -s /opt/packer/packer /usr/local/bin/packer + - uses: hashicorp/setup-packer@v3 + with: + version: ${{ steps.setup-env.outputs.packer-version }} - uses: hashicorp/setup-terraform@v3 with: terraform_version: ${{ steps.setup-env.outputs.terraform-version }} diff --git a/.github/workflows/prerelease.yml b/.github/workflows/prerelease.yml index dcc3d6a..c2e25fe 100644 --- a/.github/workflows/prerelease.yml +++ b/.github/workflows/prerelease.yml @@ -7,7 +7,6 @@ on: env: AWS_DEFAULT_REGION: us-east-1 - CURL_CACHE_DIR: ~/.cache/curl PIP_CACHE_DIR: ~/.cache/pip RUN_TMATE: ${{ secrets.RUN_TMATE }} @@ -61,26 +60,13 @@ jobs: with: path: | ${{ env.PIP_CACHE_DIR }} - ${{ env.CURL_CACHE_DIR }} key: "${{ env.BASE_CACHE_KEY }}\ ${{ hashFiles('**/requirements.txt') }}" restore-keys: | ${{ env.BASE_CACHE_KEY }} - - name: Setup curl cache - run: mkdir -p ${{ env.CURL_CACHE_DIR }} - - name: Install Packer - env: - PACKER_VERSION: ${{ steps.setup-env.outputs.packer-version }} - run: | - PACKER_ZIP="packer_${PACKER_VERSION}_linux_amd64.zip" - curl --output ${{ env.CURL_CACHE_DIR }}/"${PACKER_ZIP}" \ - --time-cond ${{ env.CURL_CACHE_DIR }}/"${PACKER_ZIP}" \ - --location \ - "https://releases.hashicorp.com/packer/${PACKER_VERSION}/${PACKER_ZIP}" - sudo unzip -d /opt/packer \ - ${{ env.CURL_CACHE_DIR }}/"${PACKER_ZIP}" - sudo mv /usr/local/bin/packer /usr/local/bin/packer-default - sudo ln -s /opt/packer/packer /usr/local/bin/packer + - uses: hashicorp/setup-packer@v3 + with: + version: ${{ steps.setup-env.outputs.packer-version }} - uses: hashicorp/setup-terraform@v3 with: terraform_version: ${{ steps.setup-env.outputs.terraform-version }} diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index f055508..9bb5e02 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -14,7 +14,6 @@ env: # COPY_REGIONS_KMS_MAP: "us-east-2:alias/cool-amis, # us-west-1:alias/cool-amis, # us-west-2:alias/cool-amis" - CURL_CACHE_DIR: ~/.cache/curl PIP_CACHE_DIR: ~/.cache/pip RUN_TMATE: ${{ secrets.RUN_TMATE }} @@ -68,26 +67,13 @@ jobs: with: path: | ${{ env.PIP_CACHE_DIR }} - ${{ env.CURL_CACHE_DIR }} key: "${{ env.BASE_CACHE_KEY }}\ ${{ hashFiles('**/requirements.txt') }}" restore-keys: | ${{ env.BASE_CACHE_KEY }} - - name: Setup curl cache - run: mkdir -p ${{ env.CURL_CACHE_DIR }} - - name: Install Packer - env: - PACKER_VERSION: ${{ steps.setup-env.outputs.packer-version }} - run: | - PACKER_ZIP="packer_${PACKER_VERSION}_linux_amd64.zip" - curl --output ${{ env.CURL_CACHE_DIR }}/"${PACKER_ZIP}" \ - --time-cond ${{ env.CURL_CACHE_DIR }}/"${PACKER_ZIP}" \ - --location \ - "https://releases.hashicorp.com/packer/${PACKER_VERSION}/${PACKER_ZIP}" - sudo unzip -d /opt/packer \ - ${{ env.CURL_CACHE_DIR }}/"${PACKER_ZIP}" - sudo mv /usr/local/bin/packer /usr/local/bin/packer-default - sudo ln -s /opt/packer/packer /usr/local/bin/packer + - uses: hashicorp/setup-packer@v3 + with: + version: ${{ steps.setup-env.outputs.packer-version }} - uses: hashicorp/setup-terraform@v3 with: terraform_version: ${{ steps.setup-env.outputs.terraform-version }} From 7caddb4884f3ddf23a4a9b5f79c337c1679cc90a Mon Sep 17 00:00:00 2001 From: Jeremy Frasier Date: Wed, 30 Oct 2024 13:24:01 -0400 Subject: [PATCH 34/75] Upgrade Bandit to 1.7.10 We should use the same version of Bandit throughout the pre-commit configuration. --- .pre-commit-config.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 12c848c..c4c8bf9 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -133,7 +133,7 @@ repos: - --config=.bandit.yml # Run bandit on everything except the "tests" tree - repo: https://github.com/PyCQA/bandit - rev: 1.7.7 + rev: 1.7.10 hooks: - id: bandit name: bandit (everything else) From e0fac13547fc7e76b8724585665476d82156f241 Mon Sep 17 00:00:00 2001 From: Jeremy Frasier Date: Wed, 30 Oct 2024 13:31:49 -0400 Subject: [PATCH 35/75] Add explicit permissions for jobs that lack them --- .github/workflows/build.yml | 4 ++++ .github/workflows/prerelease.yml | 4 ++++ .github/workflows/release.yml | 4 ++++ 3 files changed, 12 insertions(+) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index aba9570..22dcf42 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -178,6 +178,8 @@ jobs: test: needs: - diagnostics + # This job does not need any permissions + permissions: {} runs-on: ubuntu-latest steps: - id: harden-runner @@ -225,6 +227,8 @@ jobs: needs: - lint - test + # This job does not need any permissions + permissions: {} runs-on: ubuntu-latest strategy: fail-fast: false diff --git a/.github/workflows/prerelease.yml b/.github/workflows/prerelease.yml index c2e25fe..6105305 100644 --- a/.github/workflows/prerelease.yml +++ b/.github/workflows/prerelease.yml @@ -13,6 +13,8 @@ env: jobs: diagnostics: name: Run diagnostics + # This job does not need any permissions + permissions: {} runs-on: ubuntu-latest steps: # Note that a duplicate of this step must be added at the top of @@ -31,6 +33,8 @@ jobs: prerelease: needs: - diagnostics + # This job does not need any permissions + permissions: {} runs-on: ubuntu-latest strategy: fail-fast: false diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 9bb5e02..e438d59 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -20,6 +20,8 @@ env: jobs: diagnostics: name: Run diagnostics + # This job does not need any permissions + permissions: {} runs-on: ubuntu-latest steps: # Note that a duplicate of this step must be added at the top of @@ -38,6 +40,8 @@ jobs: release: needs: - diagnostics + # This job does not need any permissions + permissions: {} runs-on: ubuntu-latest strategy: fail-fast: false From c98703c8b28c6d7ead8e85731584020831b8417c Mon Sep 17 00:00:00 2001 From: Jeremy Frasier Date: Wed, 30 Oct 2024 13:35:46 -0400 Subject: [PATCH 36/75] Add GitHubSecurityLab/actions-permissions/monitor task to each job that lacks it --- .github/workflows/build.yml | 12 ++++++++++++ .github/workflows/prerelease.yml | 12 ++++++++++++ .github/workflows/release.yml | 12 ++++++++++++ 3 files changed, 36 insertions(+) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 22dcf42..91b460c 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -182,6 +182,12 @@ jobs: permissions: {} runs-on: ubuntu-latest steps: + # Note that a duplicate of this step must be added at the top of + # each job. + - uses: GitHubSecurityLab/actions-permissions/monitor@v1 + with: + # Uses the organization variable unless overridden + config: ${{ vars.ACTIONS_PERMISSIONS_CONFIG }} - id: harden-runner name: Harden the runner uses: step-security/harden-runner@v2 @@ -237,6 +243,12 @@ jobs: - arm64 - x86_64 steps: + # Note that a duplicate of this step must be added at the top of + # each job. + - uses: GitHubSecurityLab/actions-permissions/monitor@v1 + with: + # Uses the organization variable unless overridden + config: ${{ vars.ACTIONS_PERMISSIONS_CONFIG }} - id: harden-runner name: Harden the runner uses: step-security/harden-runner@v2 diff --git a/.github/workflows/prerelease.yml b/.github/workflows/prerelease.yml index 6105305..62f9ecf 100644 --- a/.github/workflows/prerelease.yml +++ b/.github/workflows/prerelease.yml @@ -19,6 +19,12 @@ jobs: steps: # Note that a duplicate of this step must be added at the top of # each job. + - uses: GitHubSecurityLab/actions-permissions/monitor@v1 + with: + # Uses the organization variable unless overridden + config: ${{ vars.ACTIONS_PERMISSIONS_CONFIG }} + # Note that a duplicate of this step must be added at the top of + # each job. - id: harden-runner name: Harden the runner uses: step-security/harden-runner@v2 @@ -43,6 +49,12 @@ jobs: - arm64 - x86_64 steps: + # Note that a duplicate of this step must be added at the top of + # each job. + - uses: GitHubSecurityLab/actions-permissions/monitor@v1 + with: + # Uses the organization variable unless overridden + config: ${{ vars.ACTIONS_PERMISSIONS_CONFIG }} - id: harden-runner name: Harden the runner uses: step-security/harden-runner@v2 diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index e438d59..ad73375 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -26,6 +26,12 @@ jobs: steps: # Note that a duplicate of this step must be added at the top of # each job. + - uses: GitHubSecurityLab/actions-permissions/monitor@v1 + with: + # Uses the organization variable unless overridden + config: ${{ vars.ACTIONS_PERMISSIONS_CONFIG }} + # Note that a duplicate of this step must be added at the top of + # each job. - id: harden-runner name: Harden the runner uses: step-security/harden-runner@v2 @@ -50,6 +56,12 @@ jobs: - arm64 - x86_64 steps: + # Note that a duplicate of this step must be added at the top of + # each job. + - uses: GitHubSecurityLab/actions-permissions/monitor@v1 + with: + # Uses the organization variable unless overridden + config: ${{ vars.ACTIONS_PERMISSIONS_CONFIG }} - id: harden-runner name: Harden the runner uses: step-security/harden-runner@v2 From 469c1661d4e6c7c7cd7fda70e3c3e4e682cc0f9b Mon Sep 17 00:00:00 2001 From: Jeremy Frasier Date: Wed, 30 Oct 2024 13:59:22 -0400 Subject: [PATCH 37/75] Remove shebang This file is not executable and hence it should not have a shebang. --- tests/test_version.py | 1 - 1 file changed, 1 deletion(-) diff --git a/tests/test_version.py b/tests/test_version.py index b4181dd..54cdd0a 100644 --- a/tests/test_version.py +++ b/tests/test_version.py @@ -1,4 +1,3 @@ -#!/usr/bin/env pytest -vs """Version tests for packer skeleton project.""" # Standard Python Libraries From b752ed86f1f59c024624e1f6f7880457f84dc82e Mon Sep 17 00:00:00 2001 From: Jeremy Frasier Date: Wed, 30 Oct 2024 22:33:26 -0400 Subject: [PATCH 38/75] Add repo read permissions for all jobs that checkout code Co-authored-by: Nick <50747025+mcdonnnj@users.noreply.github.com> --- .github/workflows/build.yml | 10 ++++++---- .github/workflows/prerelease.yml | 5 +++-- .github/workflows/release.yml | 5 +++-- 3 files changed, 12 insertions(+), 8 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 91b460c..1a95915 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -178,8 +178,9 @@ jobs: test: needs: - diagnostics - # This job does not need any permissions - permissions: {} + permissions: + # actions/checkout needs this to fetch code + contents: read runs-on: ubuntu-latest steps: # Note that a duplicate of this step must be added at the top of @@ -233,8 +234,9 @@ jobs: needs: - lint - test - # This job does not need any permissions - permissions: {} + permissions: + # actions/checkout needs this to fetch code + contents: read runs-on: ubuntu-latest strategy: fail-fast: false diff --git a/.github/workflows/prerelease.yml b/.github/workflows/prerelease.yml index 62f9ecf..5a05ad8 100644 --- a/.github/workflows/prerelease.yml +++ b/.github/workflows/prerelease.yml @@ -39,8 +39,9 @@ jobs: prerelease: needs: - diagnostics - # This job does not need any permissions - permissions: {} + permissions: + # actions/checkout needs this to fetch code + contents: read runs-on: ubuntu-latest strategy: fail-fast: false diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index ad73375..a09282c 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -46,8 +46,9 @@ jobs: release: needs: - diagnostics - # This job does not need any permissions - permissions: {} + permissions: + # actions/checkout needs this to fetch code + contents: read runs-on: ubuntu-latest strategy: fail-fast: false From 36300f3bab2ea1021997162763c30af642da284c Mon Sep 17 00:00:00 2001 From: Jeremy Frasier Date: Thu, 31 Oct 2024 14:31:13 -0400 Subject: [PATCH 39/75] Remove repeated comment --- .github/workflows/build.yml | 4 ---- .github/workflows/prerelease.yml | 2 -- .github/workflows/release.yml | 2 -- 3 files changed, 8 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 1a95915..602d9e1 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -183,8 +183,6 @@ jobs: contents: read runs-on: ubuntu-latest steps: - # Note that a duplicate of this step must be added at the top of - # each job. - uses: GitHubSecurityLab/actions-permissions/monitor@v1 with: # Uses the organization variable unless overridden @@ -245,8 +243,6 @@ jobs: - arm64 - x86_64 steps: - # Note that a duplicate of this step must be added at the top of - # each job. - uses: GitHubSecurityLab/actions-permissions/monitor@v1 with: # Uses the organization variable unless overridden diff --git a/.github/workflows/prerelease.yml b/.github/workflows/prerelease.yml index 5a05ad8..af5b325 100644 --- a/.github/workflows/prerelease.yml +++ b/.github/workflows/prerelease.yml @@ -50,8 +50,6 @@ jobs: - arm64 - x86_64 steps: - # Note that a duplicate of this step must be added at the top of - # each job. - uses: GitHubSecurityLab/actions-permissions/monitor@v1 with: # Uses the organization variable unless overridden diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index a09282c..d14d3b0 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -57,8 +57,6 @@ jobs: - arm64 - x86_64 steps: - # Note that a duplicate of this step must be added at the top of - # each job. - uses: GitHubSecurityLab/actions-permissions/monitor@v1 with: # Uses the organization variable unless overridden From 97a31b940e1b7c3177ae936d8a417c1dbc5285fd Mon Sep 17 00:00:00 2001 From: Jeremy Frasier Date: Thu, 7 Nov 2024 15:53:00 -0500 Subject: [PATCH 40/75] Update the commented out dependabot ignore directives Add a directive for hashicorp/setup-packer that was missed when it was added to the `build` workflow. Add a directive for cisagov/setup-env-github-action that is not strictly necessary since we currently just pull from the `develop` branch, but is good to have in case we were to change that in the future. --- .github/dependabot.yml | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/.github/dependabot.yml b/.github/dependabot.yml index 27bad88..632829e 100644 --- a/.github/dependabot.yml +++ b/.github/dependabot.yml @@ -7,6 +7,7 @@ updates: - directory: / +<<<<<<< HEAD ignore: # Managed by cisagov/skeleton-generic - dependency-name: actions/cache @@ -22,6 +23,23 @@ updates: - dependency-name: step-security/harden-runner # # Managed by cisagov/skeleton-packer # - dependency-name: aws-actions/configure-aws-credentials +======= + # ignore: + # # Managed by cisagov/skeleton-generic + # - dependency-name: actions/cache + # - dependency-name: actions/checkout + # - dependency-name: actions/setup-go + # - dependency-name: actions/setup-python + # - dependency-name: cisagov/setup-env-github-action + # - dependency-name: crazy-max/ghaction-dump-context + # - dependency-name: crazy-max/ghaction-github-labeler + # - dependency-name: crazy-max/ghaction-github-status + # - dependency-name: GitHubSecurityLab/actions-permissions + # - dependency-name: hashicorp/setup-packer + # - dependency-name: hashicorp/setup-terraform + # - dependency-name: mxschmitt/action-tmate + # - dependency-name: step-security/harden-runner +>>>>>>> 8824475 (Update the commented out dependabot ignore directives) package-ecosystem: github-actions schedule: interval: weekly From 52945c22c99d7102a9a9e9c68a6e43417ae2d6bc Mon Sep 17 00:00:00 2001 From: Jeremy Frasier Date: Thu, 7 Nov 2024 15:55:16 -0500 Subject: [PATCH 41/75] Resolve conflict from follow-on Lineage update --- .github/dependabot.yml | 20 ++------------------ 1 file changed, 2 insertions(+), 18 deletions(-) diff --git a/.github/dependabot.yml b/.github/dependabot.yml index 632829e..0bebd17 100644 --- a/.github/dependabot.yml +++ b/.github/dependabot.yml @@ -7,39 +7,23 @@ updates: - directory: / -<<<<<<< HEAD ignore: # Managed by cisagov/skeleton-generic - dependency-name: actions/cache - dependency-name: actions/checkout - dependency-name: actions/setup-go - dependency-name: actions/setup-python + # - dependency-name: cisagov/setup-env-github-action - dependency-name: crazy-max/ghaction-dump-context - dependency-name: crazy-max/ghaction-github-labeler - dependency-name: crazy-max/ghaction-github-status - dependency-name: GitHubSecurityLab/actions-permissions + # - dependency-name: hashicorp/setup-packer - dependency-name: hashicorp/setup-terraform - dependency-name: mxschmitt/action-tmate - dependency-name: step-security/harden-runner # # Managed by cisagov/skeleton-packer # - dependency-name: aws-actions/configure-aws-credentials -======= - # ignore: - # # Managed by cisagov/skeleton-generic - # - dependency-name: actions/cache - # - dependency-name: actions/checkout - # - dependency-name: actions/setup-go - # - dependency-name: actions/setup-python - # - dependency-name: cisagov/setup-env-github-action - # - dependency-name: crazy-max/ghaction-dump-context - # - dependency-name: crazy-max/ghaction-github-labeler - # - dependency-name: crazy-max/ghaction-github-status - # - dependency-name: GitHubSecurityLab/actions-permissions - # - dependency-name: hashicorp/setup-packer - # - dependency-name: hashicorp/setup-terraform - # - dependency-name: mxschmitt/action-tmate - # - dependency-name: step-security/harden-runner ->>>>>>> 8824475 (Update the commented out dependabot ignore directives) package-ecosystem: github-actions schedule: interval: weekly From 7748de41cc11b0516fc3f64aa9b3ff7ecdc206bb Mon Sep 17 00:00:00 2001 From: Jeremy Frasier Date: Wed, 6 Nov 2024 10:14:38 -0500 Subject: [PATCH 42/75] Uncomment new Dependabot directives from upstream --- .github/dependabot.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/dependabot.yml b/.github/dependabot.yml index 0bebd17..c0add00 100644 --- a/.github/dependabot.yml +++ b/.github/dependabot.yml @@ -13,12 +13,12 @@ updates: - dependency-name: actions/checkout - dependency-name: actions/setup-go - dependency-name: actions/setup-python - # - dependency-name: cisagov/setup-env-github-action + - dependency-name: cisagov/setup-env-github-action - dependency-name: crazy-max/ghaction-dump-context - dependency-name: crazy-max/ghaction-github-labeler - dependency-name: crazy-max/ghaction-github-status - dependency-name: GitHubSecurityLab/actions-permissions - # - dependency-name: hashicorp/setup-packer + - dependency-name: hashicorp/setup-packer - dependency-name: hashicorp/setup-terraform - dependency-name: mxschmitt/action-tmate - dependency-name: step-security/harden-runner From e9db1f59c5bd5dc791143e86373af2a1f8156d59 Mon Sep 17 00:00:00 2001 From: Jeremy Frasier Date: Thu, 7 Nov 2024 15:59:25 -0500 Subject: [PATCH 43/75] Upgrade to actions/cache@v4 everywhere --- .github/workflows/build.yml | 4 ++-- .github/workflows/prerelease.yml | 2 +- .github/workflows/release.yml | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 602d9e1..a2d99e8 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -199,7 +199,7 @@ jobs: uses: actions/setup-python@v5 with: python-version: ${{ steps.setup-env.outputs.python-version }} - - uses: actions/cache@v3 + - uses: actions/cache@v4 env: BASE_CACHE_KEY: "${{ github.job }}-${{ runner.os }}-\ py${{ steps.setup-python.outputs.python-version }}-\ @@ -259,7 +259,7 @@ jobs: uses: actions/setup-python@v5 with: python-version: ${{ steps.setup-env.outputs.python-version }} - - uses: actions/cache@v3 + - uses: actions/cache@v4 env: BASE_CACHE_KEY: "${{ github.job }}-${{ runner.os }}-\ py${{ steps.setup-python.outputs.python-version }}-\ diff --git a/.github/workflows/prerelease.yml b/.github/workflows/prerelease.yml index af5b325..98f7212 100644 --- a/.github/workflows/prerelease.yml +++ b/.github/workflows/prerelease.yml @@ -66,7 +66,7 @@ jobs: uses: actions/setup-python@v5 with: python-version: ${{ steps.setup-env.outputs.python-version }} - - uses: actions/cache@v3 + - uses: actions/cache@v4 env: BASE_CACHE_KEY: "${{ github.job }}-${{ runner.os }}-\ py${{ steps.setup-python.outputs.python-version }}-\ diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index d14d3b0..7d02eab 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -73,7 +73,7 @@ jobs: uses: actions/setup-python@v5 with: python-version: ${{ steps.setup-env.outputs.python-version }} - - uses: actions/cache@v3 + - uses: actions/cache@v4 env: BASE_CACHE_KEY: "${{ github.job }}-${{ runner.os }}-\ py${{ steps.setup-python.outputs.python-version }}-\ From 1845fd72cef9e647aeeddcdb2426a7c785e04caa Mon Sep 17 00:00:00 2001 From: Jeremy Frasier Date: Thu, 7 Nov 2024 18:38:30 -0500 Subject: [PATCH 44/75] Resolve conflict from follow-on Lineage update --- .github/dependabot.yml | 18 ------------------ 1 file changed, 18 deletions(-) diff --git a/.github/dependabot.yml b/.github/dependabot.yml index a5e74ff..c0add00 100644 --- a/.github/dependabot.yml +++ b/.github/dependabot.yml @@ -7,7 +7,6 @@ updates: - directory: / -<<<<<<< HEAD ignore: # Managed by cisagov/skeleton-generic - dependency-name: actions/cache @@ -25,23 +24,6 @@ updates: - dependency-name: step-security/harden-runner # # Managed by cisagov/skeleton-packer # - dependency-name: aws-actions/configure-aws-credentials -======= - # ignore: - # # Managed by cisagov/skeleton-generic - # - dependency-name: actions/cache - # - dependency-name: actions/checkout - # - dependency-name: actions/setup-go - # - dependency-name: actions/setup-python - # - dependency-name: cisagov/setup-env-github-action - # - dependency-name: crazy-max/ghaction-dump-context - # - dependency-name: crazy-max/ghaction-github-labeler - # - dependency-name: crazy-max/ghaction-github-status - # - dependency-name: GitHubSecurityLab/actions-permissions - # - dependency-name: hashicorp/setup-packer - # - dependency-name: hashicorp/setup-terraform - # - dependency-name: mxschmitt/action-tmate - # - dependency-name: step-security/harden-runner ->>>>>>> e6afb68083e4b6e1ec38f036dee2f5e294b5cc96 package-ecosystem: github-actions schedule: interval: weekly From 26a8bafe25f49a099a07342a1539e4dd6eb60095 Mon Sep 17 00:00:00 2001 From: Jeremy Frasier Date: Fri, 8 Nov 2024 13:22:49 -0500 Subject: [PATCH 45/75] Bump up the lower bound on ansible-core This is being done because the pip-audit pre-commit hook identifies a vulnerability in ansible-core version 2.16.13. Note that this requires that we bump up ansible to version 10 since all versions of ansible 9 have a dependency on ~=2.16.X. --- requirements.txt | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/requirements.txt b/requirements.txt index bdf26b2..0f38eb1 100644 --- a/requirements.txt +++ b/requirements.txt @@ -8,20 +8,27 @@ # as using the dnf package manager, and version 8 is currently the # oldest supported version. # -# We have tested against version 9. We want to avoid automatically +# Version 10 is required because the pip-audit pre-commit hook +# identifies a vulnerability in ansible-core 2.16.13, but all versions +# of ansible 9 have a dependency on ~=2.16.X. +# +# We have tested against version 10. We want to avoid automatically # jumping to another major version without testing, since there are # often breaking changes across major versions. This is the reason # for the upper bound. -ansible>=8,<10 +ansible>=10,<11 # ansible-core 2.16.3 through 2.16.6 suffer from the bug discussed in # ansible/ansible#82702, which breaks any symlinked files in vars, # tasks, etc. for any Ansible role installed via ansible-galaxy. # Hence we never want to install those versions. # +# Note that the pip-audit pre-commit hook identifies a vulnerability +# in ansible-core 2.16.13. +# # Note that any changes made to this dependency must also be made in # requirements-test.txt in cisagov/skeleton-ansible-role and # .pre-commit-config.yaml in cisagov/skeleton-generic. -ansible-core>=2.16.7 +ansible-core>2.16.13 boto3 docopt semver From 12a91ad97e76cd2f221fffaef4f66956533f6540 Mon Sep 17 00:00:00 2001 From: Jeremy Frasier Date: Fri, 8 Nov 2024 13:40:44 -0500 Subject: [PATCH 46/75] Bump up the lower bound on ansible-core This is being done because the pip-audit pre-commit hook identifies a vulnerability in ansible-core version 2.16.13. Note that this requires that we bump up ansible to version 10 since all versions of ansible 9 have a dependency on ~=2.16.X. --- .pre-commit-config.yaml | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index c5e1096..8b402fb 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -176,17 +176,25 @@ repos: # necessary to add the ansible package itself as an # additional dependency, with the same pinning as is done in # requirements-test.txt of cisagov/skeleton-ansible-role. - # - ansible>=9,<10 + # + # Version 10 is required because the pip-audit pre-commit + # hook identifies a vulnerability in ansible-core 2.16.13, + # but all versions of ansible 9 have a dependency on + # ~=2.16.X. + # - ansible>=10,<11 # ansible-core 2.16.3 through 2.16.6 suffer from the bug # discussed in ansible/ansible#82702, which breaks any # symlinked files in vars, tasks, etc. for any Ansible role # installed via ansible-galaxy. Hence we never want to # install those versions. # + # Note that the pip-audit pre-commit hook identifies a + # vulnerability in ansible-core 2.16.13. + # # Note that any changes made to this dependency must also be # made in requirements.txt in cisagov/skeleton-packer and # requirements-test.txt in cisagov/skeleton-ansible-role. - - ansible-core>=2.16.7 + - ansible-core>2.16.13 # Terraform hooks - repo: https://github.com/antonbabenko/pre-commit-terraform From b9f798d03afb72f33ffa625982dd5b548dea5132 Mon Sep 17 00:00:00 2001 From: Jeremy Frasier Date: Wed, 13 Nov 2024 10:29:42 -0500 Subject: [PATCH 47/75] Update the version of the ansible-lint pre-commit hook Version 24.10.0 is the first version that supports Fedora 41 as a valid platform. --- .pre-commit-config.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index c5e1096..ebd6138 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -165,7 +165,7 @@ repos: # Ansible hooks - repo: https://github.com/ansible/ansible-lint - rev: v24.9.2 + rev: v24.10.0 hooks: - id: ansible-lint additional_dependencies: From cca133a2710c5ed99e4c0ce3d06a57ec118bcf13 Mon Sep 17 00:00:00 2001 From: Jeremy Frasier Date: Wed, 13 Nov 2024 21:33:32 -0500 Subject: [PATCH 48/75] Adjust pin for ansible-core The pin of ansible-core was originally put in place because the pip-audit pre-commit hook identifies a vulnerability in ansible-core 2.16.13. Normally we would pin ansible-core to >2.16.13, but in the spirit of the earlier, optional pin of ansible>=10 we pin ansible-core to >=2.17. This effectively also pins ansible to >=10. Co-authored-by: Nick M <50747025+mcdonnnj@users.noreply.github.com> --- .pre-commit-config.yaml | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 8b402fb..b61a8f5 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -189,12 +189,14 @@ repos: # install those versions. # # Note that the pip-audit pre-commit hook identifies a - # vulnerability in ansible-core 2.16.13. + # vulnerability in ansible-core 2.16.13. The pin of + # ansible-core to >=2.17 effectively also pins ansible to + # >=10. # # Note that any changes made to this dependency must also be # made in requirements.txt in cisagov/skeleton-packer and # requirements-test.txt in cisagov/skeleton-ansible-role. - - ansible-core>2.16.13 + - ansible-core>=2.17 # Terraform hooks - repo: https://github.com/antonbabenko/pre-commit-terraform From a00c336b5f533326f97288784423db9275f2f590 Mon Sep 17 00:00:00 2001 From: Jeremy Frasier Date: Wed, 13 Nov 2024 21:41:45 -0500 Subject: [PATCH 49/75] Upgrade pin for ansible-core The pin of ansible-core was originally put in place because the pip-audit pre-commit hook identifies a vulnerability in ansible-core 2.16.13. Normally we would pin ansible-core accordingly (>2.16.13), but the earlier pin of ansible>=10 effectively pins ansible-core to >=2.17 so that's what do. Co-authored-by: Nick M <50747025+mcdonnnj@users.noreply.github.com> --- requirements.txt | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/requirements.txt b/requirements.txt index 0f38eb1..8d6bd0f 100644 --- a/requirements.txt +++ b/requirements.txt @@ -23,12 +23,14 @@ ansible>=10,<11 # Hence we never want to install those versions. # # Note that the pip-audit pre-commit hook identifies a vulnerability -# in ansible-core 2.16.13. +# in ansible-core 2.16.13. Normally we would pin ansible-core +# accordingly (>2.16.13), but the above pin of ansible>=10 effectively +# pins ansible-core to >=2.17 anyway so that's what we use. # # Note that any changes made to this dependency must also be made in # requirements-test.txt in cisagov/skeleton-ansible-role and # .pre-commit-config.yaml in cisagov/skeleton-generic. -ansible-core>2.16.13 +ansible-core>=2.17 boto3 docopt semver From b2020a838ac8851c40d8aba74af0bf2e6691d89b Mon Sep 17 00:00:00 2001 From: Jeremy Frasier Date: Mon, 18 Nov 2024 16:05:09 -0500 Subject: [PATCH 50/75] Ignore a particular ansible-core vulnerability This is being done only temporarily, and only because there is no recent version of ansible-core that does not exhibit the vulnerability. Without this change we get a failure from the pip-audit pre-commit hook that we cannot do anything about. See cisagov/skeleton-packer#380 for more details. --- .pre-commit-config.yaml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index c4c8bf9..a4686f3 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -161,6 +161,11 @@ repos: hooks: - id: pip-audit args: + # We have to ignore this particular vulnerability in + # ansible-core>=2.11 as there is currently no fix. See + # cisagov/skeleton-packer#380 for more details. + - --ignore-vuln + - GHSA-99w6-3xph-cx78 # Add any pip requirements files to scan - --requirement - requirements-dev.txt From bd852610595fdd2eee77f489d4b184f88d90643b Mon Sep 17 00:00:00 2001 From: Jeremy Frasier Date: Wed, 20 Nov 2024 12:21:14 -0500 Subject: [PATCH 51/75] Add comments about looming EOL issues for ansible and ansible-core This adds even more evidence for why it is a good idea to go ahead and upgrade ansible and ansible-core, in addition to the vulnerability that pip-audit turned up. Co-authored-by: Nick M <50747025+mcdonnnj@users.noreply.github.com> --- .pre-commit-config.yaml | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index b61a8f5..97fbf1c 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -181,6 +181,10 @@ repos: # hook identifies a vulnerability in ansible-core 2.16.13, # but all versions of ansible 9 have a dependency on # ~=2.16.X. + # + # It is also a good idea to go ahead and upgrade to version + # 10 since version 9 is going EOL at the end of November: + # https://endoflife.date/ansible # - ansible>=10,<11 # ansible-core 2.16.3 through 2.16.6 suffer from the bug # discussed in ansible/ansible#82702, which breaks any @@ -193,6 +197,11 @@ repos: # ansible-core to >=2.17 effectively also pins ansible to # >=10. # + # It is also a good idea to go ahead and upgrade to + # ansible-core 2.17 since security support for ansible-core + # 2.16 ends this month: + # https://docs.ansible.com/ansible/devel/reference_appendices/release_and_maintenance.html#ansible-core-support-matrix + # # Note that any changes made to this dependency must also be # made in requirements.txt in cisagov/skeleton-packer and # requirements-test.txt in cisagov/skeleton-ansible-role. From d077b675429cda3a937a7bcac24abffff19665a5 Mon Sep 17 00:00:00 2001 From: Jeremy Frasier Date: Wed, 20 Nov 2024 12:50:05 -0500 Subject: [PATCH 52/75] Add comments about looming EOL issues for ansible and ansible-core This adds even more evidence for why it is a good idea to go ahead and upgrade ansible and ansible-core, in addition to the vulnerability that pip-audit turned up. Co-authored-by: Nick M <50747025+mcdonnnj@users.noreply.github.com> --- requirements.txt | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/requirements.txt b/requirements.txt index 8d6bd0f..5d2d607 100644 --- a/requirements.txt +++ b/requirements.txt @@ -12,6 +12,10 @@ # identifies a vulnerability in ansible-core 2.16.13, but all versions # of ansible 9 have a dependency on ~=2.16.X. # +# It is also a good idea to go ahead and upgrade to version 10 since +# version 9 is going EOL at the end of November: +# https://endoflife.date/ansible +# # We have tested against version 10. We want to avoid automatically # jumping to another major version without testing, since there are # often breaking changes across major versions. This is the reason @@ -27,6 +31,10 @@ ansible>=10,<11 # accordingly (>2.16.13), but the above pin of ansible>=10 effectively # pins ansible-core to >=2.17 anyway so that's what we use. # +# It is also a good idea to go ahead and upgrade to ansible-core 2.17 +# since security support for ansible-core 2.16 ends this month: +# https://docs.ansible.com/ansible/devel/reference_appendices/release_and_maintenance.html#ansible-core-support-matrix +# # Note that any changes made to this dependency must also be made in # requirements-test.txt in cisagov/skeleton-ansible-role and # .pre-commit-config.yaml in cisagov/skeleton-generic. From c8ca96b611563695de19e06c119ff4dbf87922d1 Mon Sep 17 00:00:00 2001 From: Nicholas McDonnell <50747025+mcdonnnj@users.noreply.github.com> Date: Fri, 4 Oct 2024 16:49:07 -0400 Subject: [PATCH 53/75] Move the Packer template to the root of the repository --- .github/CODEOWNERS | 2 +- .github/workflows/build.yml | 6 +++--- .github/workflows/prerelease.yml | 4 ++-- .github/workflows/release.yml | 6 +++--- README.md | 6 +++--- src/packer.pkr.hcl => packer.pkr.hcl | 0 6 files changed, 12 insertions(+), 12 deletions(-) rename src/packer.pkr.hcl => packer.pkr.hcl (100%) diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index a4405d1..275fe15 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -11,8 +11,8 @@ # Let jsf9k own the sometimes-touchy AWS and Python playbooks, as well # as the packer.pkr.hcl file. +/packer.pkr.hcl @jsf9k /src/aws.yml @jsf9k -/src/packer.pkr.hcl @jsf9k /src/python.yml @jsf9k # These folks own all linting configuration files. diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index a2d99e8..a1ecbbf 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -167,7 +167,7 @@ jobs: # This must happen before pre-commit is run or the Packer format # linter will throw an error. - name: Install Packer plugins - run: packer init src + run: packer init . - name: Set up pre-commit hook environments run: pre-commit install-hooks - name: Run pre-commit on all files @@ -305,7 +305,7 @@ jobs: sudo ln -s ${{ env.pythonLocation }}/bin/python3 \ /usr/bin/python3 - name: Install Packer plugins - run: packer init src + run: packer init . - name: Create machine image # This runs through the AMI creation process but does not # actually create an AMI @@ -313,7 +313,7 @@ jobs: packer build -only amazon-ebs.${{ matrix.architecture }} \ -timestamp-ui \ -var skip_create_ami=true \ - src/packer.pkr.hcl + packer.pkr.hcl - name: Remove /usr/bin/python3 symlink to the installed Python run: | sudo mv /usr/bin/python3-default /usr/bin/python3 diff --git a/.github/workflows/prerelease.yml b/.github/workflows/prerelease.yml index 98f7212..a0cec7f 100644 --- a/.github/workflows/prerelease.yml +++ b/.github/workflows/prerelease.yml @@ -112,7 +112,7 @@ jobs: sudo ln -s ${{ env.pythonLocation }}/bin/python3 \ /usr/bin/python3 - name: Install Packer plugins - run: packer init src + run: packer init . - name: Create machine image run: | packer build -only amazon-ebs.${{ matrix.architecture }} \ @@ -120,7 +120,7 @@ jobs: -var is_prerelease=${{ github.event.release.prerelease }} \ -var release_tag=${{ github.event.release.tag_name }} \ -var release_url=${{ github.event.release.html_url }} \ - src/packer.pkr.hcl + packer.pkr.hcl - name: Remove /usr/bin/python3 symlink to the installed python run: | sudo mv /usr/bin/python3-default /usr/bin/python3 diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 7d02eab..768f478 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -106,7 +106,7 @@ jobs: # - name: Add copy regions to packer configuration # run: | # echo $COPY_REGIONS_KMS_MAP | \ - # ./patch_packer_config.py src/packer.pkr.hcl + # ./patch_packer_config.py packer.pkr.hcl - name: Assume AWS build role uses: aws-actions/configure-aws-credentials@v4 with: @@ -127,7 +127,7 @@ jobs: sudo ln -s ${{ env.pythonLocation }}/bin/python3 \ /usr/bin/python3 - name: Install Packer plugins - run: packer init src + run: packer init . - name: Create machine image run: | packer build -only amazon-ebs.${{ matrix.architecture }} \ @@ -135,7 +135,7 @@ jobs: -var is_prerelease=${{ github.event.release.prerelease }} \ -var release_tag=${{ github.event.release.tag_name }} \ -var release_url=${{ github.event.release.html_url }} \ - src/packer.pkr.hcl + packer.pkr.hcl - name: Remove /usr/bin/python3 symlink to the installed python run: | sudo mv /usr/bin/python3-default /usr/bin/python3 diff --git a/README.md b/README.md index 3100385..67b3854 100644 --- a/README.md +++ b/README.md @@ -104,7 +104,7 @@ source_profile = build-skeleton-packer role_session_name = example ``` -The [Packer template](src/packer.pkr.hcl) defines a number of variables: +The [Packer template](packer.pkr.hcl) defines a number of variables: | Name | Description | Type | Default | Required | |------|-------------|------|---------|:--------:| @@ -130,7 +130,7 @@ Here is an example of how to kick off a pre-release build: ```console pip install --requirement requirements-dev.txt ansible-galaxy install --force --force-with-deps --role-file src/requirements.yml -AWS_PROFILE=cool-images-ec2amicreate-skeleton-packer packer build --timestamp-ui -var release_tag=$(./bump_version.sh show) -var is_prerelease=true src/packer.pkr.hcl +AWS_PROFILE=cool-images-ec2amicreate-skeleton-packer packer build --timestamp-ui -var release_tag=$(./bump_version.sh show) -var is_prerelease=true packer.pkr.hcl ``` If you are satisfied with your pre-release image, you can easily create a release @@ -147,7 +147,7 @@ region_kms_keys = { ``` ```console -AWS_PROFILE=cool-images-ec2amicreate-skeleton-packer packer build --timestamp-ui -var-file release.pkrvars.hcl src/packer.pkr.hcl +AWS_PROFILE=cool-images-ec2amicreate-skeleton-packer packer build --timestamp-ui -var-file release.pkrvars.hcl packer.pkr.hcl ``` ### Giving Other AWS Accounts Permission to Launch the Image ### diff --git a/src/packer.pkr.hcl b/packer.pkr.hcl similarity index 100% rename from src/packer.pkr.hcl rename to packer.pkr.hcl From 6c478f05f50ee933d0e20ca7a09ef8a637af4bc3 Mon Sep 17 00:00:00 2001 From: Nicholas McDonnell <50747025+mcdonnnj@users.noreply.github.com> Date: Fri, 4 Oct 2024 16:50:56 -0400 Subject: [PATCH 54/75] Move the version file to the root of the repository --- bump_version.sh | 2 +- tests/test_version.py | 2 +- src/version.txt => version.txt | 0 3 files changed, 2 insertions(+), 2 deletions(-) rename src/version.txt => version.txt (100%) diff --git a/bump_version.sh b/bump_version.sh index 49e6136..1d8fc5d 100755 --- a/bump_version.sh +++ b/bump_version.sh @@ -6,7 +6,7 @@ set -o nounset set -o errexit set -o pipefail -VERSION_FILE=src/version.txt +VERSION_FILE=version.txt HELP_INFORMATION="bump_version.sh (show|major|minor|patch|prerelease|build|finalize)" diff --git a/tests/test_version.py b/tests/test_version.py index 54cdd0a..7196fa4 100644 --- a/tests/test_version.py +++ b/tests/test_version.py @@ -7,7 +7,7 @@ import pytest GITHUB_RELEASE_TAG = os.getenv("GITHUB_RELEASE_TAG") -VERSION_FILE = "src/version.txt" +VERSION_FILE = "version.txt" @pytest.mark.skipif( diff --git a/src/version.txt b/version.txt similarity index 100% rename from src/version.txt rename to version.txt From ad2b7688ea2b2b33044ff17b6e58b6d8d68db3d7 Mon Sep 17 00:00:00 2001 From: Nicholas McDonnell <50747025+mcdonnnj@users.noreply.github.com> Date: Fri, 4 Oct 2024 16:52:16 -0400 Subject: [PATCH 55/75] Rename src/ to ansible/ This aligns with what the src/ directory still contains. --- .github/CODEOWNERS | 4 ++-- .github/workflows/build.yml | 4 ++-- .github/workflows/prerelease.yml | 2 +- .github/workflows/release.yml | 2 +- README.md | 2 +- {src => ansible}/aws.yml | 0 {src => ansible}/base.yml | 0 {src => ansible}/example.yml | 0 .../install-prerequisites-for-netplan-configuration-fix.yml | 0 {src => ansible}/playbook.yml | 0 {src => ansible}/python.yml | 0 {src => ansible}/requirements.yml | 0 {src => ansible}/upgrade.yml | 0 packer.pkr.hcl | 6 +++--- 14 files changed, 10 insertions(+), 10 deletions(-) rename {src => ansible}/aws.yml (100%) rename {src => ansible}/base.yml (100%) rename {src => ansible}/example.yml (100%) rename {src => ansible}/install-prerequisites-for-netplan-configuration-fix.yml (100%) rename {src => ansible}/playbook.yml (100%) rename {src => ansible}/python.yml (100%) rename {src => ansible}/requirements.yml (100%) rename {src => ansible}/upgrade.yml (100%) diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index 275fe15..d406ce8 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -11,9 +11,9 @@ # Let jsf9k own the sometimes-touchy AWS and Python playbooks, as well # as the packer.pkr.hcl file. +/ansible/aws.yml @jsf9k +/ansible/python.yml @jsf9k /packer.pkr.hcl @jsf9k -/src/aws.yml @jsf9k -/src/python.yml @jsf9k # These folks own all linting configuration files. /.ansible-lint @dav3r @felddy @jsf9k @mcdonnnj diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index a1ecbbf..ef45a42 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -163,7 +163,7 @@ jobs: python -m pip install --upgrade pip setuptools wheel pip install --upgrade --requirement requirements-test.txt - name: Install Ansible roles - run: ansible-galaxy install --force --role-file src/requirements.yml + run: ansible-galaxy install --force --role-file ansible/requirements.yml # This must happen before pre-commit is run or the Packer format # linter will throw an error. - name: Install Packer plugins @@ -284,7 +284,7 @@ jobs: pip install --upgrade \ --requirement requirements.txt - name: Install Ansible roles - run: ansible-galaxy install --force --role-file src/requirements.yml + run: ansible-galaxy install --force --role-file ansible/requirements.yml - name: Assume AWS build role uses: aws-actions/configure-aws-credentials@v4 with: diff --git a/.github/workflows/prerelease.yml b/.github/workflows/prerelease.yml index a0cec7f..6de94fa 100644 --- a/.github/workflows/prerelease.yml +++ b/.github/workflows/prerelease.yml @@ -91,7 +91,7 @@ jobs: pip install --upgrade \ --requirement requirements.txt - name: Install ansible roles - run: ansible-galaxy install --force --role-file src/requirements.yml + run: ansible-galaxy install --force --role-file ansible/requirements.yml - name: Assume AWS build role uses: aws-actions/configure-aws-credentials@v4 with: diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 768f478..4832990 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -98,7 +98,7 @@ jobs: pip install --upgrade \ --requirement requirements.txt - name: Install ansible roles - run: ansible-galaxy install --force --role-file src/requirements.yml + run: ansible-galaxy install --force --role-file ansible/requirements.yml # Do not copy the AMI to other regions until we have figured out a # workable mechanism for creating and managing AMI KMS keys in other # regions. diff --git a/README.md b/README.md index 67b3854..a5dada9 100644 --- a/README.md +++ b/README.md @@ -129,7 +129,7 @@ Here is an example of how to kick off a pre-release build: ```console pip install --requirement requirements-dev.txt -ansible-galaxy install --force --force-with-deps --role-file src/requirements.yml +ansible-galaxy install --force --force-with-deps --role-file ansible/requirements.yml AWS_PROFILE=cool-images-ec2amicreate-skeleton-packer packer build --timestamp-ui -var release_tag=$(./bump_version.sh show) -var is_prerelease=true packer.pkr.hcl ``` diff --git a/src/aws.yml b/ansible/aws.yml similarity index 100% rename from src/aws.yml rename to ansible/aws.yml diff --git a/src/base.yml b/ansible/base.yml similarity index 100% rename from src/base.yml rename to ansible/base.yml diff --git a/src/example.yml b/ansible/example.yml similarity index 100% rename from src/example.yml rename to ansible/example.yml diff --git a/src/install-prerequisites-for-netplan-configuration-fix.yml b/ansible/install-prerequisites-for-netplan-configuration-fix.yml similarity index 100% rename from src/install-prerequisites-for-netplan-configuration-fix.yml rename to ansible/install-prerequisites-for-netplan-configuration-fix.yml diff --git a/src/playbook.yml b/ansible/playbook.yml similarity index 100% rename from src/playbook.yml rename to ansible/playbook.yml diff --git a/src/python.yml b/ansible/python.yml similarity index 100% rename from src/python.yml rename to ansible/python.yml diff --git a/src/requirements.yml b/ansible/requirements.yml similarity index 100% rename from src/requirements.yml rename to ansible/requirements.yml diff --git a/src/upgrade.yml b/ansible/upgrade.yml similarity index 100% rename from src/upgrade.yml rename to ansible/upgrade.yml diff --git a/packer.pkr.hcl b/packer.pkr.hcl index 437adf0..ed6e27a 100644 --- a/packer.pkr.hcl +++ b/packer.pkr.hcl @@ -184,20 +184,20 @@ build { ] provisioner "ansible" { - playbook_file = "src/upgrade.yml" + playbook_file = "ansible/upgrade.yml" use_proxy = false use_sftp = true } provisioner "ansible" { - playbook_file = "src/python.yml" + playbook_file = "ansible/python.yml" use_proxy = false use_sftp = true } provisioner "ansible" { ansible_env_vars = ["AWS_DEFAULT_REGION=${var.build_region}"] - playbook_file = "src/playbook.yml" + playbook_file = "ansible/playbook.yml" use_proxy = false use_sftp = true } From 354a1b070822847e3958904113c1a0f707ebfe33 Mon Sep 17 00:00:00 2001 From: Nicholas McDonnell <50747025+mcdonnnj@users.noreply.github.com> Date: Fri, 4 Oct 2024 17:01:09 -0400 Subject: [PATCH 56/75] Move the Packer template variables to their own file This also requires us to replace any Packer commands that reference the `packer.pkr.hcl` file to instead reference the directory that houses the template (the root directory `.`). --- .github/CODEOWNERS | 3 +- .github/workflows/build.yml | 2 +- .github/workflows/prerelease.yml | 2 +- .github/workflows/release.yml | 4 +-- packer.pkr.hcl | 48 -------------------------- variables.pkr.hcl | 59 ++++++++++++++++++++++++++++++++ 6 files changed, 65 insertions(+), 53 deletions(-) create mode 100644 variables.pkr.hcl diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index d406ce8..6594c19 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -10,10 +10,11 @@ /.github/ @dav3r @felddy @jsf9k @mcdonnnj # Let jsf9k own the sometimes-touchy AWS and Python playbooks, as well -# as the packer.pkr.hcl file. +# as the Packer template. /ansible/aws.yml @jsf9k /ansible/python.yml @jsf9k /packer.pkr.hcl @jsf9k +/variables.pkr.hcl @jsf9k # These folks own all linting configuration files. /.ansible-lint @dav3r @felddy @jsf9k @mcdonnnj diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index ef45a42..59e1e34 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -313,7 +313,7 @@ jobs: packer build -only amazon-ebs.${{ matrix.architecture }} \ -timestamp-ui \ -var skip_create_ami=true \ - packer.pkr.hcl + . - name: Remove /usr/bin/python3 symlink to the installed Python run: | sudo mv /usr/bin/python3-default /usr/bin/python3 diff --git a/.github/workflows/prerelease.yml b/.github/workflows/prerelease.yml index 6de94fa..e75bb2f 100644 --- a/.github/workflows/prerelease.yml +++ b/.github/workflows/prerelease.yml @@ -120,7 +120,7 @@ jobs: -var is_prerelease=${{ github.event.release.prerelease }} \ -var release_tag=${{ github.event.release.tag_name }} \ -var release_url=${{ github.event.release.html_url }} \ - packer.pkr.hcl + . - name: Remove /usr/bin/python3 symlink to the installed python run: | sudo mv /usr/bin/python3-default /usr/bin/python3 diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 4832990..3b7e98f 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -106,7 +106,7 @@ jobs: # - name: Add copy regions to packer configuration # run: | # echo $COPY_REGIONS_KMS_MAP | \ - # ./patch_packer_config.py packer.pkr.hcl + # ./patch_packer_config.py variables.pkr.hcl - name: Assume AWS build role uses: aws-actions/configure-aws-credentials@v4 with: @@ -135,7 +135,7 @@ jobs: -var is_prerelease=${{ github.event.release.prerelease }} \ -var release_tag=${{ github.event.release.tag_name }} \ -var release_url=${{ github.event.release.html_url }} \ - packer.pkr.hcl + . - name: Remove /usr/bin/python3 symlink to the installed python run: | sudo mv /usr/bin/python3-default /usr/bin/python3 diff --git a/packer.pkr.hcl b/packer.pkr.hcl index ed6e27a..e5b5d7f 100644 --- a/packer.pkr.hcl +++ b/packer.pkr.hcl @@ -15,54 +15,6 @@ packer { required_version = "~> 1.7" } -variable "ami_regions" { - default = [] - description = "The list of AWS regions to copy the AMI to once it has been created. Example: [\"us-east-1\"]" - type = list(string) -} - -variable "build_region" { - default = "us-east-1" - description = "The region in which to retrieve the base AMI from and build the new AMI." - type = string -} - -variable "build_region_kms" { - default = "alias/cool-amis" - description = "The ID or ARN of the KMS key to use for AMI encryption." - type = string -} - -variable "is_prerelease" { - default = false - description = "The pre-release status to use for the tags applied to the created AMI." - type = bool -} - -variable "region_kms_keys" { - default = {} - description = "A map of regions to copy the created AMI to and the KMS keys to use for encryption in that region. The keys for this map must match the values provided to the aws_regions variable. Example: {\"us-east-1\": \"alias/example-kms\"}" - type = map(string) -} - -variable "release_tag" { - default = "" - description = "The GitHub release tag to use for the tags applied to the created AMI." - type = string -} - -variable "release_url" { - default = "" - description = "The GitHub release URL to use for the tags applied to the created AMI." - type = string -} - -variable "skip_create_ami" { - default = false - description = "Indicate if Packer should not create the AMI." - type = bool -} - data "amazon-ami" "debian_bookworm_arm64" { filters = { architecture = "arm64" diff --git a/variables.pkr.hcl b/variables.pkr.hcl new file mode 100644 index 0000000..7df066f --- /dev/null +++ b/variables.pkr.hcl @@ -0,0 +1,59 @@ +# ------------------------------------------------------------------------------ +# Required parameters +# +# You must provide a value for each of these parameters. +# ------------------------------------------------------------------------------ + +# ------------------------------------------------------------------------------ +# Optional parameters +# +# These parameters have reasonable defaults. +# ------------------------------------------------------------------------------ + +variable "ami_regions" { + default = [] + description = "The list of AWS regions to copy the AMI to once it has been created. Example: [\"us-east-1\"]" + type = list(string) +} + +variable "build_region" { + default = "us-east-1" + description = "The region in which to retrieve the base AMI from and build the new AMI." + type = string +} + +variable "build_region_kms" { + default = "alias/cool-amis" + description = "The ID or ARN of the KMS key to use for AMI encryption." + type = string +} + +variable "is_prerelease" { + default = false + description = "The pre-release status to use for the tags applied to the created AMI." + type = bool +} + +variable "region_kms_keys" { + default = {} + description = "A map of regions to copy the created AMI to and the KMS keys to use for encryption in that region. The keys for this map must match the values provided to the aws_regions variable. Example: {\"us-east-1\": \"alias/example-kms\"}" + type = map(string) +} + +variable "release_tag" { + default = "" + description = "The GitHub release tag to use for the tags applied to the created AMI." + type = string +} + +variable "release_url" { + default = "" + description = "The GitHub release URL to use for the tags applied to the created AMI." + type = string +} + +variable "skip_create_ami" { + default = false + description = "Indicate if Packer should not create the AMI." + type = bool +} From 4ac76ee4d2a3d6e9a4692e4664f66fc2c1e953ed Mon Sep 17 00:00:00 2001 From: Nicholas McDonnell <50747025+mcdonnnj@users.noreply.github.com> Date: Fri, 4 Oct 2024 17:09:02 -0400 Subject: [PATCH 57/75] Move the Packer template locals to their own file --- .github/CODEOWNERS | 1 + locals.pkr.hcl | 3 +++ packer.pkr.hcl | 2 -- 3 files changed, 4 insertions(+), 2 deletions(-) create mode 100644 locals.pkr.hcl diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index 6594c19..3ef07d3 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -13,6 +13,7 @@ # as the Packer template. /ansible/aws.yml @jsf9k /ansible/python.yml @jsf9k +/locals.pkr.hcl @jsf9k /packer.pkr.hcl @jsf9k /variables.pkr.hcl @jsf9k diff --git a/locals.pkr.hcl b/locals.pkr.hcl new file mode 100644 index 0000000..90911c5 --- /dev/null +++ b/locals.pkr.hcl @@ -0,0 +1,3 @@ +locals { + timestamp = regex_replace(timestamp(), "[- TZ:]", "") +} diff --git a/packer.pkr.hcl b/packer.pkr.hcl index e5b5d7f..a04a07c 100644 --- a/packer.pkr.hcl +++ b/packer.pkr.hcl @@ -39,8 +39,6 @@ data "amazon-ami" "debian_bookworm_x86_64" { region = var.build_region } -locals { timestamp = regex_replace(timestamp(), "[- TZ:]", "") } - source "amazon-ebs" "arm64" { ami_name = "example-hvm-${local.timestamp}-arm64-ebs" ami_regions = var.ami_regions From 48934ca5411479a6ce48e3791a3bb49ef0396522 Mon Sep 17 00:00:00 2001 From: Nicholas McDonnell <50747025+mcdonnnj@users.noreply.github.com> Date: Fri, 4 Oct 2024 17:10:05 -0400 Subject: [PATCH 58/75] Move the Packer template's build block to its own file --- .github/CODEOWNERS | 1 + build.pkr.hcl | 31 +++++++++++++++++++++++++++++++ packer.pkr.hcl | 32 -------------------------------- 3 files changed, 32 insertions(+), 32 deletions(-) create mode 100644 build.pkr.hcl diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index 3ef07d3..c0c1357 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -13,6 +13,7 @@ # as the Packer template. /ansible/aws.yml @jsf9k /ansible/python.yml @jsf9k +/build.pkr.hcl @jsf9k /locals.pkr.hcl @jsf9k /packer.pkr.hcl @jsf9k /variables.pkr.hcl @jsf9k diff --git a/build.pkr.hcl b/build.pkr.hcl new file mode 100644 index 0000000..b05ba95 --- /dev/null +++ b/build.pkr.hcl @@ -0,0 +1,31 @@ +build { + sources = [ + "source.amazon-ebs.arm64", + "source.amazon-ebs.x86_64", + ] + + provisioner "ansible" { + playbook_file = "ansible/upgrade.yml" + use_proxy = false + use_sftp = true + } + + provisioner "ansible" { + playbook_file = "ansible/python.yml" + use_proxy = false + use_sftp = true + } + + provisioner "ansible" { + ansible_env_vars = ["AWS_DEFAULT_REGION=${var.build_region}"] + playbook_file = "ansible/playbook.yml" + use_proxy = false + use_sftp = true + } + + provisioner "shell" { + execute_command = "chmod +x {{ .Path }}; sudo env {{ .Vars }} {{ .Path }} ; rm -f {{ .Path }}" + inline = ["sed -i '/^users:/ {N; s/users:.*/users: []/g}' /etc/cloud/cloud.cfg", "rm --force /etc/sudoers.d/90-cloud-init-users", "rm --force /root/.ssh/authorized_keys", "/usr/sbin/userdel --remove --force admin"] + skip_clean = true + } +} diff --git a/packer.pkr.hcl b/packer.pkr.hcl index a04a07c..1212e56 100644 --- a/packer.pkr.hcl +++ b/packer.pkr.hcl @@ -126,35 +126,3 @@ source "amazon-ebs" "x86_64" { } } } - -build { - sources = [ - "source.amazon-ebs.arm64", - "source.amazon-ebs.x86_64", - ] - - provisioner "ansible" { - playbook_file = "ansible/upgrade.yml" - use_proxy = false - use_sftp = true - } - - provisioner "ansible" { - playbook_file = "ansible/python.yml" - use_proxy = false - use_sftp = true - } - - provisioner "ansible" { - ansible_env_vars = ["AWS_DEFAULT_REGION=${var.build_region}"] - playbook_file = "ansible/playbook.yml" - use_proxy = false - use_sftp = true - } - - provisioner "shell" { - execute_command = "chmod +x {{ .Path }}; sudo env {{ .Vars }} {{ .Path }} ; rm -f {{ .Path }}" - inline = ["sed -i '/^users:/ {N; s/users:.*/users: []/g}' /etc/cloud/cloud.cfg", "rm --force /etc/sudoers.d/90-cloud-init-users", "rm --force /root/.ssh/authorized_keys", "/usr/sbin/userdel --remove --force admin"] - skip_clean = true - } -} From 566245a857432c30668d95e3e8922db1c4f5a46c Mon Sep 17 00:00:00 2001 From: Nicholas McDonnell <50747025+mcdonnnj@users.noreply.github.com> Date: Fri, 4 Oct 2024 17:12:29 -0400 Subject: [PATCH 59/75] Move the Packer template's AMI sources to their own files --- .github/CODEOWNERS | 2 ++ ami_arm64.pkr.hcl | 43 ++++++++++++++++++++++ ami_x86_64.pkr.hcl | 43 ++++++++++++++++++++++ packer.pkr.hcl | 88 ---------------------------------------------- 4 files changed, 88 insertions(+), 88 deletions(-) create mode 100644 ami_arm64.pkr.hcl create mode 100644 ami_x86_64.pkr.hcl diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index c0c1357..f2e78c7 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -11,6 +11,8 @@ # Let jsf9k own the sometimes-touchy AWS and Python playbooks, as well # as the Packer template. +/ami_arm64.pkr.hcl @jsf9k +/ami_x86.pkr.hcl @jsf9k /ansible/aws.yml @jsf9k /ansible/python.yml @jsf9k /build.pkr.hcl @jsf9k diff --git a/ami_arm64.pkr.hcl b/ami_arm64.pkr.hcl new file mode 100644 index 0000000..9e87064 --- /dev/null +++ b/ami_arm64.pkr.hcl @@ -0,0 +1,43 @@ +source "amazon-ebs" "arm64" { + ami_name = "example-hvm-${local.timestamp}-arm64-ebs" + ami_regions = var.ami_regions + associate_public_ip_address = true + encrypt_boot = true + instance_type = "t4g.small" + kms_key_id = var.build_region_kms + launch_block_device_mappings { + delete_on_termination = true + device_name = "/dev/xvda" + encrypted = true + volume_size = 8 + volume_type = "gp3" + } + region = var.build_region + region_kms_key_ids = var.region_kms_keys + skip_create_ami = var.skip_create_ami + source_ami = data.amazon-ami.debian_bookworm_arm64.id + ssh_username = "admin" + subnet_filter { + filters = { + "tag:Name" = "AMI Build" + } + } + tags = { + Application = "Example" + Architecture = "arm64" + Base_AMI_Name = data.amazon-ami.debian_bookworm_arm64.name + GitHub_Release_URL = var.release_url + OS_Version = "Debian Bookworm" + Pre_Release = var.is_prerelease + Release = var.release_tag + Team = "VM Fusion - Development" + } + # Many Linux distributions are now disallowing the use of RSA keys, + # so it makes sense to use an ED25519 key instead. + temporary_key_pair_type = "ed25519" + vpc_filter { + filters = { + "tag:Name" = "AMI Build" + } + } +} diff --git a/ami_x86_64.pkr.hcl b/ami_x86_64.pkr.hcl new file mode 100644 index 0000000..8976d2a --- /dev/null +++ b/ami_x86_64.pkr.hcl @@ -0,0 +1,43 @@ +source "amazon-ebs" "x86_64" { + ami_name = "example-hvm-${local.timestamp}-x86_64-ebs" + ami_regions = var.ami_regions + associate_public_ip_address = true + encrypt_boot = true + instance_type = "t3.small" + kms_key_id = var.build_region_kms + launch_block_device_mappings { + delete_on_termination = true + device_name = "/dev/xvda" + encrypted = true + volume_size = 8 + volume_type = "gp3" + } + region = var.build_region + region_kms_key_ids = var.region_kms_keys + skip_create_ami = var.skip_create_ami + source_ami = data.amazon-ami.debian_bookworm_x86_64.id + ssh_username = "admin" + subnet_filter { + filters = { + "tag:Name" = "AMI Build" + } + } + tags = { + Application = "Example" + Architecture = "x86_64" + Base_AMI_Name = data.amazon-ami.debian_bookworm_x86_64.name + GitHub_Release_URL = var.release_url + OS_Version = "Debian Bookworm" + Pre_Release = var.is_prerelease + Release = var.release_tag + Team = "VM Fusion - Development" + } + # Many Linux distributions are now disallowing the use of RSA keys, + # so it makes sense to use an ED25519 key instead. + temporary_key_pair_type = "ed25519" + vpc_filter { + filters = { + "tag:Name" = "AMI Build" + } + } +} diff --git a/packer.pkr.hcl b/packer.pkr.hcl index 1212e56..8dcaeb3 100644 --- a/packer.pkr.hcl +++ b/packer.pkr.hcl @@ -38,91 +38,3 @@ data "amazon-ami" "debian_bookworm_x86_64" { owners = ["136693071363"] region = var.build_region } - -source "amazon-ebs" "arm64" { - ami_name = "example-hvm-${local.timestamp}-arm64-ebs" - ami_regions = var.ami_regions - associate_public_ip_address = true - encrypt_boot = true - instance_type = "t4g.small" - kms_key_id = var.build_region_kms - launch_block_device_mappings { - delete_on_termination = true - device_name = "/dev/xvda" - encrypted = true - volume_size = 8 - volume_type = "gp3" - } - region = var.build_region - region_kms_key_ids = var.region_kms_keys - skip_create_ami = var.skip_create_ami - source_ami = data.amazon-ami.debian_bookworm_arm64.id - ssh_username = "admin" - subnet_filter { - filters = { - "tag:Name" = "AMI Build" - } - } - tags = { - Application = "Example" - Architecture = "arm64" - Base_AMI_Name = data.amazon-ami.debian_bookworm_arm64.name - GitHub_Release_URL = var.release_url - OS_Version = "Debian Bookworm" - Pre_Release = var.is_prerelease - Release = var.release_tag - Team = "VM Fusion - Development" - } - # Many Linux distributions are now disallowing the use of RSA keys, - # so it makes sense to use an ED25519 key instead. - temporary_key_pair_type = "ed25519" - vpc_filter { - filters = { - "tag:Name" = "AMI Build" - } - } -} - -source "amazon-ebs" "x86_64" { - ami_name = "example-hvm-${local.timestamp}-x86_64-ebs" - ami_regions = var.ami_regions - associate_public_ip_address = true - encrypt_boot = true - instance_type = "t3.small" - kms_key_id = var.build_region_kms - launch_block_device_mappings { - delete_on_termination = true - device_name = "/dev/xvda" - encrypted = true - volume_size = 8 - volume_type = "gp3" - } - region = var.build_region - region_kms_key_ids = var.region_kms_keys - skip_create_ami = var.skip_create_ami - source_ami = data.amazon-ami.debian_bookworm_x86_64.id - ssh_username = "admin" - subnet_filter { - filters = { - "tag:Name" = "AMI Build" - } - } - tags = { - Application = "Example" - Architecture = "x86_64" - Base_AMI_Name = data.amazon-ami.debian_bookworm_x86_64.name - GitHub_Release_URL = var.release_url - OS_Version = "Debian Bookworm" - Pre_Release = var.is_prerelease - Release = var.release_tag - Team = "VM Fusion - Development" - } - # Many Linux distributions are now disallowing the use of RSA keys, - # so it makes sense to use an ED25519 key instead. - temporary_key_pair_type = "ed25519" - vpc_filter { - filters = { - "tag:Name" = "AMI Build" - } - } -} From 4b59dc75e813d20d9bdf3dada36f4b1746d1e4ad Mon Sep 17 00:00:00 2001 From: Nicholas McDonnell <50747025+mcdonnnj@users.noreply.github.com> Date: Fri, 4 Oct 2024 17:20:56 -0400 Subject: [PATCH 60/75] Move the Packer template's AMI data blocks to their own file --- .github/CODEOWNERS | 1 + base_amis.pkr.hcl | 23 +++++++++++++++++++++++ packer.pkr.hcl | 24 ------------------------ 3 files changed, 24 insertions(+), 24 deletions(-) create mode 100644 base_amis.pkr.hcl diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index f2e78c7..88e4a90 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -15,6 +15,7 @@ /ami_x86.pkr.hcl @jsf9k /ansible/aws.yml @jsf9k /ansible/python.yml @jsf9k +/base_amis.pkr.hcl @jsf9k /build.pkr.hcl @jsf9k /locals.pkr.hcl @jsf9k /packer.pkr.hcl @jsf9k diff --git a/base_amis.pkr.hcl b/base_amis.pkr.hcl new file mode 100644 index 0000000..8f9f472 --- /dev/null +++ b/base_amis.pkr.hcl @@ -0,0 +1,23 @@ +data "amazon-ami" "debian_bookworm_arm64" { + filters = { + architecture = "arm64" + name = "debian-12-arm64-*" + root-device-type = "ebs" + virtualization-type = "hvm" + } + most_recent = true + owners = ["136693071363"] + region = var.build_region +} + +data "amazon-ami" "debian_bookworm_x86_64" { + filters = { + architecture = "x86_64" + name = "debian-12-amd64-*" + root-device-type = "ebs" + virtualization-type = "hvm" + } + most_recent = true + owners = ["136693071363"] + region = var.build_region +} diff --git a/packer.pkr.hcl b/packer.pkr.hcl index 8dcaeb3..9017550 100644 --- a/packer.pkr.hcl +++ b/packer.pkr.hcl @@ -14,27 +14,3 @@ packer { # ready. required_version = "~> 1.7" } - -data "amazon-ami" "debian_bookworm_arm64" { - filters = { - architecture = "arm64" - name = "debian-12-arm64-*" - root-device-type = "ebs" - virtualization-type = "hvm" - } - most_recent = true - owners = ["136693071363"] - region = var.build_region -} - -data "amazon-ami" "debian_bookworm_x86_64" { - filters = { - architecture = "x86_64" - name = "debian-12-amd64-*" - root-device-type = "ebs" - virtualization-type = "hvm" - } - most_recent = true - owners = ["136693071363"] - region = var.build_region -} From 8b7217e181c0082c0df053411423543a637a655c Mon Sep 17 00:00:00 2001 From: Nicholas McDonnell <50747025+mcdonnnj@users.noreply.github.com> Date: Fri, 4 Oct 2024 17:23:33 -0400 Subject: [PATCH 61/75] Update the README Fix the link for variables in the template and update all references to a Packer configuration to refer to a Packer template instead. --- README.md | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index a5dada9..9ee7bd1 100644 --- a/README.md +++ b/README.md @@ -4,7 +4,7 @@ This is a generic skeleton project that can be used to quickly get a new [cisagov](https://github.com/cisagov) GitHub -[Packer](https://packer.io) project started. This skeleton project +[Packer](https://packer.io) template project started. This skeleton project contains [licensing information](LICENSE), as well as [pre-commit hooks](https://pre-commit.com) and [GitHub Actions](https://github.com/features/actions) configurations @@ -69,7 +69,7 @@ how the build was triggered from GitHub. 1. **Non-release test**: After a normal commit or pull request GitHub Actions will build the project, and run tests and validation on the - packer configuration. It will **not** build an image. + Packer template. It will **not** build an image. 1. **Pre-release deploy**: Publish a GitHub release with the "This is a pre-release" checkbox checked. An image will be built and deployed using the [`prerelease`](.github/workflows/prerelease.yml) @@ -104,7 +104,7 @@ source_profile = build-skeleton-packer role_session_name = example ``` -The [Packer template](packer.pkr.hcl) defines a number of variables: +The [Packer template](variables.pkr.hcl) defines a number of variables: | Name | Description | Type | Default | Required | |------|-------------|------|---------|:--------:| @@ -134,7 +134,7 @@ AWS_PROFILE=cool-images-ec2amicreate-skeleton-packer packer build --timestamp-ui ``` If you are satisfied with your pre-release image, you can easily create a release -that deploys to all regions by adding additional regions to the packer configuration. +that deploys to all regions by adding additional regions to the Packertemplate. This can be done by using a `.pkrvars.hcl` for example with `release.pkrvars.hcl`: ```hcl From 4d45cfc870ede75ec7a676d93cb69e443d60b22f Mon Sep 17 00:00:00 2001 From: Nicholas McDonnell <50747025+mcdonnnj@users.noreply.github.com> Date: Fri, 4 Oct 2024 17:25:29 -0400 Subject: [PATCH 62/75] Add a `terraform-docs` configuration Add the `terraform-docs` configuration file and markers for where to insert output in the README. --- .terraform-docs.yml | 14 ++++++++++++++ README.md | 3 +++ 2 files changed, 17 insertions(+) create mode 100644 .terraform-docs.yml diff --git a/.terraform-docs.yml b/.terraform-docs.yml new file mode 100644 index 0000000..575b15d --- /dev/null +++ b/.terraform-docs.yml @@ -0,0 +1,14 @@ +--- +formatter: markdown table +output: + file: README.md + mode: inject + template: |- + + {{ .Content }} + +settings: + anchor: false + atx-closed: true + html: false + lockfile: false diff --git a/README.md b/README.md index 9ee7bd1..ae5e1d1 100644 --- a/README.md +++ b/README.md @@ -166,6 +166,9 @@ terraform init --upgrade=true terraform apply ``` + + + ## New Repositories from a Skeleton ## Please see our [Project Setup guide](https://github.com/cisagov/development-guide/tree/develop/project_setup) From 868b3d6a27ac99d5afe76761be9799c2cc3fc838 Mon Sep 17 00:00:00 2001 From: Nicholas McDonnell <50747025+mcdonnnj@users.noreply.github.com> Date: Fri, 4 Oct 2024 17:27:34 -0400 Subject: [PATCH 63/75] Run `terraform-docs` to populate the README --- README.md | 37 +++++++++++++++++++++++++++++++++++++ 1 file changed, 37 insertions(+) diff --git a/README.md b/README.md index ae5e1d1..36cf6e2 100644 --- a/README.md +++ b/README.md @@ -167,6 +167,43 @@ terraform apply ``` +## Requirements ## + +No requirements. + +## Providers ## + +| Name | Version | +|------|---------| +| amazon-ami | n/a | + +## Modules ## + +No modules. + +## Resources ## + +| Name | Type | +|------|------| +| [amazon-ami_amazon-ami.debian_bookworm_arm64](https://registry.terraform.io/providers/hashicorp/amazon-ami/latest/docs/data-sources/amazon-ami) | data source | +| [amazon-ami_amazon-ami.debian_bookworm_x86_64](https://registry.terraform.io/providers/hashicorp/amazon-ami/latest/docs/data-sources/amazon-ami) | data source | + +## Inputs ## + +| Name | Description | Type | Default | Required | +|------|-------------|------|---------|:--------:| +| ami\_regions | The list of AWS regions to copy the AMI to once it has been created. Example: ["us-east-1"] | `list(string)` | `[]` | no | +| build\_region | The region in which to retrieve the base AMI from and build the new AMI. | `string` | `"us-east-1"` | no | +| build\_region\_kms | The ID or ARN of the KMS key to use for AMI encryption. | `string` | `"alias/cool-amis"` | no | +| is\_prerelease | The pre-release status to use for the tags applied to the created AMI. | `bool` | `false` | no | +| region\_kms\_keys | A map of regions to copy the created AMI to and the KMS keys to use for encryption in that region. The keys for this map must match the values provided to the aws\_regions variable. Example: {"us-east-1": "alias/example-kms"} | `map(string)` | `{}` | no | +| release\_tag | The GitHub release tag to use for the tags applied to the created AMI. | `string` | `""` | no | +| release\_url | The GitHub release URL to use for the tags applied to the created AMI. | `string` | `""` | no | +| skip\_create\_ami | Indicate if Packer should not create the AMI. | `bool` | `false` | no | + +## Outputs ## + +No outputs. ## New Repositories from a Skeleton ## From f069241990ba71f765a8bf242184fc95ad312e40 Mon Sep 17 00:00:00 2001 From: Nicholas McDonnell <50747025+mcdonnnj@users.noreply.github.com> Date: Fri, 4 Oct 2024 17:29:15 -0400 Subject: [PATCH 64/75] Update the README Now that we use `terraform-docs` to populate information about the template's variables we do not need a manually managed table with that information in the README. --- README.md | 16 ++-------------- 1 file changed, 2 insertions(+), 14 deletions(-) diff --git a/README.md b/README.md index 36cf6e2..de007ae 100644 --- a/README.md +++ b/README.md @@ -104,20 +104,8 @@ source_profile = build-skeleton-packer role_session_name = example ``` -The [Packer template](variables.pkr.hcl) defines a number of variables: - -| Name | Description | Type | Default | Required | -|------|-------------|------|---------|:--------:| -| ami\_regions | The list of AWS regions to copy the AMI to once it has been created. Example: ["us-east-1"] | `list(string)` | `[]` | no | -| build\_region | The region in which to retrieve the base AMI from and build the new AMI. | `string` | `"us-east-1"` | no | -| build\_region\_kms | The ID or ARN of the KMS key to use for AMI encryption. | `string` | `"alias/cool-amis"` | no | -| is\_prerelease | The pre-release status to use for the tags applied to the created AMI. | `bool` | `false` | no | -| region\_kms\_keys | A map of regions to copy the created AMI to and the KMS keys to use for encryption in that region. The keys for this map must match the values provided to the aws\_regions variable. Example: {"us-east-1": "alias/example-kms"} | `map(string)` | `{}` | no | -| release\_tag | The GitHub release tag to use for the tags applied to the created AMI. | `string` | `""` | no | -| release\_url | The GitHub release URL to use for the tags applied to the created AMI. | `string` | `""` | no | -| skip\_create\_ami | Indicate if Packer should not create the AMI. | `bool` | `false` | no | - -Changing these defaults can be done through a `.pkrvars.hcl` file: +This Packer template defines a number of variables whose defaults can be changed +through a `.pkrvars.hcl` file: ```hcl build_region = "us-east-2" From d7335783cc659c2c37183324e302e786ce44de6c Mon Sep 17 00:00:00 2001 From: Nicholas McDonnell <50747025+mcdonnnj@users.noreply.github.com> Date: Thu, 10 Oct 2024 14:23:48 -0400 Subject: [PATCH 65/75] Rename the `packer.pkr.hcl` file Calling it `versions.pkr.hcl` aligns with how we name files in our Terraform projects. --- .github/CODEOWNERS | 2 +- packer.pkr.hcl => versions.pkr.hcl | 0 2 files changed, 1 insertion(+), 1 deletion(-) rename packer.pkr.hcl => versions.pkr.hcl (100%) diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index 88e4a90..ca109ca 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -18,8 +18,8 @@ /base_amis.pkr.hcl @jsf9k /build.pkr.hcl @jsf9k /locals.pkr.hcl @jsf9k -/packer.pkr.hcl @jsf9k /variables.pkr.hcl @jsf9k +/versions.pkr.hcl @jsf9k # These folks own all linting configuration files. /.ansible-lint @dav3r @felddy @jsf9k @mcdonnnj diff --git a/packer.pkr.hcl b/versions.pkr.hcl similarity index 100% rename from packer.pkr.hcl rename to versions.pkr.hcl From ee0ffe85fdd91fe1c81ca39e1f0d001cad8c17ae Mon Sep 17 00:00:00 2001 From: Nicholas McDonnell <50747025+mcdonnnj@users.noreply.github.com> Date: Tue, 15 Oct 2024 08:58:56 -0400 Subject: [PATCH 66/75] Simply the CODEOWNERS configuration for the Packer template There is no reason to list each file if we can wildcard match on the file extension. --- .github/CODEOWNERS | 8 +------- 1 file changed, 1 insertion(+), 7 deletions(-) diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index ca109ca..8c60230 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -11,15 +11,9 @@ # Let jsf9k own the sometimes-touchy AWS and Python playbooks, as well # as the Packer template. -/ami_arm64.pkr.hcl @jsf9k -/ami_x86.pkr.hcl @jsf9k +/*.pkr.hcl @jsf9k /ansible/aws.yml @jsf9k /ansible/python.yml @jsf9k -/base_amis.pkr.hcl @jsf9k -/build.pkr.hcl @jsf9k -/locals.pkr.hcl @jsf9k -/variables.pkr.hcl @jsf9k -/versions.pkr.hcl @jsf9k # These folks own all linting configuration files. /.ansible-lint @dav3r @felddy @jsf9k @mcdonnnj From d862b89336b8476790cc58cc550635f47e2d9734 Mon Sep 17 00:00:00 2001 From: Nick <50747025+mcdonnnj@users.noreply.github.com> Date: Mon, 28 Oct 2024 11:38:08 -0400 Subject: [PATCH 67/75] Add a missed space in the README Co-authored-by: Shane Frasier --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index de007ae..36f2a56 100644 --- a/README.md +++ b/README.md @@ -122,7 +122,7 @@ AWS_PROFILE=cool-images-ec2amicreate-skeleton-packer packer build --timestamp-ui ``` If you are satisfied with your pre-release image, you can easily create a release -that deploys to all regions by adding additional regions to the Packertemplate. +that deploys to all regions by adding additional regions to the Packer template. This can be done by using a `.pkrvars.hcl` for example with `release.pkrvars.hcl`: ```hcl From 2db3797bd16a5ea65898810d2af19a0fe9f1f143 Mon Sep 17 00:00:00 2001 From: Nicholas McDonnell <50747025+mcdonnnj@users.noreply.github.com> Date: Tue, 1 Oct 2024 03:53:39 -0400 Subject: [PATCH 68/75] Pull in the `bump-version` script from cisagov/pre-commit-packer --- bump-version | 172 +++++++++++++++++++++++++++++++++++++++++++++++ requirements.txt | 2 +- 2 files changed, 173 insertions(+), 1 deletion(-) create mode 100755 bump-version diff --git a/bump-version b/bump-version new file mode 100755 index 0000000..b19230e --- /dev/null +++ b/bump-version @@ -0,0 +1,172 @@ +#!/usr/bin/env bash + +# bump-version [--push] [--label LABEL] (major | minor | patch | prerelease | build | finalize | show) +# bump-version --list-files + +set -o nounset +set -o errexit +set -o pipefail + +# Stores the canonical version for the project. +VERSION_FILE=config/version.txt +# Files that should be updated with the new version. +VERSION_FILES=("$VERSION_FILE" README.md) + +USAGE=$( + cat << END_OF_LINE +Update the version of the project. + +Usage: + ${0##*/} [--push] [--label LABEL] (major | minor | patch | prerelease | build | finalize | show) + ${0##*/} --list-files + ${0##*/} (-h | --help) + +Options: + -h | --help Show this message. + --push Perform a \`git push\` after updating the version. + --label LABEL Specify the label to use when updating the build or prerelease version. + --list-files List the files that will be updated when the version is bumped. +END_OF_LINE +) + +old_version=$(< "$VERSION_FILE") +# Comment out periods so they are interpreted as periods and don't +# just match any character +old_version_regex=${old_version//\./\\\.} +new_version="$old_version" + +bump_part="" +label="" +commit_prefix="Bump" +with_push=false +commands_with_label=("build" "prerelease") +commands_with_prerelease=("major" "minor" "patch") +with_prerelease=false + +####################################### +# Display an error message, the help information, and exit with a non-zero status. +# Arguments: +# Error message. +####################################### +function invalid_option() { + echo "$1" + echo "$USAGE" + exit 1 +} + +####################################### +# Bump the version using the provided command. +# Arguments: +# The version to bump. +# The command to bump the version. +# Returns: +# The new version. +####################################### +function bump_version() { + local temp_version + temp_version=$(python -c "import semver; print(semver.parse_version_info('$1').${2})") + echo "$temp_version" +} + +if [ $# -eq 0 ]; then + echo "$USAGE" + exit 1 +else + while [ $# -gt 0 ]; do + case $1 in + --push) + if [ "$with_push" = true ]; then + invalid_option "Push has already been set." + fi + + with_push=true + shift + ;; + --label) + if [ -n "$label" ]; then + invalid_option "Label has already been set." + fi + + label="$2" + shift 2 + ;; + build | finalize | major | minor | patch) + if [ -n "$bump_part" ]; then + invalid_option "Only one version part should be bumped at a time." + fi + + bump_part="$1" + shift + ;; + prerelease) + with_prerelease=true + shift + ;; + show) + echo "$old_version" + exit 0 + ;; + -h | --help) + echo "$USAGE" + exit 0 + ;; + --list-files) + printf '%s\n' "${VERSION_FILES[@]}" + exit 0 + ;; + *) + invalid_option "Invalid option: $1" + ;; + esac + done +fi + +if [ -n "$label" ] && [ "$with_prerelease" = false ] && [[ ! " ${commands_with_label[*]} " =~ [[:space:]]${bump_part}[[:space:]] ]]; then + invalid_option "Setting the label is only allowed for the following commands: ${commands_with_label[*]}" +fi + +if [ "$with_prerelease" = true ] && [[ ! " ${commands_with_prerelease[*]} " =~ [[:space:]]${bump_part}[[:space:]] ]]; then + invalid_option "Changing the prerelease is only allowed in conjunction with the following commands: ${commands_with_prerelease[*]}" +fi + +label_option="" +if [ -n "$label" ]; then + label_option="token='$label'" +fi + +if [ -n "$bump_part" ]; then + if [ "$bump_part" = "finalize" ]; then + commit_prefix="Finalize" + bump_command="finalize_version()" + elif [ "$bump_part" = "build" ]; then + bump_command="bump_${bump_part}($label_option)" + else + bump_command="bump_${bump_part}()" + fi + new_version=$(bump_version "$old_version" "$bump_command") + echo Changing version from "$old_version" to "$new_version" +fi + +if [ "$with_prerelease" = true ]; then + bump_command="bump_prerelease($label_option)" + temp_version=$(bump_version "$new_version" "$bump_command") + echo Changing version from "$new_version" to "$temp_version" + new_version="$temp_version" +fi + +tmp_file=/tmp/version.$$ +for version_file in "${VERSION_FILES[@]}"; do + if [ ! -f "$version_file" ]; then + echo Missing expected file: "$version_file" + exit 1 + fi + sed "s/$old_version_regex/$new_version/" "$version_file" > $tmp_file + mv $tmp_file "$version_file" +done + +git add "${VERSION_FILES[@]}" +git commit --message "$commit_prefix version from $old_version to $new_version" + +if [ "$with_push" = true ]; then + git push +fi diff --git a/requirements.txt b/requirements.txt index 5d2d607..d98d7ea 100644 --- a/requirements.txt +++ b/requirements.txt @@ -41,6 +41,6 @@ ansible>=10,<11 ansible-core>=2.17 boto3 docopt -semver +semver>=3 setuptools wheel From 50bdadc63d053f3dec874a1388ebe40c0fdbccb7 Mon Sep 17 00:00:00 2001 From: Nicholas McDonnell <50747025+mcdonnnj@users.noreply.github.com> Date: Tue, 1 Oct 2024 03:56:15 -0400 Subject: [PATCH 69/75] Modify the `bump-version` script for this repository Copy over necessary changes from the `bump_version.sh` script. --- bump-version | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/bump-version b/bump-version index b19230e..9fdddea 100755 --- a/bump-version +++ b/bump-version @@ -8,9 +8,9 @@ set -o errexit set -o pipefail # Stores the canonical version for the project. -VERSION_FILE=config/version.txt +VERSION_FILE=src/version.txt # Files that should be updated with the new version. -VERSION_FILES=("$VERSION_FILE" README.md) +VERSION_FILES=("$VERSION_FILE") USAGE=$( cat << END_OF_LINE @@ -29,7 +29,7 @@ Options: END_OF_LINE ) -old_version=$(< "$VERSION_FILE") +old_version=$(sed -n "s/^__version__ = \"\(.*\)\"$/\1/p" $VERSION_FILE) # Comment out periods so they are interpreted as periods and don't # just match any character old_version_regex=${old_version//\./\\\.} From 8c2b38426db787cdf7f3626780c4d428b7e5342d Mon Sep 17 00:00:00 2001 From: Nicholas McDonnell <50747025+mcdonnnj@users.noreply.github.com> Date: Tue, 1 Oct 2024 03:56:39 -0400 Subject: [PATCH 70/75] Remove the `bump_version.sh` script This functionality has been replaced by the `bump-version` script. --- README.md | 2 +- bump_version.sh | 49 ------------------------------------------------- 2 files changed, 1 insertion(+), 50 deletions(-) delete mode 100755 bump_version.sh diff --git a/README.md b/README.md index 36f2a56..33231a6 100644 --- a/README.md +++ b/README.md @@ -118,7 +118,7 @@ Here is an example of how to kick off a pre-release build: ```console pip install --requirement requirements-dev.txt ansible-galaxy install --force --force-with-deps --role-file ansible/requirements.yml -AWS_PROFILE=cool-images-ec2amicreate-skeleton-packer packer build --timestamp-ui -var release_tag=$(./bump_version.sh show) -var is_prerelease=true packer.pkr.hcl +AWS_PROFILE=cool-images-ec2amicreate-skeleton-packer packer build --timestamp-ui -var release_tag=$(./bump-version show) -var is_prerelease=true packer.pkr.hcl ``` If you are satisfied with your pre-release image, you can easily create a release diff --git a/bump_version.sh b/bump_version.sh deleted file mode 100755 index 1d8fc5d..0000000 --- a/bump_version.sh +++ /dev/null @@ -1,49 +0,0 @@ -#!/usr/bin/env bash - -# bump_version.sh (show|major|minor|patch|prerelease|build) - -set -o nounset -set -o errexit -set -o pipefail - -VERSION_FILE=version.txt - -HELP_INFORMATION="bump_version.sh (show|major|minor|patch|prerelease|build|finalize)" - -old_version=$(sed -n "s/^__version__ = \"\(.*\)\"$/\1/p" $VERSION_FILE) -# Comment out periods so they are interpreted as periods and don't -# just match any character -old_version_regex=${old_version//\./\\\.} - -if [ $# -ne 1 ]; then - echo "$HELP_INFORMATION" -else - case $1 in - major | minor | patch | prerelease | build) - new_version=$(python -c "import semver; print(semver.bump_$1('$old_version'))") - echo Changing version from "$old_version" to "$new_version" - tmp_file=/tmp/version.$$ - sed "s/$old_version_regex/$new_version/" $VERSION_FILE > $tmp_file - mv $tmp_file $VERSION_FILE - git add $VERSION_FILE - git commit -m"Bump version from $old_version to $new_version" - git push - ;; - finalize) - new_version=$(python -c "import semver; print(semver.finalize_version('$old_version'))") - echo Changing version from "$old_version" to "$new_version" - tmp_file=/tmp/version.$$ - sed "s/$old_version_regex/$new_version/" $VERSION_FILE > $tmp_file - mv $tmp_file $VERSION_FILE - git add $VERSION_FILE - git commit -m"Finalize version from $old_version to $new_version" - git push - ;; - show) - echo "$old_version" - ;; - *) - echo "$HELP_INFORMATION" - ;; - esac -fi From 49340059f9655678464223602ebbc16796e42e17 Mon Sep 17 00:00:00 2001 From: Nicholas McDonnell <50747025+mcdonnnj@users.noreply.github.com> Date: Thu, 10 Oct 2024 16:21:33 -0400 Subject: [PATCH 71/75] Change the format of the version tracking file There is no reason to have any information but the version in the version tracking file for Packer template projects. The current format is simply an artifact of mirroring things over from the cisagov/skeleton-python-library project. --- bump-version | 4 ++-- tests/test_version.py | 4 +--- version.txt | 2 +- 3 files changed, 4 insertions(+), 6 deletions(-) diff --git a/bump-version b/bump-version index 9fdddea..716613c 100755 --- a/bump-version +++ b/bump-version @@ -8,7 +8,7 @@ set -o errexit set -o pipefail # Stores the canonical version for the project. -VERSION_FILE=src/version.txt +VERSION_FILE=version.txt # Files that should be updated with the new version. VERSION_FILES=("$VERSION_FILE") @@ -29,7 +29,7 @@ Options: END_OF_LINE ) -old_version=$(sed -n "s/^__version__ = \"\(.*\)\"$/\1/p" $VERSION_FILE) +old_version=$(< "$VERSION_FILE") # Comment out periods so they are interpreted as periods and don't # just match any character old_version_regex=${old_version//\./\\\.} diff --git a/tests/test_version.py b/tests/test_version.py index 7196fa4..b06f751 100644 --- a/tests/test_version.py +++ b/tests/test_version.py @@ -16,10 +16,8 @@ ) def test_release_version(): """Verify that release tag version agrees with the module version.""" - pkg_vars = {} with open(VERSION_FILE) as f: - exec(f.read(), pkg_vars) # nosec - project_version = pkg_vars["__version__"] + project_version = f.read().strip() assert ( GITHUB_RELEASE_TAG == f"v{project_version}" ), "GITHUB_RELEASE_TAG does not match the project version" diff --git a/version.txt b/version.txt index 8c0d5d5..227cea2 100644 --- a/version.txt +++ b/version.txt @@ -1 +1 @@ -__version__ = "2.0.0" +2.0.0 From a3687048db92decd60cf2b9ce453577f28f9e3b0 Mon Sep 17 00:00:00 2001 From: Jeremy Frasier Date: Tue, 10 Dec 2024 10:33:57 -0500 Subject: [PATCH 72/75] Correct reference to src directory in setup-env The src directory no longer exists. --- setup-env | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup-env b/setup-env index e1340b6..4004b77 100755 --- a/setup-env +++ b/setup-env @@ -281,7 +281,7 @@ for req_file in "requirements-dev.txt" "requirements-test.txt" "requirements.txt done # Install Packer plugin dependencies -packer init -upgrade src +packer init -upgrade . # Install git pre-commit hooks now or later. pre-commit install ${INSTALL_HOOKS:+"--install-hooks"} From a770f2bed2bb25a972964e820d4f0bf3c43d3772 Mon Sep 17 00:00:00 2001 From: Jeremy Frasier Date: Wed, 11 Dec 2024 14:07:15 -0500 Subject: [PATCH 73/75] Use the correct packer build commands --- README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 33231a6..a085903 100644 --- a/README.md +++ b/README.md @@ -118,7 +118,7 @@ Here is an example of how to kick off a pre-release build: ```console pip install --requirement requirements-dev.txt ansible-galaxy install --force --force-with-deps --role-file ansible/requirements.yml -AWS_PROFILE=cool-images-ec2amicreate-skeleton-packer packer build --timestamp-ui -var release_tag=$(./bump-version show) -var is_prerelease=true packer.pkr.hcl +AWS_PROFILE=cool-images-ec2amicreate-skeleton-packer packer build --timestamp-ui -var release_tag=$(./bump-version show) -var is_prerelease=true . ``` If you are satisfied with your pre-release image, you can easily create a release @@ -135,7 +135,7 @@ region_kms_keys = { ``` ```console -AWS_PROFILE=cool-images-ec2amicreate-skeleton-packer packer build --timestamp-ui -var-file release.pkrvars.hcl packer.pkr.hcl +AWS_PROFILE=cool-images-ec2amicreate-skeleton-packer packer build --timestamp-ui -var-file release.pkrvars.hcl . ``` ### Giving Other AWS Accounts Permission to Launch the Image ### From 643e725a047e3cd20041012753ea590a4731324f Mon Sep 17 00:00:00 2001 From: Shane Frasier Date: Mon, 16 Dec 2024 11:29:37 -0500 Subject: [PATCH 74/75] Update reference to skeleton repo Co-authored-by: dav3r --- tests/test_version.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_version.py b/tests/test_version.py index 7b11f3e..cdf70cf 100644 --- a/tests/test_version.py +++ b/tests/test_version.py @@ -1,4 +1,4 @@ -"""Version tests for pca-gophish-composition packer skeleton project.""" +"""Version tests for pca-gophish-composition-packer project.""" # Standard Python Libraries import os From e37e185d6717f8d7043da1a7bda1ea55929885be Mon Sep 17 00:00:00 2001 From: Shane Frasier Date: Mon, 16 Dec 2024 11:30:17 -0500 Subject: [PATCH 75/75] Remove remaining reference to @jasonodoom from CODEOWNERS Co-authored-by: dav3r --- .github/CODEOWNERS | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index ec596ca..1e3437a 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -15,8 +15,8 @@ /ansible/aws.yml @jsf9k /ansible/python.yml @jsf9k -# Let dav3r, jasonodoom, jsf9k, and @mcdonnnj share ownership of packer.pkr.hcl. -/src/packer.pkr.hcl @dav3r @jasonodoom @jsf9k @mcdonnnj +# Let dav3r, jsf9k, and @mcdonnnj share ownership of packer.pkr.hcl. +/src/packer.pkr.hcl @dav3r @jsf9k @mcdonnnj # These folks own all linting configuration files. /.ansible-lint @dav3r @felddy @jsf9k @mcdonnnj