From b5e5c11bcd63fcf556d31e3753eb7b84e0a85e3e Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 13 Sep 2023 02:01:21 +0000 Subject: [PATCH 001/170] Bump crazy-max/ghaction-github-status from 3 to 4 Bumps [crazy-max/ghaction-github-status](https://github.com/crazy-max/ghaction-github-status) from 3 to 4. - [Release notes](https://github.com/crazy-max/ghaction-github-status/releases) - [Commits](https://github.com/crazy-max/ghaction-github-status/compare/v3...v4) --- updated-dependencies: - dependency-name: crazy-max/ghaction-github-status dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] --- .github/workflows/build.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index e7b75a4..894bb58 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -27,7 +27,7 @@ jobs: egress-policy: audit - id: github-status name: Check GitHub status - uses: crazy-max/ghaction-github-status@v3 + uses: crazy-max/ghaction-github-status@v4 - id: dump-context name: Dump context uses: crazy-max/ghaction-dump-context@v2 From 371179ef14354df4d00d7f961754ddb86c30929b Mon Sep 17 00:00:00 2001 From: Jeremy Frasier Date: Wed, 13 Sep 2023 12:48:59 -0400 Subject: [PATCH 002/170] Add a diagnostics job for the label syncing workflow Also add a runner hardening task to the labeler job. --- .github/workflows/sync-labels.yml | 24 ++++++++++++++++++++++++ 1 file changed, 24 insertions(+) diff --git a/.github/workflows/sync-labels.yml b/.github/workflows/sync-labels.yml index 44e8e19..5a20438 100644 --- a/.github/workflows/sync-labels.yml +++ b/.github/workflows/sync-labels.yml @@ -11,7 +11,26 @@ permissions: contents: read jobs: + diagnostics: + name: Run diagnostics + runs-on: ubuntu-latest + steps: + # Note that a duplicate of this step must be added at the top of + # each job. + - id: harden-runner + name: Harden the runner + uses: step-security/harden-runner@v2 + with: + egress-policy: audit + - id: github-status + name: Check GitHub status + uses: crazy-max/ghaction-github-status@v3 + - id: dump-context + name: Dump context + uses: crazy-max/ghaction-dump-context@v2 labeler: + needs: + - diagnostics permissions: # actions/checkout needs this to fetch code contents: read @@ -19,6 +38,11 @@ jobs: issues: write runs-on: ubuntu-latest steps: + - id: harden-runner + name: Harden the runner + uses: step-security/harden-runner@v2 + with: + egress-policy: audit - uses: actions/checkout@v4 - name: Sync repository labels if: success() From 1f611fc67710100e3e6efeb395d67b82e3f52eaa Mon Sep 17 00:00:00 2001 From: Jeremy Frasier Date: Thu, 14 Sep 2023 15:45:17 -0400 Subject: [PATCH 003/170] Make the dev team the owners of the linter configuration files Co-authored-by: Nick <50747025+mcdonnnj@users.noreply.github.com> --- .github/CODEOWNERS | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index 8f5c8c5..a22502d 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -8,3 +8,12 @@ # These folks own any files in the .github directory at the root of # the repository and any of its subdirectories. /.github/ @dav3r @felddy @jasonodoom @jsf9k @mcdonnnj + +# These folks own all linting configuration files. +/.*.cfg @dav3r @felddy @jasonodoom @jsf9k @mcdonnnj +/.*.yaml @dav3r @felddy @jasonodoom @jsf9k @mcdonnnj +/.*.yml @dav3r @felddy @jasonodoom @jsf9k @mcdonnnj +/.ansible-lint @dav3r @felddy @jasonodoom @jsf9k @mcdonnnj +/.flake8 @dav3r @felddy @jasonodoom @jsf9k @mcdonnnj +/.prettierignore @dav3r @felddy @jasonodoom @jsf9k @mcdonnnj +/.yamllint @dav3r @felddy @jasonodoom @jsf9k @mcdonnnj From c356768305b027d915db7ef7be1126ee687757ad Mon Sep 17 00:00:00 2001 From: Jeremy Frasier Date: Thu, 14 Sep 2023 15:59:22 -0400 Subject: [PATCH 004/170] Make dev team members the codeowners of the requirements*.txt and setup-env files Co-authored-by: Nick <50747025+mcdonnnj@users.noreply.github.com> --- .github/CODEOWNERS | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index a22502d..b4e689e 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -17,3 +17,5 @@ /.flake8 @dav3r @felddy @jasonodoom @jsf9k @mcdonnnj /.prettierignore @dav3r @felddy @jasonodoom @jsf9k @mcdonnnj /.yamllint @dav3r @felddy @jasonodoom @jsf9k @mcdonnnj +/requirements*.txt @dav3r @felddy @jasonodoom @jsf9k @mcdonnnj +/setup-env @dav3r @felddy @jasonodoom @jsf9k @mcdonnnj From 0195005cccab3e087ec8353c5049ea633d252835 Mon Sep 17 00:00:00 2001 From: Jeremy Frasier Date: Fri, 15 Sep 2023 10:26:45 -0400 Subject: [PATCH 005/170] Explicitly list the linter config files the dev team should own @mcdonnnj correctly pointed out that other projects add their own configuration files that match, e.g., the /.*.yaml pattern. We want to ensure that we only own the linter configuration files from the skeleton. Co-authored-by: Nick <50747025+mcdonnnj@users.noreply.github.com> --- .github/CODEOWNERS | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index b4e689e..229920c 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -10,12 +10,15 @@ /.github/ @dav3r @felddy @jasonodoom @jsf9k @mcdonnnj # These folks own all linting configuration files. -/.*.cfg @dav3r @felddy @jasonodoom @jsf9k @mcdonnnj -/.*.yaml @dav3r @felddy @jasonodoom @jsf9k @mcdonnnj -/.*.yml @dav3r @felddy @jasonodoom @jsf9k @mcdonnnj /.ansible-lint @dav3r @felddy @jasonodoom @jsf9k @mcdonnnj +/.bandit.yml @dav3r @felddy @jasonodoom @jsf9k @mcdonnnj /.flake8 @dav3r @felddy @jasonodoom @jsf9k @mcdonnnj +/.isort.cfg @dav3r @felddy @jasonodoom @jsf9k @mcdonnnj +/.mdl_config.yaml @dav3r @felddy @jasonodoom @jsf9k @mcdonnnj +/.pre-commit-config.yaml @dav3r @felddy @jasonodoom @jsf9k @mcdonnnj /.prettierignore @dav3r @felddy @jasonodoom @jsf9k @mcdonnnj /.yamllint @dav3r @felddy @jasonodoom @jsf9k @mcdonnnj -/requirements*.txt @dav3r @felddy @jasonodoom @jsf9k @mcdonnnj +/requirements.txt @dav3r @felddy @jasonodoom @jsf9k @mcdonnnj +/requirements-dev.txt @dav3r @felddy @jasonodoom @jsf9k @mcdonnnj +/requirements-test.txt @dav3r @felddy @jasonodoom @jsf9k @mcdonnnj /setup-env @dav3r @felddy @jasonodoom @jsf9k @mcdonnnj From b768a289e1702e9e46d81ad5a59f51abe666a976 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 30 Oct 2023 18:41:36 +0000 Subject: [PATCH 006/170] Bump hashicorp/setup-terraform from 2 to 3 Bumps [hashicorp/setup-terraform](https://github.com/hashicorp/setup-terraform) from 2 to 3. - [Release notes](https://github.com/hashicorp/setup-terraform/releases) - [Changelog](https://github.com/hashicorp/setup-terraform/blob/main/CHANGELOG.md) - [Commits](https://github.com/hashicorp/setup-terraform/compare/v2...v3) --- updated-dependencies: - dependency-name: hashicorp/setup-terraform dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] --- .github/workflows/build.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index e7b75a4..1ff72a9 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -101,7 +101,7 @@ jobs: ${{ env.CURL_CACHE_DIR }}/"${PACKER_ZIP}" sudo mv /usr/local/bin/packer /usr/local/bin/packer-default sudo ln -s /opt/packer/packer /usr/local/bin/packer - - uses: hashicorp/setup-terraform@v2 + - uses: hashicorp/setup-terraform@v3 with: terraform_version: ${{ steps.setup-env.outputs.terraform-version }} - name: Install go-critic From 9f317002ac85066adc1efa05d9c2f55275ef7d9c Mon Sep 17 00:00:00 2001 From: Nicholas McDonnell <50747025+mcdonnnj@users.noreply.github.com> Date: Thu, 2 Nov 2023 15:49:06 -0400 Subject: [PATCH 007/170] Prefer block style to flow style We prefer block style to flow style for sequences and mappings in YAML. --- .github/workflows/build.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index e7b75a4..9332263 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -5,7 +5,8 @@ on: push: pull_request: repository_dispatch: - types: [apb] + types: + - apb env: CURL_CACHE_DIR: ~/.cache/curl From 696433ae2c3b3f9f2b3613562d3889c753c7bc34 Mon Sep 17 00:00:00 2001 From: Nicholas McDonnell <50747025+mcdonnnj@users.noreply.github.com> Date: Thu, 2 Nov 2023 15:57:37 -0400 Subject: [PATCH 008/170] Alphabetize entries in the build workflow We prefer to alphabetize mapping keys in YAML documents whenever possible. --- .github/workflows/build.yml | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 9332263..2fef5d5 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -2,8 +2,8 @@ name: build on: - push: pull_request: + push: repository_dispatch: types: - apb @@ -58,8 +58,8 @@ jobs: # it relies on the existence of a go.sum file. cache: false go-version: "1.20" - - name: Lookup Go cache directory - id: go-cache + - id: go-cache + name: Lookup Go cache directory run: | echo "dir=$(go env GOCACHE)" >> $GITHUB_OUTPUT - uses: actions/cache@v3 @@ -70,6 +70,10 @@ jobs: packer${{ steps.setup-env.outputs.packer-version }}-\ tf${{ steps.setup-env.outputs.terraform-version }}-" with: + key: "${{ env.BASE_CACHE_KEY }}\ + ${{ hashFiles('**/requirements-test.txt') }}-\ + ${{ hashFiles('**/requirements.txt') }}-\ + ${{ hashFiles('**/.pre-commit-config.yaml') }}" # Note that the .terraform directory IS NOT included in the # cache because if we were caching, then we would need to use # the `-upgrade=true` option. This option blindly pulls down the @@ -81,10 +85,6 @@ jobs: ${{ env.PRE_COMMIT_CACHE_DIR }} ${{ env.CURL_CACHE_DIR }} ${{ steps.go-cache.outputs.dir }} - key: "${{ env.BASE_CACHE_KEY }}\ - ${{ hashFiles('**/requirements-test.txt') }}-\ - ${{ hashFiles('**/requirements.txt') }}-\ - ${{ hashFiles('**/.pre-commit-config.yaml') }}" restore-keys: | ${{ env.BASE_CACHE_KEY }} - name: Setup curl cache From 6503a9e7f67d908833ff5aa178a2fd28faf5e803 Mon Sep 17 00:00:00 2001 From: Nicholas McDonnell <50747025+mcdonnnj@users.noreply.github.com> Date: Thu, 2 Nov 2023 16:00:00 -0400 Subject: [PATCH 009/170] Add a `merge_group` trigger to the build workflow This should improve compatibility with merge queues. We configure it to only trigger on the `checks_requested` type which is currently the only supported type for this trigger. If additional types are added in the future they should be added if appropriate. --- .github/workflows/build.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 2fef5d5..1928cb0 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -2,6 +2,9 @@ name: build on: + merge_group: + types: + - checks_requested pull_request: push: repository_dispatch: From 193e799a2601d030a0dd7bb6608752a076dc5981 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 11 Dec 2023 18:59:21 +0000 Subject: [PATCH 010/170] Bump actions/setup-go from 4 to 5 Bumps [actions/setup-go](https://github.com/actions/setup-go) from 4 to 5. - [Release notes](https://github.com/actions/setup-go/releases) - [Commits](https://github.com/actions/setup-go/compare/v4...v5) --- updated-dependencies: - dependency-name: actions/setup-go dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] --- .github/workflows/build.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index e7b75a4..f208d82 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -51,7 +51,7 @@ jobs: # We need the Go version and Go cache location for the actions/cache step, # so the Go installation must happen before that. - id: setup-go - uses: actions/setup-go@v4 + uses: actions/setup-go@v5 with: # There is no expectation for actual Go code so we disable caching as # it relies on the existence of a go.sum file. From 5c84295bb6811bc49d3ff29de067adb2d179f879 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 11 Dec 2023 18:59:25 +0000 Subject: [PATCH 011/170] Bump actions/setup-python from 4 to 5 Bumps [actions/setup-python](https://github.com/actions/setup-python) from 4 to 5. - [Release notes](https://github.com/actions/setup-python/releases) - [Commits](https://github.com/actions/setup-python/compare/v4...v5) --- updated-dependencies: - dependency-name: actions/setup-python dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] --- .github/workflows/build.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index e7b75a4..366a16b 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -45,7 +45,7 @@ jobs: uses: cisagov/setup-env-github-action@develop - uses: actions/checkout@v4 - id: setup-python - uses: actions/setup-python@v4 + uses: actions/setup-python@v5 with: python-version: "3.11" # We need the Go version and Go cache location for the actions/cache step, From aad19725f6d4924eec9366f3e9fe6c33e0af758f Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 18 Dec 2023 17:31:17 +0000 Subject: [PATCH 012/170] Bump github/codeql-action from 2 to 3 Bumps [github/codeql-action](https://github.com/github/codeql-action) from 2 to 3. - [Release notes](https://github.com/github/codeql-action/releases) - [Changelog](https://github.com/github/codeql-action/blob/main/CHANGELOG.md) - [Commits](https://github.com/github/codeql-action/compare/v2...v3) --- updated-dependencies: - dependency-name: github/codeql-action dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] --- .github/workflows/codeql-analysis.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/codeql-analysis.yml b/.github/workflows/codeql-analysis.yml index dc49271..ad0724a 100644 --- a/.github/workflows/codeql-analysis.yml +++ b/.github/workflows/codeql-analysis.yml @@ -67,7 +67,7 @@ jobs: # Initializes the CodeQL tools for scanning. - name: Initialize CodeQL - uses: github/codeql-action/init@v2 + uses: github/codeql-action/init@v3 with: languages: ${{ matrix.language }} @@ -75,7 +75,7 @@ jobs: # Java). If this step fails, then you should remove it and run the build # manually (see below). - name: Autobuild - uses: github/codeql-action/autobuild@v2 + uses: github/codeql-action/autobuild@v3 # ℹī¸ Command-line programs to run using the OS shell. # 📚 https://git.io/JvXDl @@ -89,4 +89,4 @@ jobs: # make release - name: Perform CodeQL Analysis - uses: github/codeql-action/analyze@v2 + uses: github/codeql-action/analyze@v3 From 4a63dbe91c946d74d963c103b67d4f3c0746f35c Mon Sep 17 00:00:00 2001 From: Nicholas McDonnell <50747025+mcdonnnj@users.noreply.github.com> Date: Thu, 18 Jan 2024 16:25:35 -0500 Subject: [PATCH 013/170] Switch pre-commit hooks for running shfmt This hook bundles the binaries for shfmt with a Python package which removes the need to manually install the tool for the hook to function. --- .pre-commit-config.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 34f0253..ed254a0 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -86,8 +86,8 @@ repos: - id: nixpkgs-fmt # Shell script hooks - - repo: https://github.com/cisagov/pre-commit-shfmt - rev: v0.0.2 + - repo: https://github.com/scop/pre-commit-shfmt + rev: v3.7.0-4 hooks: - id: shfmt args: From 3236b1ba94d1c833dc2f9508944c6bc6e6a49f48 Mon Sep 17 00:00:00 2001 From: Nicholas McDonnell <50747025+mcdonnnj@users.noreply.github.com> Date: Thu, 18 Jan 2024 17:43:00 -0500 Subject: [PATCH 014/170] Remove installation of shfmt in the `build` workflow The new pre-commit hook provides `shfmt` binaries so we no longer need to ensure it is installed. --- .github/workflows/build.yml | 5 ----- 1 file changed, 5 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index e7b75a4..b5d46cc 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -114,11 +114,6 @@ jobs: PACKAGE_URL: github.com/securego/gosec/v2/cmd/gosec PACKAGE_VERSION: ${{ steps.setup-env.outputs.gosec-version }} run: go install ${PACKAGE_URL}@${PACKAGE_VERSION} - - name: Install shfmt - env: - PACKAGE_URL: mvdan.cc/sh/v3/cmd/shfmt - PACKAGE_VERSION: ${{ steps.setup-env.outputs.shfmt-version }} - run: go install ${PACKAGE_URL}@${PACKAGE_VERSION} - name: Install staticcheck env: PACKAGE_URL: honnef.co/go/tools/cmd/staticcheck From 5ddb14dab4e98019a6875278a1bc556c61e4bb69 Mon Sep 17 00:00:00 2001 From: Nicholas McDonnell <50747025+mcdonnnj@users.noreply.github.com> Date: Thu, 18 Jan 2024 16:56:43 -0500 Subject: [PATCH 015/170] Use long options for shfmt arguments Since shfmt now supports long command line options we should use them as that is our preference. The single quotes for the number of spaces to indent is changed to double quotes to align with our usual quotation style. --- .pre-commit-config.yaml | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index ed254a0..ca36e5d 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -92,14 +92,14 @@ repos: - id: shfmt args: # Indent by two spaces - - -i - - '2' + - --indent + - "2" # Binary operators may start a line - - -bn + - --binary-next-line # Switch cases are indented - - -ci + - --case-indent # Redirect operators are followed by a space - - -sr + - --space-redirects - repo: https://github.com/detailyang/pre-commit-shell rev: 1.0.5 hooks: From 8ecd95718a6d1b0db176cb85bf0790e4810364fb Mon Sep 17 00:00:00 2001 From: Nicholas McDonnell <50747025+mcdonnnj@users.noreply.github.com> Date: Thu, 18 Jan 2024 17:34:18 -0500 Subject: [PATCH 016/170] Add additional shfmt options These options are baked into the functionality of the old hook but must be explicitly declared for the new hook. --- .pre-commit-config.yaml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index ca36e5d..6b35e4a 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -91,6 +91,10 @@ repos: hooks: - id: shfmt args: + # List files that will be formatted + - --list + # Write result to file instead of stdout + - --write # Indent by two spaces - --indent - "2" From 242921b9856e3ede0fa1a941324c1a3f0ba116f7 Mon Sep 17 00:00:00 2001 From: Nicholas McDonnell <50747025+mcdonnnj@users.noreply.github.com> Date: Thu, 21 Sep 2023 17:49:55 -0400 Subject: [PATCH 017/170] Set the default shell for all run steps in the build workflow This sets the default shell for any run steps in the build workflow to mirror our standard shellscript writing practices. In addition to enabling our standard options it will also enable errtrace and print any commands that are run which should make debugging/troubleshooting more straightforward. --- .github/workflows/build.yml | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index e7b75a4..7299f6b 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -7,6 +7,14 @@ on: repository_dispatch: types: [apb] +# Set a default shell for any run steps. The `-Eueo pipefail` sets errtrace, +# nounset, errexit, and pipefail. The `-x` will print all commands as they are +# run. Please see the GitHub Actions documentation for more information: +# https://docs.github.com/en/actions/using-jobs/setting-default-values-for-jobs +defaults: + run: + shell: bash -Eueo pipefail -x {0} + env: CURL_CACHE_DIR: ~/.cache/curl PIP_CACHE_DIR: ~/.cache/pip From c7b18dc7e57c5261720324bd5b1425f6992e965e Mon Sep 17 00:00:00 2001 From: Nicholas McDonnell <50747025+mcdonnnj@users.noreply.github.com> Date: Fri, 12 Jan 2024 17:55:34 -0500 Subject: [PATCH 018/170] Add linting with goimports to the pre-commit configuration This will run the Go tool `goimports` against the repository if it contains any Go files. This tool bundles the functionality of `go fmt` with the additional benefit of sorting Go imports much like the isort tool we use for Python code. --- .github/workflows/build.yml | 5 +++++ .pre-commit-config.yaml | 6 +++++- 2 files changed, 10 insertions(+), 1 deletion(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index e7b75a4..8e14eab 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -109,6 +109,11 @@ jobs: PACKAGE_URL: github.com/go-critic/go-critic/cmd/gocritic PACKAGE_VERSION: ${{ steps.setup-env.outputs.go-critic-version }} run: go install ${PACKAGE_URL}@${PACKAGE_VERSION} + - name: Install goimports + env: + PACKAGE_URL: golang.org/x/tools/cmd/goimports + PACKAGE_VERSION: ${{ steps.setup-env.outputs.goimports-version }} + run: go install ${PACKAGE_URL}@${PACKAGE_VERSION} - name: Install gosec env: PACKAGE_URL: github.com/securego/gosec/v2/cmd/gosec diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 34f0253..7bb1f8c 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -78,7 +78,11 @@ repos: - id: go-vet-repo-mod # GoSec - id: go-sec-repo-mod - + # goimports + - id: go-imports-repo + args: + # Write changes to files + - -w # Nix hooks - repo: https://github.com/nix-community/nixpkgs-fmt rev: v1.3.0 From f6d9d6e29249c4b6246fca71c285d744c7409f38 Mon Sep 17 00:00:00 2001 From: Michael Saki Date: Mon, 22 Jan 2024 12:31:43 -0600 Subject: [PATCH 019/170] Add ATX Header Support for terraform-docs This is a temporary fix until @mcdonnnj has his PR approved and merged into the terraform-docs repo. This fix will perform a shallow clone of his forked branch, build the binary, and install it. --- .github/workflows/build.yml | 22 +++++++++++++++++----- 1 file changed, 17 insertions(+), 5 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index e7b75a4..e86b966 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -8,9 +8,12 @@ on: types: [apb] env: + BRANCH_NAME: improvement/support_atx_closed_markdown_headers CURL_CACHE_DIR: ~/.cache/curl + DEPTH: 1 PIP_CACHE_DIR: ~/.cache/pip PRE_COMMIT_CACHE_DIR: ~/.cache/pre-commit + REPO_URL: https://github.com/mcdonnnj/terraform-docs.git RUN_TMATE: ${{ secrets.RUN_TMATE }} jobs: @@ -124,11 +127,20 @@ jobs: PACKAGE_URL: honnef.co/go/tools/cmd/staticcheck PACKAGE_VERSION: ${{ steps.setup-env.outputs.staticcheck-version }} run: go install ${PACKAGE_URL}@${PACKAGE_VERSION} - - name: Install Terraform-docs - env: - PACKAGE_URL: github.com/terraform-docs/terraform-docs - PACKAGE_VERSION: ${{ steps.setup-env.outputs.terraform-docs-version }} - run: go install ${PACKAGE_URL}@${PACKAGE_VERSION} + # We are temporarily using @mcdonnnj's forked branch of terraform-docs + # until his PR: https://github.com/terraform-docs/terraform-docs/pull/745 + # is approved. This temporary fix will allow for ATX Header Support when + # terraform-docs is ran during lint. + - name: Clone ATX Headers branch from Terraform-docs Fork + run: | + git clone --branch $BRANCH_NAME --single-branch \ + --depth=$DEPTH $REPO_URL /tmp/terraform-docs + - name: Build and Install Terraform-docs Binary + run: | + cd /tmp/terraform-docs + GOBIN=$(go env GOPATH)/bin + go build -o $GOBIN/terraform-docs + echo "$GOBIN" >> $GITHUB_PATH - name: Install dependencies run: | python -m pip install --upgrade pip setuptools wheel From 544e4789e052978d00f982ef45d8d0de9f5e4748 Mon Sep 17 00:00:00 2001 From: Michael Saki Date: Mon, 22 Jan 2024 14:03:00 -0600 Subject: [PATCH 020/170] Add prepended names to variables to describe their function Co-authored-by: Nick <50747025+mcdonnnj@users.noreply.github.com> --- .github/workflows/build.yml | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index e86b966..99b42c1 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -8,13 +8,13 @@ on: types: [apb] env: - BRANCH_NAME: improvement/support_atx_closed_markdown_headers CURL_CACHE_DIR: ~/.cache/curl - DEPTH: 1 PIP_CACHE_DIR: ~/.cache/pip PRE_COMMIT_CACHE_DIR: ~/.cache/pre-commit - REPO_URL: https://github.com/mcdonnnj/terraform-docs.git RUN_TMATE: ${{ secrets.RUN_TMATE }} + TERRAFORM_DOCS_REPO_BRANCH_NAME: improvement/support_atx_closed_markdown_headers + TERRAFORM_DOCS_REPO_DEPTH: 1 + TERRAFORM_DOCS_REPO_URL: https://github.com/mcdonnnj/terraform-docs.git jobs: diagnostics: @@ -133,8 +133,10 @@ jobs: # terraform-docs is ran during lint. - name: Clone ATX Headers branch from Terraform-docs Fork run: | - git clone --branch $BRANCH_NAME --single-branch \ - --depth=$DEPTH $REPO_URL /tmp/terraform-docs + git clone --single-branch \ + --branch $TERRAFORM_DOCS_REPO_BRANCH_NAME \ + --depth $TERRAFORM_DOCS_REPO_DEPTH \ + $TERRAFORM_DOCS_REPO_URL /tmp/terraform-docs - name: Build and Install Terraform-docs Binary run: | cd /tmp/terraform-docs From f5fa0ff4ec1ded10a2906ade6300bba05cc2e81c Mon Sep 17 00:00:00 2001 From: Michael Saki Date: Mon, 22 Jan 2024 14:05:01 -0600 Subject: [PATCH 021/170] Remove unnecessary capitalizations and fix grammar Co-authored-by: Shane Frasier --- .github/workflows/build.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 99b42c1..ca719aa 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -129,15 +129,15 @@ jobs: run: go install ${PACKAGE_URL}@${PACKAGE_VERSION} # We are temporarily using @mcdonnnj's forked branch of terraform-docs # until his PR: https://github.com/terraform-docs/terraform-docs/pull/745 - # is approved. This temporary fix will allow for ATX Header Support when - # terraform-docs is ran during lint. - - name: Clone ATX Headers branch from Terraform-docs Fork + # is approved. This temporary fix will allow for ATX header support when + # terraform-docs is run during linting. + - name: Clone ATX headers branch from terraform-docs fork run: | git clone --single-branch \ --branch $TERRAFORM_DOCS_REPO_BRANCH_NAME \ --depth $TERRAFORM_DOCS_REPO_DEPTH \ $TERRAFORM_DOCS_REPO_URL /tmp/terraform-docs - - name: Build and Install Terraform-docs Binary + - name: Build and install terraform-docs binary run: | cd /tmp/terraform-docs GOBIN=$(go env GOPATH)/bin From 36361dd1c74c9855b87f2b4aec5a9be1fa7c416e Mon Sep 17 00:00:00 2001 From: Michael Saki Date: Mon, 22 Jan 2024 14:14:27 -0600 Subject: [PATCH 022/170] Simplify steps in the build/install portion of workflow PATH is handled by `setup-go` so we can refactor the code setting it. Also we are taking advantage of the -C switch to handle building from the cloned repository. Co-authored-by: Nick <50747025+mcdonnnj@users.noreply.github.com> --- .github/workflows/build.yml | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index ca719aa..6ab90de 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -139,10 +139,9 @@ jobs: $TERRAFORM_DOCS_REPO_URL /tmp/terraform-docs - name: Build and install terraform-docs binary run: | - cd /tmp/terraform-docs - GOBIN=$(go env GOPATH)/bin - go build -o $GOBIN/terraform-docs - echo "$GOBIN" >> $GITHUB_PATH + go build \ + -C /tmp/terraform-docs \ + -o $(go env GOPATH)/bin/terraform-docs - name: Install dependencies run: | python -m pip install --upgrade pip setuptools wheel From 3711ebe461d81987b4121903d2f3f1b290fccad3 Mon Sep 17 00:00:00 2001 From: Michael Saki Date: Tue, 23 Jan 2024 08:04:07 -0600 Subject: [PATCH 023/170] Add TODO label --- .github/workflows/build.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 6ab90de..f5ba773 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -51,7 +51,7 @@ jobs: uses: actions/setup-python@v4 with: python-version: "3.11" - # We need the Go version and Go cache location for the actions/cache step, + # TODO: We need the Go version and Go cache location for the actions/cache step, # so the Go installation must happen before that. - id: setup-go uses: actions/setup-go@v4 From d114fb4233f22cb9edb3e1a6555ca0a677bcfc7e Mon Sep 17 00:00:00 2001 From: Michael Saki Date: Tue, 23 Jan 2024 08:24:12 -0600 Subject: [PATCH 024/170] Move TODO and add link to the issue `TODO` was placed on the wrong comment block. Also I am adding a link to the issue for the TODO. --- .github/workflows/build.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index f5ba773..0068f1f 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -51,7 +51,7 @@ jobs: uses: actions/setup-python@v4 with: python-version: "3.11" - # TODO: We need the Go version and Go cache location for the actions/cache step, + # We need the Go version and Go cache location for the actions/cache step, # so the Go installation must happen before that. - id: setup-go uses: actions/setup-go@v4 @@ -127,6 +127,7 @@ jobs: PACKAGE_URL: honnef.co/go/tools/cmd/staticcheck PACKAGE_VERSION: ${{ steps.setup-env.outputs.staticcheck-version }} run: go install ${PACKAGE_URL}@${PACKAGE_VERSION} + # TODO: https://github.com/cisagov/skeleton-generic/issues/165 # We are temporarily using @mcdonnnj's forked branch of terraform-docs # until his PR: https://github.com/terraform-docs/terraform-docs/pull/745 # is approved. This temporary fix will allow for ATX header support when From c907cfc82a8f329fbd4ad0ee17f3500922c51711 Mon Sep 17 00:00:00 2001 From: Michael Saki Date: Tue, 23 Jan 2024 08:41:41 -0600 Subject: [PATCH 025/170] Alphabetize switches Co-authored-by: dav3r --- .github/workflows/build.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 0068f1f..35391d5 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -134,9 +134,10 @@ jobs: # terraform-docs is run during linting. - name: Clone ATX headers branch from terraform-docs fork run: | - git clone --single-branch \ + git clone \ --branch $TERRAFORM_DOCS_REPO_BRANCH_NAME \ --depth $TERRAFORM_DOCS_REPO_DEPTH \ + --single-branch \ $TERRAFORM_DOCS_REPO_URL /tmp/terraform-docs - name: Build and install terraform-docs binary run: | From 48db3e3381aa61bdbe67dab64454db3e7f332c75 Mon Sep 17 00:00:00 2001 From: Michael Saki Date: Thu, 25 Jan 2024 12:19:41 -0600 Subject: [PATCH 026/170] Allow setup-env to specify Python version This commit is introducing 2 new flags into the setup-env script. -l or --list-versions will list available Python versions and allow the user to select a version interactively. The second flag -v or --version will allow a user to set the version if installed. (e.g. ./setup-env -v 3.9.6) --- setup-env | 42 +++++++++++++++++++++++++++++++++++++++--- 1 file changed, 39 insertions(+), 3 deletions(-) diff --git a/setup-env b/setup-env index 77926bf..b99ec5f 100755 --- a/setup-env +++ b/setup-env @@ -25,6 +25,8 @@ Options: -h --help Show this message. -i --install-hooks Install hook environments for all environments in the pre-commit config file. + -v --version Specify the Python version for the virtual environment. + -l --list-versions List available Python versions and select interactively. END_OF_LINE ) @@ -35,6 +37,13 @@ FORCE=0 # Positional parameters PARAMS="" +# Flags to allow a user to specify which version of Python they want to use +PYTHON_VERSION="" +LIST_VERSIONS=0 + +# Temp file that is used to search through available installed Python versions +TMPFILE=/tmp/versions.$$ + # Parse command line arguments while (("$#")); do case "$1" in @@ -50,6 +59,14 @@ while (("$#")); do INSTALL_HOOKS=1 shift ;; + -v | --version) + PYTHON_VERSION=$2 + shift 2 + ;; + -l | --list-versions) + LIST_VERSIONS=1 + shift + ;; -*) # unsupported flags echo "Error: Unsupported flag $1" >&2 exit 1 @@ -111,6 +128,25 @@ else fi set -o nounset +# List Python versions and select one interactively +if [ $LIST_VERSIONS -ne 0 ]; then + echo Available Python versions: + pyenv versions --bare --skip-aliases --skip-envs + read -p -r "Enter the desired Python version: " PYTHON_VERSION +fi + +# Check if PYTHON_VERSION isn't empty. If it is installed, set it locally. +pyenv versions --bare --skip-aliases --skip-envs > $TMPFILE +if [ -n "$PYTHON_VERSION" ]; then + if grep --fixed-strings --quiet "$PYTHON_VERSION" $TMPFILE; then + echo Using Python version "$PYTHON_VERSION" + pyenv local "$PYTHON_VERSION" + else + echo Error: Python version "$PYTHON_VERSION" is not installed. + fi + exit 1 +fi + # Remove any lingering local configuration. if [ $FORCE -ne 0 ]; then rm -f .python-version @@ -130,10 +166,10 @@ fi # Create a new virtual environment for this project if ! pyenv virtualenv "${env_name}"; then cat << END_OF_LINE - An existing virtual environment named $env_name was found. Either delete this - environment yourself or re-run with --force option to have it deleted. + An existing virtual environment named $env_name was found. Either delete this + environment yourself or re-run with --force option to have it deleted. - pyenv virtualenv-delete ${env_name} + pyenv virtualenv-delete ${env_name} END_OF_LINE exit 1 From c10929afae693990a3d4d034faa2c1ab1b2689ee Mon Sep 17 00:00:00 2001 From: Michael Saki Date: Thu, 25 Jan 2024 14:15:40 -0600 Subject: [PATCH 027/170] Add /dev/null and remove TMPFILE This makes the code a bit cleaner and still accomplishes the same functionality Co-authored-by: Shane Frasier --- setup-env | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/setup-env b/setup-env index b99ec5f..769522a 100755 --- a/setup-env +++ b/setup-env @@ -41,9 +41,6 @@ PARAMS="" PYTHON_VERSION="" LIST_VERSIONS=0 -# Temp file that is used to search through available installed Python versions -TMPFILE=/tmp/versions.$$ - # Parse command line arguments while (("$#")); do case "$1" in @@ -136,9 +133,8 @@ if [ $LIST_VERSIONS -ne 0 ]; then fi # Check if PYTHON_VERSION isn't empty. If it is installed, set it locally. -pyenv versions --bare --skip-aliases --skip-envs > $TMPFILE if [ -n "$PYTHON_VERSION" ]; then - if grep --fixed-strings --quiet "$PYTHON_VERSION" $TMPFILE; then + if pyenv versions --bare --skip-aliases --skip-envs | grep --fixed-strings "$PYTHON_VERSION" >/dev/null; then echo Using Python version "$PYTHON_VERSION" pyenv local "$PYTHON_VERSION" else From adada40e6fa6aed6fa14f253a5b86f313b3dfa8f Mon Sep 17 00:00:00 2001 From: Michael Saki Date: Thu, 25 Jan 2024 14:19:54 -0600 Subject: [PATCH 028/170] Place flags in the correct order for -r and -p --- setup-env | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/setup-env b/setup-env index 769522a..2adbd5a 100755 --- a/setup-env +++ b/setup-env @@ -129,12 +129,12 @@ set -o nounset if [ $LIST_VERSIONS -ne 0 ]; then echo Available Python versions: pyenv versions --bare --skip-aliases --skip-envs - read -p -r "Enter the desired Python version: " PYTHON_VERSION + read -r -p "Enter the desired Python version: " PYTHON_VERSION fi # Check if PYTHON_VERSION isn't empty. If it is installed, set it locally. if [ -n "$PYTHON_VERSION" ]; then - if pyenv versions --bare --skip-aliases --skip-envs | grep --fixed-strings "$PYTHON_VERSION" >/dev/null; then + if pyenv versions --bare --skip-aliases --skip-envs | grep --fixed-strings "$PYTHON_VERSION" > /dev/null; then echo Using Python version "$PYTHON_VERSION" pyenv local "$PYTHON_VERSION" else From 1861b9b86778613bdaeff804418f2d00706f4a08 Mon Sep 17 00:00:00 2001 From: Michael Saki Date: Thu, 25 Jan 2024 14:21:49 -0600 Subject: [PATCH 029/170] Remove unneccessary spacing --- setup-env | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/setup-env b/setup-env index 2adbd5a..5abb624 100755 --- a/setup-env +++ b/setup-env @@ -162,10 +162,10 @@ fi # Create a new virtual environment for this project if ! pyenv virtualenv "${env_name}"; then cat << END_OF_LINE - An existing virtual environment named $env_name was found. Either delete this - environment yourself or re-run with --force option to have it deleted. + An existing virtual environment named $env_name was found. Either delete this + environment yourself or re-run with --force option to have it deleted. - pyenv virtualenv-delete ${env_name} + pyenv virtualenv-delete ${env_name} END_OF_LINE exit 1 From 3f623e493752c581829271a148cb9fff3ad4f4ab Mon Sep 17 00:00:00 2001 From: Michael Saki Date: Thu, 25 Jan 2024 14:25:37 -0600 Subject: [PATCH 030/170] Alphabetize flags and descriptions Co-authored-by: Shane Frasier --- setup-env | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/setup-env b/setup-env index 5abb624..5282e49 100755 --- a/setup-env +++ b/setup-env @@ -25,8 +25,8 @@ Options: -h --help Show this message. -i --install-hooks Install hook environments for all environments in the pre-commit config file. - -v --version Specify the Python version for the virtual environment. -l --list-versions List available Python versions and select interactively. + -v --version Specify the Python version for the virtual environment. END_OF_LINE ) @@ -56,14 +56,14 @@ while (("$#")); do INSTALL_HOOKS=1 shift ;; - -v | --version) - PYTHON_VERSION=$2 - shift 2 - ;; -l | --list-versions) LIST_VERSIONS=1 shift ;; + -v | --version) + PYTHON_VERSION=$2 + shift 2 + ;; -*) # unsupported flags echo "Error: Unsupported flag $1" >&2 exit 1 From 9497dc276e9f148211e32cc800342846cfa25e33 Mon Sep 17 00:00:00 2001 From: Jeremy Frasier Date: Fri, 26 Jan 2024 10:08:25 -0500 Subject: [PATCH 031/170] Move misplaced exit If the Python version exists then we want the script to continue execution. --- setup-env | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup-env b/setup-env index 5282e49..5d15679 100755 --- a/setup-env +++ b/setup-env @@ -139,8 +139,8 @@ if [ -n "$PYTHON_VERSION" ]; then pyenv local "$PYTHON_VERSION" else echo Error: Python version "$PYTHON_VERSION" is not installed. + exit 1 fi - exit 1 fi # Remove any lingering local configuration. From e1d0f28ba1d96b2da0af91e277648ca88d0c3f35 Mon Sep 17 00:00:00 2001 From: Jeremy Frasier Date: Fri, 26 Jan 2024 10:10:23 -0500 Subject: [PATCH 032/170] Remove premature pyenv local command pyenv local is run below. --- setup-env | 1 - 1 file changed, 1 deletion(-) diff --git a/setup-env b/setup-env index 5d15679..a78f940 100755 --- a/setup-env +++ b/setup-env @@ -136,7 +136,6 @@ fi if [ -n "$PYTHON_VERSION" ]; then if pyenv versions --bare --skip-aliases --skip-envs | grep --fixed-strings "$PYTHON_VERSION" > /dev/null; then echo Using Python version "$PYTHON_VERSION" - pyenv local "$PYTHON_VERSION" else echo Error: Python version "$PYTHON_VERSION" is not installed. exit 1 From 517b336210269d890c28e5c05d1c13e866fb7b15 Mon Sep 17 00:00:00 2001 From: Jeremy Frasier Date: Fri, 26 Jan 2024 10:11:50 -0500 Subject: [PATCH 033/170] Include PYTHON_VERSION when running pyenv virtualenv If PYTHON_VERSION is an empty string then the system Python will be used. --- setup-env | 19 ++++++++++++++----- 1 file changed, 14 insertions(+), 5 deletions(-) diff --git a/setup-env b/setup-env index a78f940..0aa173d 100755 --- a/setup-env +++ b/setup-env @@ -37,8 +37,8 @@ FORCE=0 # Positional parameters PARAMS="" -# Flags to allow a user to specify which version of Python they want to use -PYTHON_VERSION="" +# A flag to allow a user to specify which version of Python they want +# to use. LIST_VERSIONS=0 # Parse command line arguments @@ -132,8 +132,9 @@ if [ $LIST_VERSIONS -ne 0 ]; then read -r -p "Enter the desired Python version: " PYTHON_VERSION fi -# Check if PYTHON_VERSION isn't empty. If it is installed, set it locally. -if [ -n "$PYTHON_VERSION" ]; then +# Check if PYTHON_VERSION is defined. If it is defined then check that +# it is a valid value. +if [ -n "${PYTHON_VERSION+x}" ]; then if pyenv versions --bare --skip-aliases --skip-envs | grep --fixed-strings "$PYTHON_VERSION" > /dev/null; then echo Using Python version "$PYTHON_VERSION" else @@ -159,7 +160,15 @@ END_OF_LINE fi # Create a new virtual environment for this project -if ! pyenv virtualenv "${env_name}"; then +# +# If $PYTHON_VERSION is undefined then the system Python will be used. +# +# We can't quote ${PYTHON_VERSION:=} below since if the variable is +# undefined then we want nothing to appear; this is the reason for the +# "shellcheck disable" line below. +# +# shellcheck disable=SC2086 +if ! pyenv virtualenv ${PYTHON_VERSION:=} "${env_name}"; then cat << END_OF_LINE An existing virtual environment named $env_name was found. Either delete this environment yourself or re-run with --force option to have it deleted. From 2e5794cb2aeaa76e8c94e3113f903954a9564eb4 Mon Sep 17 00:00:00 2001 From: Michael Saki Date: Tue, 30 Jan 2024 12:51:51 -0600 Subject: [PATCH 034/170] Add getopt variables and short flags --- setup-env | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/setup-env b/setup-env index 0aa173d..63cebd3 100755 --- a/setup-env +++ b/setup-env @@ -34,9 +34,21 @@ END_OF_LINE # Flag to force deletion and creation of virtual environment FORCE=0 +# Initialize the all other flags +INSTALL_HOOKS=0 +LIST_VERSIONS=0 +PYTHON_VERSION="" +VENV_NAME="" + # Positional parameters PARAMS="" +# Define short options for getopt +OPTS="fhilv:n:" + +# Parse options using BSD getopt +OPTIND=1 + # A flag to allow a user to specify which version of Python they want # to use. LIST_VERSIONS=0 From 8a5003195b6cc3a069fb0c4ec678f20372c4e10d Mon Sep 17 00:00:00 2001 From: Michael Saki Date: Tue, 30 Jan 2024 12:54:12 -0600 Subject: [PATCH 035/170] Remove redundant flag initialization --- setup-env | 4 ---- 1 file changed, 4 deletions(-) diff --git a/setup-env b/setup-env index 63cebd3..da640f0 100755 --- a/setup-env +++ b/setup-env @@ -49,10 +49,6 @@ OPTS="fhilv:n:" # Parse options using BSD getopt OPTIND=1 -# A flag to allow a user to specify which version of Python they want -# to use. -LIST_VERSIONS=0 - # Parse command line arguments while (("$#")); do case "$1" in From 0df0e6aca8252e6356762b9284f3165ee39e2017 Mon Sep 17 00:00:00 2001 From: Michael Saki Date: Tue, 30 Jan 2024 13:59:04 -0600 Subject: [PATCH 036/170] Add getopt functionality and -n flag This commit makes a couple changes. The first change is adding the BSD getopt tool to simplify the parsing of flags and arguments. Second, we are adding the -n flag so the user can specify the name of the virtual environment if they choose. --- setup-env | 57 +++++++++++++++++++++++++++++++++++-------------------- 1 file changed, 36 insertions(+), 21 deletions(-) diff --git a/setup-env b/setup-env index da640f0..e097c82 100755 --- a/setup-env +++ b/setup-env @@ -49,40 +49,55 @@ OPTS="fhilv:n:" # Parse options using BSD getopt OPTIND=1 +# Display installed python versions +python_versions() { + pyenv versions --bare --skip-aliases --skip-envs +} + # Parse command line arguments -while (("$#")); do - case "$1" in - -f | --force) +while getopts :$OPTS opt; do + case $opt in + f) FORCE=1 - shift ;; - -h | --help) - echo "${USAGE}" + h) + echo "$USAGE" exit 0 ;; - -i | --install-hooks) + i) INSTALL_HOOKS=1 - shift ;; - -l | --list-versions) + l) LIST_VERSIONS=1 - shift ;; - -v | --version) - PYTHON_VERSION=$2 - shift 2 + n) + VENV_NAME="$OPTARG" + ;; + v) + PYTHON_VERSION="$OPTARG" + # Check if Python version is valid and installed + if ! python_versions | grep --fixed-strings "$PYTHON_VERSION" > /dev/null; then + echo "Error: Python version $PYTHON_VERSION is not installed. Versions available:" + python_versions + exit 1 + fi ;; - -*) # unsupported flags - echo "Error: Unsupported flag $1" >&2 + \:) + echo Error: Option -$OPTARG requires an argument. + echo "$USAGE" exit 1 ;; - *) # preserve positional arguments - PARAMS="$PARAMS $1" - shift + \?) + echo -e "Invalid option please look through usage: \n" + echo "$USAGE" + exit 1 ;; + esac done +shift $((OPTIND-1)) + # set positional arguments in their proper place eval set -- "$PARAMS" @@ -124,9 +139,9 @@ fi set +o nounset # Determine the virtual environment name -if [ "$1" ]; then +if [ -n "$VENV_NAME" ]; then # Use the user-provided environment name - env_name=$1 + env_name="$VENV_NAME" else # Set the environment name to the last part of the working directory. env_name=${PWD##*/} @@ -136,7 +151,7 @@ set -o nounset # List Python versions and select one interactively if [ $LIST_VERSIONS -ne 0 ]; then echo Available Python versions: - pyenv versions --bare --skip-aliases --skip-envs + python_versions read -r -p "Enter the desired Python version: " PYTHON_VERSION fi From 60cad12a2a64508c1e5c80092118747b64bb32e6 Mon Sep 17 00:00:00 2001 From: Michael Saki Date: Tue, 30 Jan 2024 14:16:52 -0600 Subject: [PATCH 037/170] Update the usage and force documentation This commit will update the usage documentation so now it will display all the new flags, how to use them, and what they do. Since the long option isn't available for the BSD version of getopt we have to use short options only. The force documentation has been updated to reflect this change. Also removed some redundant code that wasn't necessary. --- setup-env | 47 +++++++++++++++++++---------------------------- 1 file changed, 19 insertions(+), 28 deletions(-) diff --git a/setup-env b/setup-env index e097c82..4ed12d9 100755 --- a/setup-env +++ b/setup-env @@ -17,16 +17,18 @@ It does the following: - Configures git upstream remote "lineage" repositories. Usage: - setup-env [options] [virt_env_name] - setup-env (-h | --help) + setup-env [-n] [virt_env_name] + setup-env [-v] [python_version] + setup-env (-h) Options: - -f --force Delete virtual enviroment if it already exists. - -h --help Show this message. - -i --install-hooks Install hook environments for all environments in the - pre-commit config file. - -l --list-versions List available Python versions and select interactively. - -v --version Specify the Python version for the virtual environment. + -f Delete virtual enviroment if it already exists. + -h Show this message. + -i Install hook environments for all environments in the + pre-commit config file. + -l List available Python versions and select interactively. + -n Choose the name of the virtual environment. + -v Specify the Python version for the virtual environment. END_OF_LINE ) @@ -49,9 +51,9 @@ OPTS="fhilv:n:" # Parse options using BSD getopt OPTIND=1 -# Display installed python versions +# Display installed python versions python_versions() { - pyenv versions --bare --skip-aliases --skip-envs + pyenv versions --bare --skip-aliases --skip-envs } # Parse command line arguments @@ -82,21 +84,21 @@ while getopts :$OPTS opt; do exit 1 fi ;; - \:) - echo Error: Option -$OPTARG requires an argument. + :) + echo -e "Error: Option -$OPTARG requires an argument. \n" echo "$USAGE" exit 1 ;; \?) - echo -e "Invalid option please look through usage: \n" + echo -e "Invalid option please look through usage: \n" echo "$USAGE" exit 1 ;; - + esac done -shift $((OPTIND-1)) +shift $((OPTIND - 1)) # set positional arguments in their proper place eval set -- "$PARAMS" @@ -155,17 +157,6 @@ if [ $LIST_VERSIONS -ne 0 ]; then read -r -p "Enter the desired Python version: " PYTHON_VERSION fi -# Check if PYTHON_VERSION is defined. If it is defined then check that -# it is a valid value. -if [ -n "${PYTHON_VERSION+x}" ]; then - if pyenv versions --bare --skip-aliases --skip-envs | grep --fixed-strings "$PYTHON_VERSION" > /dev/null; then - echo Using Python version "$PYTHON_VERSION" - else - echo Error: Python version "$PYTHON_VERSION" is not installed. - exit 1 - fi -fi - # Remove any lingering local configuration. if [ $FORCE -ne 0 ]; then rm -f .python-version @@ -173,7 +164,7 @@ if [ $FORCE -ne 0 ]; then elif [[ -f .python-version ]]; then cat << 'END_OF_LINE' An existing .python-version file was found. Either remove this file yourself - or re-run with --force option to have it deleted along with the associated + or re-run with -f (force) option to have it deleted along with the associated virtual environment. rm .python-version @@ -194,7 +185,7 @@ fi if ! pyenv virtualenv ${PYTHON_VERSION:=} "${env_name}"; then cat << END_OF_LINE An existing virtual environment named $env_name was found. Either delete this - environment yourself or re-run with --force option to have it deleted. + environment yourself or re-run with -f (force) option to have it deleted. pyenv virtualenv-delete ${env_name} From b6ab6d82fb63bece7926b9a524bbb59a96c0cb06 Mon Sep 17 00:00:00 2001 From: Michael Saki Date: Tue, 6 Feb 2024 19:58:49 -0600 Subject: [PATCH 038/170] Update usage with long options --- setup-env | 22 ++++++++++++---------- 1 file changed, 12 insertions(+), 10 deletions(-) diff --git a/setup-env b/setup-env index 4ed12d9..898907b 100755 --- a/setup-env +++ b/setup-env @@ -9,6 +9,8 @@ USAGE=$( Configure a development environment for this repository. It does the following: + - Allows user to specify Python version. + - Allows user to choose name for their virtual environment. - Verifies pyenv and pyenv-virtualenv are installed. - Creates a Python virtual environment. - Configures the activation of the virtual enviroment for the repo directory. @@ -17,18 +19,18 @@ It does the following: - Configures git upstream remote "lineage" repositories. Usage: - setup-env [-n] [virt_env_name] - setup-env [-v] [python_version] - setup-env (-h) + setup-env (-n | --name) [virt_env_name] + setup-env (-v | --version) [python_version] + setup-env (-h | --help) Options: - -f Delete virtual enviroment if it already exists. - -h Show this message. - -i Install hook environments for all environments in the - pre-commit config file. - -l List available Python versions and select interactively. - -n Choose the name of the virtual environment. - -v Specify the Python version for the virtual environment. + -f | --force Delete virtual enviroment if it already exists. + -h | --help Show this message. + -i | --install-hooks Install hook environments for all environments in the + pre-commit config file. + -l | --list-versions List available Python versions and select interactively. + -n | --name Choose the name of the virtual environment. + -v | --version Specify the Python version for the virtual environment. END_OF_LINE ) From d362614a13731f3884856fa2ee37ade0089fda47 Mon Sep 17 00:00:00 2001 From: Michael Saki Date: Tue, 6 Feb 2024 20:28:05 -0600 Subject: [PATCH 039/170] Add gnu-getopt functionality and error handling This commit will make it so long options are supported. It also includes some improved error handling. It will display a message if a Mac OS user doesn't have gnu-getopt installed on their system and suggest steps to take to get it installed via brew. --- setup-env | 107 ++++++++++++++++++++++++++++++++++-------------------- 1 file changed, 68 insertions(+), 39 deletions(-) diff --git a/setup-env b/setup-env index 898907b..f4cdf75 100755 --- a/setup-env +++ b/setup-env @@ -35,6 +35,11 @@ Options: END_OF_LINE ) +# Display installed python versions +python_versions() { + pyenv versions --bare --skip-aliases --skip-envs +} + # Flag to force deletion and creation of virtual environment FORCE=0 @@ -44,67 +49,82 @@ LIST_VERSIONS=0 PYTHON_VERSION="" VENV_NAME="" -# Positional parameters -PARAMS="" +# Define long options +LONGOPTS="force,help,install-hooks,list-versions,name:,version:" # Define short options for getopt -OPTS="fhilv:n:" +SHORTOPTS="fhiln:v:" -# Parse options using BSD getopt -OPTIND=1 +# Check if GNU getopt is available +if [[ $(getopt --version 2> /dev/null) != *"getopt from util-linux"* ]]; then + cat << 'END_OF_LINE' -# Display installed python versions -python_versions() { - pyenv versions --bare --skip-aliases --skip-envs -} + Gnu-getopt is not detected and is a dependency to run this script. + On the Mac, we recommend installing brew, https://brew.sh/. Then installation + is as simple as `brew install gnu-getopt` and adding this to your + profile: + + export PATH="/usr/local/opt/gnu-getopt/bin:$PATH" + +END_OF_LINE + exit 1 +fi -# Parse command line arguments -while getopts :$OPTS opt; do - case $opt in - f) +# Use GNU getopt to parse options +if ! PARSED=$(getopt --options $SHORTOPTS --longoptions $LONGOPTS --name "$0" -- "$@"); then + echo "Error parsing options" + exit 2 +fi +eval set -- "$PARSED" + +while true; do + case "$1" in + -f | --force) FORCE=1 + shift ;; - h) + -h | --help) echo "$USAGE" exit 0 ;; - i) + -i | --install-hooks) INSTALL_HOOKS=1 + shift ;; - l) + -l | --list-versions) LIST_VERSIONS=1 + shift ;; - n) - VENV_NAME="$OPTARG" + -n | --name) + VENV_NAME="$2" + shift 2 ;; - v) - PYTHON_VERSION="$OPTARG" - # Check if Python version is valid and installed - if ! python_versions | grep --fixed-strings "$PYTHON_VERSION" > /dev/null; then - echo "Error: Python version $PYTHON_VERSION is not installed. Versions available:" - python_versions - exit 1 + -v | --version) + PYTHON_VERSION="$2" + shift 2 + # Check the Python versions being passed in. + if [ -n "${PYTHON_VERSION+x}" ]; then + if python_versions | grep -E "^${PYTHON_VERSION}$" > /dev/null; then + echo Using Python version "$PYTHON_VERSION" + else + echo Error: Python version "$PYTHON_VERSION" is not installed. + echo Installed Python versions are: + python_versions + exit 1 + fi fi ;; - :) - echo -e "Error: Option -$OPTARG requires an argument. \n" - echo "$USAGE" - exit 1 + --) + shift + break ;; - \?) - echo -e "Invalid option please look through usage: \n" - echo "$USAGE" - exit 1 + *) + echo "Programming error" + exit 3 ;; - esac done -shift $((OPTIND - 1)) - -# set positional arguments in their proper place -eval set -- "$PARAMS" - # Check to see if pyenv is installed if [ -z "$(command -v pyenv)" ] || { [ -z "$(command -v pyenv-virtualenv)" ] && [ ! -f "$(pyenv root)/plugins/pyenv-virtualenv/bin/pyenv-virtualenv" ]; }; then echo "pyenv and pyenv-virtualenv are required." @@ -157,6 +177,15 @@ if [ $LIST_VERSIONS -ne 0 ]; then echo Available Python versions: python_versions read -r -p "Enter the desired Python version: " PYTHON_VERSION + # Check the Python versions being passed in. + if [ -n "${PYTHON_VERSION+x}" ]; then + if python_versions | grep -E "^${PYTHON_VERSION}$" > /dev/null; then + echo Using Python version "$PYTHON_VERSION" + else + echo Error: Python version "$PYTHON_VERSION" is not installed. + exit 1 + fi + fi fi # Remove any lingering local configuration. From f92458454394db61e8080b72b42fd340b3b64aa1 Mon Sep 17 00:00:00 2001 From: Michael Saki Date: Tue, 6 Feb 2024 20:49:34 -0600 Subject: [PATCH 040/170] Add documentation in CONTRIBUTING.md for gnu-getopt --- CONTRIBUTING.md | 16 +++++++++++----- 1 file changed, 11 insertions(+), 5 deletions(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 27bb9e7..dbbe417 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -46,9 +46,12 @@ There are a few ways to do this, but we prefer to use create and manage a Python virtual environment specific to this project. -If you already have `pyenv` and `pyenv-virtualenv` configured you can -take advantage of the `setup-env` tool in this repo to automate the -entire environment configuration process. +The `setup-env` tool in this repository is our recommended method +for automating the entire environment configuration process. The +dependencies required to run this tool are +[`gnu-getopt`](https://manned.org/getopt.1), `pyenv`, and +`pyenv-virtualenv`. If these tools are already configured on +your system, you can simply run the following command. ```console ./setup-env @@ -57,13 +60,14 @@ entire environment configuration process. Otherwise, follow the steps below to manually configure your environment. -#### Installing and using `pyenv` and `pyenv-virtualenv` #### +#### Installing and using `gnu-getopt`, `pyenv`, and `pyenv-virtualenv` #### On the Mac, we recommend installing [brew](https://brew.sh/). Then -installation is as simple as `brew install pyenv pyenv-virtualenv` and +installation is as simple as `brew install gnu-getopt pyenv pyenv-virtualenv` and adding this to your profile: ```bash +export PATH="/usr/local/opt/gnu-getopt/bin:$PATH" export PYENV_ROOT="$HOME/.pyenv" export PATH="$PYENV_ROOT/bin:$PATH" eval "$(pyenv init --path)" @@ -78,6 +82,8 @@ install the necessary tools. Before running this ensure that you have installed the prerequisites for your platform according to the [`pyenv` wiki page](https://github.com/pyenv/pyenv/wiki/common-build-problems). +`Gnu-getopt` is generally included in the core utilities of most +Linux distributions. On WSL you should treat your platform as whatever Linux distribution you've chosen to install. From ba86ead6c4f757f03610c6065df46e5ba4ad4dbd Mon Sep 17 00:00:00 2001 From: Michael Saki Date: Wed, 7 Feb 2024 09:43:30 -0600 Subject: [PATCH 041/170] Fix grammar and capitalization errors Co-authored-by: Shane Frasier --- CONTRIBUTING.md | 2 +- setup-env | 12 ++++++------ 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index dbbe417..d96a0e3 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -82,7 +82,7 @@ install the necessary tools. Before running this ensure that you have installed the prerequisites for your platform according to the [`pyenv` wiki page](https://github.com/pyenv/pyenv/wiki/common-build-problems). -`Gnu-getopt` is generally included in the core utilities of most +`gnu-getopt` is generally included in the core utilities of most Linux distributions. On WSL you should treat your platform as whatever Linux distribution diff --git a/setup-env b/setup-env index f4cdf75..1054a64 100755 --- a/setup-env +++ b/setup-env @@ -9,10 +9,10 @@ USAGE=$( Configure a development environment for this repository. It does the following: - - Allows user to specify Python version. - - Allows user to choose name for their virtual environment. + - Allows the user to specify the Python version to use for the virtual environment. + - Allows the user to specify a name for the virtual environment. - Verifies pyenv and pyenv-virtualenv are installed. - - Creates a Python virtual environment. + - Creates the Python virtual environment. - Configures the activation of the virtual enviroment for the repo directory. - Installs the requirements needed for development. - Installs git pre-commit hooks. @@ -29,7 +29,7 @@ Options: -i | --install-hooks Install hook environments for all environments in the pre-commit config file. -l | --list-versions List available Python versions and select interactively. - -n | --name Choose the name of the virtual environment. + -n | --name Specify the name of the virtual environment. -v | --version Specify the Python version for the virtual environment. END_OF_LINE @@ -195,7 +195,7 @@ if [ $FORCE -ne 0 ]; then elif [[ -f .python-version ]]; then cat << 'END_OF_LINE' An existing .python-version file was found. Either remove this file yourself - or re-run with -f (force) option to have it deleted along with the associated + or re-run with the --force option to have it deleted along with the associated virtual environment. rm .python-version @@ -216,7 +216,7 @@ fi if ! pyenv virtualenv ${PYTHON_VERSION:=} "${env_name}"; then cat << END_OF_LINE An existing virtual environment named $env_name was found. Either delete this - environment yourself or re-run with -f (force) option to have it deleted. + environment yourself or re-run with the --force option to have it deleted. pyenv virtualenv-delete ${env_name} From ba0fc19811d2b9bcb2cc043660f6dcd9d6d9c64b Mon Sep 17 00:00:00 2001 From: Michael Saki Date: Wed, 7 Feb 2024 09:48:01 -0600 Subject: [PATCH 042/170] Combine PATH exports to single line Co-authored-by: Shane Frasier --- CONTRIBUTING.md | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index d96a0e3..1e6cc74 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -67,9 +67,8 @@ installation is as simple as `brew install gnu-getopt pyenv pyenv-virtualenv` an adding this to your profile: ```bash -export PATH="/usr/local/opt/gnu-getopt/bin:$PATH" export PYENV_ROOT="$HOME/.pyenv" -export PATH="$PYENV_ROOT/bin:$PATH" +export PATH="$PYENV_ROOT/bin:/usr/local/opt/gnu-getopt/bin:$PATH" eval "$(pyenv init --path)" eval "$(pyenv init -)" eval "$(pyenv virtualenv-init -)" From 1240bdd98fe3523dada222782a55df8c2b258e5c Mon Sep 17 00:00:00 2001 From: Michael Saki Date: Wed, 7 Feb 2024 09:48:39 -0600 Subject: [PATCH 043/170] Improve usage instructions Co-authored-by: Shane Frasier --- setup-env | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/setup-env b/setup-env index 1054a64..a0230ed 100755 --- a/setup-env +++ b/setup-env @@ -19,8 +19,7 @@ It does the following: - Configures git upstream remote "lineage" repositories. Usage: - setup-env (-n | --name) [virt_env_name] - setup-env (-v | --version) [python_version] + setup-env (-n | --name) [virt_env_name] (-v | --version) [python_version] setup-env (-h | --help) Options: From 297b5bd850f716372eb385c25ef29936b4bd6d7c Mon Sep 17 00:00:00 2001 From: Michael Saki Date: Wed, 7 Feb 2024 11:03:35 -0600 Subject: [PATCH 044/170] Add $(brew --prefix) to PATH for getopt Co-authored-by: dav3r --- CONTRIBUTING.md | 2 +- setup-env | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 1e6cc74..899f4ff 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -68,7 +68,7 @@ adding this to your profile: ```bash export PYENV_ROOT="$HOME/.pyenv" -export PATH="$PYENV_ROOT/bin:/usr/local/opt/gnu-getopt/bin:$PATH" +export PATH="$PYENV_ROOT/bin:$(brew --prefix)/opt/gnu-getopt/bin:$PATH" eval "$(pyenv init --path)" eval "$(pyenv init -)" eval "$(pyenv virtualenv-init -)" diff --git a/setup-env b/setup-env index a0230ed..3ddc9c9 100755 --- a/setup-env +++ b/setup-env @@ -63,7 +63,7 @@ if [[ $(getopt --version 2> /dev/null) != *"getopt from util-linux"* ]]; then is as simple as `brew install gnu-getopt` and adding this to your profile: - export PATH="/usr/local/opt/gnu-getopt/bin:$PATH" + export PATH="$(brew --prefix)/opt/gnu-getopt/bin:$PATH" END_OF_LINE exit 1 From 7af70f514fba159ed6e7c8ae85af7a0e14a667a1 Mon Sep 17 00:00:00 2001 From: Michael Saki Date: Wed, 7 Feb 2024 11:04:27 -0600 Subject: [PATCH 045/170] Fix confusing wording Co-authored-by: dav3r --- setup-env | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup-env b/setup-env index 3ddc9c9..1e2fd79 100755 --- a/setup-env +++ b/setup-env @@ -42,7 +42,7 @@ python_versions() { # Flag to force deletion and creation of virtual environment FORCE=0 -# Initialize the all other flags +# Initialize the other flags INSTALL_HOOKS=0 LIST_VERSIONS=0 PYTHON_VERSION="" From e5a2d14964a65c0f5b74e53e5f7e606778b5f003 Mon Sep 17 00:00:00 2001 From: Michael Saki Date: Wed, 7 Feb 2024 11:06:50 -0600 Subject: [PATCH 046/170] Replace virt_env_name w/ virtual_env_name for clarity Co-authored-by: dav3r --- setup-env | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup-env b/setup-env index 1e2fd79..ac20852 100755 --- a/setup-env +++ b/setup-env @@ -19,7 +19,7 @@ It does the following: - Configures git upstream remote "lineage" repositories. Usage: - setup-env (-n | --name) [virt_env_name] (-v | --version) [python_version] + setup-env (-n | --name) [virtual_env_name] (-v | --version) [python_version] setup-env (-h | --help) Options: From 82c70e089bfd43c84273a83228da38030d340c14 Mon Sep 17 00:00:00 2001 From: Michael Saki Date: Tue, 13 Feb 2024 14:31:35 -0600 Subject: [PATCH 047/170] Differentiate between GNU getopt and gnu-getopt brew formula This commit will clearly differentiate between GNU getopt the tool and gnu-getopt the Homebrew formula. Also updating the URL so that getopt points at the source repository and pyenv and pyenv-virtualenv point at their respective repositories. Updated the ending punctuation. Co-authored-by: Nick <50747025+mcdonnnj@users.noreply.github.com> --- CONTRIBUTING.md | 13 +++++++------ setup-env | 2 +- 2 files changed, 8 insertions(+), 7 deletions(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 899f4ff..afb1ae8 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -49,9 +49,10 @@ project. The `setup-env` tool in this repository is our recommended method for automating the entire environment configuration process. The dependencies required to run this tool are -[`gnu-getopt`](https://manned.org/getopt.1), `pyenv`, and -`pyenv-virtualenv`. If these tools are already configured on -your system, you can simply run the following command. +[GNU `getopt`](https://github.com/util-linux/util-linux/blob/master/misc-utils/getopt.1.adoc), +[`pyenv`](https://github.com/pyenv/pyenv), and [`pyenv-virtualenv`](https://github.com/pyenv/pyenv-virtualenv). +If these tools are already configured on your system, you can simply run the +following command: ```console ./setup-env @@ -60,7 +61,7 @@ your system, you can simply run the following command. Otherwise, follow the steps below to manually configure your environment. -#### Installing and using `gnu-getopt`, `pyenv`, and `pyenv-virtualenv` #### +#### Installing and using GNU `getopt`, `pyenv`, and `pyenv-virtualenv` #### On the Mac, we recommend installing [brew](https://brew.sh/). Then installation is as simple as `brew install gnu-getopt pyenv pyenv-virtualenv` and @@ -81,8 +82,8 @@ install the necessary tools. Before running this ensure that you have installed the prerequisites for your platform according to the [`pyenv` wiki page](https://github.com/pyenv/pyenv/wiki/common-build-problems). -`gnu-getopt` is generally included in the core utilities of most -Linux distributions. +GNU `getopt` is included in most Linux distributions as part of the +[`util-linux`](https://github.com/util-linux/util-linux) package. On WSL you should treat your platform as whatever Linux distribution you've chosen to install. diff --git a/setup-env b/setup-env index ac20852..0e9598b 100755 --- a/setup-env +++ b/setup-env @@ -58,7 +58,7 @@ SHORTOPTS="fhiln:v:" if [[ $(getopt --version 2> /dev/null) != *"getopt from util-linux"* ]]; then cat << 'END_OF_LINE' - Gnu-getopt is not detected and is a dependency to run this script. + GNU getopt is not detected and is a dependency to run this script. On the Mac, we recommend installing brew, https://brew.sh/. Then installation is as simple as `brew install gnu-getopt` and adding this to your profile: From 493a4a3e5dfe4c705998395797c75eda756c6dc4 Mon Sep 17 00:00:00 2001 From: Michael Saki Date: Tue, 13 Feb 2024 14:58:43 -0600 Subject: [PATCH 048/170] Add parenthesis over brew link Co-authored-by: Nick <50747025+mcdonnnj@users.noreply.github.com> --- setup-env | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup-env b/setup-env index 0e9598b..f0d4b36 100755 --- a/setup-env +++ b/setup-env @@ -59,7 +59,7 @@ if [[ $(getopt --version 2> /dev/null) != *"getopt from util-linux"* ]]; then cat << 'END_OF_LINE' GNU getopt is not detected and is a dependency to run this script. - On the Mac, we recommend installing brew, https://brew.sh/. Then installation + On the Mac, we recommend installing brew (https://brew.sh/). Then installation is as simple as `brew install gnu-getopt` and adding this to your profile: From 3bc9aeb12fb7a270fa6bc54fcc4deceb897906bc Mon Sep 17 00:00:00 2001 From: Michael Saki Date: Wed, 14 Feb 2024 07:37:24 -0600 Subject: [PATCH 049/170] Refactor flag names for clarity and accuracy This commit will remove the previous flags -v or --version for -p or --python-version. Also it will replace -n or --name for -v or --venv-name. The usage menu has been updated to reflect these changes as well. --- setup-env | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/setup-env b/setup-env index f0d4b36..b7b08e6 100755 --- a/setup-env +++ b/setup-env @@ -19,7 +19,7 @@ It does the following: - Configures git upstream remote "lineage" repositories. Usage: - setup-env (-n | --name) [virtual_env_name] (-v | --version) [python_version] + setup-env [--venv-name virtual_env_name] [--python-version python_version] setup-env (-h | --help) Options: @@ -28,8 +28,8 @@ Options: -i | --install-hooks Install hook environments for all environments in the pre-commit config file. -l | --list-versions List available Python versions and select interactively. - -n | --name Specify the name of the virtual environment. - -v | --version Specify the Python version for the virtual environment. + -v | --venv-name Specify the name of the virtual environment. + -p | --python-version Specify the Python version for the virtual environment. END_OF_LINE ) @@ -49,10 +49,10 @@ PYTHON_VERSION="" VENV_NAME="" # Define long options -LONGOPTS="force,help,install-hooks,list-versions,name:,version:" +LONGOPTS="force,help,install-hooks,list-versions,python-version:,venv-name:" # Define short options for getopt -SHORTOPTS="fhiln:v:" +SHORTOPTS="fhilp:v:" # Check if GNU getopt is available if [[ $(getopt --version 2> /dev/null) != *"getopt from util-linux"* ]]; then @@ -94,11 +94,7 @@ while true; do LIST_VERSIONS=1 shift ;; - -n | --name) - VENV_NAME="$2" - shift 2 - ;; - -v | --version) + -p | --python-version) PYTHON_VERSION="$2" shift 2 # Check the Python versions being passed in. @@ -113,6 +109,10 @@ while true; do fi fi ;; + -v | --venv-name) + VENV_NAME="$2" + shift 2 + ;; --) shift break From 0be1f6376dc914be4eff9ce5fde7649d085a2582 Mon Sep 17 00:00:00 2001 From: Michael Saki Date: Wed, 14 Feb 2024 08:08:07 -0600 Subject: [PATCH 050/170] Elaborate on message when checking for GNU getopt --- setup-env | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/setup-env b/setup-env index b7b08e6..6649b4d 100755 --- a/setup-env +++ b/setup-env @@ -58,7 +58,13 @@ SHORTOPTS="fhilp:v:" if [[ $(getopt --version 2> /dev/null) != *"getopt from util-linux"* ]]; then cat << 'END_OF_LINE' - GNU getopt is not detected and is a dependency to run this script. + Please note, this script requires GNU getopt due to its enhanced + functionality and compatibility with certain script features that + are not supported by the POSIX getopt found in some systems, particularly + those with a non-GNU version of getopt. This distinction is crucial + as a system might have a non-GNU version of getopt installed by default, + which could lead to unexpected behavior or script failure. + On the Mac, we recommend installing brew (https://brew.sh/). Then installation is as simple as `brew install gnu-getopt` and adding this to your profile: From c8f0b1b996958a5fa965343fa7e6295b7160cbf4 Mon Sep 17 00:00:00 2001 From: Michael Saki Date: Wed, 14 Feb 2024 08:10:21 -0600 Subject: [PATCH 051/170] Remove unnecessary nounset flipping logic --- setup-env | 2 -- 1 file changed, 2 deletions(-) diff --git a/setup-env b/setup-env index 6649b4d..05b010b 100755 --- a/setup-env +++ b/setup-env @@ -166,7 +166,6 @@ END_OF_LINE exit 1 fi -set +o nounset # Determine the virtual environment name if [ -n "$VENV_NAME" ]; then # Use the user-provided environment name @@ -175,7 +174,6 @@ else # Set the environment name to the last part of the working directory. env_name=${PWD##*/} fi -set -o nounset # List Python versions and select one interactively if [ $LIST_VERSIONS -ne 0 ]; then From 495862aeaf3430b0a69f92cf54b6415818139db2 Mon Sep 17 00:00:00 2001 From: Michael Saki Date: Wed, 14 Feb 2024 08:20:56 -0600 Subject: [PATCH 052/170] Separate pyenv PATH from GNU getopt PATH --- CONTRIBUTING.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index afb1ae8..3a5e087 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -69,7 +69,8 @@ adding this to your profile: ```bash export PYENV_ROOT="$HOME/.pyenv" -export PATH="$PYENV_ROOT/bin:$(brew --prefix)/opt/gnu-getopt/bin:$PATH" +export PATH="$PYENV_ROOT/bin:$PATH" +export PATH="$(brew --prefix)/opt/gnu-getopt/bin:$PATH" eval "$(pyenv init --path)" eval "$(pyenv init -)" eval "$(pyenv virtualenv-init -)" From 33582a11f73d4e5c3dc588a040e59941c381d09a Mon Sep 17 00:00:00 2001 From: Michael Saki Date: Wed, 14 Feb 2024 12:59:30 -0600 Subject: [PATCH 053/170] Add checks for semantic python versions --- setup-env | 39 +++++++++++++++++++++------------------ 1 file changed, 21 insertions(+), 18 deletions(-) diff --git a/setup-env b/setup-env index 05b010b..44368b3 100755 --- a/setup-env +++ b/setup-env @@ -39,6 +39,14 @@ python_versions() { pyenv versions --bare --skip-aliases --skip-envs } +check_semantic_version() { + local version=$1 + local regex="^(0|[1-9]\\d*)\\.(0|[1-9]\\d*)\\.(0|[1-9]\\d*)(?:-((?:0|[1-9]\\d*|\\d*[a-zA-Z-][0-9a-zA-Z-]*)(?:\\.(?:0|[1-9]\\d*|\\d*[a-zA-Z-][0-9a-zA-Z-]*))*))?(?:\\+([0-9a-zA-Z-]+(?:\\.[0-9a-zA-Z-]+)*))?\$" + + # Use Perl for regex matching and output true or false + echo "$version" | perl -ne "exit(!/$regex/)" +} + # Flag to force deletion and creation of virtual environment FORCE=0 @@ -103,16 +111,18 @@ while true; do -p | --python-version) PYTHON_VERSION="$2" shift 2 - # Check the Python versions being passed in. - if [ -n "${PYTHON_VERSION+x}" ]; then - if python_versions | grep -E "^${PYTHON_VERSION}$" > /dev/null; then - echo Using Python version "$PYTHON_VERSION" - else - echo Error: Python version "$PYTHON_VERSION" is not installed. - echo Installed Python versions are: - python_versions - exit 1 - fi + # Validate the semantic version format + if ! check_semantic_version "$PYTHON_VERSION"; then + echo "Error: The specified Python version $PYTHON_VERSION does not follow the semantic versioning standard." + echo "Example of a valid version: 3.8.1, 3.8.1-alpha.1, or 3.8.1+20130313144700" + exit 1 + elif ! python_versions | grep -E "^${PYTHON_VERSION}$" > /dev/null; then + echo "Error: Python version $PYTHON_VERSION is not installed." + echo "Installed Python versions are:" + python_versions + exit 1 + else + echo "Using Python version $PYTHON_VERSION" fi ;; -v | --venv-name) @@ -181,14 +191,7 @@ if [ $LIST_VERSIONS -ne 0 ]; then python_versions read -r -p "Enter the desired Python version: " PYTHON_VERSION # Check the Python versions being passed in. - if [ -n "${PYTHON_VERSION+x}" ]; then - if python_versions | grep -E "^${PYTHON_VERSION}$" > /dev/null; then - echo Using Python version "$PYTHON_VERSION" - else - echo Error: Python version "$PYTHON_VERSION" is not installed. - exit 1 - fi - fi + check_semantic_version "$PYTHON_VERSION" fi # Remove any lingering local configuration. From 94381940a9d28f87da2b85c5e1647a5a80d4a18d Mon Sep 17 00:00:00 2001 From: Michael Saki Date: Wed, 14 Feb 2024 14:29:13 -0600 Subject: [PATCH 054/170] Refactor code for the semantic check This commit will make a few changes. The orginal version of the semantic checking function was a bit more difficult to read. It is now somewhat easier to follow how the regex is structured. Also the function has been renamed to check_python_version since it has 2 functions, making sure that the version is semantically correct and the second is to make sure that it is installed on the user's machine. This makes it easier to follow the logic for the flags, -p or --python-version and -l or --list-versions --- setup-env | 54 ++++++++++++++++++++++++++++++++++++------------------ 1 file changed, 36 insertions(+), 18 deletions(-) diff --git a/setup-env b/setup-env index 44368b3..11ec170 100755 --- a/setup-env +++ b/setup-env @@ -39,12 +39,41 @@ python_versions() { pyenv versions --bare --skip-aliases --skip-envs } -check_semantic_version() { +check_python_version() { local version=$1 - local regex="^(0|[1-9]\\d*)\\.(0|[1-9]\\d*)\\.(0|[1-9]\\d*)(?:-((?:0|[1-9]\\d*|\\d*[a-zA-Z-][0-9a-zA-Z-]*)(?:\\.(?:0|[1-9]\\d*|\\d*[a-zA-Z-][0-9a-zA-Z-]*))*))?(?:\\+([0-9a-zA-Z-]+(?:\\.[0-9a-zA-Z-]+)*))?\$" - # Use Perl for regex matching and output true or false - echo "$version" | perl -ne "exit(!/$regex/)" + # Break down the regex into readable parts major.minor.patch + local major="0|[1-9]\\d*" + local minor="0|[1-9]\\d*" + local patch="0|[1-9]\\d*" + + # Splitting the prerelease part for readability + # Start of prerelease + local prerelease="(?:-" + # Numeric or alphanumeric identifiers + local prerelease+="(?:0|[1-9]\\d*|\\d*[a-zA-Z-][0-9a-zA-Z-]*)" + # Additional dot-separated identifiers + local prerelease+="(?:\\.(?:0|[1-9]\\d*|\\d*[a-zA-Z-][0-9a-zA-Z-]*))*" + # End of prerelease, making it optional + local prerelease+=")?" + # Optional build metadata + local build="(?:\\+([0-9a-zA-Z-]+(?:\\.[0-9a-zA-Z-]+)*))?" + + # Final regex composed of parts + local regex="^($major)\\.($minor)\\.($patch)$prerelease$build\$" + + if ! echo "$version" | perl -ne "exit(!/$regex/)"; then + echo "Error: The specified Python version $version does not follow the semantic versioning standard." + echo "Example of a valid version: 3.8.1, 3.8.1-alpha.1, or 3.8.1+20130313144700" + exit 1 + elif ! python_versions | grep -E "^${version}$" > /dev/null; then + echo "Error: Python version $version is not installed." + echo "Installed Python versions are:" + python_versions + exit 1 + else + echo "Using Python version $version" + fi } # Flag to force deletion and creation of virtual environment @@ -111,19 +140,8 @@ while true; do -p | --python-version) PYTHON_VERSION="$2" shift 2 - # Validate the semantic version format - if ! check_semantic_version "$PYTHON_VERSION"; then - echo "Error: The specified Python version $PYTHON_VERSION does not follow the semantic versioning standard." - echo "Example of a valid version: 3.8.1, 3.8.1-alpha.1, or 3.8.1+20130313144700" - exit 1 - elif ! python_versions | grep -E "^${PYTHON_VERSION}$" > /dev/null; then - echo "Error: Python version $PYTHON_VERSION is not installed." - echo "Installed Python versions are:" - python_versions - exit 1 - else - echo "Using Python version $PYTHON_VERSION" - fi + # Check the Python version being passed in. + check_python_version "$PYTHON_VERSION" ;; -v | --venv-name) VENV_NAME="$2" @@ -191,7 +209,7 @@ if [ $LIST_VERSIONS -ne 0 ]; then python_versions read -r -p "Enter the desired Python version: " PYTHON_VERSION # Check the Python versions being passed in. - check_semantic_version "$PYTHON_VERSION" + check_python_version "$PYTHON_VERSION" fi # Remove any lingering local configuration. From 4752b37ec887a65e967f01a1f9b7633e02a02af8 Mon Sep 17 00:00:00 2001 From: Michael Saki Date: Wed, 21 Feb 2024 10:57:38 -0600 Subject: [PATCH 055/170] Improve verbiage in comments Co-authored-by: Shane Frasier --- CONTRIBUTING.md | 4 ++-- setup-env | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 3a5e087..9ff54e4 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -46,8 +46,8 @@ There are a few ways to do this, but we prefer to use create and manage a Python virtual environment specific to this project. -The `setup-env` tool in this repository is our recommended method -for automating the entire environment configuration process. The +We recommend using the `setup-env` tool located in this repository, +as it automates the entire environment configuration process. The dependencies required to run this tool are [GNU `getopt`](https://github.com/util-linux/util-linux/blob/master/misc-utils/getopt.1.adoc), [`pyenv`](https://github.com/pyenv/pyenv), and [`pyenv-virtualenv`](https://github.com/pyenv/pyenv-virtualenv). diff --git a/setup-env b/setup-env index 05b010b..6d9fec0 100755 --- a/setup-env +++ b/setup-env @@ -27,7 +27,7 @@ Options: -h | --help Show this message. -i | --install-hooks Install hook environments for all environments in the pre-commit config file. - -l | --list-versions List available Python versions and select interactively. + -l | --list-versions List available Python versions and select one interactively. -v | --venv-name Specify the name of the virtual environment. -p | --python-version Specify the Python version for the virtual environment. @@ -63,7 +63,7 @@ if [[ $(getopt --version 2> /dev/null) != *"getopt from util-linux"* ]]; then are not supported by the POSIX getopt found in some systems, particularly those with a non-GNU version of getopt. This distinction is crucial as a system might have a non-GNU version of getopt installed by default, - which could lead to unexpected behavior or script failure. + which could lead to unexpected behavior. On the Mac, we recommend installing brew (https://brew.sh/). Then installation is as simple as `brew install gnu-getopt` and adding this to your From 2e389979b0b0b34a6f32443d88f8ed0ba8c647cd Mon Sep 17 00:00:00 2001 From: Michael Saki Date: Wed, 21 Feb 2024 11:22:53 -0600 Subject: [PATCH 056/170] Clarify between pyenv and GNU getopt setup Co-authored-by: Nick <50747025+mcdonnnj@users.noreply.github.com> --- CONTRIBUTING.md | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 9ff54e4..a844f86 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -68,9 +68,12 @@ installation is as simple as `brew install gnu-getopt pyenv pyenv-virtualenv` an adding this to your profile: ```bash +# Enable GNU getopt since it is keg-only +export PATH="$(brew --prefix)/opt/gnu-getopt/bin:$PATH" + +# Setup pyenv export PYENV_ROOT="$HOME/.pyenv" export PATH="$PYENV_ROOT/bin:$PATH" -export PATH="$(brew --prefix)/opt/gnu-getopt/bin:$PATH" eval "$(pyenv init --path)" eval "$(pyenv init -)" eval "$(pyenv virtualenv-init -)" From f8824c8aa157cbee0ec95367b1754eeac67934c9 Mon Sep 17 00:00:00 2001 From: Michael Saki Date: Wed, 21 Feb 2024 12:14:23 -0600 Subject: [PATCH 057/170] Improve comment on conditional check for regex --- setup-env | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/setup-env b/setup-env index 6d9fec0..6d16958 100755 --- a/setup-env +++ b/setup-env @@ -54,7 +54,9 @@ LONGOPTS="force,help,install-hooks,list-versions,python-version:,venv-name:" # Define short options for getopt SHORTOPTS="fhilp:v:" -# Check if GNU getopt is available +# Check for GNU getopt by matching a specific pattern ("getopt from util-linux") +# in its version output. This approach presumes the output format remains stable. +# Be aware that format changes could invalidate this check. if [[ $(getopt --version 2> /dev/null) != *"getopt from util-linux"* ]]; then cat << 'END_OF_LINE' From 88724e70167a5864495eccff5532f4c59fbefc1c Mon Sep 17 00:00:00 2001 From: Michael Saki Date: Wed, 21 Feb 2024 12:25:49 -0600 Subject: [PATCH 058/170] Add comment explaining that GNU getopt is keg-only --- setup-env | 2 ++ 1 file changed, 2 insertions(+) diff --git a/setup-env b/setup-env index 6d16958..84a8305 100755 --- a/setup-env +++ b/setup-env @@ -73,6 +73,8 @@ if [[ $(getopt --version 2> /dev/null) != *"getopt from util-linux"* ]]; then export PATH="$(brew --prefix)/opt/gnu-getopt/bin:$PATH" + This will enable GNU getopt since it is keg-only. + END_OF_LINE exit 1 fi From c1870be7a16f5e88f6e6540c290f674da4861e07 Mon Sep 17 00:00:00 2001 From: Michael Saki Date: Wed, 21 Feb 2024 16:28:54 -0600 Subject: [PATCH 059/170] Improve comments to better describe `keg-only` terminology Co-authored-by: Shane Frasier --- CONTRIBUTING.md | 3 ++- setup-env | 3 ++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index a844f86..7bb8881 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -68,7 +68,8 @@ installation is as simple as `brew install gnu-getopt pyenv pyenv-virtualenv` an adding this to your profile: ```bash -# Enable GNU getopt since it is keg-only +# GNU getopt must be explicitly added to the path since it is +# keg-only export PATH="$(brew --prefix)/opt/gnu-getopt/bin:$PATH" # Setup pyenv diff --git a/setup-env b/setup-env index 84a8305..3fa8c47 100755 --- a/setup-env +++ b/setup-env @@ -73,7 +73,8 @@ if [[ $(getopt --version 2> /dev/null) != *"getopt from util-linux"* ]]; then export PATH="$(brew --prefix)/opt/gnu-getopt/bin:$PATH" - This will enable GNU getopt since it is keg-only. + GNU getopt must be explicitly added to the PATH since it + is keg-only. END_OF_LINE exit 1 From a3f69cd24e58077ee6f084bce900ed176c88cff7 Mon Sep 17 00:00:00 2001 From: Michael Saki Date: Mon, 26 Feb 2024 08:04:19 -0600 Subject: [PATCH 060/170] Change "'setup-env' tool" to "'setup-env' script" Co-authored-by: Nick <50747025+mcdonnnj@users.noreply.github.com> --- CONTRIBUTING.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 7bb8881..fca0cf7 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -46,7 +46,7 @@ There are a few ways to do this, but we prefer to use create and manage a Python virtual environment specific to this project. -We recommend using the `setup-env` tool located in this repository, +We recommend using the `setup-env` script located in this repository, as it automates the entire environment configuration process. The dependencies required to run this tool are [GNU `getopt`](https://github.com/util-linux/util-linux/blob/master/misc-utils/getopt.1.adoc), From 8ff51796ec530e68d35362fab761608e676dcd3a Mon Sep 17 00:00:00 2001 From: Michael Saki Date: Mon, 26 Feb 2024 08:07:16 -0600 Subject: [PATCH 061/170] Remove build-in error exit for generic error exit Exit code 2 usually relates to build-ins which does not fit what is failing here. Not sure what alternative would be best so we can just use the generic failure code Co-authored-by: Nick <50747025+mcdonnnj@users.noreply.github.com> --- setup-env | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup-env b/setup-env index 3fa8c47..f7ce620 100755 --- a/setup-env +++ b/setup-env @@ -83,7 +83,7 @@ fi # Use GNU getopt to parse options if ! PARSED=$(getopt --options $SHORTOPTS --longoptions $LONGOPTS --name "$0" -- "$@"); then echo "Error parsing options" - exit 2 + exit 1 fi eval set -- "$PARSED" From 1c21e2b9839980c75410584c69d0ac0a0651f097 Mon Sep 17 00:00:00 2001 From: Michael Saki Date: Mon, 26 Feb 2024 08:08:39 -0600 Subject: [PATCH 062/170] Change verbiage from 'tool' to 'script' for clarity Co-authored-by: Nick <50747025+mcdonnnj@users.noreply.github.com> --- CONTRIBUTING.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index fca0cf7..cdab589 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -48,7 +48,7 @@ project. We recommend using the `setup-env` script located in this repository, as it automates the entire environment configuration process. The -dependencies required to run this tool are +dependencies required to run this script are [GNU `getopt`](https://github.com/util-linux/util-linux/blob/master/misc-utils/getopt.1.adoc), [`pyenv`](https://github.com/pyenv/pyenv), and [`pyenv-virtualenv`](https://github.com/pyenv/pyenv-virtualenv). If these tools are already configured on your system, you can simply run the From 3acc8d6a17360221d6c59c306db1a14a98b48b8b Mon Sep 17 00:00:00 2001 From: Michael Saki Date: Mon, 26 Feb 2024 10:04:42 -0600 Subject: [PATCH 063/170] Check for pyenv earlier in the script Since the new function, python_versions expects pyenv to be installed it makes sense to have it checked earlier in the script. --- setup-env | 72 +++++++++++++++++++++++++++---------------------------- 1 file changed, 36 insertions(+), 36 deletions(-) diff --git a/setup-env b/setup-env index f7ce620..3898ca8 100755 --- a/setup-env +++ b/setup-env @@ -80,6 +80,42 @@ END_OF_LINE exit 1 fi +# Check to see if pyenv is installed +if [ -z "$(command -v pyenv)" ] || { [ -z "$(command -v pyenv-virtualenv)" ] && [ ! -f "$(pyenv root)/plugins/pyenv-virtualenv/bin/pyenv-virtualenv" ]; }; then + echo "pyenv and pyenv-virtualenv are required." + if [[ "$OSTYPE" == "darwin"* ]]; then + cat << 'END_OF_LINE' + + On the Mac, we recommend installing brew, https://brew.sh/. Then installation + is as simple as `brew install pyenv pyenv-virtualenv` and adding this to your + profile: + + eval "$(pyenv init -)" + eval "$(pyenv virtualenv-init -)" + +END_OF_LINE + + fi + cat << 'END_OF_LINE' + For Linux, Windows Subsystem for Linux (WSL), or on the Mac (if you don't want + to use "brew") you can use https://github.com/pyenv/pyenv-installer to install + the necessary tools. Before running this ensure that you have installed the + prerequisites for your platform according to the pyenv wiki page, + https://github.com/pyenv/pyenv/wiki/common-build-problems. + + On WSL you should treat your platform as whatever Linux distribution you've + chosen to install. + + Once you have installed "pyenv" you will need to add the following lines to + your ".bashrc": + + export PATH="$PATH:$HOME/.pyenv/bin" + eval "$(pyenv init -)" + eval "$(pyenv virtualenv-init -)" +END_OF_LINE + exit 1 +fi + # Use GNU getopt to parse options if ! PARSED=$(getopt --options $SHORTOPTS --longoptions $LONGOPTS --name "$0" -- "$@"); then echo "Error parsing options" @@ -135,42 +171,6 @@ while true; do esac done -# Check to see if pyenv is installed -if [ -z "$(command -v pyenv)" ] || { [ -z "$(command -v pyenv-virtualenv)" ] && [ ! -f "$(pyenv root)/plugins/pyenv-virtualenv/bin/pyenv-virtualenv" ]; }; then - echo "pyenv and pyenv-virtualenv are required." - if [[ "$OSTYPE" == "darwin"* ]]; then - cat << 'END_OF_LINE' - - On the Mac, we recommend installing brew, https://brew.sh/. Then installation - is as simple as `brew install pyenv pyenv-virtualenv` and adding this to your - profile: - - eval "$(pyenv init -)" - eval "$(pyenv virtualenv-init -)" - -END_OF_LINE - - fi - cat << 'END_OF_LINE' - For Linux, Windows Subsystem for Linux (WSL), or on the Mac (if you don't want - to use "brew") you can use https://github.com/pyenv/pyenv-installer to install - the necessary tools. Before running this ensure that you have installed the - prerequisites for your platform according to the pyenv wiki page, - https://github.com/pyenv/pyenv/wiki/common-build-problems. - - On WSL you should treat your platform as whatever Linux distribution you've - chosen to install. - - Once you have installed "pyenv" you will need to add the following lines to - your ".bashrc": - - export PATH="$PATH:$HOME/.pyenv/bin" - eval "$(pyenv init -)" - eval "$(pyenv virtualenv-init -)" -END_OF_LINE - exit 1 -fi - # Determine the virtual environment name if [ -n "$VENV_NAME" ]; then # Use the user-provided environment name From b377ce71fa3dd5f56697d415c5230f3fe18e8090 Mon Sep 17 00:00:00 2001 From: Michael Saki Date: Mon, 26 Feb 2024 11:02:39 -0600 Subject: [PATCH 064/170] Explain -r and -p in Python version prompt --- setup-env | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/setup-env b/setup-env index 3898ca8..5c2b580 100755 --- a/setup-env +++ b/setup-env @@ -180,10 +180,12 @@ else env_name=${PWD##*/} fi -# List Python versions and select one interactively +# List Python versions and select one interactively. if [ $LIST_VERSIONS -ne 0 ]; then echo Available Python versions: python_versions + # Read the user's desired Python version. + # -r: treat backslashes as literal, -p: display prompt before input. read -r -p "Enter the desired Python version: " PYTHON_VERSION # Check the Python versions being passed in. if [ -n "${PYTHON_VERSION+x}" ]; then From 74838a256e134e00208a23bb6dded7012cfb8534 Mon Sep 17 00:00:00 2001 From: Michael Saki Date: Mon, 26 Feb 2024 11:58:44 -0600 Subject: [PATCH 065/170] Refine exit code to 64 with gnu-getopt note - Set exit code to 64 per sysexits.h for clarity. - Added comment on gnu-getopt handling. Aligns with Unix standards and enhances readability. --- setup-env | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/setup-env b/setup-env index 5c2b580..549c079 100755 --- a/setup-env +++ b/setup-env @@ -165,8 +165,9 @@ while true; do break ;; *) + # Unreachable due to gnu-getopt handling all options echo "Programming error" - exit 3 + exit 64 ;; esac done From 487126e34bd85e231b7021b70d225154694ad24d Mon Sep 17 00:00:00 2001 From: Michael Saki Date: Wed, 28 Feb 2024 06:32:35 -0600 Subject: [PATCH 066/170] Rename gnu-getopt tool to GNU getopt formula Co-authored-by: Nick <50747025+mcdonnnj@users.noreply.github.com> --- setup-env | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup-env b/setup-env index 549c079..d2db6e2 100755 --- a/setup-env +++ b/setup-env @@ -165,7 +165,7 @@ while true; do break ;; *) - # Unreachable due to gnu-getopt handling all options + # Unreachable due to GNU getopt handling all options echo "Programming error" exit 64 ;; From 6c82a8d1bb86a90f0e903dd2d90efdb6198fba33 Mon Sep 17 00:00:00 2001 From: Michael Saki Date: Wed, 28 Feb 2024 07:49:16 -0600 Subject: [PATCH 067/170] Fix whitespace for usage menu Co-authored-by: Shane Frasier --- setup-env | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/setup-env b/setup-env index d2db6e2..c93eb25 100755 --- a/setup-env +++ b/setup-env @@ -28,8 +28,8 @@ Options: -i | --install-hooks Install hook environments for all environments in the pre-commit config file. -l | --list-versions List available Python versions and select one interactively. - -v | --venv-name Specify the name of the virtual environment. - -p | --python-version Specify the Python version for the virtual environment. + -v | --venv-name Specify the name of the virtual environment. + -p | --python-version Specify the Python version for the virtual environment. END_OF_LINE ) From 324f6d426a149cb2c7bfc30ffcf7c1810b8ae1fa Mon Sep 17 00:00:00 2001 From: Michael Saki Date: Wed, 28 Feb 2024 11:26:24 -0600 Subject: [PATCH 068/170] Add link to brew terminology Co-authored-by: dav3r --- CONTRIBUTING.md | 2 +- setup-env | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index cdab589..bc32a55 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -69,7 +69,7 @@ adding this to your profile: ```bash # GNU getopt must be explicitly added to the path since it is -# keg-only +# keg-only (https://docs.brew.sh/FAQ#what-does-keg-only-mean) export PATH="$(brew --prefix)/opt/gnu-getopt/bin:$PATH" # Setup pyenv diff --git a/setup-env b/setup-env index c93eb25..bb689f3 100755 --- a/setup-env +++ b/setup-env @@ -74,7 +74,7 @@ if [[ $(getopt --version 2> /dev/null) != *"getopt from util-linux"* ]]; then export PATH="$(brew --prefix)/opt/gnu-getopt/bin:$PATH" GNU getopt must be explicitly added to the PATH since it - is keg-only. + is keg-only (https://docs.brew.sh/FAQ#what-does-keg-only-mean). END_OF_LINE exit 1 From a26d0e36f0c33a7f472b91d66697e4cecb8f9557 Mon Sep 17 00:00:00 2001 From: Michael Saki Date: Wed, 28 Feb 2024 15:52:20 -0600 Subject: [PATCH 069/170] Rephrase comment to improve clarity Co-authored-by: Nick <50747025+mcdonnnj@users.noreply.github.com> --- setup-env | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup-env b/setup-env index bb689f3..3fddb36 100755 --- a/setup-env +++ b/setup-env @@ -16,7 +16,7 @@ It does the following: - Configures the activation of the virtual enviroment for the repo directory. - Installs the requirements needed for development. - Installs git pre-commit hooks. - - Configures git upstream remote "lineage" repositories. + - Configures git remotes for upstream "lineage" repositories. Usage: setup-env [--venv-name virtual_env_name] [--python-version python_version] From 05108705fa4120db51f261ad59347bb01134eafc Mon Sep 17 00:00:00 2001 From: Michael Saki Date: Wed, 28 Feb 2024 15:58:43 -0600 Subject: [PATCH 070/170] Improve comment for clarity Co-authored-by: Nick <50747025+mcdonnnj@users.noreply.github.com> --- setup-env | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup-env b/setup-env index 3fddb36..c8680ea 100755 --- a/setup-env +++ b/setup-env @@ -34,7 +34,7 @@ Options: END_OF_LINE ) -# Display installed python versions +# Display pyenv's installed Python versions python_versions() { pyenv versions --bare --skip-aliases --skip-envs } From 01abde6f580cf549d77c81af9378d4a18812fa8f Mon Sep 17 00:00:00 2001 From: Michael Saki Date: Wed, 28 Feb 2024 16:03:47 -0600 Subject: [PATCH 071/170] Improve verbiage in comment --- setup-env | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup-env b/setup-env index c8680ea..534d47f 100755 --- a/setup-env +++ b/setup-env @@ -217,7 +217,7 @@ fi # Create a new virtual environment for this project # -# If $PYTHON_VERSION is undefined then the system Python will be used. +# If $PYTHON_VERSION is undefined then the global version of Python will be used. # # We can't quote ${PYTHON_VERSION:=} below since if the variable is # undefined then we want nothing to appear; this is the reason for the From 0989d17f49a089e03e23b72ab638595d538add3f Mon Sep 17 00:00:00 2001 From: Michael Saki Date: Wed, 28 Feb 2024 16:09:15 -0600 Subject: [PATCH 072/170] Change comments for macOS and venv_name --- CONTRIBUTING.md | 4 ++-- setup-env | 8 ++++---- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index bc32a55..617db85 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -63,7 +63,7 @@ environment. #### Installing and using GNU `getopt`, `pyenv`, and `pyenv-virtualenv` #### -On the Mac, we recommend installing [brew](https://brew.sh/). Then +On macOS, we recommend installing [brew](https://brew.sh/). Then installation is as simple as `brew install gnu-getopt pyenv pyenv-virtualenv` and adding this to your profile: @@ -80,7 +80,7 @@ eval "$(pyenv init -)" eval "$(pyenv virtualenv-init -)" ``` -For Linux, Windows Subsystem for Linux (WSL), or on the Mac (if you +For Linux, Windows Subsystem for Linux (WSL), or on macOS (if you don't want to use `brew`) you can use [pyenv/pyenv-installer](https://github.com/pyenv/pyenv-installer) to install the necessary tools. Before running this ensure that you have diff --git a/setup-env b/setup-env index 534d47f..d2cd111 100755 --- a/setup-env +++ b/setup-env @@ -19,7 +19,7 @@ It does the following: - Configures git remotes for upstream "lineage" repositories. Usage: - setup-env [--venv-name virtual_env_name] [--python-version python_version] + setup-env [--venv-name venv_name] [--python-version python_version] setup-env (-h | --help) Options: @@ -67,7 +67,7 @@ if [[ $(getopt --version 2> /dev/null) != *"getopt from util-linux"* ]]; then as a system might have a non-GNU version of getopt installed by default, which could lead to unexpected behavior. - On the Mac, we recommend installing brew (https://brew.sh/). Then installation + On macOS, we recommend installing brew (https://brew.sh/). Then installation is as simple as `brew install gnu-getopt` and adding this to your profile: @@ -86,7 +86,7 @@ if [ -z "$(command -v pyenv)" ] || { [ -z "$(command -v pyenv-virtualenv)" ] && if [[ "$OSTYPE" == "darwin"* ]]; then cat << 'END_OF_LINE' - On the Mac, we recommend installing brew, https://brew.sh/. Then installation + On macOS, we recommend installing brew, https://brew.sh/. Then installation is as simple as `brew install pyenv pyenv-virtualenv` and adding this to your profile: @@ -97,7 +97,7 @@ END_OF_LINE fi cat << 'END_OF_LINE' - For Linux, Windows Subsystem for Linux (WSL), or on the Mac (if you don't want + For Linux, Windows Subsystem for Linux (WSL), or on mac OS (if you don't want to use "brew") you can use https://github.com/pyenv/pyenv-installer to install the necessary tools. Before running this ensure that you have installed the prerequisites for your platform according to the pyenv wiki page, From a9c6ed88c3c993db890a09ece5433b33e5d5d206 Mon Sep 17 00:00:00 2001 From: Michael Saki Date: Thu, 29 Feb 2024 09:19:01 -0600 Subject: [PATCH 073/170] Improve comments for clarity Co-authored-by: Nick <50747025+mcdonnnj@users.noreply.github.com> --- CONTRIBUTING.md | 2 +- setup-env | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 617db85..f9fa53a 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -80,7 +80,7 @@ eval "$(pyenv init -)" eval "$(pyenv virtualenv-init -)" ``` -For Linux, Windows Subsystem for Linux (WSL), or on macOS (if you +For Linux, Windows Subsystem for Linux (WSL), or macOS (if you don't want to use `brew`) you can use [pyenv/pyenv-installer](https://github.com/pyenv/pyenv-installer) to install the necessary tools. Before running this ensure that you have diff --git a/setup-env b/setup-env index d2cd111..3a22d43 100755 --- a/setup-env +++ b/setup-env @@ -97,7 +97,7 @@ END_OF_LINE fi cat << 'END_OF_LINE' - For Linux, Windows Subsystem for Linux (WSL), or on mac OS (if you don't want + For Linux, Windows Subsystem for Linux (WSL), or macOS (if you don't want to use "brew") you can use https://github.com/pyenv/pyenv-installer to install the necessary tools. Before running this ensure that you have installed the prerequisites for your platform according to the pyenv wiki page, @@ -217,7 +217,7 @@ fi # Create a new virtual environment for this project # -# If $PYTHON_VERSION is undefined then the global version of Python will be used. +# If $PYTHON_VERSION is undefined then the current pyenv Python version will be used. # # We can't quote ${PYTHON_VERSION:=} below since if the variable is # undefined then we want nothing to appear; this is the reason for the From b9c729f6159defe55490d2198390ba2ceab5924f Mon Sep 17 00:00:00 2001 From: Nicholas McDonnell <50747025+mcdonnnj@users.noreply.github.com> Date: Thu, 4 Jan 2024 15:03:15 -0500 Subject: [PATCH 074/170] Update pre-commit hook versions This is done automatically with the `pre-commit autoupdate` command. The pre-commit/mirrors-prettier was manually held back because the latest tags are for alpha releases of the next major version. --- .pre-commit-config.yaml | 26 +++++++++++++------------- 1 file changed, 13 insertions(+), 13 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 34f0253..e7f5d0a 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -5,7 +5,7 @@ default_language_version: repos: - repo: https://github.com/pre-commit/pre-commit-hooks - rev: v4.4.0 + rev: v4.5.0 hooks: - id: check-case-conflict - id: check-executables-have-shebangs @@ -31,7 +31,7 @@ repos: # Text file hooks - repo: https://github.com/igorshubovych/markdownlint-cli - rev: v0.36.0 + rev: v0.39.0 hooks: - id: markdownlint args: @@ -41,7 +41,7 @@ repos: hooks: - id: prettier - repo: https://github.com/adrienverge/yamllint - rev: v1.32.0 + rev: v1.35.1 hooks: - id: yamllint args: @@ -49,14 +49,14 @@ repos: # GitHub Actions hooks - repo: https://github.com/python-jsonschema/check-jsonschema - rev: 0.26.3 + rev: 0.28.0 hooks: - id: check-github-actions - id: check-github-workflows # pre-commit hooks - repo: https://github.com/pre-commit/pre-commit - rev: v3.4.0 + rev: v3.6.2 hooks: - id: validate_manifest @@ -107,44 +107,44 @@ repos: # Python hooks - repo: https://github.com/PyCQA/bandit - rev: 1.7.5 + rev: 1.7.7 hooks: - id: bandit args: - --config=.bandit.yml - repo: https://github.com/psf/black-pre-commit-mirror - rev: 23.9.1 + rev: 24.2.0 hooks: - id: black - repo: https://github.com/PyCQA/flake8 - rev: 6.1.0 + rev: 7.0.0 hooks: - id: flake8 additional_dependencies: - flake8-docstrings - repo: https://github.com/PyCQA/isort - rev: 5.12.0 + rev: 5.13.2 hooks: - id: isort - repo: https://github.com/pre-commit/mirrors-mypy - rev: v1.5.1 + rev: v1.8.0 hooks: - id: mypy - repo: https://github.com/asottile/pyupgrade - rev: v3.10.1 + rev: v3.15.1 hooks: - id: pyupgrade # Ansible hooks - repo: https://github.com/ansible/ansible-lint - rev: v6.19.0 + rev: v24.2.0 hooks: - id: ansible-lint # files: molecule/default/playbook.yml # Terraform hooks - repo: https://github.com/antonbabenko/pre-commit-terraform - rev: v1.83.2 + rev: v1.88.0 hooks: - id: terraform_fmt - id: terraform_validate From 4c933958bca75c291bcb3b28fa1ba821d1d1f6f5 Mon Sep 17 00:00:00 2001 From: Nicholas McDonnell <50747025+mcdonnnj@users.noreply.github.com> Date: Thu, 4 Jan 2024 15:08:30 -0500 Subject: [PATCH 075/170] Manually update the prettier hook The hook is manually updated to the last v3 tag available from the pre-commit/mirrors-prettier repository. We then add the latest release of v3 available from NPM as an additional depdency. --- .pre-commit-config.yaml | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index e7f5d0a..8e3941d 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -37,9 +37,16 @@ repos: args: - --config=.mdl_config.yaml - repo: https://github.com/pre-commit/mirrors-prettier - rev: v3.0.3 + # This is the last version of v3 available from the mirror. We should hold + # here until v4, which is currently in alpha, is more stable. + rev: v3.1.0 hooks: - id: prettier + # This is the latest version of v3 available from NPM. The pre-commit + # mirror does not pull tags for old major versions once a new major + # version tag is published. + additional_dependencies: + - prettier@3.2.5 - repo: https://github.com/adrienverge/yamllint rev: v1.35.1 hooks: From 7169dcf65670f1d42b9498b68624a1638de08531 Mon Sep 17 00:00:00 2001 From: Nicholas McDonnell <50747025+mcdonnnj@users.noreply.github.com> Date: Sat, 11 Nov 2023 16:20:14 -0500 Subject: [PATCH 076/170] Use Python and Go versions provided by cisagov/setup-env-github-action --- .github/workflows/build.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 4911745..de6937d 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -59,7 +59,7 @@ jobs: - id: setup-python uses: actions/setup-python@v5 with: - python-version: "3.11" + python-version: ${{ steps.setup-env.outputs.python-version }} # We need the Go version and Go cache location for the actions/cache step, # so the Go installation must happen before that. - id: setup-go @@ -68,7 +68,7 @@ jobs: # There is no expectation for actual Go code so we disable caching as # it relies on the existence of a go.sum file. cache: false - go-version: "1.20" + go-version: ${{ steps.setup-env.outputs.go-version }} - id: go-cache name: Lookup Go cache directory run: | From 035cf86cf76f40ed8b4bf67ff1b14c420701e145 Mon Sep 17 00:00:00 2001 From: Nicholas McDonnell <50747025+mcdonnnj@users.noreply.github.com> Date: Tue, 27 Feb 2024 17:17:39 -0500 Subject: [PATCH 077/170] Switch pre-commit hooks for running shellcheck This hook bundles the binaries for shellcheck with a Python package which removes the need to ensure the tool is installed for the hook to function. It also ties the version of shellcheck used to the hook which will help guarantee consistency. --- .pre-commit-config.yaml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 1d12a0d..2c5b3c8 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -115,10 +115,10 @@ repos: - --case-indent # Redirect operators are followed by a space - --space-redirects - - repo: https://github.com/detailyang/pre-commit-shell - rev: 1.0.5 + - repo: https://github.com/shellcheck-py/shellcheck-py + rev: v0.9.0.6 hooks: - - id: shell-lint + - id: shellcheck # Python hooks - repo: https://github.com/PyCQA/bandit From cea8edc5bcdcec8a06b6b810514b0222fc03f42e Mon Sep 17 00:00:00 2001 From: Michael Saki Date: Wed, 14 Feb 2024 12:59:30 -0600 Subject: [PATCH 078/170] Add checks for semantic python versions --- setup-env | 39 +++++++++++++++++++++------------------ 1 file changed, 21 insertions(+), 18 deletions(-) diff --git a/setup-env b/setup-env index 3a22d43..5e537bc 100755 --- a/setup-env +++ b/setup-env @@ -39,6 +39,14 @@ python_versions() { pyenv versions --bare --skip-aliases --skip-envs } +check_semantic_version() { + local version=$1 + local regex="^(0|[1-9]\\d*)\\.(0|[1-9]\\d*)\\.(0|[1-9]\\d*)(?:-((?:0|[1-9]\\d*|\\d*[a-zA-Z-][0-9a-zA-Z-]*)(?:\\.(?:0|[1-9]\\d*|\\d*[a-zA-Z-][0-9a-zA-Z-]*))*))?(?:\\+([0-9a-zA-Z-]+(?:\\.[0-9a-zA-Z-]+)*))?\$" + + # Use Perl for regex matching and output true or false + echo "$version" | perl -ne "exit(!/$regex/)" +} + # Flag to force deletion and creation of virtual environment FORCE=0 @@ -144,16 +152,18 @@ while true; do -p | --python-version) PYTHON_VERSION="$2" shift 2 - # Check the Python versions being passed in. - if [ -n "${PYTHON_VERSION+x}" ]; then - if python_versions | grep -E "^${PYTHON_VERSION}$" > /dev/null; then - echo Using Python version "$PYTHON_VERSION" - else - echo Error: Python version "$PYTHON_VERSION" is not installed. - echo Installed Python versions are: - python_versions - exit 1 - fi + # Validate the semantic version format + if ! check_semantic_version "$PYTHON_VERSION"; then + echo "Error: The specified Python version $PYTHON_VERSION does not follow the semantic versioning standard." + echo "Example of a valid version: 3.8.1, 3.8.1-alpha.1, or 3.8.1+20130313144700" + exit 1 + elif ! python_versions | grep -E "^${PYTHON_VERSION}$" > /dev/null; then + echo "Error: Python version $PYTHON_VERSION is not installed." + echo "Installed Python versions are:" + python_versions + exit 1 + else + echo "Using Python version $PYTHON_VERSION" fi ;; -v | --venv-name) @@ -189,14 +199,7 @@ if [ $LIST_VERSIONS -ne 0 ]; then # -r: treat backslashes as literal, -p: display prompt before input. read -r -p "Enter the desired Python version: " PYTHON_VERSION # Check the Python versions being passed in. - if [ -n "${PYTHON_VERSION+x}" ]; then - if python_versions | grep -E "^${PYTHON_VERSION}$" > /dev/null; then - echo Using Python version "$PYTHON_VERSION" - else - echo Error: Python version "$PYTHON_VERSION" is not installed. - exit 1 - fi - fi + check_semantic_version "$PYTHON_VERSION" fi # Remove any lingering local configuration. From d5c7c4a566f88f7575f06ff2e0829f257a00cb08 Mon Sep 17 00:00:00 2001 From: Michael Saki Date: Wed, 14 Feb 2024 14:29:13 -0600 Subject: [PATCH 079/170] Refactor code for the semantic check This commit will make a few changes. The orginal version of the semantic checking function was a bit more difficult to read. It is now somewhat easier to follow how the regex is structured. Also the function has been renamed to check_python_version since it has 2 functions, making sure that the version is semantically correct and the second is to make sure that it is installed on the user's machine. This makes it easier to follow the logic for the flags, -p or --python-version and -l or --list-versions --- setup-env | 54 ++++++++++++++++++++++++++++++++++++------------------ 1 file changed, 36 insertions(+), 18 deletions(-) diff --git a/setup-env b/setup-env index 5e537bc..92540d1 100755 --- a/setup-env +++ b/setup-env @@ -39,12 +39,41 @@ python_versions() { pyenv versions --bare --skip-aliases --skip-envs } -check_semantic_version() { +check_python_version() { local version=$1 - local regex="^(0|[1-9]\\d*)\\.(0|[1-9]\\d*)\\.(0|[1-9]\\d*)(?:-((?:0|[1-9]\\d*|\\d*[a-zA-Z-][0-9a-zA-Z-]*)(?:\\.(?:0|[1-9]\\d*|\\d*[a-zA-Z-][0-9a-zA-Z-]*))*))?(?:\\+([0-9a-zA-Z-]+(?:\\.[0-9a-zA-Z-]+)*))?\$" - # Use Perl for regex matching and output true or false - echo "$version" | perl -ne "exit(!/$regex/)" + # Break down the regex into readable parts major.minor.patch + local major="0|[1-9]\\d*" + local minor="0|[1-9]\\d*" + local patch="0|[1-9]\\d*" + + # Splitting the prerelease part for readability + # Start of prerelease + local prerelease="(?:-" + # Numeric or alphanumeric identifiers + local prerelease+="(?:0|[1-9]\\d*|\\d*[a-zA-Z-][0-9a-zA-Z-]*)" + # Additional dot-separated identifiers + local prerelease+="(?:\\.(?:0|[1-9]\\d*|\\d*[a-zA-Z-][0-9a-zA-Z-]*))*" + # End of prerelease, making it optional + local prerelease+=")?" + # Optional build metadata + local build="(?:\\+([0-9a-zA-Z-]+(?:\\.[0-9a-zA-Z-]+)*))?" + + # Final regex composed of parts + local regex="^($major)\\.($minor)\\.($patch)$prerelease$build\$" + + if ! echo "$version" | perl -ne "exit(!/$regex/)"; then + echo "Error: The specified Python version $version does not follow the semantic versioning standard." + echo "Example of a valid version: 3.8.1, 3.8.1-alpha.1, or 3.8.1+20130313144700" + exit 1 + elif ! python_versions | grep -E "^${version}$" > /dev/null; then + echo "Error: Python version $version is not installed." + echo "Installed Python versions are:" + python_versions + exit 1 + else + echo "Using Python version $version" + fi } # Flag to force deletion and creation of virtual environment @@ -152,19 +181,8 @@ while true; do -p | --python-version) PYTHON_VERSION="$2" shift 2 - # Validate the semantic version format - if ! check_semantic_version "$PYTHON_VERSION"; then - echo "Error: The specified Python version $PYTHON_VERSION does not follow the semantic versioning standard." - echo "Example of a valid version: 3.8.1, 3.8.1-alpha.1, or 3.8.1+20130313144700" - exit 1 - elif ! python_versions | grep -E "^${PYTHON_VERSION}$" > /dev/null; then - echo "Error: Python version $PYTHON_VERSION is not installed." - echo "Installed Python versions are:" - python_versions - exit 1 - else - echo "Using Python version $PYTHON_VERSION" - fi + # Check the Python version being passed in. + check_python_version "$PYTHON_VERSION" ;; -v | --venv-name) VENV_NAME="$2" @@ -199,7 +217,7 @@ if [ $LIST_VERSIONS -ne 0 ]; then # -r: treat backslashes as literal, -p: display prompt before input. read -r -p "Enter the desired Python version: " PYTHON_VERSION # Check the Python versions being passed in. - check_semantic_version "$PYTHON_VERSION" + check_python_version "$PYTHON_VERSION" fi # Remove any lingering local configuration. From 327ab733aeaaad6a4916eb86b20d86618c9351e3 Mon Sep 17 00:00:00 2001 From: Michael Saki Date: Mon, 18 Mar 2024 12:36:02 -0500 Subject: [PATCH 080/170] Remove example of correct semantic version --- setup-env | 1 - 1 file changed, 1 deletion(-) diff --git a/setup-env b/setup-env index 92540d1..bacd2d5 100755 --- a/setup-env +++ b/setup-env @@ -64,7 +64,6 @@ check_python_version() { if ! echo "$version" | perl -ne "exit(!/$regex/)"; then echo "Error: The specified Python version $version does not follow the semantic versioning standard." - echo "Example of a valid version: 3.8.1, 3.8.1-alpha.1, or 3.8.1+20130313144700" exit 1 elif ! python_versions | grep -E "^${version}$" > /dev/null; then echo "Error: Python version $version is not installed." From 4dedf50886fd47c67895deb07367fca5c36ca33f Mon Sep 17 00:00:00 2001 From: Michael Saki Date: Wed, 20 Mar 2024 12:58:03 -0500 Subject: [PATCH 081/170] Refactor the error message for the user --- setup-env | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/setup-env b/setup-env index bacd2d5..d7824cb 100755 --- a/setup-env +++ b/setup-env @@ -63,7 +63,9 @@ check_python_version() { local regex="^($major)\\.($minor)\\.($patch)$prerelease$build\$" if ! echo "$version" | perl -ne "exit(!/$regex/)"; then - echo "Error: The specified Python version $version does not follow the semantic versioning standard." + echo "Invalid version of Python: Python follows semantic versioning, " \ + "so any version string that is not a valid semantic version is an " \ + "invalid version of Python." exit 1 elif ! python_versions | grep -E "^${version}$" > /dev/null; then echo "Error: Python version $version is not installed." From e84deea5181f27471f01343113c91dc2b13e159e Mon Sep 17 00:00:00 2001 From: Michael Saki Date: Wed, 20 Mar 2024 14:52:16 -0500 Subject: [PATCH 082/170] Improve the semantic error message --- setup-env | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/setup-env b/setup-env index d7824cb..bba5f9e 100755 --- a/setup-env +++ b/setup-env @@ -63,8 +63,8 @@ check_python_version() { local regex="^($major)\\.($minor)\\.($patch)$prerelease$build\$" if ! echo "$version" | perl -ne "exit(!/$regex/)"; then - echo "Invalid version of Python: Python follows semantic versioning, " \ - "so any version string that is not a valid semantic version is an " \ + echo "Invalid version of Python: Python follows semantic versioning," \ + "so any version string that is not a valid semantic version is an" \ "invalid version of Python." exit 1 elif ! python_versions | grep -E "^${version}$" > /dev/null; then From 5fdc7befc1d1d4811c4550ca1e4c65a711971c21 Mon Sep 17 00:00:00 2001 From: Michael Saki Date: Wed, 20 Mar 2024 15:39:07 -0500 Subject: [PATCH 083/170] Fix grammar Co-authored-by: dav3r --- setup-env | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup-env b/setup-env index bba5f9e..b93810c 100755 --- a/setup-env +++ b/setup-env @@ -217,7 +217,7 @@ if [ $LIST_VERSIONS -ne 0 ]; then # Read the user's desired Python version. # -r: treat backslashes as literal, -p: display prompt before input. read -r -p "Enter the desired Python version: " PYTHON_VERSION - # Check the Python versions being passed in. + # Check the Python version being passed in. check_python_version "$PYTHON_VERSION" fi From 42ef8c2d7b54cde82d4390a0050622cddfccf92a Mon Sep 17 00:00:00 2001 From: Michael Saki Date: Thu, 21 Mar 2024 09:19:42 -0500 Subject: [PATCH 084/170] Refactor regex, add link, and improve comments --- setup-env | 25 ++++++++++++++++--------- 1 file changed, 16 insertions(+), 9 deletions(-) diff --git a/setup-env b/setup-env index b93810c..2f30021 100755 --- a/setup-env +++ b/setup-env @@ -42,31 +42,38 @@ python_versions() { check_python_version() { local version=$1 + # This is a valid regex for semantically correct Python version strings. + # For more information see here: https://regex101.com/r/vkijKf/1/. # Break down the regex into readable parts major.minor.patch - local major="0|[1-9]\\d*" - local minor="0|[1-9]\\d*" - local patch="0|[1-9]\\d*" + local major="0|[1-9]\d*" + local minor="0|[1-9]\d*" + local patch="0|[1-9]\d*" # Splitting the prerelease part for readability - # Start of prerelease + # Start of the prerelease local prerelease="(?:-" # Numeric or alphanumeric identifiers - local prerelease+="(?:0|[1-9]\\d*|\\d*[a-zA-Z-][0-9a-zA-Z-]*)" + local prerelease+="(?:0|[1-9]\d*|\d*[a-zA-Z-][0-9a-zA-Z-]*)" # Additional dot-separated identifiers - local prerelease+="(?:\\.(?:0|[1-9]\\d*|\\d*[a-zA-Z-][0-9a-zA-Z-]*))*" - # End of prerelease, making it optional + local prerelease+="(?:\.(?:0|[1-9]\d*|\d*[a-zA-Z-][0-9a-zA-Z-]*))*" + # End of the prerelease, making it optional local prerelease+=")?" # Optional build metadata - local build="(?:\\+([0-9a-zA-Z-]+(?:\\.[0-9a-zA-Z-]+)*))?" + local build="(?:\+([0-9a-zA-Z-]+(?:\.[0-9a-zA-Z-]+)*))?" # Final regex composed of parts - local regex="^($major)\\.($minor)\\.($patch)$prerelease$build\$" + local regex="^($major)\.($minor)\.($patch)$prerelease$build$" + # This checks if the Python version does not match the regex pattern specified in $regex, + # using Perl for regex matching. If the pattern is not found, then prompt the user with + # the invalid version message. if ! echo "$version" | perl -ne "exit(!/$regex/)"; then echo "Invalid version of Python: Python follows semantic versioning," \ "so any version string that is not a valid semantic version is an" \ "invalid version of Python." exit 1 + # Else if the Python version isn't installed then notify the user. + # grep -E is used for searching through text lines that match the specific verison. elif ! python_versions | grep -E "^${version}$" > /dev/null; then echo "Error: Python version $version is not installed." echo "Installed Python versions are:" From a77e5e1c9a8752a2072a6a974d4164be116069e9 Mon Sep 17 00:00:00 2001 From: Michael Saki Date: Thu, 21 Mar 2024 10:13:11 -0500 Subject: [PATCH 085/170] Update link to use semver.org over regex101.com --- setup-env | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/setup-env b/setup-env index 2f30021..8d7b347 100755 --- a/setup-env +++ b/setup-env @@ -43,7 +43,8 @@ check_python_version() { local version=$1 # This is a valid regex for semantically correct Python version strings. - # For more information see here: https://regex101.com/r/vkijKf/1/. + # For more information see here: + # https://semver.org/#is-there-a-suggested-regular-expression-regex-to-check-a-semver-string. # Break down the regex into readable parts major.minor.patch local major="0|[1-9]\d*" local minor="0|[1-9]\d*" From 5fe14c7c6066d30381f6746eb313a56e4d447ac5 Mon Sep 17 00:00:00 2001 From: Michael Saki Date: Thu, 21 Mar 2024 10:29:58 -0500 Subject: [PATCH 086/170] Remove unnecessary period Co-authored-by: dav3r --- setup-env | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup-env b/setup-env index 8d7b347..059ccad 100755 --- a/setup-env +++ b/setup-env @@ -44,7 +44,7 @@ check_python_version() { # This is a valid regex for semantically correct Python version strings. # For more information see here: - # https://semver.org/#is-there-a-suggested-regular-expression-regex-to-check-a-semver-string. + # https://semver.org/#is-there-a-suggested-regular-expression-regex-to-check-a-semver-string # Break down the regex into readable parts major.minor.patch local major="0|[1-9]\d*" local minor="0|[1-9]\d*" From b7896a0a2790cc121842c6ac1602734bbd5dd726 Mon Sep 17 00:00:00 2001 From: Nicholas McDonnell <50747025+mcdonnnj@users.noreply.github.com> Date: Sat, 20 Apr 2024 04:11:57 -0400 Subject: [PATCH 087/170] Add a meta hook to the pre-commit configuration Add the `check-useless-excludes` meta hook to verify that any defined `exclude` directives apply to at least one file in the repository. --- .pre-commit-config.yaml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 2c5b3c8..de8c587 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -4,6 +4,11 @@ default_language_version: python: python3 repos: + # Check the pre-commit configuration + - repo: meta + hooks: + - id: check-useless-excludes + - repo: https://github.com/pre-commit/pre-commit-hooks rev: v4.5.0 hooks: From 260566f177520175530963c469e50d124e5bc0e4 Mon Sep 17 00:00:00 2001 From: Nicholas McDonnell <50747025+mcdonnnj@users.noreply.github.com> Date: Sat, 20 Apr 2024 04:15:52 -0400 Subject: [PATCH 088/170] Remove `exclude` directive that does not apply to any files --- .pre-commit-config.yaml | 1 - 1 file changed, 1 deletion(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index de8c587..5ec468e 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -24,7 +24,6 @@ repos: - --allow-missing-credentials - id: detect-private-key - id: end-of-file-fixer - exclude: files/(issue|motd) - id: mixed-line-ending args: - --fix=lf From 07e2b60f912ada6de4de0dcf0573bbecc0f2037c Mon Sep 17 00:00:00 2001 From: Jeremy Frasier Date: Sat, 4 May 2024 10:09:12 -0400 Subject: [PATCH 089/170] Pin ansible-core when running the ansible-lint linter New versions of ansible-core (2.16.7 and 2.17.0) have been released that do not suffer from the bug discussed in ansible/ansible#82702. This bug broke any symlinked files in vars, tasks, etc. for any Ansible role installed via ansible-galaxy. All versions later than ansible-core 2.16.7 and 2.17.0 should function as expected. Co-authored-by: Nick <50747025+mcdonnnj@users.noreply.github.com> --- .pre-commit-config.yaml | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 2c5b3c8..895384c 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -155,6 +155,17 @@ repos: rev: v24.2.0 hooks: - id: ansible-lint + additional_dependencies: + # ansible-core 2.16.3 through 2.16.6 suffer from the bug + # discussed in ansible/ansible#82702, which breaks any + # symlinked files in vars, tasks, etc. for any Ansible role + # installed via ansible-galaxy. Hence we never want to + # install those versions. + # + # Note that any changes made to this dependency must also be + # made in requirements.txt in cisagov/skeleton-packer and + # requirements-test.txt in cisagov/skeleton-ansible-role. + - ansible-core>=2.16.7 # files: molecule/default/playbook.yml # Terraform hooks From c74e5db75b7e3785a3f7196365b7ba99f9a004ea Mon Sep 17 00:00:00 2001 From: Jeremy Frasier Date: Sat, 4 May 2024 10:11:08 -0400 Subject: [PATCH 090/170] Remove unnecessary line The line is not only unnecessary, it was commented out to boot! --- .pre-commit-config.yaml | 1 - 1 file changed, 1 deletion(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 895384c..11772d9 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -166,7 +166,6 @@ repos: # made in requirements.txt in cisagov/skeleton-packer and # requirements-test.txt in cisagov/skeleton-ansible-role. - ansible-core>=2.16.7 - # files: molecule/default/playbook.yml # Terraform hooks - repo: https://github.com/antonbabenko/pre-commit-terraform From 2e53e0de98caa5d5a42320618d8e604c770da7d9 Mon Sep 17 00:00:00 2001 From: Jeremy Frasier Date: Tue, 7 May 2024 15:21:06 -0400 Subject: [PATCH 091/170] Explain why ansible may need to be added as a dependency for ansible-lint On its own ansible-lint does not pull in ansible, only ansible-core. Therefore, if an Ansible module lives in ansible instead of ansible-core, the linter will complain that the module is unknown. In these cases it is necessary to add the ansible package itself as an additional dependency, with the same pinning as is done in requirements-test.txt of cisagov/skeleton-ansible-role. --- .pre-commit-config.yaml | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 11772d9..a48e196 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -156,6 +156,14 @@ repos: hooks: - id: ansible-lint additional_dependencies: + # On its own ansible-lint does not pull in ansible, only + # ansible-core. Therefore, if an Ansible module lives in + # ansible instead of ansible-core, the linter will complain + # that the module is unknown. In these cases it is + # necessary to add the ansible package itself as an + # additional dependency, with the same pinning as is done in + # requirements-test.txt of cisagov/skeleton-ansible-role. + # - ansible>=8,<10 # ansible-core 2.16.3 through 2.16.6 suffer from the bug # discussed in ansible/ansible#82702, which breaks any # symlinked files in vars, tasks, etc. for any Ansible role From f51fe623bb34e68bb874cd30756fee2692d34b5a Mon Sep 17 00:00:00 2001 From: Nicholas McDonnell <50747025+mcdonnnj@users.noreply.github.com> Date: Sat, 11 May 2024 00:38:50 -0400 Subject: [PATCH 092/170] Update pre-commit hook versions This is done automatically with the `pre-commit autoupdate` command. The pre-commit/mirrors-prettier was manually held back because the latest tags are for alpha releases of the next major version. --- .pre-commit-config.yaml | 24 ++++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 2c5b3c8..d315345 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -5,7 +5,7 @@ default_language_version: repos: - repo: https://github.com/pre-commit/pre-commit-hooks - rev: v4.5.0 + rev: v4.6.0 hooks: - id: check-case-conflict - id: check-executables-have-shebangs @@ -31,7 +31,7 @@ repos: # Text file hooks - repo: https://github.com/igorshubovych/markdownlint-cli - rev: v0.39.0 + rev: v0.41.0 hooks: - id: markdownlint args: @@ -56,14 +56,14 @@ repos: # GitHub Actions hooks - repo: https://github.com/python-jsonschema/check-jsonschema - rev: 0.28.0 + rev: 0.28.4 hooks: - id: check-github-actions - id: check-github-workflows # pre-commit hooks - repo: https://github.com/pre-commit/pre-commit - rev: v3.6.2 + rev: v3.7.1 hooks: - id: validate_manifest @@ -98,7 +98,7 @@ repos: # Shell script hooks - repo: https://github.com/scop/pre-commit-shfmt - rev: v3.7.0-4 + rev: v3.8.0-1 hooks: - id: shfmt args: @@ -116,19 +116,19 @@ repos: # Redirect operators are followed by a space - --space-redirects - repo: https://github.com/shellcheck-py/shellcheck-py - rev: v0.9.0.6 + rev: v0.10.0.1 hooks: - id: shellcheck # Python hooks - repo: https://github.com/PyCQA/bandit - rev: 1.7.7 + rev: 1.7.8 hooks: - id: bandit args: - --config=.bandit.yml - repo: https://github.com/psf/black-pre-commit-mirror - rev: 24.2.0 + rev: 24.4.2 hooks: - id: black - repo: https://github.com/PyCQA/flake8 @@ -142,24 +142,24 @@ repos: hooks: - id: isort - repo: https://github.com/pre-commit/mirrors-mypy - rev: v1.8.0 + rev: v1.10.0 hooks: - id: mypy - repo: https://github.com/asottile/pyupgrade - rev: v3.15.1 + rev: v3.15.2 hooks: - id: pyupgrade # Ansible hooks - repo: https://github.com/ansible/ansible-lint - rev: v24.2.0 + rev: v24.6.0 hooks: - id: ansible-lint # files: molecule/default/playbook.yml # Terraform hooks - repo: https://github.com/antonbabenko/pre-commit-terraform - rev: v1.88.0 + rev: v1.90.0 hooks: - id: terraform_fmt - id: terraform_validate From 8e55b8e24e047008b1fa80bc87a76163888e4e5e Mon Sep 17 00:00:00 2001 From: Nicholas McDonnell <50747025+mcdonnnj@users.noreply.github.com> Date: Thu, 6 Jun 2024 13:58:38 -0400 Subject: [PATCH 093/170] Manually update the prettier hook Use the latest v3 release available from NPM. --- .pre-commit-config.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index d315345..2ece2c0 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -46,7 +46,7 @@ repos: # mirror does not pull tags for old major versions once a new major # version tag is published. additional_dependencies: - - prettier@3.2.5 + - prettier@3.3.1 - repo: https://github.com/adrienverge/yamllint rev: v1.35.1 hooks: From c617bb92af0bd01d0ef9bb7e51c007e34f91a915 Mon Sep 17 00:00:00 2001 From: Shane Frasier Date: Thu, 6 Jun 2024 16:42:24 -0400 Subject: [PATCH 094/170] Correct commented-out ansible pin The pin now agrees with what is in cisagov/skeleton-ansible-role. Co-authored-by: Nick <50747025+mcdonnnj@users.noreply.github.com> --- .pre-commit-config.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index a48e196..3071c44 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -163,7 +163,7 @@ repos: # necessary to add the ansible package itself as an # additional dependency, with the same pinning as is done in # requirements-test.txt of cisagov/skeleton-ansible-role. - # - ansible>=8,<10 + # - ansible>=9,<10 # ansible-core 2.16.3 through 2.16.6 suffer from the bug # discussed in ansible/ansible#82702, which breaks any # symlinked files in vars, tasks, etc. for any Ansible role From a68994d17dcc11e9b90132c50fe52732d5fda07b Mon Sep 17 00:00:00 2001 From: Jeremy Frasier Date: Mon, 1 Jul 2024 16:19:46 -0400 Subject: [PATCH 095/170] Add a lower-bound pin for flake8-docstrings --- .pre-commit-config.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 386c83f..74c9c76 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -136,7 +136,7 @@ repos: hooks: - id: flake8 additional_dependencies: - - flake8-docstrings + - flake8-docstrings>=1.7.0 - repo: https://github.com/PyCQA/isort rev: 5.13.2 hooks: From 43b91c74754e912172c702e20f12ba9f767ac202 Mon Sep 17 00:00:00 2001 From: Nicholas McDonnell <50747025+mcdonnnj@users.noreply.github.com> Date: Mon, 12 Aug 2024 06:24:06 -0400 Subject: [PATCH 096/170] Use the hashicorp/setup-packer GitHub Action Instead of manually installing Packer we can instead leverage the hashicorp/setup-packer Action just as we do for Terraform. --- .github/workflows/build.yml | 20 +++----------------- 1 file changed, 3 insertions(+), 17 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 9bb221a..e12b842 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -20,7 +20,6 @@ defaults: shell: bash -Eueo pipefail -x {0} env: - CURL_CACHE_DIR: ~/.cache/curl PIP_CACHE_DIR: ~/.cache/pip PRE_COMMIT_CACHE_DIR: ~/.cache/pre-commit RUN_TMATE: ${{ secrets.RUN_TMATE }} @@ -97,25 +96,12 @@ jobs: path: | ${{ env.PIP_CACHE_DIR }} ${{ env.PRE_COMMIT_CACHE_DIR }} - ${{ env.CURL_CACHE_DIR }} ${{ steps.go-cache.outputs.dir }} restore-keys: | ${{ env.BASE_CACHE_KEY }} - - name: Setup curl cache - run: mkdir -p ${{ env.CURL_CACHE_DIR }} - - name: Install Packer - env: - PACKER_VERSION: ${{ steps.setup-env.outputs.packer-version }} - run: | - PACKER_ZIP="packer_${PACKER_VERSION}_linux_amd64.zip" - curl --output ${{ env.CURL_CACHE_DIR }}/"${PACKER_ZIP}" \ - --time-cond ${{ env.CURL_CACHE_DIR }}/"${PACKER_ZIP}" \ - --location \ - "https://releases.hashicorp.com/packer/${PACKER_VERSION}/${PACKER_ZIP}" - sudo unzip -d /opt/packer \ - ${{ env.CURL_CACHE_DIR }}/"${PACKER_ZIP}" - sudo mv /usr/local/bin/packer /usr/local/bin/packer-default - sudo ln -s /opt/packer/packer /usr/local/bin/packer + - uses: hashicorp/setup-packer@v3 + with: + version: ${{ steps.setup-env.outputs.packer-version }} - uses: hashicorp/setup-terraform@v3 with: terraform_version: ${{ steps.setup-env.outputs.terraform-version }} From 8ada75d419c3ea546843fc0772d9d0b678beeea4 Mon Sep 17 00:00:00 2001 From: Jeremy Frasier Date: Fri, 23 Aug 2024 00:54:54 -0400 Subject: [PATCH 097/170] Remove @jasonodoom as a codeowner He is no longer a member of @cisagov/vm-dev. --- .github/CODEOWNERS | 28 ++++++++++++++-------------- 1 file changed, 14 insertions(+), 14 deletions(-) diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index 229920c..3af99ba 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -3,22 +3,22 @@ # These owners will be the default owners for everything in the # repo. Unless a later match takes precedence, these owners will be # requested for review when someone opens a pull request. -* @dav3r @felddy @jasonodoom @jsf9k @mcdonnnj +* @dav3r @felddy @jsf9k @mcdonnnj # These folks own any files in the .github directory at the root of # the repository and any of its subdirectories. -/.github/ @dav3r @felddy @jasonodoom @jsf9k @mcdonnnj +/.github/ @dav3r @felddy @jsf9k @mcdonnnj # These folks own all linting configuration files. -/.ansible-lint @dav3r @felddy @jasonodoom @jsf9k @mcdonnnj -/.bandit.yml @dav3r @felddy @jasonodoom @jsf9k @mcdonnnj -/.flake8 @dav3r @felddy @jasonodoom @jsf9k @mcdonnnj -/.isort.cfg @dav3r @felddy @jasonodoom @jsf9k @mcdonnnj -/.mdl_config.yaml @dav3r @felddy @jasonodoom @jsf9k @mcdonnnj -/.pre-commit-config.yaml @dav3r @felddy @jasonodoom @jsf9k @mcdonnnj -/.prettierignore @dav3r @felddy @jasonodoom @jsf9k @mcdonnnj -/.yamllint @dav3r @felddy @jasonodoom @jsf9k @mcdonnnj -/requirements.txt @dav3r @felddy @jasonodoom @jsf9k @mcdonnnj -/requirements-dev.txt @dav3r @felddy @jasonodoom @jsf9k @mcdonnnj -/requirements-test.txt @dav3r @felddy @jasonodoom @jsf9k @mcdonnnj -/setup-env @dav3r @felddy @jasonodoom @jsf9k @mcdonnnj +/.ansible-lint @dav3r @felddy @jsf9k @mcdonnnj +/.bandit.yml @dav3r @felddy @jsf9k @mcdonnnj +/.flake8 @dav3r @felddy @jsf9k @mcdonnnj +/.isort.cfg @dav3r @felddy @jsf9k @mcdonnnj +/.mdl_config.yaml @dav3r @felddy @jsf9k @mcdonnnj +/.pre-commit-config.yaml @dav3r @felddy @jsf9k @mcdonnnj +/.prettierignore @dav3r @felddy @jsf9k @mcdonnnj +/.yamllint @dav3r @felddy @jsf9k @mcdonnnj +/requirements.txt @dav3r @felddy @jsf9k @mcdonnnj +/requirements-dev.txt @dav3r @felddy @jsf9k @mcdonnnj +/requirements-test.txt @dav3r @felddy @jsf9k @mcdonnnj +/setup-env @dav3r @felddy @jsf9k @mcdonnnj From 293020830fb6830a7324b5eacb8c3122979d9882 Mon Sep 17 00:00:00 2001 From: Shane Frasier Date: Mon, 26 Aug 2024 09:27:58 -0400 Subject: [PATCH 098/170] Pin to a specific version Previously we only provided a lower bound for the version, but pinning to a specific version aligns with what has been done with the prettier hook and how pre-commit hooks are pinned in general. The flake8-docstrings package is rarely updated, so there is no real downside to pinning to a specific version. Co-authored-by: Nick <50747025+mcdonnnj@users.noreply.github.com> --- .pre-commit-config.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 74c9c76..236eeda 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -136,7 +136,7 @@ repos: hooks: - id: flake8 additional_dependencies: - - flake8-docstrings>=1.7.0 + - flake8-docstrings==1.7.0 - repo: https://github.com/PyCQA/isort rev: 5.13.2 hooks: From 46e055367c1e34711ed0980b2934b9df54bf33fe Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 24 Sep 2024 18:23:01 +0000 Subject: [PATCH 099/170] Bump actions/cache from 3 to 4 Bumps [actions/cache](https://github.com/actions/cache) from 3 to 4. - [Release notes](https://github.com/actions/cache/releases) - [Changelog](https://github.com/actions/cache/blob/main/RELEASES.md) - [Commits](https://github.com/actions/cache/compare/v3...v4) --- updated-dependencies: - dependency-name: actions/cache dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] --- .github/workflows/build.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 9bb221a..a403ea9 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -76,7 +76,7 @@ jobs: name: Lookup Go cache directory run: | echo "dir=$(go env GOCACHE)" >> $GITHUB_OUTPUT - - uses: actions/cache@v3 + - uses: actions/cache@v4 env: BASE_CACHE_KEY: "${{ github.job }}-${{ runner.os }}-\ py${{ steps.setup-python.outputs.python-version }}-\ From 3167421109abf3fe94dc801203587e1bf3ce33a3 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 24 Sep 2024 18:23:14 +0000 Subject: [PATCH 100/170] Bump crazy-max/ghaction-github-status from 3 to 4 Bumps [crazy-max/ghaction-github-status](https://github.com/crazy-max/ghaction-github-status) from 3 to 4. - [Release notes](https://github.com/crazy-max/ghaction-github-status/releases) - [Commits](https://github.com/crazy-max/ghaction-github-status/compare/v3...v4) --- updated-dependencies: - dependency-name: crazy-max/ghaction-github-status dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] --- .github/workflows/sync-labels.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/sync-labels.yml b/.github/workflows/sync-labels.yml index 5a20438..e83bd41 100644 --- a/.github/workflows/sync-labels.yml +++ b/.github/workflows/sync-labels.yml @@ -24,7 +24,7 @@ jobs: egress-policy: audit - id: github-status name: Check GitHub status - uses: crazy-max/ghaction-github-status@v3 + uses: crazy-max/ghaction-github-status@v4 - id: dump-context name: Dump context uses: crazy-max/ghaction-dump-context@v2 From 6a58c2c24ef1eb15c7a69a44f16c63964f1c7f82 Mon Sep 17 00:00:00 2001 From: Nicholas McDonnell <50747025+mcdonnnj@users.noreply.github.com> Date: Mon, 12 Aug 2024 16:23:58 -0400 Subject: [PATCH 101/170] Update pre-commit hook versions This is done automatically with the `pre-commit autoupdate` command. The pre-commit/mirrors-prettier hook was manually held back because the latest tags are for alpha releases of the next major version. --- .pre-commit-config.yaml | 24 ++++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 386c83f..81f3276 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -31,7 +31,7 @@ repos: # Text file hooks - repo: https://github.com/igorshubovych/markdownlint-cli - rev: v0.41.0 + rev: v0.42.0 hooks: - id: markdownlint args: @@ -56,14 +56,14 @@ repos: # GitHub Actions hooks - repo: https://github.com/python-jsonschema/check-jsonschema - rev: 0.28.4 + rev: 0.29.2 hooks: - id: check-github-actions - id: check-github-workflows # pre-commit hooks - repo: https://github.com/pre-commit/pre-commit - rev: v3.7.1 + rev: v3.8.0 hooks: - id: validate_manifest @@ -98,7 +98,7 @@ repos: # Shell script hooks - repo: https://github.com/scop/pre-commit-shfmt - rev: v3.8.0-1 + rev: v3.9.0-1 hooks: - id: shfmt args: @@ -122,17 +122,17 @@ repos: # Python hooks - repo: https://github.com/PyCQA/bandit - rev: 1.7.8 + rev: 1.7.10 hooks: - id: bandit args: - --config=.bandit.yml - repo: https://github.com/psf/black-pre-commit-mirror - rev: 24.4.2 + rev: 24.8.0 hooks: - id: black - repo: https://github.com/PyCQA/flake8 - rev: 7.0.0 + rev: 7.1.1 hooks: - id: flake8 additional_dependencies: @@ -142,17 +142,17 @@ repos: hooks: - id: isort - repo: https://github.com/pre-commit/mirrors-mypy - rev: v1.10.0 + rev: v1.11.2 hooks: - id: mypy - repo: https://github.com/asottile/pyupgrade - rev: v3.15.2 + rev: v3.17.0 hooks: - id: pyupgrade # Ansible hooks - repo: https://github.com/ansible/ansible-lint - rev: v24.6.0 + rev: v24.9.2 hooks: - id: ansible-lint additional_dependencies: @@ -177,7 +177,7 @@ repos: # Terraform hooks - repo: https://github.com/antonbabenko/pre-commit-terraform - rev: v1.90.0 + rev: v1.96.1 hooks: - id: terraform_fmt - id: terraform_validate @@ -190,7 +190,7 @@ repos: # Packer hooks - repo: https://github.com/cisagov/pre-commit-packer - rev: v0.0.2 + rev: v0.1.0 hooks: - id: packer_validate - id: packer_fmt From 553efcb0d4e755ebd47abb49c865367ed6d0a236 Mon Sep 17 00:00:00 2001 From: Nicholas McDonnell <50747025+mcdonnnj@users.noreply.github.com> Date: Mon, 12 Aug 2024 16:30:49 -0400 Subject: [PATCH 102/170] Manually update the prettier hook Use the latest v3 release available from NPM. --- .pre-commit-config.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 81f3276..2104775 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -46,7 +46,7 @@ repos: # mirror does not pull tags for old major versions once a new major # version tag is published. additional_dependencies: - - prettier@3.3.1 + - prettier@3.3.3 - repo: https://github.com/adrienverge/yamllint rev: v1.35.1 hooks: From 045a998dcf14dc7e3de9301ba7ee2103272b0ac4 Mon Sep 17 00:00:00 2001 From: Nicholas McDonnell <50747025+mcdonnnj@users.noreply.github.com> Date: Thu, 18 Jan 2024 16:11:15 -0500 Subject: [PATCH 103/170] Add a pre-commit hook to run pip-audit The pip-audit tool will audit any supplied pip requirements files for vulnerable packages. --- .pre-commit-config.yaml | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 2c5b3c8..78140ff 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -145,6 +145,18 @@ repos: rev: v1.8.0 hooks: - id: mypy + - repo: https://github.com/pypa/pip-audit + rev: v2.7.3 + hooks: + - id: pip-audit + args: + # Add any pip requirements files to scan + - --requirement + - requirements-dev.txt + - --requirement + - requirements-test.txt + - --requirement + - requirements.txt - repo: https://github.com/asottile/pyupgrade rev: v3.15.1 hooks: From c502f1ab7cca8bd383a34360ce456b50fd6e8b21 Mon Sep 17 00:00:00 2001 From: Nicholas McDonnell <50747025+mcdonnnj@users.noreply.github.com> Date: Mon, 12 Aug 2024 16:32:02 -0400 Subject: [PATCH 104/170] Use the rbubley/mirrors-prettier hook for prettier This replaces the now archived pre-commit/mirrors-prettier hook. --- .pre-commit-config.yaml | 11 ++--------- 1 file changed, 2 insertions(+), 9 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index ca59d6f..3cb1f85 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -40,17 +40,10 @@ repos: - id: markdownlint args: - --config=.mdl_config.yaml - - repo: https://github.com/pre-commit/mirrors-prettier - # This is the last version of v3 available from the mirror. We should hold - # here until v4, which is currently in alpha, is more stable. - rev: v3.1.0 + - repo: https://github.com/rbubley/mirrors-prettier + rev: v3.3.3 hooks: - id: prettier - # This is the latest version of v3 available from NPM. The pre-commit - # mirror does not pull tags for old major versions once a new major - # version tag is published. - additional_dependencies: - - prettier@3.3.3 - repo: https://github.com/adrienverge/yamllint rev: v1.35.1 hooks: From 4f86062dfc30c6a5e2d6ae0f6f9564e08112b964 Mon Sep 17 00:00:00 2001 From: Nicholas McDonnell <50747025+mcdonnnj@users.noreply.github.com> Date: Tue, 1 Oct 2024 03:43:21 -0400 Subject: [PATCH 105/170] Pull in the `bump-version` script from cisagov/pre-commit-packer --- bump-version | 172 +++++++++++++++++++++++++++++++++++++++++++ requirements-dev.txt | 2 +- 2 files changed, 173 insertions(+), 1 deletion(-) create mode 100755 bump-version diff --git a/bump-version b/bump-version new file mode 100755 index 0000000..b19230e --- /dev/null +++ b/bump-version @@ -0,0 +1,172 @@ +#!/usr/bin/env bash + +# bump-version [--push] [--label LABEL] (major | minor | patch | prerelease | build | finalize | show) +# bump-version --list-files + +set -o nounset +set -o errexit +set -o pipefail + +# Stores the canonical version for the project. +VERSION_FILE=config/version.txt +# Files that should be updated with the new version. +VERSION_FILES=("$VERSION_FILE" README.md) + +USAGE=$( + cat << END_OF_LINE +Update the version of the project. + +Usage: + ${0##*/} [--push] [--label LABEL] (major | minor | patch | prerelease | build | finalize | show) + ${0##*/} --list-files + ${0##*/} (-h | --help) + +Options: + -h | --help Show this message. + --push Perform a \`git push\` after updating the version. + --label LABEL Specify the label to use when updating the build or prerelease version. + --list-files List the files that will be updated when the version is bumped. +END_OF_LINE +) + +old_version=$(< "$VERSION_FILE") +# Comment out periods so they are interpreted as periods and don't +# just match any character +old_version_regex=${old_version//\./\\\.} +new_version="$old_version" + +bump_part="" +label="" +commit_prefix="Bump" +with_push=false +commands_with_label=("build" "prerelease") +commands_with_prerelease=("major" "minor" "patch") +with_prerelease=false + +####################################### +# Display an error message, the help information, and exit with a non-zero status. +# Arguments: +# Error message. +####################################### +function invalid_option() { + echo "$1" + echo "$USAGE" + exit 1 +} + +####################################### +# Bump the version using the provided command. +# Arguments: +# The version to bump. +# The command to bump the version. +# Returns: +# The new version. +####################################### +function bump_version() { + local temp_version + temp_version=$(python -c "import semver; print(semver.parse_version_info('$1').${2})") + echo "$temp_version" +} + +if [ $# -eq 0 ]; then + echo "$USAGE" + exit 1 +else + while [ $# -gt 0 ]; do + case $1 in + --push) + if [ "$with_push" = true ]; then + invalid_option "Push has already been set." + fi + + with_push=true + shift + ;; + --label) + if [ -n "$label" ]; then + invalid_option "Label has already been set." + fi + + label="$2" + shift 2 + ;; + build | finalize | major | minor | patch) + if [ -n "$bump_part" ]; then + invalid_option "Only one version part should be bumped at a time." + fi + + bump_part="$1" + shift + ;; + prerelease) + with_prerelease=true + shift + ;; + show) + echo "$old_version" + exit 0 + ;; + -h | --help) + echo "$USAGE" + exit 0 + ;; + --list-files) + printf '%s\n' "${VERSION_FILES[@]}" + exit 0 + ;; + *) + invalid_option "Invalid option: $1" + ;; + esac + done +fi + +if [ -n "$label" ] && [ "$with_prerelease" = false ] && [[ ! " ${commands_with_label[*]} " =~ [[:space:]]${bump_part}[[:space:]] ]]; then + invalid_option "Setting the label is only allowed for the following commands: ${commands_with_label[*]}" +fi + +if [ "$with_prerelease" = true ] && [[ ! " ${commands_with_prerelease[*]} " =~ [[:space:]]${bump_part}[[:space:]] ]]; then + invalid_option "Changing the prerelease is only allowed in conjunction with the following commands: ${commands_with_prerelease[*]}" +fi + +label_option="" +if [ -n "$label" ]; then + label_option="token='$label'" +fi + +if [ -n "$bump_part" ]; then + if [ "$bump_part" = "finalize" ]; then + commit_prefix="Finalize" + bump_command="finalize_version()" + elif [ "$bump_part" = "build" ]; then + bump_command="bump_${bump_part}($label_option)" + else + bump_command="bump_${bump_part}()" + fi + new_version=$(bump_version "$old_version" "$bump_command") + echo Changing version from "$old_version" to "$new_version" +fi + +if [ "$with_prerelease" = true ]; then + bump_command="bump_prerelease($label_option)" + temp_version=$(bump_version "$new_version" "$bump_command") + echo Changing version from "$new_version" to "$temp_version" + new_version="$temp_version" +fi + +tmp_file=/tmp/version.$$ +for version_file in "${VERSION_FILES[@]}"; do + if [ ! -f "$version_file" ]; then + echo Missing expected file: "$version_file" + exit 1 + fi + sed "s/$old_version_regex/$new_version/" "$version_file" > $tmp_file + mv $tmp_file "$version_file" +done + +git add "${VERSION_FILES[@]}" +git commit --message "$commit_prefix version from $old_version to $new_version" + +if [ "$with_push" = true ]; then + git push +fi diff --git a/requirements-dev.txt b/requirements-dev.txt index cb51627..de5eb3b 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -1,3 +1,3 @@ --requirement requirements-test.txt ipython -semver +semver>=3 From b780601c2ba9949edccc128dfd0df3c749701ff6 Mon Sep 17 00:00:00 2001 From: Nicholas McDonnell <50747025+mcdonnnj@users.noreply.github.com> Date: Tue, 1 Oct 2024 03:47:01 -0400 Subject: [PATCH 106/170] Modify the `bump-version` script for this repository Copy over necessary changes from the `bump_version.sh` script. --- bump-version | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/bump-version b/bump-version index b19230e..600265b 100755 --- a/bump-version +++ b/bump-version @@ -8,7 +8,7 @@ set -o errexit set -o pipefail # Stores the canonical version for the project. -VERSION_FILE=config/version.txt +VERSION_FILE=src/version.txt # Files that should be updated with the new version. VERSION_FILES=("$VERSION_FILE" README.md) @@ -29,7 +29,7 @@ Options: END_OF_LINE ) -old_version=$(< "$VERSION_FILE") +old_version=$(sed -n "s/^__version__ = \"\(.*\)\"$/\1/p" $VERSION_FILE) # Comment out periods so they are interpreted as periods and don't # just match any character old_version_regex=${old_version//\./\\\.} From e3bc34a02d32849341e255ba35026c4e1f935bd8 Mon Sep 17 00:00:00 2001 From: Nicholas McDonnell <50747025+mcdonnnj@users.noreply.github.com> Date: Tue, 1 Oct 2024 03:47:33 -0400 Subject: [PATCH 107/170] Remove the `bump_version.sh` script This functionality has been replaced by the `bump-version` script. --- .github/workflows/build.yml | 4 +- bump_version.sh | 81 ------------------------------------- tag.sh | 2 +- 3 files changed, 3 insertions(+), 84 deletions(-) delete mode 100755 bump_version.sh diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index d8ba132..8078b05 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -173,7 +173,7 @@ jobs: # repometa: # The json metadata describing this repository. # source_version: - # The source version as reported by the `bump_version.sh show` command. + # The source version as reported by the `bump-version show` command. # tags: # A comma separated list of Docker tags to be applied to the images on # Docker Hub. The tags will vary depending on: @@ -265,7 +265,7 @@ jobs: TAGS="${TAGS},ghcr.io/${i}" done echo "created=$(date -u +'%Y-%m-%dT%H:%M:%SZ')" >> $GITHUB_OUTPUT - echo "source_version=$(./bump_version.sh show)" >> $GITHUB_OUTPUT + echo "source_version=$(./bump-version show)" >> $GITHUB_OUTPUT echo "tags=${TAGS}" >> $GITHUB_OUTPUT echo tags=${TAGS} - name: Setup tmate debug session diff --git a/bump_version.sh b/bump_version.sh deleted file mode 100755 index 0071670..0000000 --- a/bump_version.sh +++ /dev/null @@ -1,81 +0,0 @@ -#!/usr/bin/env bash - -# Usage: -# bump_version.sh (show|major|minor|patch|finalize) -# bump_version.sh (build|prerelease) [token] -# Notes: -# - If you specify a token it will only be used if the current version is -# tokenless or if the provided token matches the token used in the current -# version. - -set -o nounset -set -o errexit -set -o pipefail - -VERSION_FILE=src/version.txt -README_FILE=README.md - -function usage { - cat << HELP -Usage: - ${0##*/} (show|major|minor|patch|finalize) - ${0##*/} (build|prerelease) [token] - -Notes: - - If you specify a token it will only be used if the current version is - tokenless or if the provided token matches the token used in the current - version. -HELP - exit 1 -} - -function update_version { - # Comment out periods so they are interpreted as periods and don't - # just match any character - old_version_regex=${1//\./\\\.} - - echo Changing version from "$1" to "$2" - tmp_file=/tmp/version.$$ - sed "s/$old_version_regex/$2/" $VERSION_FILE > $tmp_file - mv $tmp_file $VERSION_FILE - sed "s/$old_version_regex/$2/" $README_FILE > $tmp_file - mv $tmp_file $README_FILE - git add $VERSION_FILE $README_FILE - git commit --message "$3" -} - -if [ $# -lt 1 ] || [ $# -gt 2 ]; then - usage -else - old_version=$(sed -n "s/^__version__ = \"\(.*\)\"$/\1/p" $VERSION_FILE) - case $1 in - major | minor | patch) - if [ $# -ne 1 ]; then - usage - fi - new_version=$(python -c "import semver; print(semver.bump_$1('$old_version'))") - update_version "$old_version" "$new_version" "Bump version from $old_version to $new_version" - ;; - build | prerelease) - if [ $# -eq 2 ]; then - new_version=$(python -c "import semver; print(semver.bump_$1('$old_version', token='$2'))") - else - new_version=$(python -c "import semver; print(semver.bump_$1('$old_version'))") - fi - update_version "$old_version" "$new_version" "Bump version from $old_version to $new_version" - ;; - finalize) - if [ $# -ne 1 ]; then - usage - fi - new_version=$(python -c "import semver; print(semver.finalize_version('$old_version'))") - update_version "$old_version" "$new_version" "Finalize version from $old_version to $new_version" - ;; - show) - echo "$old_version" - ;; - *) - usage - ;; - esac -fi diff --git a/tag.sh b/tag.sh index e1f7447..0a0e607 100755 --- a/tag.sh +++ b/tag.sh @@ -4,6 +4,6 @@ set -o nounset set -o errexit set -o pipefail -version=$(./bump_version.sh show) +version=$(./bump-version show) git tag "v$version" && git push --tags From 942c0dc98f605282fdf3c0ac6b9a549647f89f41 Mon Sep 17 00:00:00 2001 From: Nicholas McDonnell <50747025+mcdonnnj@users.noreply.github.com> Date: Tue, 13 Aug 2024 06:17:33 -0400 Subject: [PATCH 108/170] Add a new trigger for the sync-labels GitHub Actions workflow Add a `workflow_dispatch` trigger so we can manually run the workflow if needed. --- .github/workflows/sync-labels.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/sync-labels.yml b/.github/workflows/sync-labels.yml index e83bd41..59aefe4 100644 --- a/.github/workflows/sync-labels.yml +++ b/.github/workflows/sync-labels.yml @@ -6,6 +6,7 @@ on: paths: - '.github/labels.yml' - '.github/workflows/sync-labels.yml' + workflow_dispatch: permissions: contents: read From a267662455c30986086d4ca14173cc20af7161d4 Mon Sep 17 00:00:00 2001 From: Nicholas McDonnell <50747025+mcdonnnj@users.noreply.github.com> Date: Tue, 13 Aug 2024 06:19:38 -0400 Subject: [PATCH 109/170] Remove unnecessary quotes in the sync-labels workflow --- .github/workflows/sync-labels.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/sync-labels.yml b/.github/workflows/sync-labels.yml index 59aefe4..5d5ab41 100644 --- a/.github/workflows/sync-labels.yml +++ b/.github/workflows/sync-labels.yml @@ -4,8 +4,8 @@ name: sync-labels on: push: paths: - - '.github/labels.yml' - - '.github/workflows/sync-labels.yml' + - .github/labels.yml + - .github/workflows/sync-labels.yml workflow_dispatch: permissions: From dc7f09e29b8466af0fa2f788761e22dd2fcbd0ce Mon Sep 17 00:00:00 2001 From: Nicholas McDonnell <50747025+mcdonnnj@users.noreply.github.com> Date: Sat, 14 Sep 2024 18:44:01 -0400 Subject: [PATCH 110/170] Add four new hooks from pre-commit/pre-commit-hooks --- .pre-commit-config.yaml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 3cb1f85..c98ded8 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -16,9 +16,13 @@ repos: - id: check-executables-have-shebangs - id: check-json - id: check-merge-conflict + - id: check-shebang-scripts-are-executable + - id: check-symlinks - id: check-toml + - id: check-vcs-permalinks - id: check-xml - id: debug-statements + - id: destroyed-symlinks - id: detect-aws-credentials args: - --allow-missing-credentials From 343d2ccbd1cd983374235e5d3bfcecd3187c00d5 Mon Sep 17 00:00:00 2001 From: Nicholas McDonnell <50747025+mcdonnnj@users.noreply.github.com> Date: Mon, 28 Oct 2024 17:47:53 -0400 Subject: [PATCH 111/170] Add the GitHubSecurityLab/actions-permissions/monitor Action This Action will provide information about the usage of GITHUB_TOKEN in the workflow. It should be added to _every_ job in _any_ workflow to provide information for analysis. --- .github/dependabot.yml | 1 + .github/workflows/build.yml | 10 ++++++++++ .github/workflows/sync-labels.yml | 10 ++++++++++ 3 files changed, 21 insertions(+) diff --git a/.github/dependabot.yml b/.github/dependabot.yml index 17220c6..4a6667f 100644 --- a/.github/dependabot.yml +++ b/.github/dependabot.yml @@ -16,6 +16,7 @@ updates: # - dependency-name: crazy-max/ghaction-dump-context # - dependency-name: crazy-max/ghaction-github-labeler # - dependency-name: crazy-max/ghaction-github-status + # - dependency-name: GitHubSecurityLab/actions-permissions # - dependency-name: hashicorp/setup-terraform # - dependency-name: mxschmitt/action-tmate # - dependency-name: step-security/harden-runner diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index e7a60b2..2cdd921 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -34,6 +34,12 @@ jobs: steps: # Note that a duplicate of this step must be added at the top of # each job. + - uses: GitHubSecurityLab/actions-permissions/monitor@v1 + with: + # Uses the organization variable unless overridden + config: ${{ vars.ACTIONS_PERMISSIONS_CONFIG }} + # Note that a duplicate of this step must be added at the top of + # each job. - id: harden-runner name: Harden the runner uses: step-security/harden-runner@v2 @@ -50,6 +56,10 @@ jobs: - diagnostics runs-on: ubuntu-latest steps: + - uses: GitHubSecurityLab/actions-permissions/monitor@v1 + with: + # Uses the organization variable unless overridden + config: ${{ vars.ACTIONS_PERMISSIONS_CONFIG }} - id: harden-runner name: Harden the runner uses: step-security/harden-runner@v2 diff --git a/.github/workflows/sync-labels.yml b/.github/workflows/sync-labels.yml index e83bd41..d2458d1 100644 --- a/.github/workflows/sync-labels.yml +++ b/.github/workflows/sync-labels.yml @@ -17,6 +17,12 @@ jobs: steps: # Note that a duplicate of this step must be added at the top of # each job. + - uses: GitHubSecurityLab/actions-permissions/monitor@v1 + with: + # Uses the organization variable unless overridden + config: ${{ vars.ACTIONS_PERMISSIONS_CONFIG }} + # Note that a duplicate of this step must be added at the top of + # each job. - id: harden-runner name: Harden the runner uses: step-security/harden-runner@v2 @@ -38,6 +44,10 @@ jobs: issues: write runs-on: ubuntu-latest steps: + - uses: GitHubSecurityLab/actions-permissions/monitor@v1 + with: + # Uses the organization variable unless overridden + config: ${{ vars.ACTIONS_PERMISSIONS_CONFIG }} - id: harden-runner name: Harden the runner uses: step-security/harden-runner@v2 From 8a77a8b77a7d5e5247e8ff563d93a14510e09b9a Mon Sep 17 00:00:00 2001 From: Nicholas McDonnell <50747025+mcdonnnj@users.noreply.github.com> Date: Mon, 28 Oct 2024 17:09:15 -0400 Subject: [PATCH 112/170] Restrict permissions of GITHUB_TOKEN This changes the default permissions for the GITHUB_TOKEN used in our GitHub Actions configuration to the minimum required to successfully run. --- .github/workflows/build.yml | 5 +++++ .github/workflows/sync-labels.yml | 2 ++ 2 files changed, 7 insertions(+) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index e7a60b2..d4340af 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -30,6 +30,8 @@ env: jobs: diagnostics: name: Run diagnostics + # This job does not need any permissions + permissions: {} runs-on: ubuntu-latest steps: # Note that a duplicate of this step must be added at the top of @@ -48,6 +50,9 @@ jobs: lint: needs: - diagnostics + permissions: + # actions/checkout needs this to fetch code + contents: read runs-on: ubuntu-latest steps: - id: harden-runner diff --git a/.github/workflows/sync-labels.yml b/.github/workflows/sync-labels.yml index e83bd41..39e7379 100644 --- a/.github/workflows/sync-labels.yml +++ b/.github/workflows/sync-labels.yml @@ -13,6 +13,8 @@ permissions: jobs: diagnostics: name: Run diagnostics + # This job does not need any permissions + permissions: {} runs-on: ubuntu-latest steps: # Note that a duplicate of this step must be added at the top of From 3b1d4ef0fae08e6444e9b414ce1315841e681322 Mon Sep 17 00:00:00 2001 From: Nicholas McDonnell <50747025+mcdonnnj@users.noreply.github.com> Date: Wed, 16 Oct 2024 12:53:42 -0400 Subject: [PATCH 113/170] Update pre-commit hook versions This is done automatically with the `pre-commit autoupdate` command. --- .pre-commit-config.yaml | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 3cb1f85..26b399d 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -10,7 +10,7 @@ repos: - id: check-useless-excludes - repo: https://github.com/pre-commit/pre-commit-hooks - rev: v4.6.0 + rev: v5.0.0 hooks: - id: check-case-conflict - id: check-executables-have-shebangs @@ -53,14 +53,14 @@ repos: # GitHub Actions hooks - repo: https://github.com/python-jsonschema/check-jsonschema - rev: 0.29.2 + rev: 0.29.4 hooks: - id: check-github-actions - id: check-github-workflows # pre-commit hooks - repo: https://github.com/pre-commit/pre-commit - rev: v3.8.0 + rev: v4.0.1 hooks: - id: validate_manifest @@ -95,7 +95,7 @@ repos: # Shell script hooks - repo: https://github.com/scop/pre-commit-shfmt - rev: v3.9.0-1 + rev: v3.10.0-1 hooks: - id: shfmt args: @@ -125,7 +125,7 @@ repos: args: - --config=.bandit.yml - repo: https://github.com/psf/black-pre-commit-mirror - rev: 24.8.0 + rev: 24.10.0 hooks: - id: black - repo: https://github.com/PyCQA/flake8 @@ -139,7 +139,7 @@ repos: hooks: - id: isort - repo: https://github.com/pre-commit/mirrors-mypy - rev: v1.11.2 + rev: v1.13.0 hooks: - id: mypy - repo: https://github.com/pypa/pip-audit @@ -155,7 +155,7 @@ repos: - --requirement - requirements.txt - repo: https://github.com/asottile/pyupgrade - rev: v3.17.0 + rev: v3.19.0 hooks: - id: pyupgrade @@ -199,7 +199,7 @@ repos: # Packer hooks - repo: https://github.com/cisagov/pre-commit-packer - rev: v0.1.0 + rev: v0.3.0 hooks: - id: packer_validate - id: packer_fmt From 1d285f2d851926effdbfbdcf58853ce70d1bf016 Mon Sep 17 00:00:00 2001 From: Nicholas McDonnell <50747025+mcdonnnj@users.noreply.github.com> Date: Tue, 29 Oct 2024 16:36:27 -0400 Subject: [PATCH 114/170] Sort hook ids in each pre-commit hook entry Ensure that all hook ids are sorted alphabetically in each hook entry in our pre-commit configuration. --- .pre-commit-config.yaml | 24 ++++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 3cb1f85..0fd3234 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -68,25 +68,25 @@ repos: - repo: https://github.com/TekWizely/pre-commit-golang rev: v1.0.0-rc.1 hooks: - # Style Checkers - - id: go-critic - # StaticCheck - - id: go-staticcheck-repo-mod # Go Build - id: go-build-repo-mod + # Style Checkers + - id: go-critic + # goimports + - id: go-imports-repo + args: + # Write changes to files + - -w # Go Mod Tidy - id: go-mod-tidy-repo + # GoSec + - id: go-sec-repo-mod + # StaticCheck + - id: go-staticcheck-repo-mod # Go Test - id: go-test-repo-mod # Go Vet - id: go-vet-repo-mod - # GoSec - - id: go-sec-repo-mod - # goimports - - id: go-imports-repo - args: - # Write changes to files - - -w # Nix hooks - repo: https://github.com/nix-community/nixpkgs-fmt rev: v1.3.0 @@ -201,5 +201,5 @@ repos: - repo: https://github.com/cisagov/pre-commit-packer rev: v0.1.0 hooks: - - id: packer_validate - id: packer_fmt + - id: packer_validate From 8824475dfadd1a9cbc9ce9bd1c9f31e4a688994b Mon Sep 17 00:00:00 2001 From: Nicholas McDonnell <50747025+mcdonnnj@users.noreply.github.com> Date: Fri, 1 Nov 2024 12:25:35 -0400 Subject: [PATCH 115/170] Update the commented out dependabot ignore directives Add a directive for hashicorp/setup-packer that was missed when it was added to the `build` workflow. Add a directive for cisagov/setup-env-github-action that is not strictly necessary since we currently just pull from the `develop` branch, but is good to have in case we were to change that in the future. --- .github/dependabot.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/dependabot.yml b/.github/dependabot.yml index 4a6667f..81cd6bd 100644 --- a/.github/dependabot.yml +++ b/.github/dependabot.yml @@ -13,10 +13,12 @@ updates: # - dependency-name: actions/checkout # - dependency-name: actions/setup-go # - dependency-name: actions/setup-python + # - dependency-name: cisagov/setup-env-github-action # - dependency-name: crazy-max/ghaction-dump-context # - dependency-name: crazy-max/ghaction-github-labeler # - dependency-name: crazy-max/ghaction-github-status # - dependency-name: GitHubSecurityLab/actions-permissions + # - dependency-name: hashicorp/setup-packer # - dependency-name: hashicorp/setup-terraform # - dependency-name: mxschmitt/action-tmate # - dependency-name: step-security/harden-runner From 12a91ad97e76cd2f221fffaef4f66956533f6540 Mon Sep 17 00:00:00 2001 From: Jeremy Frasier Date: Fri, 8 Nov 2024 13:40:44 -0500 Subject: [PATCH 116/170] Bump up the lower bound on ansible-core This is being done because the pip-audit pre-commit hook identifies a vulnerability in ansible-core version 2.16.13. Note that this requires that we bump up ansible to version 10 since all versions of ansible 9 have a dependency on ~=2.16.X. --- .pre-commit-config.yaml | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index c5e1096..8b402fb 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -176,17 +176,25 @@ repos: # necessary to add the ansible package itself as an # additional dependency, with the same pinning as is done in # requirements-test.txt of cisagov/skeleton-ansible-role. - # - ansible>=9,<10 + # + # Version 10 is required because the pip-audit pre-commit + # hook identifies a vulnerability in ansible-core 2.16.13, + # but all versions of ansible 9 have a dependency on + # ~=2.16.X. + # - ansible>=10,<11 # ansible-core 2.16.3 through 2.16.6 suffer from the bug # discussed in ansible/ansible#82702, which breaks any # symlinked files in vars, tasks, etc. for any Ansible role # installed via ansible-galaxy. Hence we never want to # install those versions. # + # Note that the pip-audit pre-commit hook identifies a + # vulnerability in ansible-core 2.16.13. + # # Note that any changes made to this dependency must also be # made in requirements.txt in cisagov/skeleton-packer and # requirements-test.txt in cisagov/skeleton-ansible-role. - - ansible-core>=2.16.7 + - ansible-core>2.16.13 # Terraform hooks - repo: https://github.com/antonbabenko/pre-commit-terraform From b9f798d03afb72f33ffa625982dd5b548dea5132 Mon Sep 17 00:00:00 2001 From: Jeremy Frasier Date: Wed, 13 Nov 2024 10:29:42 -0500 Subject: [PATCH 117/170] Update the version of the ansible-lint pre-commit hook Version 24.10.0 is the first version that supports Fedora 41 as a valid platform. --- .pre-commit-config.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index c5e1096..ebd6138 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -165,7 +165,7 @@ repos: # Ansible hooks - repo: https://github.com/ansible/ansible-lint - rev: v24.9.2 + rev: v24.10.0 hooks: - id: ansible-lint additional_dependencies: From cca133a2710c5ed99e4c0ce3d06a57ec118bcf13 Mon Sep 17 00:00:00 2001 From: Jeremy Frasier Date: Wed, 13 Nov 2024 21:33:32 -0500 Subject: [PATCH 118/170] Adjust pin for ansible-core The pin of ansible-core was originally put in place because the pip-audit pre-commit hook identifies a vulnerability in ansible-core 2.16.13. Normally we would pin ansible-core to >2.16.13, but in the spirit of the earlier, optional pin of ansible>=10 we pin ansible-core to >=2.17. This effectively also pins ansible to >=10. Co-authored-by: Nick M <50747025+mcdonnnj@users.noreply.github.com> --- .pre-commit-config.yaml | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 8b402fb..b61a8f5 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -189,12 +189,14 @@ repos: # install those versions. # # Note that the pip-audit pre-commit hook identifies a - # vulnerability in ansible-core 2.16.13. + # vulnerability in ansible-core 2.16.13. The pin of + # ansible-core to >=2.17 effectively also pins ansible to + # >=10. # # Note that any changes made to this dependency must also be # made in requirements.txt in cisagov/skeleton-packer and # requirements-test.txt in cisagov/skeleton-ansible-role. - - ansible-core>2.16.13 + - ansible-core>=2.17 # Terraform hooks - repo: https://github.com/antonbabenko/pre-commit-terraform From bd852610595fdd2eee77f489d4b184f88d90643b Mon Sep 17 00:00:00 2001 From: Jeremy Frasier Date: Wed, 20 Nov 2024 12:21:14 -0500 Subject: [PATCH 119/170] Add comments about looming EOL issues for ansible and ansible-core This adds even more evidence for why it is a good idea to go ahead and upgrade ansible and ansible-core, in addition to the vulnerability that pip-audit turned up. Co-authored-by: Nick M <50747025+mcdonnnj@users.noreply.github.com> --- .pre-commit-config.yaml | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index b61a8f5..97fbf1c 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -181,6 +181,10 @@ repos: # hook identifies a vulnerability in ansible-core 2.16.13, # but all versions of ansible 9 have a dependency on # ~=2.16.X. + # + # It is also a good idea to go ahead and upgrade to version + # 10 since version 9 is going EOL at the end of November: + # https://endoflife.date/ansible # - ansible>=10,<11 # ansible-core 2.16.3 through 2.16.6 suffer from the bug # discussed in ansible/ansible#82702, which breaks any @@ -193,6 +197,11 @@ repos: # ansible-core to >=2.17 effectively also pins ansible to # >=10. # + # It is also a good idea to go ahead and upgrade to + # ansible-core 2.17 since security support for ansible-core + # 2.16 ends this month: + # https://docs.ansible.com/ansible/devel/reference_appendices/release_and_maintenance.html#ansible-core-support-matrix + # # Note that any changes made to this dependency must also be # made in requirements.txt in cisagov/skeleton-packer and # requirements-test.txt in cisagov/skeleton-ansible-role. From 26ccf289eb4c09a4be02bd7e18c018c6c4b77a9b Mon Sep 17 00:00:00 2001 From: Nicholas McDonnell <50747025+mcdonnnj@users.noreply.github.com> Date: Thu, 21 Nov 2024 16:58:03 -0500 Subject: [PATCH 120/170] Enable new dependabot ignore directives --- .github/dependabot.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/dependabot.yml b/.github/dependabot.yml index b4a8084..d01227a 100644 --- a/.github/dependabot.yml +++ b/.github/dependabot.yml @@ -18,12 +18,12 @@ updates: - dependency-name: actions/checkout - dependency-name: actions/setup-go - dependency-name: actions/setup-python - # - dependency-name: cisagov/setup-env-github-action + - dependency-name: cisagov/setup-env-github-action - dependency-name: crazy-max/ghaction-dump-context - dependency-name: crazy-max/ghaction-github-labeler - dependency-name: crazy-max/ghaction-github-status - # - dependency-name: GitHubSecurityLab/actions-permissions - # - dependency-name: hashicorp/setup-packer + - dependency-name: GitHubSecurityLab/actions-permissions + - dependency-name: hashicorp/setup-packer - dependency-name: hashicorp/setup-terraform - dependency-name: mxschmitt/action-tmate - dependency-name: step-security/harden-runner From ec72a4b015fc418a0be625fb1f0ad2316e028811 Mon Sep 17 00:00:00 2001 From: Jeremy Frasier Date: Wed, 25 Sep 2024 09:51:30 -0400 Subject: [PATCH 121/170] Upgrade instances of actions/cache to v4 --- .github/workflows/build.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 2b2bd00..ad412e0 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -320,7 +320,7 @@ jobs: - name: Set up Docker Buildx uses: docker/setup-buildx-action@v3 - name: Cache Docker layers - uses: actions/cache@v3 + uses: actions/cache@v4 env: BASE_CACHE_KEY: buildx-${{ runner.os }}- with: @@ -401,7 +401,7 @@ jobs: with: python-version: "3.11" - name: Cache testing environments - uses: actions/cache@v3 + uses: actions/cache@v4 env: BASE_CACHE_KEY: "${{ github.job }}-${{ runner.os }}-\ py${{ steps.setup-python.outputs.python-version }}-" @@ -473,7 +473,7 @@ jobs: - name: Set up Docker Buildx uses: docker/setup-buildx-action@v3 - name: Cache Docker layers - uses: actions/cache@v3 + uses: actions/cache@v4 env: BASE_CACHE_KEY: buildx-${{ runner.os }}- with: From 4301e1c8f1475f7b179c325a1ce765b46065e4ec Mon Sep 17 00:00:00 2001 From: Jeremy Frasier Date: Wed, 25 Sep 2024 09:52:05 -0400 Subject: [PATCH 122/170] Upgrade instance of actions/setup-python to v5 Also pull the Python version from steps.setup-env.outputs instead of hardcoding it. --- .github/workflows/build.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index ad412e0..fa6bc81 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -397,9 +397,9 @@ jobs: egress-policy: audit - uses: actions/checkout@v4 - id: setup-python - uses: actions/setup-python@v4 + uses: actions/setup-python@v5 with: - python-version: "3.11" + python-version: ${{ steps.setup-env.outputs.python-version }} - name: Cache testing environments uses: actions/cache@v4 env: From 8aeb3b6d708ab1a064c5577cf867269dfe281122 Mon Sep 17 00:00:00 2001 From: Nicholas McDonnell <50747025+mcdonnnj@users.noreply.github.com> Date: Thu, 21 Nov 2024 17:01:14 -0500 Subject: [PATCH 123/170] Upgrade instances of crazy-max/ghaction-github-status to v4 --- .github/workflows/codeql-analysis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/codeql-analysis.yml b/.github/workflows/codeql-analysis.yml index dc49271..382d3d1 100644 --- a/.github/workflows/codeql-analysis.yml +++ b/.github/workflows/codeql-analysis.yml @@ -33,7 +33,7 @@ jobs: egress-policy: audit - id: github-status name: Check GitHub status - uses: crazy-max/ghaction-github-status@v3 + uses: crazy-max/ghaction-github-status@v4 - id: dump-context name: Dump context uses: crazy-max/ghaction-dump-context@v2 From 31adcba8d17211611389315dc92da966a319687d Mon Sep 17 00:00:00 2001 From: Nicholas McDonnell <50747025+mcdonnnj@users.noreply.github.com> Date: Thu, 21 Nov 2024 17:14:08 -0500 Subject: [PATCH 124/170] Alphabetize entries in the build workflow We prefer to alphabetize mapping keys in YAML documents whenever possible. This ensures any parts specific to this repository are now alphabetized. --- .github/workflows/build.yml | 64 ++++++++++++++++++------------------- 1 file changed, 32 insertions(+), 32 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index fa6bc81..53f65cf 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -5,28 +5,28 @@ on: merge_group: types: - checks_requested + pull_request: push: branches: - "**" tags: - "v*.*.*" - pull_request: - schedule: - - cron: "0 10 * * *" # everyday at 10am repository_dispatch: # Respond to rebuild requests. See: https://github.com/cisagov/action-apb/ types: - apb + schedule: + - cron: "0 10 * * *" # everyday at 10am workflow_dispatch: inputs: - remote-shell: - description: "Debug with remote shell" - required: true - default: "false" image-tag: + default: "dispatch" description: "Tag to apply to pushed images" required: true - default: "dispatch" + remote-shell: + default: "false" + description: "Debug with remote shell" + required: true # Set a default shell for any run steps. The `-Eueo pipefail` sets errtrace, # nounset, errexit, and pipefail. The `-x` will print all commands as they are @@ -324,8 +324,8 @@ jobs: env: BASE_CACHE_KEY: buildx-${{ runner.os }}- with: - path: ${{ env.BUILDX_CACHE_DIR }} key: ${{ env.BASE_CACHE_KEY }}${{ github.sha }} + path: ${{ env.BUILDX_CACHE_DIR }} restore-keys: | ${{ env.BASE_CACHE_KEY }} - name: Create dist directory @@ -340,15 +340,6 @@ jobs: cache-to: type=local,dest=${{ env.BUILDX_CACHE_DIR }} context: . file: ./Dockerfile - outputs: type=docker,dest=dist/image.tar - # Uncomment the following option if you are building an image for use - # on Google Cloud Run or AWS Lambda. The current default image output - # is unable to run on either. Please see the following issue for more - # information: https://github.com/docker/buildx/issues/1533 - # provenance: false - tags: ${{ env.IMAGE_NAME }}:latest # not to be pushed - # For a list of pre-defined annotation keys and value types see: - # https://github.com/opencontainers/image-spec/blob/master/annotations.md labels: "\ org.opencontainers.image.created=${{ needs.prepare.outputs.created }} @@ -372,6 +363,15 @@ jobs: org.opencontainers.image.version=${{ needs.prepare.outputs.source_version }}" + outputs: type=docker,dest=dist/image.tar + # Uncomment the following option if you are building an image for use + # on Google Cloud Run or AWS Lambda. The current default image output + # is unable to run on either. Please see the following issue for more + # information: https://github.com/docker/buildx/issues/1533 + # provenance: false + tags: ${{ env.IMAGE_NAME }}:latest # not to be pushed + # For a list of pre-defined annotation keys and value types see: + # https://github.com/opencontainers/image-spec/blob/master/annotations.md - name: Compress image run: gzip dist/image.tar - name: Upload artifacts @@ -406,10 +406,10 @@ jobs: BASE_CACHE_KEY: "${{ github.job }}-${{ runner.os }}-\ py${{ steps.setup-python.outputs.python-version }}-" with: - path: ${{ env.PIP_CACHE_DIR }} key: "${{ env.BASE_CACHE_KEY }}\ ${{ hashFiles('**/requirements-test.txt') }}-\ ${{ hashFiles('**/requirements.txt') }}" + path: ${{ env.PIP_CACHE_DIR }} restore-keys: | ${{ env.BASE_CACHE_KEY }} - name: Install dependencies @@ -458,14 +458,14 @@ jobs: - name: Login to Docker Hub uses: docker/login-action@v3 with: - username: ${{ secrets.DOCKER_USERNAME }} password: ${{ secrets.DOCKER_PASSWORD }} + username: ${{ secrets.DOCKER_USERNAME }} - name: Login to GitHub Container Registry uses: docker/login-action@v3 with: + password: ${{ secrets.GITHUB_TOKEN }} registry: ghcr.io username: ${{ github.actor }} - password: ${{ secrets.GITHUB_TOKEN }} - name: Checkout uses: actions/checkout@v4 - name: Set up QEMU @@ -477,8 +477,8 @@ jobs: env: BASE_CACHE_KEY: buildx-${{ runner.os }}- with: - path: ${{ env.BUILDX_CACHE_DIR }} key: ${{ env.BASE_CACHE_KEY }}${{ github.sha }} + path: ${{ env.BUILDX_CACHE_DIR }} restore-keys: | ${{ env.BASE_CACHE_KEY }} - name: Create cross-platform support Dockerfile-x @@ -493,16 +493,6 @@ jobs: cache-to: type=local,dest=${{ env.BUILDX_CACHE_DIR }} context: . file: ./Dockerfile-x - platforms: ${{ env.PLATFORMS }} - # Uncomment the following option if you are building an image for use - # on Google Cloud Run or AWS Lambda. The current default image output - # is unable to run on either. Please see the following issue for more - # information: https://github.com/docker/buildx/issues/1533 - # provenance: false - push: true - tags: ${{ needs.prepare.outputs.tags }} - # For a list of pre-defined annotation keys and value types see: - # https://github.com/opencontainers/image-spec/blob/master/annotations.md labels: "\ org.opencontainers.image.created=${{ needs.prepare.outputs.created }} @@ -526,6 +516,16 @@ jobs: org.opencontainers.image.version=${{ needs.prepare.outputs.source_version }}" + platforms: ${{ env.PLATFORMS }} + # Uncomment the following option if you are building an image for use + # on Google Cloud Run or AWS Lambda. The current default image output + # is unable to run on either. Please see the following issue for more + # information: https://github.com/docker/buildx/issues/1533 + # provenance: false + push: true + tags: ${{ needs.prepare.outputs.tags }} + # For a list of pre-defined annotation keys and value types see: + # https://github.com/opencontainers/image-spec/blob/master/annotations.md - name: Publish README.md to Docker Hub env: DOCKER_PASSWORD: ${{ secrets.DOCKER_PASSWORD }} From 458bbf5a874dfcffc9a40217b1bf81bba98d01cf Mon Sep 17 00:00:00 2001 From: Nicholas McDonnell <50747025+mcdonnnj@users.noreply.github.com> Date: Thu, 21 Nov 2024 17:31:33 -0500 Subject: [PATCH 125/170] Alphabetize entries in the CodeQL workflow We prefer to alphabetize mapping keys in YAML documents whenever possible. A comment was updated to reflect the change in order. --- .github/workflows/codeql-analysis.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/codeql-analysis.yml b/.github/workflows/codeql-analysis.yml index 382d3d1..2fdf6e9 100644 --- a/.github/workflows/codeql-analysis.yml +++ b/.github/workflows/codeql-analysis.yml @@ -7,15 +7,15 @@ name: CodeQL on: + pull_request: + # The branches here must be a subset of the ones in the push key + branches: + - develop push: # Dependabot triggered push events have read-only access, but uploading code # scanning requires write access. branches-ignore: - dependabot/** - pull_request: - # The branches below must be a subset of the branches above - branches: - - develop schedule: - cron: '0 21 * * 6' From 9342f0cd6709d53e3dd55cf9b53ee761bc4fdffd Mon Sep 17 00:00:00 2001 From: Nicholas McDonnell <50747025+mcdonnnj@users.noreply.github.com> Date: Thu, 21 Nov 2024 17:33:56 -0500 Subject: [PATCH 126/170] Add a merge_group trigger to the CodeQL workflow This should improve compatibility with merge queues. We configure it to only trigger on the `checks_requested` type which is currently the only supported type for this trigger. If additional types are added in the future they should be added if appropriate. --- .github/workflows/codeql-analysis.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.github/workflows/codeql-analysis.yml b/.github/workflows/codeql-analysis.yml index 2fdf6e9..f0df652 100644 --- a/.github/workflows/codeql-analysis.yml +++ b/.github/workflows/codeql-analysis.yml @@ -7,6 +7,9 @@ name: CodeQL on: + merge_group: + types: + - checks_requested pull_request: # The branches here must be a subset of the ones in the push key branches: From 7f04a42ba010435f6d0a0de763692bc4f034d238 Mon Sep 17 00:00:00 2001 From: Nicholas McDonnell <50747025+mcdonnnj@users.noreply.github.com> Date: Thu, 21 Nov 2024 17:36:09 -0500 Subject: [PATCH 127/170] Monitor GITHUB_TOKEN permissions for all jobs in the build workflow This ensures the GitHubSecurityLab/actions-permissions/monitor Action is used in every job in the workflow. --- .github/workflows/build.yml | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 53f65cf..aa071dc 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -243,6 +243,10 @@ jobs: tags: ${{ steps.prep.outputs.tags }} runs-on: ubuntu-latest steps: + - uses: GitHubSecurityLab/actions-permissions/monitor@v1 + with: + # Uses the organization variable unless overridden + config: ${{ vars.ACTIONS_PERMISSIONS_CONFIG }} - id: harden-runner name: Harden the runner uses: step-security/harden-runner@v2 @@ -308,6 +312,10 @@ jobs: - prepare runs-on: ubuntu-latest steps: + - uses: GitHubSecurityLab/actions-permissions/monitor@v1 + with: + # Uses the organization variable unless overridden + config: ${{ vars.ACTIONS_PERMISSIONS_CONFIG }} - id: harden-runner name: Harden the runner uses: step-security/harden-runner@v2 @@ -390,6 +398,10 @@ jobs: - build runs-on: ubuntu-latest steps: + - uses: GitHubSecurityLab/actions-permissions/monitor@v1 + with: + # Uses the organization variable unless overridden + config: ${{ vars.ACTIONS_PERMISSIONS_CONFIG }} - id: harden-runner name: Harden the runner uses: step-security/harden-runner@v2 @@ -450,6 +462,10 @@ jobs: packages: write runs-on: ubuntu-latest steps: + - uses: GitHubSecurityLab/actions-permissions/monitor@v1 + with: + # Uses the organization variable unless overridden + config: ${{ vars.ACTIONS_PERMISSIONS_CONFIG }} - id: harden-runner name: Harden the runner uses: step-security/harden-runner@v2 From b9639f279ddd273bdcfc8f2be1e9f9f17ef8de9d Mon Sep 17 00:00:00 2001 From: Nicholas McDonnell <50747025+mcdonnnj@users.noreply.github.com> Date: Thu, 21 Nov 2024 17:41:43 -0500 Subject: [PATCH 128/170] Monitor GITHUB_TOKEN permissions for all jobs in the CodeQL workflow This ensures the GitHubSecurityLab/actions-permissions/monitor Action is used in every job in the workflow. --- .github/workflows/codeql-analysis.yml | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/.github/workflows/codeql-analysis.yml b/.github/workflows/codeql-analysis.yml index f0df652..3844efc 100644 --- a/.github/workflows/codeql-analysis.yml +++ b/.github/workflows/codeql-analysis.yml @@ -29,6 +29,12 @@ jobs: steps: # Note that a duplicate of this step must be added at the top of # each job. + - uses: GitHubSecurityLab/actions-permissions/monitor@v1 + with: + # Uses the organization variable unless overridden + config: ${{ vars.ACTIONS_PERMISSIONS_CONFIG }} + # Note that a duplicate of this step must be added at the top of + # each job. - id: harden-runner name: Harden the runner uses: step-security/harden-runner@v2 @@ -59,6 +65,10 @@ jobs: # https://docs.github.com/en/github/finding-security-vulnerabilities-and-errors-in-your-code/configuring-code-scanning#overriding-automatic-language-detection steps: + - uses: GitHubSecurityLab/actions-permissions/monitor@v1 + with: + # Uses the organization variable unless overridden + config: ${{ vars.ACTIONS_PERMISSIONS_CONFIG }} - id: harden-runner name: Harden the runner uses: step-security/harden-runner@v2 From 805d0fd2569f8fdae405b95bdd51f9bd91b09e17 Mon Sep 17 00:00:00 2001 From: Jeremy Frasier Date: Wed, 25 Sep 2024 09:49:19 -0400 Subject: [PATCH 129/170] Upgrade bandit to 1.7.10 We should use the same version of bandit throughout the pre-commit configuration. --- .pre-commit-config.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index a4c10db..4d0c4a9 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -133,7 +133,7 @@ repos: - --config=.bandit.yml # Run bandit on everything except the "tests" tree - repo: https://github.com/PyCQA/bandit - rev: 1.7.5 + rev: 1.7.10 hooks: - id: bandit name: bandit (everything else) From 05169bcf069a715ffa007c79457a587b146ebe6d Mon Sep 17 00:00:00 2001 From: Jeremy Frasier Date: Wed, 25 Sep 2024 09:50:10 -0400 Subject: [PATCH 130/170] Add whitespace to placate black pre-commit hook --- tests/conftest.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/conftest.py b/tests/conftest.py index 054f09d..29a8b92 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -2,6 +2,7 @@ https://docs.pytest.org/en/latest/writing_plugins.html#conftest-py-plugins """ + # Third-Party Libraries import pytest from python_on_whales import docker From 0bfb66241cb555b1e1b32adf53b7068aca32949a Mon Sep 17 00:00:00 2001 From: Nicholas McDonnell <50747025+mcdonnnj@users.noreply.github.com> Date: Fri, 22 Nov 2024 04:04:24 -0500 Subject: [PATCH 131/170] Remove needless shebang --- tests/container_test.py | 1 - 1 file changed, 1 deletion(-) diff --git a/tests/container_test.py b/tests/container_test.py index c2e1874..00c8e82 100644 --- a/tests/container_test.py +++ b/tests/container_test.py @@ -1,4 +1,3 @@ -#!/usr/bin/env pytest -vs """Tests for example container.""" # Standard Python Libraries From f5dd0463743633e9e8f202a2dddfc1c9e522688d Mon Sep 17 00:00:00 2001 From: Nicholas McDonnell <50747025+mcdonnnj@users.noreply.github.com> Date: Fri, 22 Nov 2024 04:10:14 -0500 Subject: [PATCH 132/170] Add missing use of the cisagov/setup-env-github-action Action Now that the actions/setup-python Action in the test job expects the Python version to be supplied by cisagov/setup-env-github-action we need to make sure the Action is run in the job. --- .github/workflows/build.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index aa071dc..eb883ea 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -407,6 +407,8 @@ jobs: uses: step-security/harden-runner@v2 with: egress-policy: audit + - id: setup-env + uses: cisagov/setup-env-github-action@develop - uses: actions/checkout@v4 - id: setup-python uses: actions/setup-python@v5 From ee5a0af9c7ddadb1d22c9529a540b37159b1b4d3 Mon Sep 17 00:00:00 2001 From: Nicholas McDonnell <50747025+mcdonnnj@users.noreply.github.com> Date: Fri, 22 Nov 2024 04:29:48 -0500 Subject: [PATCH 133/170] Change the port used as an example in the Docker composition Since no actual networking is going on inside the image this change is non-destructive. I am making it because we are now using the GitHubSecurityLab/actions-permissions/monitor Action and the proxy it configures uses port 8080. There are currently no options for configuring this in the Action and the easiest path forward in this case is to simply change the port number used in the Docker composition. --- docker-compose.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docker-compose.yml b/docker-compose.yml index e746ae9..fa067b7 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -21,8 +21,8 @@ services: environment: - ECHO_MESSAGE=Hello World from docker compose! ports: - - target: 8080 - published: 8080 + - target: 8421 + published: 8421 protocol: tcp mode: host secrets: From b342704ebf0892e198332048fdd2de9d13d9aca3 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Fri, 22 Nov 2024 18:29:48 +0000 Subject: [PATCH 134/170] Bump docker/build-push-action from 5 to 6 Bumps [docker/build-push-action](https://github.com/docker/build-push-action) from 5 to 6. - [Release notes](https://github.com/docker/build-push-action/releases) - [Commits](https://github.com/docker/build-push-action/compare/v5...v6) --- updated-dependencies: - dependency-name: docker/build-push-action dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] --- .github/workflows/build.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index d8ba132..56c9a79 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -304,7 +304,7 @@ jobs: run: mkdir -p dist - name: Build image id: docker_build - uses: docker/build-push-action@v5 + uses: docker/build-push-action@v6 with: build-args: | VERSION=${{ needs.prepare.outputs.source_version }} @@ -457,7 +457,7 @@ jobs: run: ./buildx-dockerfile.sh - name: Build and push platform images to registries id: docker_build - uses: docker/build-push-action@v5 + uses: docker/build-push-action@v6 with: build-args: | VERSION=${{ needs.prepare.outputs.source_version }} From 26360aaca08a23c5738d96abb5527c4907d75534 Mon Sep 17 00:00:00 2001 From: Nicholas McDonnell <50747025+mcdonnnj@users.noreply.github.com> Date: Fri, 22 Nov 2024 13:31:43 -0500 Subject: [PATCH 135/170] Update job permissions in the `build` workflow Adjust permissions based on the output of the GitHubSecurityLab/actions-permissions/monitor Action. We have been explicitly declaring the `read` permission for `content` if a job uses actions/checkout. --- .github/workflows/build.yml | 15 +++++++++++++-- 1 file changed, 13 insertions(+), 2 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index eb883ea..6edf624 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -241,6 +241,9 @@ jobs: repometa: ${{ steps.repo.outputs.result }} source_version: ${{ steps.prep.outputs.source_version }} tags: ${{ steps.prep.outputs.tags }} + permissions: + # actions/checkout needs this to fetch code + contents: read runs-on: ubuntu-latest steps: - uses: GitHubSecurityLab/actions-permissions/monitor@v1 @@ -310,6 +313,9 @@ jobs: needs: - diagnostics - prepare + permissions: + # actions/checkout needs this to fetch code + contents: read runs-on: ubuntu-latest steps: - uses: GitHubSecurityLab/actions-permissions/monitor@v1 @@ -396,6 +402,9 @@ jobs: needs: - diagnostics - build + permissions: + # actions/checkout needs this to fetch code + contents: read runs-on: ubuntu-latest steps: - uses: GitHubSecurityLab/actions-permissions/monitor@v1 @@ -458,9 +467,11 @@ jobs: - lint - prepare - test - # When Dependabot creates a PR it requires this permission in - # order to push Docker images to ghcr.io. permissions: + # actions/checkout needs this to fetch code + contents: read + # When Dependabot creates a PR it requires this permission in + # order to push Docker images to ghcr.io. packages: write runs-on: ubuntu-latest steps: From 77f9251baf3a3d0af3cdc4e065e21c88fff9021c Mon Sep 17 00:00:00 2001 From: Nicholas McDonnell <50747025+mcdonnnj@users.noreply.github.com> Date: Fri, 22 Nov 2024 13:34:45 -0500 Subject: [PATCH 136/170] Update job permissions in the `CodeQL` workflow Adjust permissions based on the output of the GitHubSecurityLab/actions-permissions/monitor Action. We have been explicitly declaring the `read` permission for `content` if a job uses actions/checkout. --- .github/workflows/codeql-analysis.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.github/workflows/codeql-analysis.yml b/.github/workflows/codeql-analysis.yml index 3844efc..19439d9 100644 --- a/.github/workflows/codeql-analysis.yml +++ b/.github/workflows/codeql-analysis.yml @@ -25,6 +25,8 @@ on: jobs: diagnostics: name: Run diagnostics + # This job does not need any permissions + permissions: {} runs-on: ubuntu-latest steps: # Note that a duplicate of this step must be added at the top of @@ -52,6 +54,8 @@ jobs: - diagnostics runs-on: ubuntu-latest permissions: + # actions/checkout needs this to fetch code + contents: read # required for all workflows security-events: write strategy: From cd1ee1d8bf9af0c47fb609193514e66cb2e6a98a Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 18 Dec 2023 17:31:21 +0000 Subject: [PATCH 137/170] Bump actions/download-artifact from 3 to 4 Bumps [actions/download-artifact](https://github.com/actions/download-artifact) from 3 to 4. - [Release notes](https://github.com/actions/download-artifact/releases) - [Commits](https://github.com/actions/download-artifact/compare/v3...v4) --- updated-dependencies: - dependency-name: actions/download-artifact dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] --- .github/workflows/build.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index d8ba132..3bbdb1b 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -389,7 +389,7 @@ jobs: python -m pip install --upgrade pip setuptools wheel pip install --upgrade --requirement requirements-test.txt - name: Download docker image artifact - uses: actions/download-artifact@v3 + uses: actions/download-artifact@v4 with: name: dist path: dist From 18815c74096728cf40a12b82eef25cd14b0e3d17 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 18 Dec 2023 17:31:13 +0000 Subject: [PATCH 138/170] Bump actions/upload-artifact from 3 to 4 Bumps [actions/upload-artifact](https://github.com/actions/upload-artifact) from 3 to 4. - [Release notes](https://github.com/actions/upload-artifact/releases) - [Commits](https://github.com/actions/upload-artifact/compare/v3...v4) --- updated-dependencies: - dependency-name: actions/upload-artifact dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] --- .github/workflows/build.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 3bbdb1b..9e79e04 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -347,7 +347,7 @@ jobs: - name: Compress image run: gzip dist/image.tar - name: Upload artifacts - uses: actions/upload-artifact@v3 + uses: actions/upload-artifact@v4 with: name: dist path: dist From 03e668f4ca285e5868d7513dc8137c69883f9d58 Mon Sep 17 00:00:00 2001 From: Nicholas McDonnell <50747025+mcdonnnj@users.noreply.github.com> Date: Fri, 6 Dec 2024 04:39:31 -0500 Subject: [PATCH 139/170] Change the format of the version tracking file There is no reason to have any information but the version in the version tracking file for Docker image projects. The current format is simply an artifact of mirroring things over from the cisagov/skeleton-python-library project. --- bump-version | 2 +- src/version.txt | 2 +- tests/container_test.py | 12 +++--------- 3 files changed, 5 insertions(+), 11 deletions(-) diff --git a/bump-version b/bump-version index 600265b..414552a 100755 --- a/bump-version +++ b/bump-version @@ -29,7 +29,7 @@ Options: END_OF_LINE ) -old_version=$(sed -n "s/^__version__ = \"\(.*\)\"$/\1/p" $VERSION_FILE) +old_version=$(< "$VERSION_FILE") # Comment out periods so they are interpreted as periods and don't # just match any character old_version_regex=${old_version//\./\\\.} diff --git a/src/version.txt b/src/version.txt index f102a9c..8acdd82 100644 --- a/src/version.txt +++ b/src/version.txt @@ -1 +1 @@ -__version__ = "0.0.1" +0.0.1 diff --git a/tests/container_test.py b/tests/container_test.py index 00c8e82..f58d325 100644 --- a/tests/container_test.py +++ b/tests/container_test.py @@ -62,10 +62,8 @@ def test_output(dockerc, main_container): ) def test_release_version(): """Verify that release tag version agrees with the module version.""" - pkg_vars = {} with open(VERSION_FILE) as f: - exec(f.read(), pkg_vars) # nosec - project_version = pkg_vars["__version__"] + project_version = f.read().strip() assert ( RELEASE_TAG == f"v{project_version}" ), "RELEASE_TAG does not match the project version" @@ -76,10 +74,8 @@ def test_log_version(dockerc, version_container): # make sure container exited if running test isolated dockerc.wait(version_container.id) log_output = version_container.logs().strip() - pkg_vars = {} with open(VERSION_FILE) as f: - exec(f.read(), pkg_vars) # nosec - project_version = pkg_vars["__version__"] + project_version = f.read().strip() assert ( log_output == project_version ), f"Container version output to log does not match project version file {VERSION_FILE}" @@ -87,10 +83,8 @@ def test_log_version(dockerc, version_container): def test_container_version_label_matches(version_container): """Verify the container version label is the correct version.""" - pkg_vars = {} with open(VERSION_FILE) as f: - exec(f.read(), pkg_vars) # nosec - project_version = pkg_vars["__version__"] + project_version = f.read().strip() assert ( version_container.config.labels["org.opencontainers.image.version"] == project_version From 414efb4c4e60c86083d416bdfd8e9c92a9898b0c Mon Sep 17 00:00:00 2001 From: Nicholas McDonnell <50747025+mcdonnnj@users.noreply.github.com> Date: Fri, 6 Dec 2024 10:22:06 -0500 Subject: [PATCH 140/170] DRY out getting the project version in tests We were previously getting the version in three different tests. It makes more sense to turn getting the project version into a fixture and to use that fixture anywhere the project version is needed. --- tests/conftest.py | 10 ++++++++++ tests/container_test.py | 12 +++--------- 2 files changed, 13 insertions(+), 9 deletions(-) diff --git a/tests/conftest.py b/tests/conftest.py index 29a8b92..6092a48 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -10,6 +10,8 @@ MAIN_SERVICE_NAME = "example" VERSION_SERVICE_NAME = f"{MAIN_SERVICE_NAME}-version" +VERSION_FILE = "src/version.txt" + @pytest.fixture(scope="session") def dockerc(): @@ -36,6 +38,14 @@ def version_container(dockerc): return dockerc.compose.ps(services=[VERSION_SERVICE_NAME], all=True)[0] +@pytest.fixture(scope="session") +def project_version(): + """Return the version of the project.""" + with open(VERSION_FILE) as f: + project_version = f.read().strip() + return project_version + + def pytest_addoption(parser): """Add new commandline options to pytest.""" parser.addoption( diff --git a/tests/container_test.py b/tests/container_test.py index f58d325..cf18333 100644 --- a/tests/container_test.py +++ b/tests/container_test.py @@ -60,31 +60,25 @@ def test_output(dockerc, main_container): @pytest.mark.skipif( RELEASE_TAG in [None, ""], reason="this is not a release (RELEASE_TAG not set)" ) -def test_release_version(): +def test_release_version(project_version): """Verify that release tag version agrees with the module version.""" - with open(VERSION_FILE) as f: - project_version = f.read().strip() assert ( RELEASE_TAG == f"v{project_version}" ), "RELEASE_TAG does not match the project version" -def test_log_version(dockerc, version_container): +def test_log_version(dockerc, project_version, version_container): """Verify the container outputs the correct version to the logs.""" # make sure container exited if running test isolated dockerc.wait(version_container.id) log_output = version_container.logs().strip() - with open(VERSION_FILE) as f: - project_version = f.read().strip() assert ( log_output == project_version ), f"Container version output to log does not match project version file {VERSION_FILE}" -def test_container_version_label_matches(version_container): +def test_container_version_label_matches(project_version, version_container): """Verify the container version label is the correct version.""" - with open(VERSION_FILE) as f: - project_version = f.read().strip() assert ( version_container.config.labels["org.opencontainers.image.version"] == project_version From b6b0b7c27b9c1e12f321dc16b89acbf90c00af19 Mon Sep 17 00:00:00 2001 From: Nicholas McDonnell <50747025+mcdonnnj@users.noreply.github.com> Date: Fri, 2 Feb 2024 05:10:50 -0500 Subject: [PATCH 141/170] Use the full identifier for source Docker images This helps ensure that when a Docker image is built the expacted source image is used regardless of what repository is configured as the default on the host system. It also makes our Dockerfiles more seamlessly convertible to using the GitHub Container Registry or any other Open Container Initiative (OCI) compatible registry. --- Dockerfile | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index fc7b5c2..e4a610d 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,6 +1,8 @@ ARG VERSION=unspecified -FROM python:3.12.0-alpine +# Official Docker images are in the form library/ while non-official +# images are in the form /. +FROM docker.io/library/python:3.12.0-alpine ARG VERSION From 3b7a9ccdf02878c1d68c2fa8a26db02df0ba3b8c Mon Sep 17 00:00:00 2001 From: Nicholas McDonnell <50747025+mcdonnnj@users.noreply.github.com> Date: Tue, 20 Feb 2024 10:02:56 -0500 Subject: [PATCH 142/170] Install cisagov/skeleton-python-library directly Instead of downloading the source archive, extracting it, and then installing it with pip we instead just let pip directly install the package. --- Dockerfile | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/Dockerfile b/Dockerfile index 9c939b7..7492a54 100644 --- a/Dockerfile +++ b/Dockerfile @@ -79,12 +79,7 @@ WORKDIR ${CISA_HOME} # cache. This results in a smaller final image, at the cost of # slightly longer install times. ### -RUN wget --output-document sourcecode.tgz \ - https://github.com/cisagov/skeleton-python-library/archive/v${VERSION}.tar.gz \ - && tar --extract --gzip --file sourcecode.tgz --strip-components=1 \ - && pip3 install --no-cache-dir --requirement requirements.txt \ - && ln -snf /run/secrets/quote.txt src/example/data/secret.txt \ - && rm sourcecode.tgz +RUN pip3 install --no-cache-dir https://github.com/cisagov/skeleton-python-library/archive/v${VERSION}.tar.gz ### # Prepare to run From db1970697171c6e530b8328f79071b022c14c848 Mon Sep 17 00:00:00 2001 From: Nicholas McDonnell <50747025+mcdonnnj@users.noreply.github.com> Date: Tue, 13 Feb 2024 05:05:50 -0500 Subject: [PATCH 143/170] Use a specific version of Alpine Linux Use the full tag that includes the Alpine Linux version to ensure the pulled image is always the same. --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index e4a610d..9c939b7 100644 --- a/Dockerfile +++ b/Dockerfile @@ -2,7 +2,7 @@ ARG VERSION=unspecified # Official Docker images are in the form library/ while non-official # images are in the form /. -FROM docker.io/library/python:3.12.0-alpine +FROM docker.io/library/python:3.12.0-alpine3.18 ARG VERSION From 9e6eef290a6dc61f0cb95357383770c635dec4c3 Mon Sep 17 00:00:00 2001 From: Nicholas McDonnell <50747025+mcdonnnj@users.noreply.github.com> Date: Tue, 20 Feb 2024 10:05:26 -0500 Subject: [PATCH 144/170] Remove unused OS package dependencies Since we are now installing cisagov/skeleton-python-library directly with pip we no longer need these OS packages. --- Dockerfile | 13 ------------- 1 file changed, 13 deletions(-) diff --git a/Dockerfile b/Dockerfile index 7492a54..ce1aa88 100644 --- a/Dockerfile +++ b/Dockerfile @@ -45,19 +45,6 @@ RUN apk --update --no-cache --quiet upgrade RUN addgroup --system --gid ${CISA_GID} ${CISA_GROUP} \ && adduser --system --uid ${CISA_UID} --ingroup ${CISA_GROUP} ${CISA_USER} -### -# Dependencies -# -# Note that we use apk --no-cache to avoid writing to a local cache. -# This results in a smaller final image, at the cost of slightly -# longer install times. -### -ENV DEPS \ - ca-certificates \ - openssl \ - py-pip -RUN apk --no-cache --quiet add ${DEPS} - ### # Make sure pip, setuptools, and wheel are the latest versions # From c516e44a361bdb3ddf285203bfe487e2c9f94ef6 Mon Sep 17 00:00:00 2001 From: Nicholas McDonnell <50747025+mcdonnnj@users.noreply.github.com> Date: Tue, 20 Feb 2024 13:16:14 -0500 Subject: [PATCH 145/170] Remove package upgrading We should not blindly upgrade all pre-installed packages. This can create inconsistent build results due to changes in installed versions. --- Dockerfile | 9 --------- 1 file changed, 9 deletions(-) diff --git a/Dockerfile b/Dockerfile index ce1aa88..296d03d 100644 --- a/Dockerfile +++ b/Dockerfile @@ -30,15 +30,6 @@ ARG CISA_USER="cisa" ENV CISA_GROUP=${CISA_USER} ENV CISA_HOME="/home/${CISA_USER}" -### -# Upgrade the system -# -# Note that we use apk --no-cache to avoid writing to a local cache. -# This results in a smaller final image, at the cost of slightly -# longer install times. -### -RUN apk --update --no-cache --quiet upgrade - ### # Create unprivileged user ### From 460eeec5f3861847966f525d6388adbfed8fc49d Mon Sep 17 00:00:00 2001 From: Nicholas McDonnell <50747025+mcdonnnj@users.noreply.github.com> Date: Tue, 20 Feb 2024 10:11:44 -0500 Subject: [PATCH 146/170] Change the secret message being checks in tests Now that we are not overwriting the internal Python package file the text we look for must match what is output by default. The Docker Compose secret configuration is left in place to continue to serve as an example and to be leveraged for a future update to cisagov/skeleton-python-library that can provide similar functionality to what was removed in this project. --- tests/container_test.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/tests/container_test.py b/tests/container_test.py index cf18333..6451766 100644 --- a/tests/container_test.py +++ b/tests/container_test.py @@ -10,9 +10,7 @@ ENV_VAR = "ECHO_MESSAGE" ENV_VAR_VAL = "Hello World from docker compose!" READY_MESSAGE = "This is a debug message" -SECRET_QUOTE = ( - "There are no secrets better kept than the secrets everybody guesses." # nosec -) +SECRET_QUOTE = "Three may keep a secret, if two of them are dead." # nosec RELEASE_TAG = os.getenv("RELEASE_TAG") VERSION_FILE = "src/version.txt" From 001e85e96753b00cfc4909e691ee29930b834787 Mon Sep 17 00:00:00 2001 From: Nicholas McDonnell <50747025+mcdonnnj@users.noreply.github.com> Date: Tue, 20 Feb 2024 13:21:51 -0500 Subject: [PATCH 147/170] Pin Python packages directly installed Pin the versions of the pip, setuptools, and wheel packages that are installed. --- Dockerfile | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/Dockerfile b/Dockerfile index 296d03d..7ba2012 100644 --- a/Dockerfile +++ b/Dockerfile @@ -30,6 +30,11 @@ ARG CISA_USER="cisa" ENV CISA_GROUP=${CISA_USER} ENV CISA_HOME="/home/${CISA_USER}" +# Versions of the Python packages installed directly +ENV PYTHON_PIP_VERSION=24.0 +ENV PYTHON_SETUPTOOLS_VERSION=69.1.0 +ENV PYTHON_WHEEL_VERSION=0.42.0 + ### # Create unprivileged user ### @@ -37,16 +42,16 @@ RUN addgroup --system --gid ${CISA_GID} ${CISA_GROUP} \ && adduser --system --uid ${CISA_UID} --ingroup ${CISA_GROUP} ${CISA_USER} ### -# Make sure pip, setuptools, and wheel are the latest versions +# Make sure the specified versions of pip, setuptools, and wheel are installed # # Note that we use pip3 --no-cache-dir to avoid writing to a local # cache. This results in a smaller final image, at the cost of # slightly longer install times. ### RUN pip3 install --no-cache-dir --upgrade \ - pip \ - setuptools \ - wheel + pip==${PYTHON_PIP_VERSION} \ + setuptools==${PYTHON_SETUPTOOLS_VERSION} \ + wheel==${PYTHON_WHEEL_VERSION} WORKDIR ${CISA_HOME} From 48fa1a5c61a0ef05db5bc352b68bd3d8cdcd2c94 Mon Sep 17 00:00:00 2001 From: Nicholas McDonnell <50747025+mcdonnnj@users.noreply.github.com> Date: Mon, 26 Feb 2024 17:02:05 -0500 Subject: [PATCH 148/170] Prefer calling pip as a module Instead of relying on `pip3` being on the PATH we instead call the module through the Python executable. This ensures that the `pip` being used is in the same environment as the `python3` being used. --- Dockerfile | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/Dockerfile b/Dockerfile index ca97b41..b07e712 100644 --- a/Dockerfile +++ b/Dockerfile @@ -44,11 +44,11 @@ RUN addgroup --system --gid ${CISA_GID} ${CISA_GROUP} \ ### # Make sure the specified versions of pip, setuptools, and wheel are installed # -# Note that we use pip3 --no-cache-dir to avoid writing to a local +# Note that we use the --no-cache-dir flag to avoid writing to a local # cache. This results in a smaller final image, at the cost of # slightly longer install times. ### -RUN pip3 install --no-cache-dir --upgrade \ +RUN python3 -m pip install --no-cache-dir --upgrade \ pip==${PYTHON_PIP_VERSION} \ setuptools==${PYTHON_SETUPTOOLS_VERSION} \ wheel==${PYTHON_WHEEL_VERSION} @@ -56,11 +56,11 @@ RUN pip3 install --no-cache-dir --upgrade \ ### # Install Python dependencies # -# Note that we use pip3 --no-cache-dir to avoid writing to a local +# Note that we use the --no-cache-dir flag to avoid writing to a local # cache. This results in a smaller final image, at the cost of # slightly longer install times. ### -RUN pip3 install --no-cache-dir https://github.com/cisagov/skeleton-python-library/archive/v${VERSION}.tar.gz +RUN python3 -m pip install --no-cache-dir https://github.com/cisagov/skeleton-python-library/archive/v${VERSION}.tar.gz ### # Prepare to run From e6f5798b9bdbe140aab760f0cedee7ee01324907 Mon Sep 17 00:00:00 2001 From: Nicholas McDonnell <50747025+mcdonnnj@users.noreply.github.com> Date: Fri, 23 Feb 2024 13:14:34 -0500 Subject: [PATCH 149/170] Move WORKDIR instruction We can move this instruction to the end of the Dockerfile now that we are no longer working with files in the Docker container when building. --- Dockerfile | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/Dockerfile b/Dockerfile index 7ba2012..ca97b41 100644 --- a/Dockerfile +++ b/Dockerfile @@ -53,8 +53,6 @@ RUN pip3 install --no-cache-dir --upgrade \ setuptools==${PYTHON_SETUPTOOLS_VERSION} \ wheel==${PYTHON_WHEEL_VERSION} -WORKDIR ${CISA_HOME} - ### # Install Python dependencies # @@ -68,6 +66,7 @@ RUN pip3 install --no-cache-dir https://github.com/cisagov/skeleton-python-libra # Prepare to run ### ENV ECHO_MESSAGE="Hello World from Dockerfile" +WORKDIR ${CISA_HOME} USER ${CISA_USER}:${CISA_GROUP} EXPOSE 8080/TCP VOLUME ["/var/log"] From 95d4a7a6f49a61216547f9f49c363f847bf99107 Mon Sep 17 00:00:00 2001 From: Nicholas McDonnell <50747025+mcdonnnj@users.noreply.github.com> Date: Mon, 26 Feb 2024 17:13:30 -0500 Subject: [PATCH 150/170] Use a Python virtual environment in the Docker image Using a virtual environment is a Python best practice. We also consolidate all of the Python dependency installation steps into a single RUN instruction. This ensures that Python setup is cached in one layer and mirrors the logical organization of this being a single step. --- Dockerfile | 28 ++++++++++++++++++---------- 1 file changed, 18 insertions(+), 10 deletions(-) diff --git a/Dockerfile b/Dockerfile index b07e712..7416591 100644 --- a/Dockerfile +++ b/Dockerfile @@ -29,6 +29,7 @@ ARG CISA_GID=${CISA_UID} ARG CISA_USER="cisa" ENV CISA_GROUP=${CISA_USER} ENV CISA_HOME="/home/${CISA_USER}" +ENV VIRTUAL_ENV="${CISA_HOME}/.venv" # Versions of the Python packages installed directly ENV PYTHON_PIP_VERSION=24.0 @@ -42,25 +43,32 @@ RUN addgroup --system --gid ${CISA_GID} ${CISA_GROUP} \ && adduser --system --uid ${CISA_UID} --ingroup ${CISA_GROUP} ${CISA_USER} ### -# Make sure the specified versions of pip, setuptools, and wheel are installed +# Set up a Python virtual environment (venv); install the specified versions of pip, +# setuptools, and wheel into it; and then install the Python dependencies for +# the application. # # Note that we use the --no-cache-dir flag to avoid writing to a local # cache. This results in a smaller final image, at the cost of # slightly longer install times. ### -RUN python3 -m pip install --no-cache-dir --upgrade \ - pip==${PYTHON_PIP_VERSION} \ - setuptools==${PYTHON_SETUPTOOLS_VERSION} \ - wheel==${PYTHON_WHEEL_VERSION} +RUN python3 -m venv ${VIRTUAL_ENV} \ + && ${VIRTUAL_ENV}/bin/python3 -m pip install --no-cache-dir --upgrade \ + pip==${PYTHON_PIP_VERSION} \ + setuptools==${PYTHON_SETUPTOOLS_VERSION} \ + wheel==${PYTHON_WHEEL_VERSION} \ + && ${VIRTUAL_ENV}/bin/python3 -m pip install --no-cache-dir --upgrade \ + https://github.com/cisagov/skeleton-python-library/archive/v${VERSION}.tar.gz ### -# Install Python dependencies +# Sym-link the Python binary in the venv to the system-wide Python and add the venv to +# the PATH. # -# Note that we use the --no-cache-dir flag to avoid writing to a local -# cache. This results in a smaller final image, at the cost of -# slightly longer install times. +# Note that we sym-link the Python binary in the venv to the system-wide Python so that +# any calls to `python3` will use our virtual environment. We are using short flags +# because the ln binary in Alpine Linux does not support long flags. ### -RUN python3 -m pip install --no-cache-dir https://github.com/cisagov/skeleton-python-library/archive/v${VERSION}.tar.gz +RUN ln -sf "$(command -v python3)" "${VIRTUAL_ENV}"/bin/python3 +ENV PATH="${VIRTUAL_ENV}/bin:$PATH" ### # Prepare to run From 79a4adf8a55a81c8360b1a49c55ea5fd91ebdc49 Mon Sep 17 00:00:00 2001 From: Nicholas McDonnell <50747025+mcdonnnj@users.noreply.github.com> Date: Tue, 27 Feb 2024 18:12:50 -0500 Subject: [PATCH 151/170] Add a pipenv configuration This configuration includes a Pipfile configuration file and the generated Pipfile.lock file that pins to specific versions for the Python dependencies for this project. This will help us ensure repeatable builds. The pipenv package is added as a developmental requirement to support these files. --- requirements-dev.txt | 1 + src/Pipfile | 13 +++++++++++++ src/Pipfile.lock | 38 ++++++++++++++++++++++++++++++++++++++ 3 files changed, 52 insertions(+) create mode 100644 src/Pipfile create mode 100644 src/Pipfile.lock diff --git a/requirements-dev.txt b/requirements-dev.txt index de5eb3b..d7a04ed 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -1,3 +1,4 @@ --requirement requirements-test.txt ipython +pipenv semver>=3 diff --git a/src/Pipfile b/src/Pipfile new file mode 100644 index 0000000..56f2fc9 --- /dev/null +++ b/src/Pipfile @@ -0,0 +1,13 @@ +[[source]] +url = "https://pypi.org/simple" +verify_ssl = true +name = "pypi" + +# List any Python dependencies for the image here +[packages] +# This should match the version of the image +example = {file = "https://github.com/cisagov/skeleton-python-library/archive/v0.0.1.tar.gz"} + +# This version should match the version of Python in the image +[requires] +python_full_version = "3.12.0" diff --git a/src/Pipfile.lock b/src/Pipfile.lock new file mode 100644 index 0000000..d39d053 --- /dev/null +++ b/src/Pipfile.lock @@ -0,0 +1,38 @@ +{ + "_meta": { + "hash": { + "sha256": "654452851fea1eb2c8811649e5efe8873c8ff51f5c14dd27a4a8ebb5b15a27c4" + }, + "pipfile-spec": 6, + "requires": { + "python_full_version": "3.12.0" + }, + "sources": [ + { + "name": "pypi", + "url": "https://pypi.org/simple", + "verify_ssl": true + } + ] + }, + "default": { + "docopt": { + "hashes": [ + "sha256:49b3a825280bd66b3aa83585ef59c4a8c82f2c8a522dbe754a8bc8d08c85c491" + ], + "version": "==0.6.2" + }, + "example": { + "file": "https://github.com/cisagov/skeleton-python-library/archive/v0.0.1.tar.gz" + }, + "setuptools": { + "hashes": [ + "sha256:02fa291a0471b3a18b2b2481ed902af520c69e8ae0919c13da936542754b4c56", + "sha256:5c0806c7d9af348e6dd3777b4f4dbb42c7ad85b190104837488eab9a7c945cf8" + ], + "markers": "python_version >= '3.8'", + "version": "==69.1.1" + } + }, + "develop": {} +} From d3895efb85ba686d8d4bfac0bbf6f0a674aeb145 Mon Sep 17 00:00:00 2001 From: Nicholas McDonnell <50747025+mcdonnnj@users.noreply.github.com> Date: Mon, 26 Feb 2024 22:16:54 -0500 Subject: [PATCH 152/170] Explain `ln` options being used Since we cannot use long options on Alpine Linux we should explain what the short options we are using do. I also changed the order of options so that they are in alphabetical order. Co-authored-by: Shane Frasier --- Dockerfile | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/Dockerfile b/Dockerfile index 7416591..1aae8db 100644 --- a/Dockerfile +++ b/Dockerfile @@ -65,9 +65,10 @@ RUN python3 -m venv ${VIRTUAL_ENV} \ # # Note that we sym-link the Python binary in the venv to the system-wide Python so that # any calls to `python3` will use our virtual environment. We are using short flags -# because the ln binary in Alpine Linux does not support long flags. +# because the ln binary in Alpine Linux does not support long flags. The -f instructs +# ln to remove the existing file and the -s instructs ln to create a symbolic link. ### -RUN ln -sf "$(command -v python3)" "${VIRTUAL_ENV}"/bin/python3 +RUN ln -fs "$(command -v python3)" "${VIRTUAL_ENV}"/bin/python3 ENV PATH="${VIRTUAL_ENV}/bin:$PATH" ### From 6b869bb2ee9ec0423008973746a498305787d23a Mon Sep 17 00:00:00 2001 From: Nicholas McDonnell <50747025+mcdonnnj@users.noreply.github.com> Date: Tue, 27 Feb 2024 19:01:45 -0500 Subject: [PATCH 153/170] Install Python dependencies using pipenv Now that we have a pipenv configuration we will use it to install the Python dependencies for the image. The `build` workflow is updated to no longer pass the VERSION build argument in line with this change. --- .github/workflows/build.yml | 4 ---- Dockerfile | 34 +++++++++++++++++++++++----------- README.md | 2 -- 3 files changed, 23 insertions(+), 17 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index f643b86..612806d 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -348,8 +348,6 @@ jobs: id: docker_build uses: docker/build-push-action@v6 with: - build-args: | - VERSION=${{ needs.prepare.outputs.source_version }} cache-from: type=local,src=${{ env.BUILDX_CACHE_DIR }} cache-to: type=local,dest=${{ env.BUILDX_CACHE_DIR }} context: . @@ -516,8 +514,6 @@ jobs: id: docker_build uses: docker/build-push-action@v6 with: - build-args: | - VERSION=${{ needs.prepare.outputs.source_version }} cache-from: type=local,src=${{ env.BUILDX_CACHE_DIR }} cache-to: type=local,dest=${{ env.BUILDX_CACHE_DIR }} context: . diff --git a/Dockerfile b/Dockerfile index 1aae8db..c36ae46 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,11 +1,7 @@ -ARG VERSION=unspecified - # Official Docker images are in the form library/ while non-official # images are in the form /. FROM docker.io/library/python:3.12.0-alpine3.18 -ARG VERSION - ### # For a list of pre-defined annotation keys and value types see: # https://github.com/opencontainers/image-spec/blob/master/annotations.md @@ -33,6 +29,7 @@ ENV VIRTUAL_ENV="${CISA_HOME}/.venv" # Versions of the Python packages installed directly ENV PYTHON_PIP_VERSION=24.0 +ENV PYTHON_PIPENV_VERSION=2023.12.1 ENV PYTHON_SETUPTOOLS_VERSION=69.1.0 ENV PYTHON_WHEEL_VERSION=0.42.0 @@ -43,21 +40,36 @@ RUN addgroup --system --gid ${CISA_GID} ${CISA_GROUP} \ && adduser --system --uid ${CISA_UID} --ingroup ${CISA_GROUP} ${CISA_USER} ### -# Set up a Python virtual environment (venv); install the specified versions of pip, -# setuptools, and wheel into it; and then install the Python dependencies for -# the application. +# Install the specified version of pipenv; set up a Python virtual environment (venv); +# and install the specified versions of pip, setuptools, and wheel into the venv. # # Note that we use the --no-cache-dir flag to avoid writing to a local # cache. This results in a smaller final image, at the cost of # slightly longer install times. ### -RUN python3 -m venv ${VIRTUAL_ENV} \ +RUN python3 -m pip install --no-cache-dir --upgrade pipenv==${PYTHON_PIPENV_VERSION} \ + # Manueally create the virtual environment + && python3 -m venv ${VIRTUAL_ENV} \ + # Ensure the core Python packages are installed in the virtual environment && ${VIRTUAL_ENV}/bin/python3 -m pip install --no-cache-dir --upgrade \ pip==${PYTHON_PIP_VERSION} \ setuptools==${PYTHON_SETUPTOOLS_VERSION} \ - wheel==${PYTHON_WHEEL_VERSION} \ - && ${VIRTUAL_ENV}/bin/python3 -m pip install --no-cache-dir --upgrade \ - https://github.com/cisagov/skeleton-python-library/archive/v${VERSION}.tar.gz + wheel==${PYTHON_WHEEL_VERSION} + +### +# Check the Pipfile configuration and then install the Python dependencies into +# the virtual environment. +# +# Note that pipenv will install into a virtual environment if the VIRTUAL_ENV +# environment variable is set. We are using short flags because the rm binary +# in Alpine Linux does not support long flags. The -f instructs rm to remove +# files without prompting. +### +WORKDIR /tmp +COPY src/Pipfile src/Pipfile.lock ./ +RUN pipenv check --verbose \ + && pipenv install --clear --deploy --extra-pip-args "--no-cache-dir" --verbose \ + && rm -f Pipfile* ### # Sym-link the Python binary in the venv to the system-wide Python and add the venv to diff --git a/README.md b/README.md index 57f8c30..8d926e4 100644 --- a/README.md +++ b/README.md @@ -196,7 +196,6 @@ Build the image locally using this git repository as the [build context](https:/ ```console docker build \ - --build-arg VERSION=0.0.1 \ --tag cisagov/example:0.0.1 \ https://github.com/cisagov/example.git#develop ``` @@ -227,7 +226,6 @@ Docker: docker buildx build \ --file Dockerfile-x \ --platform linux/amd64 \ - --build-arg VERSION=0.0.1 \ --output type=docker \ --tag cisagov/example:0.0.1 . ``` From 5cbb75beff797cc24c469cb24941d2541473923b Mon Sep 17 00:00:00 2001 From: Nicholas McDonnell <50747025+mcdonnnj@users.noreply.github.com> Date: Tue, 27 Feb 2024 19:09:29 -0500 Subject: [PATCH 154/170] Use a multi-stage Docker build Switch to using a multi-stage build in the Dockerfile. This reduces image size since pipenv and its dependencices are not needed in the final image. It also ensures that the system Python environment is unmodified. --- Dockerfile | 71 +++++++++++++++++++++++++++++++----------------------- 1 file changed, 41 insertions(+), 30 deletions(-) diff --git a/Dockerfile b/Dockerfile index c36ae46..ca0742e 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,29 +1,11 @@ # Official Docker images are in the form library/ while non-official # images are in the form /. -FROM docker.io/library/python:3.12.0-alpine3.18 +FROM docker.io/library/python:3.12.0-alpine3.18 as compile-stage ### -# For a list of pre-defined annotation keys and value types see: -# https://github.com/opencontainers/image-spec/blob/master/annotations.md -# -# Note: Additional labels are added by the build workflow. -### -# github@cisa.dhs.gov is a very generic email distribution, and it is -# unlikely that anyone on that distribution is familiar with the -# particulars of your repository. It is therefore *strongly* -# suggested that you use an email address here that is specific to the -# person or group that maintains this repository; for example: -# LABEL org.opencontainers.image.authors="vm-fusion-dev-group@trio.dhs.gov" -LABEL org.opencontainers.image.authors="github@cisa.dhs.gov" -LABEL org.opencontainers.image.vendor="Cybersecurity and Infrastructure Security Agency" - -### -# Unprivileged user setup variables +# Unprivileged user variables ### -ARG CISA_UID=421 -ARG CISA_GID=${CISA_UID} ARG CISA_USER="cisa" -ENV CISA_GROUP=${CISA_USER} ENV CISA_HOME="/home/${CISA_USER}" ENV VIRTUAL_ENV="${CISA_HOME}/.venv" @@ -33,12 +15,6 @@ ENV PYTHON_PIPENV_VERSION=2023.12.1 ENV PYTHON_SETUPTOOLS_VERSION=69.1.0 ENV PYTHON_WHEEL_VERSION=0.42.0 -### -# Create unprivileged user -### -RUN addgroup --system --gid ${CISA_GID} ${CISA_GROUP} \ - && adduser --system --uid ${CISA_UID} --ingroup ${CISA_GROUP} ${CISA_USER} - ### # Install the specified version of pipenv; set up a Python virtual environment (venv); # and install the specified versions of pip, setuptools, and wheel into the venv. @@ -68,18 +44,53 @@ RUN python3 -m pip install --no-cache-dir --upgrade pipenv==${PYTHON_PIPENV_VERS WORKDIR /tmp COPY src/Pipfile src/Pipfile.lock ./ RUN pipenv check --verbose \ - && pipenv install --clear --deploy --extra-pip-args "--no-cache-dir" --verbose \ - && rm -f Pipfile* + && pipenv install --clear --deploy --extra-pip-args "--no-cache-dir" --verbose + +# Official Docker images are in the form library/ while non-official +# images are in the form /. +FROM docker.io/library/python:3.12.0-alpine3.18 as build-stage + +### +# For a list of pre-defined annotation keys and value types see: +# https://github.com/opencontainers/image-spec/blob/master/annotations.md +# +# Note: Additional labels are added by the build workflow. +### +# github@cisa.dhs.gov is a very generic email distribution, and it is +# unlikely that anyone on that distribution is familiar with the +# particulars of your repository. It is therefore *strongly* +# suggested that you use an email address here that is specific to the +# person or group that maintains this repository; for example: +# LABEL org.opencontainers.image.authors="vm-fusion-dev-group@trio.dhs.gov" +LABEL org.opencontainers.image.authors="github@cisa.dhs.gov" +LABEL org.opencontainers.image.vendor="Cybersecurity and Infrastructure Security Agency" + +### +# Unprivileged user setup variables +### +ARG CISA_UID=421 +ARG CISA_GID=${CISA_UID} +ARG CISA_USER="cisa" +ENV CISA_GROUP=${CISA_USER} +ENV CISA_HOME="/home/${CISA_USER}" +ENV VIRTUAL_ENV="${CISA_HOME}/.venv" + +### +# Create unprivileged user +### +RUN addgroup --system --gid ${CISA_GID} ${CISA_GROUP} \ + && adduser --system --uid ${CISA_UID} --ingroup ${CISA_GROUP} ${CISA_USER} ### -# Sym-link the Python binary in the venv to the system-wide Python and add the venv to -# the PATH. +# Copy in the Python virtual environment created in compile-stage, Sym-link the +# Python binary in the venv to the system-wide Python and add the venv to the PATH. # # Note that we sym-link the Python binary in the venv to the system-wide Python so that # any calls to `python3` will use our virtual environment. We are using short flags # because the ln binary in Alpine Linux does not support long flags. The -f instructs # ln to remove the existing file and the -s instructs ln to create a symbolic link. ### +COPY --from=compile-stage --chown=${CISA_USER}:${CISA_GROUP} ${VIRTUAL_ENV} ${VIRTUAL_ENV} RUN ln -fs "$(command -v python3)" "${VIRTUAL_ENV}"/bin/python3 ENV PATH="${VIRTUAL_ENV}/bin:$PATH" From 5dca6d92e268ce2cf1e75efae4e04703d9ec35b4 Mon Sep 17 00:00:00 2001 From: Nicholas McDonnell <50747025+mcdonnnj@users.noreply.github.com> Date: Tue, 27 Feb 2024 19:13:33 -0500 Subject: [PATCH 155/170] Install core Python packages into the system Python environment Install the core Python packages (pip, setuptools, and wheel) into the system Python environment before installing pipenv. This keeps things consistent with our usual approach to Python environments. --- Dockerfile | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/Dockerfile b/Dockerfile index ca0742e..96a5126 100644 --- a/Dockerfile +++ b/Dockerfile @@ -16,14 +16,21 @@ ENV PYTHON_SETUPTOOLS_VERSION=69.1.0 ENV PYTHON_WHEEL_VERSION=0.42.0 ### -# Install the specified version of pipenv; set up a Python virtual environment (venv); -# and install the specified versions of pip, setuptools, and wheel into the venv. +# Install the specified versions of pip, setuptools, and wheel into the system +# Python environment; install the specified version of pipenv into the system Python +# environment; set up a Python virtual environment (venv); and install the specified +# versions of pip, setuptools, and wheel into the venv. # # Note that we use the --no-cache-dir flag to avoid writing to a local # cache. This results in a smaller final image, at the cost of # slightly longer install times. ### -RUN python3 -m pip install --no-cache-dir --upgrade pipenv==${PYTHON_PIPENV_VERSION} \ +RUN python3 -m pip install --no-cache-dir --upgrade \ + pip==${PYTHON_PIP_VERSION} \ + setuptools==${PYTHON_SETUPTOOLS_VERSION} \ + wheel==${PYTHON_WHEEL_VERSION} \ + && python3 -m pip install --no-cache-dir --upgrade \ + pipenv==${PYTHON_PIPENV_VERSION} \ # Manueally create the virtual environment && python3 -m venv ${VIRTUAL_ENV} \ # Ensure the core Python packages are installed in the virtual environment From 79d8de9ae144f57326b4b6a620a55545328e8a38 Mon Sep 17 00:00:00 2001 From: Nick <50747025+mcdonnnj@users.noreply.github.com> Date: Wed, 28 Feb 2024 12:04:09 -0500 Subject: [PATCH 156/170] Fix outdated comment in the Dockerfile The comment references a command that is no longer being run. Co-authored-by: Shane Frasier --- Dockerfile | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/Dockerfile b/Dockerfile index 96a5126..38cf0a4 100644 --- a/Dockerfile +++ b/Dockerfile @@ -44,9 +44,7 @@ RUN python3 -m pip install --no-cache-dir --upgrade \ # the virtual environment. # # Note that pipenv will install into a virtual environment if the VIRTUAL_ENV -# environment variable is set. We are using short flags because the rm binary -# in Alpine Linux does not support long flags. The -f instructs rm to remove -# files without prompting. +# environment variable is set. ### WORKDIR /tmp COPY src/Pipfile src/Pipfile.lock ./ From 440ba709fe36c8a86d9880995b46a56f6741229d Mon Sep 17 00:00:00 2001 From: Nick <50747025+mcdonnnj@users.noreply.github.com> Date: Wed, 28 Feb 2024 12:04:51 -0500 Subject: [PATCH 157/170] Fix typo in Dockerfile comment Co-authored-by: dav3r --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index 38cf0a4..6a00b36 100644 --- a/Dockerfile +++ b/Dockerfile @@ -31,7 +31,7 @@ RUN python3 -m pip install --no-cache-dir --upgrade \ wheel==${PYTHON_WHEEL_VERSION} \ && python3 -m pip install --no-cache-dir --upgrade \ pipenv==${PYTHON_PIPENV_VERSION} \ - # Manueally create the virtual environment + # Manually create the virtual environment && python3 -m venv ${VIRTUAL_ENV} \ # Ensure the core Python packages are installed in the virtual environment && ${VIRTUAL_ENV}/bin/python3 -m pip install --no-cache-dir --upgrade \ From 78f4f78b2047cd29c5edac51fe0fd218ceb296cb Mon Sep 17 00:00:00 2001 From: Nicholas McDonnell <50747025+mcdonnnj@users.noreply.github.com> Date: Thu, 29 Feb 2024 13:17:59 -0500 Subject: [PATCH 158/170] Update image tag information in the README Change the tags used in the table to match the version of the project. Previously "1.2.3" was used as an example version but there is no reason not to use the real version of the image. --- README.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index 8d926e4..7ff4648 100644 --- a/README.md +++ b/README.md @@ -139,9 +139,9 @@ containerize. It is recommended that most users use a version tag (e.g. | Image:tag | Description | |-----------|-------------| -|`cisagov/example:1.2.3`| An exact release version. | -|`cisagov/example:1.2`| The most recent release matching the major and minor version numbers. | -|`cisagov/example:1`| The most recent release matching the major version number. | +|`cisagov/example:0.0.1`| An exact release version. | +|`cisagov/example:0.0`| The most recent release matching the major and minor version numbers. | +|`cisagov/example:0`| The most recent release matching the major version number. | |`cisagov/example:edge` | The most recent image built from a merge into the `develop` branch of this repository. | |`cisagov/example:nightly` | A nightly build of the `develop` branch of this repository. | |`cisagov/example:latest`| The most recent release image pushed to a container registry. Pulling an image using the `:latest` tag [should be avoided.](https://vsupalov.com/docker-latest-tag/) | From 1774267011a35e29623648a9fa631e499c3fd36f Mon Sep 17 00:00:00 2001 From: Nicholas McDonnell <50747025+mcdonnnj@users.noreply.github.com> Date: Thu, 29 Feb 2024 15:15:07 -0500 Subject: [PATCH 159/170] Add instructions for managing Python dependencies --- README.md | 31 +++++++++++++++++++++++++++++++ 1 file changed, 31 insertions(+) diff --git a/README.md b/README.md index 7ff4648..a74bba4 100644 --- a/README.md +++ b/README.md @@ -130,6 +130,35 @@ environment variables. See the 1. Recreate and run the container by following the [previous instructions](#running-with-docker). +## Updating Python dependencies ## + +This image uses [Pipenv] to manage Python dependencies using a [Pipfile](https://github.com/pypa/pipfile). +Both updating dependencies and changing the [Pipenv] configuration in `src/Pipfile` +will result in a modified `src/Pipfile.lock` file that should be committed to the +repository. + +> [!WARNING] +> The `src/Pipfile.lock` as generated will fail `pre-commit` checks due to JSON formatting. + +### Updating dependencies ### + +If you want to update existing dependencies you would run the following command +in the `src/` subdirectory: + +```console +pipenv lock +``` + +### Modifying dependencies ### + +If you want to add or remove dependencies you would update the `src/Pipfile` file +and then update dependencies as you would above. + +> [!NOTE] +> You should only specify packages that are explicitly needed for your Docker +> configuration. Allow [Pipenv] to manage the dependencies of the specified +> packages. + ## Image tags ## The images of this container are tagged with [semantic @@ -254,3 +283,5 @@ dedication](https://creativecommons.org/publicdomain/zero/1.0/). All contributions to this project will be released under the CC0 dedication. By submitting a pull request, you are agreeing to comply with this waiver of copyright interest. + +[Pipenv]: https://pypi.org/project/pipenv/ From af716cc3d2f9dd7bcf4181856f8666cab7188cf2 Mon Sep 17 00:00:00 2001 From: Nicholas McDonnell <50747025+mcdonnnj@users.noreply.github.com> Date: Wed, 28 Feb 2024 14:20:07 -0500 Subject: [PATCH 160/170] Bump setuptools from 69.1.0 to 69.1.1 --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index 6a00b36..4af43cc 100644 --- a/Dockerfile +++ b/Dockerfile @@ -12,7 +12,7 @@ ENV VIRTUAL_ENV="${CISA_HOME}/.venv" # Versions of the Python packages installed directly ENV PYTHON_PIP_VERSION=24.0 ENV PYTHON_PIPENV_VERSION=2023.12.1 -ENV PYTHON_SETUPTOOLS_VERSION=69.1.0 +ENV PYTHON_SETUPTOOLS_VERSION=69.1.1 ENV PYTHON_WHEEL_VERSION=0.42.0 ### From a65ebdc8b38221690b006cea5b07d086c40a8539 Mon Sep 17 00:00:00 2001 From: Nicholas McDonnell <50747025+mcdonnnj@users.noreply.github.com> Date: Wed, 28 Feb 2024 14:21:44 -0500 Subject: [PATCH 161/170] Bump Python from 3.12.0 to 3.12.2 --- Dockerfile | 4 ++-- src/Pipfile | 2 +- src/Pipfile.lock | 4 ++-- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/Dockerfile b/Dockerfile index 4af43cc..e78325e 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,6 +1,6 @@ # Official Docker images are in the form library/ while non-official # images are in the form /. -FROM docker.io/library/python:3.12.0-alpine3.18 as compile-stage +FROM docker.io/library/python:3.12.2-alpine3.18 as compile-stage ### # Unprivileged user variables @@ -53,7 +53,7 @@ RUN pipenv check --verbose \ # Official Docker images are in the form library/ while non-official # images are in the form /. -FROM docker.io/library/python:3.12.0-alpine3.18 as build-stage +FROM docker.io/library/python:3.12.2-alpine3.18 as build-stage ### # For a list of pre-defined annotation keys and value types see: diff --git a/src/Pipfile b/src/Pipfile index 56f2fc9..4df9a57 100644 --- a/src/Pipfile +++ b/src/Pipfile @@ -10,4 +10,4 @@ example = {file = "https://github.com/cisagov/skeleton-python-library/archive/v0 # This version should match the version of Python in the image [requires] -python_full_version = "3.12.0" +python_full_version = "3.12.2" diff --git a/src/Pipfile.lock b/src/Pipfile.lock index d39d053..95333e6 100644 --- a/src/Pipfile.lock +++ b/src/Pipfile.lock @@ -1,11 +1,11 @@ { "_meta": { "hash": { - "sha256": "654452851fea1eb2c8811649e5efe8873c8ff51f5c14dd27a4a8ebb5b15a27c4" + "sha256": "18956d92f029b8a57000e268dee96409725be2949018c193cedd3d1e9744d878" }, "pipfile-spec": 6, "requires": { - "python_full_version": "3.12.0" + "python_full_version": "3.12.2" }, "sources": [ { From 5d52367dc42d929a86546f1ae52ee339c86faf43 Mon Sep 17 00:00:00 2001 From: Nicholas McDonnell <50747025+mcdonnnj@users.noreply.github.com> Date: Wed, 28 Feb 2024 14:22:25 -0500 Subject: [PATCH 162/170] Bump Alpine Linux from 3.18 to 3.19 --- Dockerfile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Dockerfile b/Dockerfile index e78325e..836dbf7 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,6 +1,6 @@ # Official Docker images are in the form library/ while non-official # images are in the form /. -FROM docker.io/library/python:3.12.2-alpine3.18 as compile-stage +FROM docker.io/library/python:3.12.2-alpine3.19 as compile-stage ### # Unprivileged user variables @@ -53,7 +53,7 @@ RUN pipenv check --verbose \ # Official Docker images are in the form library/ while non-official # images are in the form /. -FROM docker.io/library/python:3.12.2-alpine3.18 as build-stage +FROM docker.io/library/python:3.12.2-alpine3.19 as build-stage ### # For a list of pre-defined annotation keys and value types see: From 56afca0d991c05010d22981ba424e631fd88cb12 Mon Sep 17 00:00:00 2001 From: Nicholas McDonnell <50747025+mcdonnnj@users.noreply.github.com> Date: Wed, 28 Feb 2024 14:27:43 -0500 Subject: [PATCH 163/170] Bump cisagov/skeleton-python-library from 0.0.1 to 0.2.0 Update the Dockerfile and testing to accommodate changes in the new version. --- Dockerfile | 2 +- src/Pipfile | 2 +- src/Pipfile.lock | 19 +++++++++++++++++-- tests/container_test.py | 2 ++ 4 files changed, 21 insertions(+), 4 deletions(-) diff --git a/Dockerfile b/Dockerfile index 836dbf7..c40e756 100644 --- a/Dockerfile +++ b/Dockerfile @@ -108,4 +108,4 @@ USER ${CISA_USER}:${CISA_GROUP} EXPOSE 8080/TCP VOLUME ["/var/log"] ENTRYPOINT ["example"] -CMD ["--log-level", "DEBUG"] +CMD ["--log-level", "DEBUG", "8", "2"] diff --git a/src/Pipfile b/src/Pipfile index 4df9a57..6480562 100644 --- a/src/Pipfile +++ b/src/Pipfile @@ -6,7 +6,7 @@ name = "pypi" # List any Python dependencies for the image here [packages] # This should match the version of the image -example = {file = "https://github.com/cisagov/skeleton-python-library/archive/v0.0.1.tar.gz"} +example = {file = "https://github.com/cisagov/skeleton-python-library/archive/v0.2.0.tar.gz"} # This version should match the version of Python in the image [requires] diff --git a/src/Pipfile.lock b/src/Pipfile.lock index 95333e6..6afbec3 100644 --- a/src/Pipfile.lock +++ b/src/Pipfile.lock @@ -1,7 +1,7 @@ { "_meta": { "hash": { - "sha256": "18956d92f029b8a57000e268dee96409725be2949018c193cedd3d1e9744d878" + "sha256": "ad65a437d348bca066601e0d05a14c0ee23e38b59d8d8e71558cb17c57259f6b" }, "pipfile-spec": 6, "requires": { @@ -16,6 +16,14 @@ ] }, "default": { + "contextlib2": { + "hashes": [ + "sha256:3fbdb64466afd23abaf6c977627b75b6139a5a3e8ce38405c5b413aed7a0471f", + "sha256:ab1e2bfe1d01d968e1b7e8d9023bc51ef3509bba217bb730cee3827e1ee82869" + ], + "markers": "python_version >= '3.6'", + "version": "==21.6.0" + }, "docopt": { "hashes": [ "sha256:49b3a825280bd66b3aa83585ef59c4a8c82f2c8a522dbe754a8bc8d08c85c491" @@ -23,7 +31,14 @@ "version": "==0.6.2" }, "example": { - "file": "https://github.com/cisagov/skeleton-python-library/archive/v0.0.1.tar.gz" + "file": "https://github.com/cisagov/skeleton-python-library/archive/v0.2.0.tar.gz" + }, + "schema": { + "hashes": [ + "sha256:f06717112c61895cabc4707752b88716e8420a8819d71404501e114f91043197", + "sha256:f3ffdeeada09ec34bf40d7d79996d9f7175db93b7a5065de0faa7f41083c1e6c" + ], + "version": "==0.7.5" }, "setuptools": { "hashes": [ diff --git a/tests/container_test.py b/tests/container_test.py index 6451766..bddee4b 100644 --- a/tests/container_test.py +++ b/tests/container_test.py @@ -10,6 +10,7 @@ ENV_VAR = "ECHO_MESSAGE" ENV_VAR_VAL = "Hello World from docker compose!" READY_MESSAGE = "This is a debug message" +DIVISION_MESSAGE = "8 / 2 == 4.000000" SECRET_QUOTE = "Three may keep a secret, if two of them are dead." # nosec RELEASE_TAG = os.getenv("RELEASE_TAG") VERSION_FILE = "src/version.txt" @@ -52,6 +53,7 @@ def test_output(dockerc, main_container): # make sure container exited if running test isolated dockerc.wait(main_container.id) log_output = main_container.logs() + assert DIVISION_MESSAGE in log_output, "Division message not found in log output." assert SECRET_QUOTE in log_output, "Secret not found in log output." From ae0d4a886c28645fec794e30ee0c7c57eeffd46b Mon Sep 17 00:00:00 2001 From: Nicholas McDonnell <50747025+mcdonnnj@users.noreply.github.com> Date: Mon, 4 Mar 2024 16:23:04 -0500 Subject: [PATCH 164/170] Bump version from 0.0.1 to 0.2.0 --- README.md | 18 +++++++++--------- src/version.txt | 2 +- 2 files changed, 10 insertions(+), 10 deletions(-) diff --git a/README.md b/README.md index a74bba4..306552f 100644 --- a/README.md +++ b/README.md @@ -24,7 +24,7 @@ appropriate for Docker containers and the major languages that we use. To run the `cisagov/example` image via Docker: ```console -docker run cisagov/example:0.0.1 +docker run cisagov/example:0.2.0 ``` ### Running with Docker Compose ### @@ -37,7 +37,7 @@ docker run cisagov/example:0.0.1 services: example: - image: cisagov/example:0.0.1 + image: cisagov/example:0.2.0 volumes: - type: bind source: @@ -82,7 +82,7 @@ environment variables. See the services: example: - image: cisagov/example:0.0.1 + image: cisagov/example:0.2.0 volumes: - type: bind source: @@ -125,7 +125,7 @@ environment variables. See the 1. Pull the new image: ```console - docker pull cisagov/example:0.0.1 + docker pull cisagov/example:0.2.0 ``` 1. Recreate and run the container by following the [previous instructions](#running-with-docker). @@ -164,12 +164,12 @@ and then update dependencies as you would above. The images of this container are tagged with [semantic versions](https://semver.org) of the underlying example project that they containerize. It is recommended that most users use a version tag (e.g. -`:0.0.1`). +`:0.2.0`). | Image:tag | Description | |-----------|-------------| -|`cisagov/example:0.0.1`| An exact release version. | -|`cisagov/example:0.0`| The most recent release matching the major and minor version numbers. | +|`cisagov/example:0.2.0`| An exact release version. | +|`cisagov/example:0.2`| The most recent release matching the major and minor version numbers. | |`cisagov/example:0`| The most recent release matching the major version number. | |`cisagov/example:edge` | The most recent image built from a merge into the `develop` branch of this repository. | |`cisagov/example:nightly` | A nightly build of the `develop` branch of this repository. | @@ -225,7 +225,7 @@ Build the image locally using this git repository as the [build context](https:/ ```console docker build \ - --tag cisagov/example:0.0.1 \ + --tag cisagov/example:0.2.0 \ https://github.com/cisagov/example.git#develop ``` @@ -256,7 +256,7 @@ Docker: --file Dockerfile-x \ --platform linux/amd64 \ --output type=docker \ - --tag cisagov/example:0.0.1 . + --tag cisagov/example:0.2.0 . ``` ## New repositories from a skeleton ## diff --git a/src/version.txt b/src/version.txt index 8acdd82..0ea3a94 100644 --- a/src/version.txt +++ b/src/version.txt @@ -1 +1 @@ -0.0.1 +0.2.0 From 965da2cea1aebc7681459817c981c715449b43c0 Mon Sep 17 00:00:00 2001 From: Nick <50747025+mcdonnnj@users.noreply.github.com> Date: Wed, 13 Mar 2024 17:17:44 -0400 Subject: [PATCH 165/170] Correct usage of the term "symlink" Co-authored-by: dav3r --- Dockerfile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Dockerfile b/Dockerfile index c40e756..8dc0ad5 100644 --- a/Dockerfile +++ b/Dockerfile @@ -87,10 +87,10 @@ RUN addgroup --system --gid ${CISA_GID} ${CISA_GROUP} \ && adduser --system --uid ${CISA_UID} --ingroup ${CISA_GROUP} ${CISA_USER} ### -# Copy in the Python virtual environment created in compile-stage, Sym-link the +# Copy in the Python virtual environment created in compile-stage, symlink the # Python binary in the venv to the system-wide Python and add the venv to the PATH. # -# Note that we sym-link the Python binary in the venv to the system-wide Python so that +# Note that we symlink the Python binary in the venv to the system-wide Python so that # any calls to `python3` will use our virtual environment. We are using short flags # because the ln binary in Alpine Linux does not support long flags. The -f instructs # ln to remove the existing file and the -s instructs ln to create a symbolic link. From 570b7d5538da33d5379a4220a55cb4aa84757ad2 Mon Sep 17 00:00:00 2001 From: Nicholas McDonnell <50747025+mcdonnnj@users.noreply.github.com> Date: Fri, 22 Nov 2024 19:16:35 -0500 Subject: [PATCH 166/170] Bump python from 3.12.2-alpine3.19 to 3.13.1-alpine3.20 The version of Python listed in the Pipfile is updated to match the new Docker image tag. --- Dockerfile | 4 ++-- src/Pipfile | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/Dockerfile b/Dockerfile index 8dc0ad5..d234755 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,6 +1,6 @@ # Official Docker images are in the form library/ while non-official # images are in the form /. -FROM docker.io/library/python:3.12.2-alpine3.19 as compile-stage +FROM docker.io/library/python:3.13.1-alpine3.20 as compile-stage ### # Unprivileged user variables @@ -53,7 +53,7 @@ RUN pipenv check --verbose \ # Official Docker images are in the form library/ while non-official # images are in the form /. -FROM docker.io/library/python:3.12.2-alpine3.19 as build-stage +FROM docker.io/library/python:3.13.1-alpine3.20 as build-stage ### # For a list of pre-defined annotation keys and value types see: diff --git a/src/Pipfile b/src/Pipfile index 6480562..fdd19e6 100644 --- a/src/Pipfile +++ b/src/Pipfile @@ -10,4 +10,4 @@ example = {file = "https://github.com/cisagov/skeleton-python-library/archive/v0 # This version should match the version of Python in the image [requires] -python_full_version = "3.12.2" +python_full_version = "3.13.1" From b3c5df532bb8f7dac5119574ce9b50bf9e52f1e1 Mon Sep 17 00:00:00 2001 From: Nicholas McDonnell <50747025+mcdonnnj@users.noreply.github.com> Date: Fri, 22 Nov 2024 19:20:40 -0500 Subject: [PATCH 167/170] Bump Python packages installed in the Dockerfile - pip from 24.0 to 24.3.1 - pipenv from 2023.12.1 to 2024.4.0 - setuptools from 69.1.1 to 75.6.0 - wheel from 0.42.0 to 0.45.1 --- Dockerfile | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/Dockerfile b/Dockerfile index d234755..444ee90 100644 --- a/Dockerfile +++ b/Dockerfile @@ -10,10 +10,10 @@ ENV CISA_HOME="/home/${CISA_USER}" ENV VIRTUAL_ENV="${CISA_HOME}/.venv" # Versions of the Python packages installed directly -ENV PYTHON_PIP_VERSION=24.0 -ENV PYTHON_PIPENV_VERSION=2023.12.1 -ENV PYTHON_SETUPTOOLS_VERSION=69.1.1 -ENV PYTHON_WHEEL_VERSION=0.42.0 +ENV PYTHON_PIP_VERSION=24.3.1 +ENV PYTHON_PIPENV_VERSION=2024.4.0 +ENV PYTHON_SETUPTOOLS_VERSION=75.6.0 +ENV PYTHON_WHEEL_VERSION=0.45.1 ### # Install the specified versions of pip, setuptools, and wheel into the system From c09b35263a6191aea522f9c67d770f54340156c1 Mon Sep 17 00:00:00 2001 From: Nicholas McDonnell <50747025+mcdonnnj@users.noreply.github.com> Date: Fri, 22 Nov 2024 19:25:00 -0500 Subject: [PATCH 168/170] Update Python virtual environment dependencies Update the dependencies installed in the Python virtual environment by running `pipenv lock` in the `src/` directory. --- src/Pipfile.lock | 26 +++++++++----------------- 1 file changed, 9 insertions(+), 17 deletions(-) diff --git a/src/Pipfile.lock b/src/Pipfile.lock index 6afbec3..408d508 100644 --- a/src/Pipfile.lock +++ b/src/Pipfile.lock @@ -1,11 +1,11 @@ { "_meta": { "hash": { - "sha256": "ad65a437d348bca066601e0d05a14c0ee23e38b59d8d8e71558cb17c57259f6b" + "sha256": "8a376df6f25cf8583d5da89da420c5e51660f33a081c1f85236643ef31601833" }, "pipfile-spec": 6, "requires": { - "python_full_version": "3.12.2" + "python_full_version": "3.13.1" }, "sources": [ { @@ -16,14 +16,6 @@ ] }, "default": { - "contextlib2": { - "hashes": [ - "sha256:3fbdb64466afd23abaf6c977627b75b6139a5a3e8ce38405c5b413aed7a0471f", - "sha256:ab1e2bfe1d01d968e1b7e8d9023bc51ef3509bba217bb730cee3827e1ee82869" - ], - "markers": "python_version >= '3.6'", - "version": "==21.6.0" - }, "docopt": { "hashes": [ "sha256:49b3a825280bd66b3aa83585ef59c4a8c82f2c8a522dbe754a8bc8d08c85c491" @@ -35,18 +27,18 @@ }, "schema": { "hashes": [ - "sha256:f06717112c61895cabc4707752b88716e8420a8819d71404501e114f91043197", - "sha256:f3ffdeeada09ec34bf40d7d79996d9f7175db93b7a5065de0faa7f41083c1e6c" + "sha256:5d976a5b50f36e74e2157b47097b60002bd4d42e65425fcc9c9befadb4255dde", + "sha256:7da553abd2958a19dc2547c388cde53398b39196175a9be59ea1caf5ab0a1807" ], - "version": "==0.7.5" + "version": "==0.7.7" }, "setuptools": { "hashes": [ - "sha256:02fa291a0471b3a18b2b2481ed902af520c69e8ae0919c13da936542754b4c56", - "sha256:5c0806c7d9af348e6dd3777b4f4dbb42c7ad85b190104837488eab9a7c945cf8" + "sha256:8199222558df7c86216af4f84c30e9b34a61d8ba19366cc914424cdbd28252f6", + "sha256:ce74b49e8f7110f9bf04883b730f4765b774ef3ef28f722cce7c273d253aaf7d" ], - "markers": "python_version >= '3.8'", - "version": "==69.1.1" + "markers": "python_version >= '3.9'", + "version": "==75.6.0" } }, "develop": {} From a1ece8f523bb39bb0c559d67c6ce2558357adaac Mon Sep 17 00:00:00 2001 From: Nicholas McDonnell <50747025+mcdonnnj@users.noreply.github.com> Date: Fri, 6 Dec 2024 04:32:58 -0500 Subject: [PATCH 169/170] Add a missing Oxford comma to a comment in the Dockerfile --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index 444ee90..b69a3f8 100644 --- a/Dockerfile +++ b/Dockerfile @@ -88,7 +88,7 @@ RUN addgroup --system --gid ${CISA_GID} ${CISA_GROUP} \ ### # Copy in the Python virtual environment created in compile-stage, symlink the -# Python binary in the venv to the system-wide Python and add the venv to the PATH. +# Python binary in the venv to the system-wide Python, and add the venv to the PATH. # # Note that we symlink the Python binary in the venv to the system-wide Python so that # any calls to `python3` will use our virtual environment. We are using short flags From fd69f453785c6b660fd3c55b01112a9a7aa2a5b2 Mon Sep 17 00:00:00 2001 From: Nicholas McDonnell <50747025+mcdonnnj@users.noreply.github.com> Date: Fri, 6 Dec 2024 10:26:13 -0500 Subject: [PATCH 170/170] Ensure FROM/AS keywords use the same casing This resolves the following warning from Docker when building the image: FromAsCasing: 'as' and 'FROM' keywords' casing do not match Co-authored-by: Shane Frasier --- Dockerfile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Dockerfile b/Dockerfile index b69a3f8..ffcae61 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,6 +1,6 @@ # Official Docker images are in the form library/ while non-official # images are in the form /. -FROM docker.io/library/python:3.13.1-alpine3.20 as compile-stage +FROM docker.io/library/python:3.13.1-alpine3.20 AS compile-stage ### # Unprivileged user variables @@ -53,7 +53,7 @@ RUN pipenv check --verbose \ # Official Docker images are in the form library/ while non-official # images are in the form /. -FROM docker.io/library/python:3.13.1-alpine3.20 as build-stage +FROM docker.io/library/python:3.13.1-alpine3.20 AS build-stage ### # For a list of pre-defined annotation keys and value types see: