From b5e5c11bcd63fcf556d31e3753eb7b84e0a85e3e Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 13 Sep 2023 02:01:21 +0000 Subject: [PATCH 001/171] Bump crazy-max/ghaction-github-status from 3 to 4 Bumps [crazy-max/ghaction-github-status](https://github.com/crazy-max/ghaction-github-status) from 3 to 4. - [Release notes](https://github.com/crazy-max/ghaction-github-status/releases) - [Commits](https://github.com/crazy-max/ghaction-github-status/compare/v3...v4) --- updated-dependencies: - dependency-name: crazy-max/ghaction-github-status dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] --- .github/workflows/build.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index e7b75a4..894bb58 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -27,7 +27,7 @@ jobs: egress-policy: audit - id: github-status name: Check GitHub status - uses: crazy-max/ghaction-github-status@v3 + uses: crazy-max/ghaction-github-status@v4 - id: dump-context name: Dump context uses: crazy-max/ghaction-dump-context@v2 From 371179ef14354df4d00d7f961754ddb86c30929b Mon Sep 17 00:00:00 2001 From: Jeremy Frasier Date: Wed, 13 Sep 2023 12:48:59 -0400 Subject: [PATCH 002/171] Add a diagnostics job for the label syncing workflow Also add a runner hardening task to the labeler job. --- .github/workflows/sync-labels.yml | 24 ++++++++++++++++++++++++ 1 file changed, 24 insertions(+) diff --git a/.github/workflows/sync-labels.yml b/.github/workflows/sync-labels.yml index 44e8e19..5a20438 100644 --- a/.github/workflows/sync-labels.yml +++ b/.github/workflows/sync-labels.yml @@ -11,7 +11,26 @@ permissions: contents: read jobs: + diagnostics: + name: Run diagnostics + runs-on: ubuntu-latest + steps: + # Note that a duplicate of this step must be added at the top of + # each job. + - id: harden-runner + name: Harden the runner + uses: step-security/harden-runner@v2 + with: + egress-policy: audit + - id: github-status + name: Check GitHub status + uses: crazy-max/ghaction-github-status@v3 + - id: dump-context + name: Dump context + uses: crazy-max/ghaction-dump-context@v2 labeler: + needs: + - diagnostics permissions: # actions/checkout needs this to fetch code contents: read @@ -19,6 +38,11 @@ jobs: issues: write runs-on: ubuntu-latest steps: + - id: harden-runner + name: Harden the runner + uses: step-security/harden-runner@v2 + with: + egress-policy: audit - uses: actions/checkout@v4 - name: Sync repository labels if: success() From 1f611fc67710100e3e6efeb395d67b82e3f52eaa Mon Sep 17 00:00:00 2001 From: Jeremy Frasier Date: Thu, 14 Sep 2023 15:45:17 -0400 Subject: [PATCH 003/171] Make the dev team the owners of the linter configuration files Co-authored-by: Nick <50747025+mcdonnnj@users.noreply.github.com> --- .github/CODEOWNERS | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index 8f5c8c5..a22502d 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -8,3 +8,12 @@ # These folks own any files in the .github directory at the root of # the repository and any of its subdirectories. /.github/ @dav3r @felddy @jasonodoom @jsf9k @mcdonnnj + +# These folks own all linting configuration files. +/.*.cfg @dav3r @felddy @jasonodoom @jsf9k @mcdonnnj +/.*.yaml @dav3r @felddy @jasonodoom @jsf9k @mcdonnnj +/.*.yml @dav3r @felddy @jasonodoom @jsf9k @mcdonnnj +/.ansible-lint @dav3r @felddy @jasonodoom @jsf9k @mcdonnnj +/.flake8 @dav3r @felddy @jasonodoom @jsf9k @mcdonnnj +/.prettierignore @dav3r @felddy @jasonodoom @jsf9k @mcdonnnj +/.yamllint @dav3r @felddy @jasonodoom @jsf9k @mcdonnnj From c356768305b027d915db7ef7be1126ee687757ad Mon Sep 17 00:00:00 2001 From: Jeremy Frasier Date: Thu, 14 Sep 2023 15:59:22 -0400 Subject: [PATCH 004/171] Make dev team members the codeowners of the requirements*.txt and setup-env files Co-authored-by: Nick <50747025+mcdonnnj@users.noreply.github.com> --- .github/CODEOWNERS | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index a22502d..b4e689e 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -17,3 +17,5 @@ /.flake8 @dav3r @felddy @jasonodoom @jsf9k @mcdonnnj /.prettierignore @dav3r @felddy @jasonodoom @jsf9k @mcdonnnj /.yamllint @dav3r @felddy @jasonodoom @jsf9k @mcdonnnj +/requirements*.txt @dav3r @felddy @jasonodoom @jsf9k @mcdonnnj +/setup-env @dav3r @felddy @jasonodoom @jsf9k @mcdonnnj From 0195005cccab3e087ec8353c5049ea633d252835 Mon Sep 17 00:00:00 2001 From: Jeremy Frasier Date: Fri, 15 Sep 2023 10:26:45 -0400 Subject: [PATCH 005/171] Explicitly list the linter config files the dev team should own @mcdonnnj correctly pointed out that other projects add their own configuration files that match, e.g., the /.*.yaml pattern. We want to ensure that we only own the linter configuration files from the skeleton. Co-authored-by: Nick <50747025+mcdonnnj@users.noreply.github.com> --- .github/CODEOWNERS | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index b4e689e..229920c 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -10,12 +10,15 @@ /.github/ @dav3r @felddy @jasonodoom @jsf9k @mcdonnnj # These folks own all linting configuration files. -/.*.cfg @dav3r @felddy @jasonodoom @jsf9k @mcdonnnj -/.*.yaml @dav3r @felddy @jasonodoom @jsf9k @mcdonnnj -/.*.yml @dav3r @felddy @jasonodoom @jsf9k @mcdonnnj /.ansible-lint @dav3r @felddy @jasonodoom @jsf9k @mcdonnnj +/.bandit.yml @dav3r @felddy @jasonodoom @jsf9k @mcdonnnj /.flake8 @dav3r @felddy @jasonodoom @jsf9k @mcdonnnj +/.isort.cfg @dav3r @felddy @jasonodoom @jsf9k @mcdonnnj +/.mdl_config.yaml @dav3r @felddy @jasonodoom @jsf9k @mcdonnnj +/.pre-commit-config.yaml @dav3r @felddy @jasonodoom @jsf9k @mcdonnnj /.prettierignore @dav3r @felddy @jasonodoom @jsf9k @mcdonnnj /.yamllint @dav3r @felddy @jasonodoom @jsf9k @mcdonnnj -/requirements*.txt @dav3r @felddy @jasonodoom @jsf9k @mcdonnnj +/requirements.txt @dav3r @felddy @jasonodoom @jsf9k @mcdonnnj +/requirements-dev.txt @dav3r @felddy @jasonodoom @jsf9k @mcdonnnj +/requirements-test.txt @dav3r @felddy @jasonodoom @jsf9k @mcdonnnj /setup-env @dav3r @felddy @jasonodoom @jsf9k @mcdonnnj From b768a289e1702e9e46d81ad5a59f51abe666a976 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 30 Oct 2023 18:41:36 +0000 Subject: [PATCH 006/171] Bump hashicorp/setup-terraform from 2 to 3 Bumps [hashicorp/setup-terraform](https://github.com/hashicorp/setup-terraform) from 2 to 3. - [Release notes](https://github.com/hashicorp/setup-terraform/releases) - [Changelog](https://github.com/hashicorp/setup-terraform/blob/main/CHANGELOG.md) - [Commits](https://github.com/hashicorp/setup-terraform/compare/v2...v3) --- updated-dependencies: - dependency-name: hashicorp/setup-terraform dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] --- .github/workflows/build.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index e7b75a4..1ff72a9 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -101,7 +101,7 @@ jobs: ${{ env.CURL_CACHE_DIR }}/"${PACKER_ZIP}" sudo mv /usr/local/bin/packer /usr/local/bin/packer-default sudo ln -s /opt/packer/packer /usr/local/bin/packer - - uses: hashicorp/setup-terraform@v2 + - uses: hashicorp/setup-terraform@v3 with: terraform_version: ${{ steps.setup-env.outputs.terraform-version }} - name: Install go-critic From 9f317002ac85066adc1efa05d9c2f55275ef7d9c Mon Sep 17 00:00:00 2001 From: Nicholas McDonnell <50747025+mcdonnnj@users.noreply.github.com> Date: Thu, 2 Nov 2023 15:49:06 -0400 Subject: [PATCH 007/171] Prefer block style to flow style We prefer block style to flow style for sequences and mappings in YAML. --- .github/workflows/build.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index e7b75a4..9332263 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -5,7 +5,8 @@ on: push: pull_request: repository_dispatch: - types: [apb] + types: + - apb env: CURL_CACHE_DIR: ~/.cache/curl From 696433ae2c3b3f9f2b3613562d3889c753c7bc34 Mon Sep 17 00:00:00 2001 From: Nicholas McDonnell <50747025+mcdonnnj@users.noreply.github.com> Date: Thu, 2 Nov 2023 15:57:37 -0400 Subject: [PATCH 008/171] Alphabetize entries in the build workflow We prefer to alphabetize mapping keys in YAML documents whenever possible. --- .github/workflows/build.yml | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 9332263..2fef5d5 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -2,8 +2,8 @@ name: build on: - push: pull_request: + push: repository_dispatch: types: - apb @@ -58,8 +58,8 @@ jobs: # it relies on the existence of a go.sum file. cache: false go-version: "1.20" - - name: Lookup Go cache directory - id: go-cache + - id: go-cache + name: Lookup Go cache directory run: | echo "dir=$(go env GOCACHE)" >> $GITHUB_OUTPUT - uses: actions/cache@v3 @@ -70,6 +70,10 @@ jobs: packer${{ steps.setup-env.outputs.packer-version }}-\ tf${{ steps.setup-env.outputs.terraform-version }}-" with: + key: "${{ env.BASE_CACHE_KEY }}\ + ${{ hashFiles('**/requirements-test.txt') }}-\ + ${{ hashFiles('**/requirements.txt') }}-\ + ${{ hashFiles('**/.pre-commit-config.yaml') }}" # Note that the .terraform directory IS NOT included in the # cache because if we were caching, then we would need to use # the `-upgrade=true` option. This option blindly pulls down the @@ -81,10 +85,6 @@ jobs: ${{ env.PRE_COMMIT_CACHE_DIR }} ${{ env.CURL_CACHE_DIR }} ${{ steps.go-cache.outputs.dir }} - key: "${{ env.BASE_CACHE_KEY }}\ - ${{ hashFiles('**/requirements-test.txt') }}-\ - ${{ hashFiles('**/requirements.txt') }}-\ - ${{ hashFiles('**/.pre-commit-config.yaml') }}" restore-keys: | ${{ env.BASE_CACHE_KEY }} - name: Setup curl cache From 6503a9e7f67d908833ff5aa178a2fd28faf5e803 Mon Sep 17 00:00:00 2001 From: Nicholas McDonnell <50747025+mcdonnnj@users.noreply.github.com> Date: Thu, 2 Nov 2023 16:00:00 -0400 Subject: [PATCH 009/171] Add a `merge_group` trigger to the build workflow This should improve compatibility with merge queues. We configure it to only trigger on the `checks_requested` type which is currently the only supported type for this trigger. If additional types are added in the future they should be added if appropriate. --- .github/workflows/build.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 2fef5d5..1928cb0 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -2,6 +2,9 @@ name: build on: + merge_group: + types: + - checks_requested pull_request: push: repository_dispatch: From 5e600a025310b1f790561cb748d61e8c18cac1f0 Mon Sep 17 00:00:00 2001 From: Jeremy Frasier Date: Tue, 5 Dec 2023 09:46:50 -0500 Subject: [PATCH 010/171] Bump the AWS Terraform provider version to 4.9 This agrees with the changes in cisagov/skeleton-tf-module#188. --- terraform-build-user/versions.tf | 14 ++++++++++---- terraform-post-packer/versions.tf | 14 ++++++++++---- 2 files changed, 20 insertions(+), 8 deletions(-) diff --git a/terraform-build-user/versions.tf b/terraform-build-user/versions.tf index 3919fa5..80277d5 100644 --- a/terraform-build-user/versions.tf +++ b/terraform-build-user/versions.tf @@ -6,12 +6,18 @@ terraform { # major version currently being used. This practice will help us # avoid unwelcome surprises. required_providers { - # Version 3.38.0 of the Terraform AWS provider is the first - # version to support default tags. - # https://www.hashicorp.com/blog/default-tags-in-the-terraform-aws-provider + # Version 4.9 of the Terraform AWS provider made changes to the S3 bucket + # refactor that is in place for versions 4.0-4.8 of the provider. With v4.9 + # only non-breaking changes and deprecation notices are introduced. Using + # this version will simplify migration to the new, broken out AWS S3 bucket + # configuration resources. Please see + # https://github.com/hashicorp/terraform-provider-aws/pull/23985 + # for more information about the changes in v4.9 and + # https://www.hashicorp.com/blog/terraform-aws-provider-4-0-refactors-s3-bucket-resource + # for more information about the S3 bucket refactor. aws = { source = "hashicorp/aws" - version = "~> 3.38" + version = "~> 4.9" } } } diff --git a/terraform-post-packer/versions.tf b/terraform-post-packer/versions.tf index 3919fa5..80277d5 100644 --- a/terraform-post-packer/versions.tf +++ b/terraform-post-packer/versions.tf @@ -6,12 +6,18 @@ terraform { # major version currently being used. This practice will help us # avoid unwelcome surprises. required_providers { - # Version 3.38.0 of the Terraform AWS provider is the first - # version to support default tags. - # https://www.hashicorp.com/blog/default-tags-in-the-terraform-aws-provider + # Version 4.9 of the Terraform AWS provider made changes to the S3 bucket + # refactor that is in place for versions 4.0-4.8 of the provider. With v4.9 + # only non-breaking changes and deprecation notices are introduced. Using + # this version will simplify migration to the new, broken out AWS S3 bucket + # configuration resources. Please see + # https://github.com/hashicorp/terraform-provider-aws/pull/23985 + # for more information about the changes in v4.9 and + # https://www.hashicorp.com/blog/terraform-aws-provider-4-0-refactors-s3-bucket-resource + # for more information about the S3 bucket refactor. aws = { source = "hashicorp/aws" - version = "~> 3.38" + version = "~> 4.9" } } } From 193e799a2601d030a0dd7bb6608752a076dc5981 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 11 Dec 2023 18:59:21 +0000 Subject: [PATCH 011/171] Bump actions/setup-go from 4 to 5 Bumps [actions/setup-go](https://github.com/actions/setup-go) from 4 to 5. - [Release notes](https://github.com/actions/setup-go/releases) - [Commits](https://github.com/actions/setup-go/compare/v4...v5) --- updated-dependencies: - dependency-name: actions/setup-go dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] --- .github/workflows/build.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index e7b75a4..f208d82 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -51,7 +51,7 @@ jobs: # We need the Go version and Go cache location for the actions/cache step, # so the Go installation must happen before that. - id: setup-go - uses: actions/setup-go@v4 + uses: actions/setup-go@v5 with: # There is no expectation for actual Go code so we disable caching as # it relies on the existence of a go.sum file. From 5c84295bb6811bc49d3ff29de067adb2d179f879 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 11 Dec 2023 18:59:25 +0000 Subject: [PATCH 012/171] Bump actions/setup-python from 4 to 5 Bumps [actions/setup-python](https://github.com/actions/setup-python) from 4 to 5. - [Release notes](https://github.com/actions/setup-python/releases) - [Commits](https://github.com/actions/setup-python/compare/v4...v5) --- updated-dependencies: - dependency-name: actions/setup-python dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] --- .github/workflows/build.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index e7b75a4..366a16b 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -45,7 +45,7 @@ jobs: uses: cisagov/setup-env-github-action@develop - uses: actions/checkout@v4 - id: setup-python - uses: actions/setup-python@v4 + uses: actions/setup-python@v5 with: python-version: "3.11" # We need the Go version and Go cache location for the actions/cache step, From 4a63dbe91c946d74d963c103b67d4f3c0746f35c Mon Sep 17 00:00:00 2001 From: Nicholas McDonnell <50747025+mcdonnnj@users.noreply.github.com> Date: Thu, 18 Jan 2024 16:25:35 -0500 Subject: [PATCH 013/171] Switch pre-commit hooks for running shfmt This hook bundles the binaries for shfmt with a Python package which removes the need to manually install the tool for the hook to function. --- .pre-commit-config.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 34f0253..ed254a0 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -86,8 +86,8 @@ repos: - id: nixpkgs-fmt # Shell script hooks - - repo: https://github.com/cisagov/pre-commit-shfmt - rev: v0.0.2 + - repo: https://github.com/scop/pre-commit-shfmt + rev: v3.7.0-4 hooks: - id: shfmt args: From 3236b1ba94d1c833dc2f9508944c6bc6e6a49f48 Mon Sep 17 00:00:00 2001 From: Nicholas McDonnell <50747025+mcdonnnj@users.noreply.github.com> Date: Thu, 18 Jan 2024 17:43:00 -0500 Subject: [PATCH 014/171] Remove installation of shfmt in the `build` workflow The new pre-commit hook provides `shfmt` binaries so we no longer need to ensure it is installed. --- .github/workflows/build.yml | 5 ----- 1 file changed, 5 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index e7b75a4..b5d46cc 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -114,11 +114,6 @@ jobs: PACKAGE_URL: github.com/securego/gosec/v2/cmd/gosec PACKAGE_VERSION: ${{ steps.setup-env.outputs.gosec-version }} run: go install ${PACKAGE_URL}@${PACKAGE_VERSION} - - name: Install shfmt - env: - PACKAGE_URL: mvdan.cc/sh/v3/cmd/shfmt - PACKAGE_VERSION: ${{ steps.setup-env.outputs.shfmt-version }} - run: go install ${PACKAGE_URL}@${PACKAGE_VERSION} - name: Install staticcheck env: PACKAGE_URL: honnef.co/go/tools/cmd/staticcheck From 5ddb14dab4e98019a6875278a1bc556c61e4bb69 Mon Sep 17 00:00:00 2001 From: Nicholas McDonnell <50747025+mcdonnnj@users.noreply.github.com> Date: Thu, 18 Jan 2024 16:56:43 -0500 Subject: [PATCH 015/171] Use long options for shfmt arguments Since shfmt now supports long command line options we should use them as that is our preference. The single quotes for the number of spaces to indent is changed to double quotes to align with our usual quotation style. --- .pre-commit-config.yaml | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index ed254a0..ca36e5d 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -92,14 +92,14 @@ repos: - id: shfmt args: # Indent by two spaces - - -i - - '2' + - --indent + - "2" # Binary operators may start a line - - -bn + - --binary-next-line # Switch cases are indented - - -ci + - --case-indent # Redirect operators are followed by a space - - -sr + - --space-redirects - repo: https://github.com/detailyang/pre-commit-shell rev: 1.0.5 hooks: From 8ecd95718a6d1b0db176cb85bf0790e4810364fb Mon Sep 17 00:00:00 2001 From: Nicholas McDonnell <50747025+mcdonnnj@users.noreply.github.com> Date: Thu, 18 Jan 2024 17:34:18 -0500 Subject: [PATCH 016/171] Add additional shfmt options These options are baked into the functionality of the old hook but must be explicitly declared for the new hook. --- .pre-commit-config.yaml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index ca36e5d..6b35e4a 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -91,6 +91,10 @@ repos: hooks: - id: shfmt args: + # List files that will be formatted + - --list + # Write result to file instead of stdout + - --write # Indent by two spaces - --indent - "2" From 242921b9856e3ede0fa1a941324c1a3f0ba116f7 Mon Sep 17 00:00:00 2001 From: Nicholas McDonnell <50747025+mcdonnnj@users.noreply.github.com> Date: Thu, 21 Sep 2023 17:49:55 -0400 Subject: [PATCH 017/171] Set the default shell for all run steps in the build workflow This sets the default shell for any run steps in the build workflow to mirror our standard shellscript writing practices. In addition to enabling our standard options it will also enable errtrace and print any commands that are run which should make debugging/troubleshooting more straightforward. --- .github/workflows/build.yml | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index e7b75a4..7299f6b 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -7,6 +7,14 @@ on: repository_dispatch: types: [apb] +# Set a default shell for any run steps. The `-Eueo pipefail` sets errtrace, +# nounset, errexit, and pipefail. The `-x` will print all commands as they are +# run. Please see the GitHub Actions documentation for more information: +# https://docs.github.com/en/actions/using-jobs/setting-default-values-for-jobs +defaults: + run: + shell: bash -Eueo pipefail -x {0} + env: CURL_CACHE_DIR: ~/.cache/curl PIP_CACHE_DIR: ~/.cache/pip From c7b18dc7e57c5261720324bd5b1425f6992e965e Mon Sep 17 00:00:00 2001 From: Nicholas McDonnell <50747025+mcdonnnj@users.noreply.github.com> Date: Fri, 12 Jan 2024 17:55:34 -0500 Subject: [PATCH 018/171] Add linting with goimports to the pre-commit configuration This will run the Go tool `goimports` against the repository if it contains any Go files. This tool bundles the functionality of `go fmt` with the additional benefit of sorting Go imports much like the isort tool we use for Python code. --- .github/workflows/build.yml | 5 +++++ .pre-commit-config.yaml | 6 +++++- 2 files changed, 10 insertions(+), 1 deletion(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index e7b75a4..8e14eab 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -109,6 +109,11 @@ jobs: PACKAGE_URL: github.com/go-critic/go-critic/cmd/gocritic PACKAGE_VERSION: ${{ steps.setup-env.outputs.go-critic-version }} run: go install ${PACKAGE_URL}@${PACKAGE_VERSION} + - name: Install goimports + env: + PACKAGE_URL: golang.org/x/tools/cmd/goimports + PACKAGE_VERSION: ${{ steps.setup-env.outputs.goimports-version }} + run: go install ${PACKAGE_URL}@${PACKAGE_VERSION} - name: Install gosec env: PACKAGE_URL: github.com/securego/gosec/v2/cmd/gosec diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 34f0253..7bb1f8c 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -78,7 +78,11 @@ repos: - id: go-vet-repo-mod # GoSec - id: go-sec-repo-mod - + # goimports + - id: go-imports-repo + args: + # Write changes to files + - -w # Nix hooks - repo: https://github.com/nix-community/nixpkgs-fmt rev: v1.3.0 From f6d9d6e29249c4b6246fca71c285d744c7409f38 Mon Sep 17 00:00:00 2001 From: Michael Saki Date: Mon, 22 Jan 2024 12:31:43 -0600 Subject: [PATCH 019/171] Add ATX Header Support for terraform-docs This is a temporary fix until @mcdonnnj has his PR approved and merged into the terraform-docs repo. This fix will perform a shallow clone of his forked branch, build the binary, and install it. --- .github/workflows/build.yml | 22 +++++++++++++++++----- 1 file changed, 17 insertions(+), 5 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index e7b75a4..e86b966 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -8,9 +8,12 @@ on: types: [apb] env: + BRANCH_NAME: improvement/support_atx_closed_markdown_headers CURL_CACHE_DIR: ~/.cache/curl + DEPTH: 1 PIP_CACHE_DIR: ~/.cache/pip PRE_COMMIT_CACHE_DIR: ~/.cache/pre-commit + REPO_URL: https://github.com/mcdonnnj/terraform-docs.git RUN_TMATE: ${{ secrets.RUN_TMATE }} jobs: @@ -124,11 +127,20 @@ jobs: PACKAGE_URL: honnef.co/go/tools/cmd/staticcheck PACKAGE_VERSION: ${{ steps.setup-env.outputs.staticcheck-version }} run: go install ${PACKAGE_URL}@${PACKAGE_VERSION} - - name: Install Terraform-docs - env: - PACKAGE_URL: github.com/terraform-docs/terraform-docs - PACKAGE_VERSION: ${{ steps.setup-env.outputs.terraform-docs-version }} - run: go install ${PACKAGE_URL}@${PACKAGE_VERSION} + # We are temporarily using @mcdonnnj's forked branch of terraform-docs + # until his PR: https://github.com/terraform-docs/terraform-docs/pull/745 + # is approved. This temporary fix will allow for ATX Header Support when + # terraform-docs is ran during lint. + - name: Clone ATX Headers branch from Terraform-docs Fork + run: | + git clone --branch $BRANCH_NAME --single-branch \ + --depth=$DEPTH $REPO_URL /tmp/terraform-docs + - name: Build and Install Terraform-docs Binary + run: | + cd /tmp/terraform-docs + GOBIN=$(go env GOPATH)/bin + go build -o $GOBIN/terraform-docs + echo "$GOBIN" >> $GITHUB_PATH - name: Install dependencies run: | python -m pip install --upgrade pip setuptools wheel From 544e4789e052978d00f982ef45d8d0de9f5e4748 Mon Sep 17 00:00:00 2001 From: Michael Saki Date: Mon, 22 Jan 2024 14:03:00 -0600 Subject: [PATCH 020/171] Add prepended names to variables to describe their function Co-authored-by: Nick <50747025+mcdonnnj@users.noreply.github.com> --- .github/workflows/build.yml | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index e86b966..99b42c1 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -8,13 +8,13 @@ on: types: [apb] env: - BRANCH_NAME: improvement/support_atx_closed_markdown_headers CURL_CACHE_DIR: ~/.cache/curl - DEPTH: 1 PIP_CACHE_DIR: ~/.cache/pip PRE_COMMIT_CACHE_DIR: ~/.cache/pre-commit - REPO_URL: https://github.com/mcdonnnj/terraform-docs.git RUN_TMATE: ${{ secrets.RUN_TMATE }} + TERRAFORM_DOCS_REPO_BRANCH_NAME: improvement/support_atx_closed_markdown_headers + TERRAFORM_DOCS_REPO_DEPTH: 1 + TERRAFORM_DOCS_REPO_URL: https://github.com/mcdonnnj/terraform-docs.git jobs: diagnostics: @@ -133,8 +133,10 @@ jobs: # terraform-docs is ran during lint. - name: Clone ATX Headers branch from Terraform-docs Fork run: | - git clone --branch $BRANCH_NAME --single-branch \ - --depth=$DEPTH $REPO_URL /tmp/terraform-docs + git clone --single-branch \ + --branch $TERRAFORM_DOCS_REPO_BRANCH_NAME \ + --depth $TERRAFORM_DOCS_REPO_DEPTH \ + $TERRAFORM_DOCS_REPO_URL /tmp/terraform-docs - name: Build and Install Terraform-docs Binary run: | cd /tmp/terraform-docs From f5fa0ff4ec1ded10a2906ade6300bba05cc2e81c Mon Sep 17 00:00:00 2001 From: Michael Saki Date: Mon, 22 Jan 2024 14:05:01 -0600 Subject: [PATCH 021/171] Remove unnecessary capitalizations and fix grammar Co-authored-by: Shane Frasier --- .github/workflows/build.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 99b42c1..ca719aa 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -129,15 +129,15 @@ jobs: run: go install ${PACKAGE_URL}@${PACKAGE_VERSION} # We are temporarily using @mcdonnnj's forked branch of terraform-docs # until his PR: https://github.com/terraform-docs/terraform-docs/pull/745 - # is approved. This temporary fix will allow for ATX Header Support when - # terraform-docs is ran during lint. - - name: Clone ATX Headers branch from Terraform-docs Fork + # is approved. This temporary fix will allow for ATX header support when + # terraform-docs is run during linting. + - name: Clone ATX headers branch from terraform-docs fork run: | git clone --single-branch \ --branch $TERRAFORM_DOCS_REPO_BRANCH_NAME \ --depth $TERRAFORM_DOCS_REPO_DEPTH \ $TERRAFORM_DOCS_REPO_URL /tmp/terraform-docs - - name: Build and Install Terraform-docs Binary + - name: Build and install terraform-docs binary run: | cd /tmp/terraform-docs GOBIN=$(go env GOPATH)/bin From 36361dd1c74c9855b87f2b4aec5a9be1fa7c416e Mon Sep 17 00:00:00 2001 From: Michael Saki Date: Mon, 22 Jan 2024 14:14:27 -0600 Subject: [PATCH 022/171] Simplify steps in the build/install portion of workflow PATH is handled by `setup-go` so we can refactor the code setting it. Also we are taking advantage of the -C switch to handle building from the cloned repository. Co-authored-by: Nick <50747025+mcdonnnj@users.noreply.github.com> --- .github/workflows/build.yml | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index ca719aa..6ab90de 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -139,10 +139,9 @@ jobs: $TERRAFORM_DOCS_REPO_URL /tmp/terraform-docs - name: Build and install terraform-docs binary run: | - cd /tmp/terraform-docs - GOBIN=$(go env GOPATH)/bin - go build -o $GOBIN/terraform-docs - echo "$GOBIN" >> $GITHUB_PATH + go build \ + -C /tmp/terraform-docs \ + -o $(go env GOPATH)/bin/terraform-docs - name: Install dependencies run: | python -m pip install --upgrade pip setuptools wheel From 3711ebe461d81987b4121903d2f3f1b290fccad3 Mon Sep 17 00:00:00 2001 From: Michael Saki Date: Tue, 23 Jan 2024 08:04:07 -0600 Subject: [PATCH 023/171] Add TODO label --- .github/workflows/build.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 6ab90de..f5ba773 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -51,7 +51,7 @@ jobs: uses: actions/setup-python@v4 with: python-version: "3.11" - # We need the Go version and Go cache location for the actions/cache step, + # TODO: We need the Go version and Go cache location for the actions/cache step, # so the Go installation must happen before that. - id: setup-go uses: actions/setup-go@v4 From d114fb4233f22cb9edb3e1a6555ca0a677bcfc7e Mon Sep 17 00:00:00 2001 From: Michael Saki Date: Tue, 23 Jan 2024 08:24:12 -0600 Subject: [PATCH 024/171] Move TODO and add link to the issue `TODO` was placed on the wrong comment block. Also I am adding a link to the issue for the TODO. --- .github/workflows/build.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index f5ba773..0068f1f 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -51,7 +51,7 @@ jobs: uses: actions/setup-python@v4 with: python-version: "3.11" - # TODO: We need the Go version and Go cache location for the actions/cache step, + # We need the Go version and Go cache location for the actions/cache step, # so the Go installation must happen before that. - id: setup-go uses: actions/setup-go@v4 @@ -127,6 +127,7 @@ jobs: PACKAGE_URL: honnef.co/go/tools/cmd/staticcheck PACKAGE_VERSION: ${{ steps.setup-env.outputs.staticcheck-version }} run: go install ${PACKAGE_URL}@${PACKAGE_VERSION} + # TODO: https://github.com/cisagov/skeleton-generic/issues/165 # We are temporarily using @mcdonnnj's forked branch of terraform-docs # until his PR: https://github.com/terraform-docs/terraform-docs/pull/745 # is approved. This temporary fix will allow for ATX header support when From c907cfc82a8f329fbd4ad0ee17f3500922c51711 Mon Sep 17 00:00:00 2001 From: Michael Saki Date: Tue, 23 Jan 2024 08:41:41 -0600 Subject: [PATCH 025/171] Alphabetize switches Co-authored-by: dav3r --- .github/workflows/build.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 0068f1f..35391d5 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -134,9 +134,10 @@ jobs: # terraform-docs is run during linting. - name: Clone ATX headers branch from terraform-docs fork run: | - git clone --single-branch \ + git clone \ --branch $TERRAFORM_DOCS_REPO_BRANCH_NAME \ --depth $TERRAFORM_DOCS_REPO_DEPTH \ + --single-branch \ $TERRAFORM_DOCS_REPO_URL /tmp/terraform-docs - name: Build and install terraform-docs binary run: | From 48db3e3381aa61bdbe67dab64454db3e7f332c75 Mon Sep 17 00:00:00 2001 From: Michael Saki Date: Thu, 25 Jan 2024 12:19:41 -0600 Subject: [PATCH 026/171] Allow setup-env to specify Python version This commit is introducing 2 new flags into the setup-env script. -l or --list-versions will list available Python versions and allow the user to select a version interactively. The second flag -v or --version will allow a user to set the version if installed. (e.g. ./setup-env -v 3.9.6) --- setup-env | 42 +++++++++++++++++++++++++++++++++++++++--- 1 file changed, 39 insertions(+), 3 deletions(-) diff --git a/setup-env b/setup-env index 77926bf..b99ec5f 100755 --- a/setup-env +++ b/setup-env @@ -25,6 +25,8 @@ Options: -h --help Show this message. -i --install-hooks Install hook environments for all environments in the pre-commit config file. + -v --version Specify the Python version for the virtual environment. + -l --list-versions List available Python versions and select interactively. END_OF_LINE ) @@ -35,6 +37,13 @@ FORCE=0 # Positional parameters PARAMS="" +# Flags to allow a user to specify which version of Python they want to use +PYTHON_VERSION="" +LIST_VERSIONS=0 + +# Temp file that is used to search through available installed Python versions +TMPFILE=/tmp/versions.$$ + # Parse command line arguments while (("$#")); do case "$1" in @@ -50,6 +59,14 @@ while (("$#")); do INSTALL_HOOKS=1 shift ;; + -v | --version) + PYTHON_VERSION=$2 + shift 2 + ;; + -l | --list-versions) + LIST_VERSIONS=1 + shift + ;; -*) # unsupported flags echo "Error: Unsupported flag $1" >&2 exit 1 @@ -111,6 +128,25 @@ else fi set -o nounset +# List Python versions and select one interactively +if [ $LIST_VERSIONS -ne 0 ]; then + echo Available Python versions: + pyenv versions --bare --skip-aliases --skip-envs + read -p -r "Enter the desired Python version: " PYTHON_VERSION +fi + +# Check if PYTHON_VERSION isn't empty. If it is installed, set it locally. +pyenv versions --bare --skip-aliases --skip-envs > $TMPFILE +if [ -n "$PYTHON_VERSION" ]; then + if grep --fixed-strings --quiet "$PYTHON_VERSION" $TMPFILE; then + echo Using Python version "$PYTHON_VERSION" + pyenv local "$PYTHON_VERSION" + else + echo Error: Python version "$PYTHON_VERSION" is not installed. + fi + exit 1 +fi + # Remove any lingering local configuration. if [ $FORCE -ne 0 ]; then rm -f .python-version @@ -130,10 +166,10 @@ fi # Create a new virtual environment for this project if ! pyenv virtualenv "${env_name}"; then cat << END_OF_LINE - An existing virtual environment named $env_name was found. Either delete this - environment yourself or re-run with --force option to have it deleted. + An existing virtual environment named $env_name was found. Either delete this + environment yourself or re-run with --force option to have it deleted. - pyenv virtualenv-delete ${env_name} + pyenv virtualenv-delete ${env_name} END_OF_LINE exit 1 From c10929afae693990a3d4d034faa2c1ab1b2689ee Mon Sep 17 00:00:00 2001 From: Michael Saki Date: Thu, 25 Jan 2024 14:15:40 -0600 Subject: [PATCH 027/171] Add /dev/null and remove TMPFILE This makes the code a bit cleaner and still accomplishes the same functionality Co-authored-by: Shane Frasier --- setup-env | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/setup-env b/setup-env index b99ec5f..769522a 100755 --- a/setup-env +++ b/setup-env @@ -41,9 +41,6 @@ PARAMS="" PYTHON_VERSION="" LIST_VERSIONS=0 -# Temp file that is used to search through available installed Python versions -TMPFILE=/tmp/versions.$$ - # Parse command line arguments while (("$#")); do case "$1" in @@ -136,9 +133,8 @@ if [ $LIST_VERSIONS -ne 0 ]; then fi # Check if PYTHON_VERSION isn't empty. If it is installed, set it locally. -pyenv versions --bare --skip-aliases --skip-envs > $TMPFILE if [ -n "$PYTHON_VERSION" ]; then - if grep --fixed-strings --quiet "$PYTHON_VERSION" $TMPFILE; then + if pyenv versions --bare --skip-aliases --skip-envs | grep --fixed-strings "$PYTHON_VERSION" >/dev/null; then echo Using Python version "$PYTHON_VERSION" pyenv local "$PYTHON_VERSION" else From adada40e6fa6aed6fa14f253a5b86f313b3dfa8f Mon Sep 17 00:00:00 2001 From: Michael Saki Date: Thu, 25 Jan 2024 14:19:54 -0600 Subject: [PATCH 028/171] Place flags in the correct order for -r and -p --- setup-env | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/setup-env b/setup-env index 769522a..2adbd5a 100755 --- a/setup-env +++ b/setup-env @@ -129,12 +129,12 @@ set -o nounset if [ $LIST_VERSIONS -ne 0 ]; then echo Available Python versions: pyenv versions --bare --skip-aliases --skip-envs - read -p -r "Enter the desired Python version: " PYTHON_VERSION + read -r -p "Enter the desired Python version: " PYTHON_VERSION fi # Check if PYTHON_VERSION isn't empty. If it is installed, set it locally. if [ -n "$PYTHON_VERSION" ]; then - if pyenv versions --bare --skip-aliases --skip-envs | grep --fixed-strings "$PYTHON_VERSION" >/dev/null; then + if pyenv versions --bare --skip-aliases --skip-envs | grep --fixed-strings "$PYTHON_VERSION" > /dev/null; then echo Using Python version "$PYTHON_VERSION" pyenv local "$PYTHON_VERSION" else From 1861b9b86778613bdaeff804418f2d00706f4a08 Mon Sep 17 00:00:00 2001 From: Michael Saki Date: Thu, 25 Jan 2024 14:21:49 -0600 Subject: [PATCH 029/171] Remove unneccessary spacing --- setup-env | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/setup-env b/setup-env index 2adbd5a..5abb624 100755 --- a/setup-env +++ b/setup-env @@ -162,10 +162,10 @@ fi # Create a new virtual environment for this project if ! pyenv virtualenv "${env_name}"; then cat << END_OF_LINE - An existing virtual environment named $env_name was found. Either delete this - environment yourself or re-run with --force option to have it deleted. + An existing virtual environment named $env_name was found. Either delete this + environment yourself or re-run with --force option to have it deleted. - pyenv virtualenv-delete ${env_name} + pyenv virtualenv-delete ${env_name} END_OF_LINE exit 1 From 3f623e493752c581829271a148cb9fff3ad4f4ab Mon Sep 17 00:00:00 2001 From: Michael Saki Date: Thu, 25 Jan 2024 14:25:37 -0600 Subject: [PATCH 030/171] Alphabetize flags and descriptions Co-authored-by: Shane Frasier --- setup-env | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/setup-env b/setup-env index 5abb624..5282e49 100755 --- a/setup-env +++ b/setup-env @@ -25,8 +25,8 @@ Options: -h --help Show this message. -i --install-hooks Install hook environments for all environments in the pre-commit config file. - -v --version Specify the Python version for the virtual environment. -l --list-versions List available Python versions and select interactively. + -v --version Specify the Python version for the virtual environment. END_OF_LINE ) @@ -56,14 +56,14 @@ while (("$#")); do INSTALL_HOOKS=1 shift ;; - -v | --version) - PYTHON_VERSION=$2 - shift 2 - ;; -l | --list-versions) LIST_VERSIONS=1 shift ;; + -v | --version) + PYTHON_VERSION=$2 + shift 2 + ;; -*) # unsupported flags echo "Error: Unsupported flag $1" >&2 exit 1 From 9497dc276e9f148211e32cc800342846cfa25e33 Mon Sep 17 00:00:00 2001 From: Jeremy Frasier Date: Fri, 26 Jan 2024 10:08:25 -0500 Subject: [PATCH 031/171] Move misplaced exit If the Python version exists then we want the script to continue execution. --- setup-env | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup-env b/setup-env index 5282e49..5d15679 100755 --- a/setup-env +++ b/setup-env @@ -139,8 +139,8 @@ if [ -n "$PYTHON_VERSION" ]; then pyenv local "$PYTHON_VERSION" else echo Error: Python version "$PYTHON_VERSION" is not installed. + exit 1 fi - exit 1 fi # Remove any lingering local configuration. From e1d0f28ba1d96b2da0af91e277648ca88d0c3f35 Mon Sep 17 00:00:00 2001 From: Jeremy Frasier Date: Fri, 26 Jan 2024 10:10:23 -0500 Subject: [PATCH 032/171] Remove premature pyenv local command pyenv local is run below. --- setup-env | 1 - 1 file changed, 1 deletion(-) diff --git a/setup-env b/setup-env index 5d15679..a78f940 100755 --- a/setup-env +++ b/setup-env @@ -136,7 +136,6 @@ fi if [ -n "$PYTHON_VERSION" ]; then if pyenv versions --bare --skip-aliases --skip-envs | grep --fixed-strings "$PYTHON_VERSION" > /dev/null; then echo Using Python version "$PYTHON_VERSION" - pyenv local "$PYTHON_VERSION" else echo Error: Python version "$PYTHON_VERSION" is not installed. exit 1 From 517b336210269d890c28e5c05d1c13e866fb7b15 Mon Sep 17 00:00:00 2001 From: Jeremy Frasier Date: Fri, 26 Jan 2024 10:11:50 -0500 Subject: [PATCH 033/171] Include PYTHON_VERSION when running pyenv virtualenv If PYTHON_VERSION is an empty string then the system Python will be used. --- setup-env | 19 ++++++++++++++----- 1 file changed, 14 insertions(+), 5 deletions(-) diff --git a/setup-env b/setup-env index a78f940..0aa173d 100755 --- a/setup-env +++ b/setup-env @@ -37,8 +37,8 @@ FORCE=0 # Positional parameters PARAMS="" -# Flags to allow a user to specify which version of Python they want to use -PYTHON_VERSION="" +# A flag to allow a user to specify which version of Python they want +# to use. LIST_VERSIONS=0 # Parse command line arguments @@ -132,8 +132,9 @@ if [ $LIST_VERSIONS -ne 0 ]; then read -r -p "Enter the desired Python version: " PYTHON_VERSION fi -# Check if PYTHON_VERSION isn't empty. If it is installed, set it locally. -if [ -n "$PYTHON_VERSION" ]; then +# Check if PYTHON_VERSION is defined. If it is defined then check that +# it is a valid value. +if [ -n "${PYTHON_VERSION+x}" ]; then if pyenv versions --bare --skip-aliases --skip-envs | grep --fixed-strings "$PYTHON_VERSION" > /dev/null; then echo Using Python version "$PYTHON_VERSION" else @@ -159,7 +160,15 @@ END_OF_LINE fi # Create a new virtual environment for this project -if ! pyenv virtualenv "${env_name}"; then +# +# If $PYTHON_VERSION is undefined then the system Python will be used. +# +# We can't quote ${PYTHON_VERSION:=} below since if the variable is +# undefined then we want nothing to appear; this is the reason for the +# "shellcheck disable" line below. +# +# shellcheck disable=SC2086 +if ! pyenv virtualenv ${PYTHON_VERSION:=} "${env_name}"; then cat << END_OF_LINE An existing virtual environment named $env_name was found. Either delete this environment yourself or re-run with --force option to have it deleted. From 2e5794cb2aeaa76e8c94e3113f903954a9564eb4 Mon Sep 17 00:00:00 2001 From: Michael Saki Date: Tue, 30 Jan 2024 12:51:51 -0600 Subject: [PATCH 034/171] Add getopt variables and short flags --- setup-env | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/setup-env b/setup-env index 0aa173d..63cebd3 100755 --- a/setup-env +++ b/setup-env @@ -34,9 +34,21 @@ END_OF_LINE # Flag to force deletion and creation of virtual environment FORCE=0 +# Initialize the all other flags +INSTALL_HOOKS=0 +LIST_VERSIONS=0 +PYTHON_VERSION="" +VENV_NAME="" + # Positional parameters PARAMS="" +# Define short options for getopt +OPTS="fhilv:n:" + +# Parse options using BSD getopt +OPTIND=1 + # A flag to allow a user to specify which version of Python they want # to use. LIST_VERSIONS=0 From 8a5003195b6cc3a069fb0c4ec678f20372c4e10d Mon Sep 17 00:00:00 2001 From: Michael Saki Date: Tue, 30 Jan 2024 12:54:12 -0600 Subject: [PATCH 035/171] Remove redundant flag initialization --- setup-env | 4 ---- 1 file changed, 4 deletions(-) diff --git a/setup-env b/setup-env index 63cebd3..da640f0 100755 --- a/setup-env +++ b/setup-env @@ -49,10 +49,6 @@ OPTS="fhilv:n:" # Parse options using BSD getopt OPTIND=1 -# A flag to allow a user to specify which version of Python they want -# to use. -LIST_VERSIONS=0 - # Parse command line arguments while (("$#")); do case "$1" in From 0df0e6aca8252e6356762b9284f3165ee39e2017 Mon Sep 17 00:00:00 2001 From: Michael Saki Date: Tue, 30 Jan 2024 13:59:04 -0600 Subject: [PATCH 036/171] Add getopt functionality and -n flag This commit makes a couple changes. The first change is adding the BSD getopt tool to simplify the parsing of flags and arguments. Second, we are adding the -n flag so the user can specify the name of the virtual environment if they choose. --- setup-env | 57 +++++++++++++++++++++++++++++++++++-------------------- 1 file changed, 36 insertions(+), 21 deletions(-) diff --git a/setup-env b/setup-env index da640f0..e097c82 100755 --- a/setup-env +++ b/setup-env @@ -49,40 +49,55 @@ OPTS="fhilv:n:" # Parse options using BSD getopt OPTIND=1 +# Display installed python versions +python_versions() { + pyenv versions --bare --skip-aliases --skip-envs +} + # Parse command line arguments -while (("$#")); do - case "$1" in - -f | --force) +while getopts :$OPTS opt; do + case $opt in + f) FORCE=1 - shift ;; - -h | --help) - echo "${USAGE}" + h) + echo "$USAGE" exit 0 ;; - -i | --install-hooks) + i) INSTALL_HOOKS=1 - shift ;; - -l | --list-versions) + l) LIST_VERSIONS=1 - shift ;; - -v | --version) - PYTHON_VERSION=$2 - shift 2 + n) + VENV_NAME="$OPTARG" + ;; + v) + PYTHON_VERSION="$OPTARG" + # Check if Python version is valid and installed + if ! python_versions | grep --fixed-strings "$PYTHON_VERSION" > /dev/null; then + echo "Error: Python version $PYTHON_VERSION is not installed. Versions available:" + python_versions + exit 1 + fi ;; - -*) # unsupported flags - echo "Error: Unsupported flag $1" >&2 + \:) + echo Error: Option -$OPTARG requires an argument. + echo "$USAGE" exit 1 ;; - *) # preserve positional arguments - PARAMS="$PARAMS $1" - shift + \?) + echo -e "Invalid option please look through usage: \n" + echo "$USAGE" + exit 1 ;; + esac done +shift $((OPTIND-1)) + # set positional arguments in their proper place eval set -- "$PARAMS" @@ -124,9 +139,9 @@ fi set +o nounset # Determine the virtual environment name -if [ "$1" ]; then +if [ -n "$VENV_NAME" ]; then # Use the user-provided environment name - env_name=$1 + env_name="$VENV_NAME" else # Set the environment name to the last part of the working directory. env_name=${PWD##*/} @@ -136,7 +151,7 @@ set -o nounset # List Python versions and select one interactively if [ $LIST_VERSIONS -ne 0 ]; then echo Available Python versions: - pyenv versions --bare --skip-aliases --skip-envs + python_versions read -r -p "Enter the desired Python version: " PYTHON_VERSION fi From 60cad12a2a64508c1e5c80092118747b64bb32e6 Mon Sep 17 00:00:00 2001 From: Michael Saki Date: Tue, 30 Jan 2024 14:16:52 -0600 Subject: [PATCH 037/171] Update the usage and force documentation This commit will update the usage documentation so now it will display all the new flags, how to use them, and what they do. Since the long option isn't available for the BSD version of getopt we have to use short options only. The force documentation has been updated to reflect this change. Also removed some redundant code that wasn't necessary. --- setup-env | 47 +++++++++++++++++++---------------------------- 1 file changed, 19 insertions(+), 28 deletions(-) diff --git a/setup-env b/setup-env index e097c82..4ed12d9 100755 --- a/setup-env +++ b/setup-env @@ -17,16 +17,18 @@ It does the following: - Configures git upstream remote "lineage" repositories. Usage: - setup-env [options] [virt_env_name] - setup-env (-h | --help) + setup-env [-n] [virt_env_name] + setup-env [-v] [python_version] + setup-env (-h) Options: - -f --force Delete virtual enviroment if it already exists. - -h --help Show this message. - -i --install-hooks Install hook environments for all environments in the - pre-commit config file. - -l --list-versions List available Python versions and select interactively. - -v --version Specify the Python version for the virtual environment. + -f Delete virtual enviroment if it already exists. + -h Show this message. + -i Install hook environments for all environments in the + pre-commit config file. + -l List available Python versions and select interactively. + -n Choose the name of the virtual environment. + -v Specify the Python version for the virtual environment. END_OF_LINE ) @@ -49,9 +51,9 @@ OPTS="fhilv:n:" # Parse options using BSD getopt OPTIND=1 -# Display installed python versions +# Display installed python versions python_versions() { - pyenv versions --bare --skip-aliases --skip-envs + pyenv versions --bare --skip-aliases --skip-envs } # Parse command line arguments @@ -82,21 +84,21 @@ while getopts :$OPTS opt; do exit 1 fi ;; - \:) - echo Error: Option -$OPTARG requires an argument. + :) + echo -e "Error: Option -$OPTARG requires an argument. \n" echo "$USAGE" exit 1 ;; \?) - echo -e "Invalid option please look through usage: \n" + echo -e "Invalid option please look through usage: \n" echo "$USAGE" exit 1 ;; - + esac done -shift $((OPTIND-1)) +shift $((OPTIND - 1)) # set positional arguments in their proper place eval set -- "$PARAMS" @@ -155,17 +157,6 @@ if [ $LIST_VERSIONS -ne 0 ]; then read -r -p "Enter the desired Python version: " PYTHON_VERSION fi -# Check if PYTHON_VERSION is defined. If it is defined then check that -# it is a valid value. -if [ -n "${PYTHON_VERSION+x}" ]; then - if pyenv versions --bare --skip-aliases --skip-envs | grep --fixed-strings "$PYTHON_VERSION" > /dev/null; then - echo Using Python version "$PYTHON_VERSION" - else - echo Error: Python version "$PYTHON_VERSION" is not installed. - exit 1 - fi -fi - # Remove any lingering local configuration. if [ $FORCE -ne 0 ]; then rm -f .python-version @@ -173,7 +164,7 @@ if [ $FORCE -ne 0 ]; then elif [[ -f .python-version ]]; then cat << 'END_OF_LINE' An existing .python-version file was found. Either remove this file yourself - or re-run with --force option to have it deleted along with the associated + or re-run with -f (force) option to have it deleted along with the associated virtual environment. rm .python-version @@ -194,7 +185,7 @@ fi if ! pyenv virtualenv ${PYTHON_VERSION:=} "${env_name}"; then cat << END_OF_LINE An existing virtual environment named $env_name was found. Either delete this - environment yourself or re-run with --force option to have it deleted. + environment yourself or re-run with -f (force) option to have it deleted. pyenv virtualenv-delete ${env_name} From b6ab6d82fb63bece7926b9a524bbb59a96c0cb06 Mon Sep 17 00:00:00 2001 From: Michael Saki Date: Tue, 6 Feb 2024 19:58:49 -0600 Subject: [PATCH 038/171] Update usage with long options --- setup-env | 22 ++++++++++++---------- 1 file changed, 12 insertions(+), 10 deletions(-) diff --git a/setup-env b/setup-env index 4ed12d9..898907b 100755 --- a/setup-env +++ b/setup-env @@ -9,6 +9,8 @@ USAGE=$( Configure a development environment for this repository. It does the following: + - Allows user to specify Python version. + - Allows user to choose name for their virtual environment. - Verifies pyenv and pyenv-virtualenv are installed. - Creates a Python virtual environment. - Configures the activation of the virtual enviroment for the repo directory. @@ -17,18 +19,18 @@ It does the following: - Configures git upstream remote "lineage" repositories. Usage: - setup-env [-n] [virt_env_name] - setup-env [-v] [python_version] - setup-env (-h) + setup-env (-n | --name) [virt_env_name] + setup-env (-v | --version) [python_version] + setup-env (-h | --help) Options: - -f Delete virtual enviroment if it already exists. - -h Show this message. - -i Install hook environments for all environments in the - pre-commit config file. - -l List available Python versions and select interactively. - -n Choose the name of the virtual environment. - -v Specify the Python version for the virtual environment. + -f | --force Delete virtual enviroment if it already exists. + -h | --help Show this message. + -i | --install-hooks Install hook environments for all environments in the + pre-commit config file. + -l | --list-versions List available Python versions and select interactively. + -n | --name Choose the name of the virtual environment. + -v | --version Specify the Python version for the virtual environment. END_OF_LINE ) From d362614a13731f3884856fa2ee37ade0089fda47 Mon Sep 17 00:00:00 2001 From: Michael Saki Date: Tue, 6 Feb 2024 20:28:05 -0600 Subject: [PATCH 039/171] Add gnu-getopt functionality and error handling This commit will make it so long options are supported. It also includes some improved error handling. It will display a message if a Mac OS user doesn't have gnu-getopt installed on their system and suggest steps to take to get it installed via brew. --- setup-env | 107 ++++++++++++++++++++++++++++++++++-------------------- 1 file changed, 68 insertions(+), 39 deletions(-) diff --git a/setup-env b/setup-env index 898907b..f4cdf75 100755 --- a/setup-env +++ b/setup-env @@ -35,6 +35,11 @@ Options: END_OF_LINE ) +# Display installed python versions +python_versions() { + pyenv versions --bare --skip-aliases --skip-envs +} + # Flag to force deletion and creation of virtual environment FORCE=0 @@ -44,67 +49,82 @@ LIST_VERSIONS=0 PYTHON_VERSION="" VENV_NAME="" -# Positional parameters -PARAMS="" +# Define long options +LONGOPTS="force,help,install-hooks,list-versions,name:,version:" # Define short options for getopt -OPTS="fhilv:n:" +SHORTOPTS="fhiln:v:" -# Parse options using BSD getopt -OPTIND=1 +# Check if GNU getopt is available +if [[ $(getopt --version 2> /dev/null) != *"getopt from util-linux"* ]]; then + cat << 'END_OF_LINE' -# Display installed python versions -python_versions() { - pyenv versions --bare --skip-aliases --skip-envs -} + Gnu-getopt is not detected and is a dependency to run this script. + On the Mac, we recommend installing brew, https://brew.sh/. Then installation + is as simple as `brew install gnu-getopt` and adding this to your + profile: + + export PATH="/usr/local/opt/gnu-getopt/bin:$PATH" + +END_OF_LINE + exit 1 +fi -# Parse command line arguments -while getopts :$OPTS opt; do - case $opt in - f) +# Use GNU getopt to parse options +if ! PARSED=$(getopt --options $SHORTOPTS --longoptions $LONGOPTS --name "$0" -- "$@"); then + echo "Error parsing options" + exit 2 +fi +eval set -- "$PARSED" + +while true; do + case "$1" in + -f | --force) FORCE=1 + shift ;; - h) + -h | --help) echo "$USAGE" exit 0 ;; - i) + -i | --install-hooks) INSTALL_HOOKS=1 + shift ;; - l) + -l | --list-versions) LIST_VERSIONS=1 + shift ;; - n) - VENV_NAME="$OPTARG" + -n | --name) + VENV_NAME="$2" + shift 2 ;; - v) - PYTHON_VERSION="$OPTARG" - # Check if Python version is valid and installed - if ! python_versions | grep --fixed-strings "$PYTHON_VERSION" > /dev/null; then - echo "Error: Python version $PYTHON_VERSION is not installed. Versions available:" - python_versions - exit 1 + -v | --version) + PYTHON_VERSION="$2" + shift 2 + # Check the Python versions being passed in. + if [ -n "${PYTHON_VERSION+x}" ]; then + if python_versions | grep -E "^${PYTHON_VERSION}$" > /dev/null; then + echo Using Python version "$PYTHON_VERSION" + else + echo Error: Python version "$PYTHON_VERSION" is not installed. + echo Installed Python versions are: + python_versions + exit 1 + fi fi ;; - :) - echo -e "Error: Option -$OPTARG requires an argument. \n" - echo "$USAGE" - exit 1 + --) + shift + break ;; - \?) - echo -e "Invalid option please look through usage: \n" - echo "$USAGE" - exit 1 + *) + echo "Programming error" + exit 3 ;; - esac done -shift $((OPTIND - 1)) - -# set positional arguments in their proper place -eval set -- "$PARAMS" - # Check to see if pyenv is installed if [ -z "$(command -v pyenv)" ] || { [ -z "$(command -v pyenv-virtualenv)" ] && [ ! -f "$(pyenv root)/plugins/pyenv-virtualenv/bin/pyenv-virtualenv" ]; }; then echo "pyenv and pyenv-virtualenv are required." @@ -157,6 +177,15 @@ if [ $LIST_VERSIONS -ne 0 ]; then echo Available Python versions: python_versions read -r -p "Enter the desired Python version: " PYTHON_VERSION + # Check the Python versions being passed in. + if [ -n "${PYTHON_VERSION+x}" ]; then + if python_versions | grep -E "^${PYTHON_VERSION}$" > /dev/null; then + echo Using Python version "$PYTHON_VERSION" + else + echo Error: Python version "$PYTHON_VERSION" is not installed. + exit 1 + fi + fi fi # Remove any lingering local configuration. From f92458454394db61e8080b72b42fd340b3b64aa1 Mon Sep 17 00:00:00 2001 From: Michael Saki Date: Tue, 6 Feb 2024 20:49:34 -0600 Subject: [PATCH 040/171] Add documentation in CONTRIBUTING.md for gnu-getopt --- CONTRIBUTING.md | 16 +++++++++++----- 1 file changed, 11 insertions(+), 5 deletions(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 27bb9e7..dbbe417 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -46,9 +46,12 @@ There are a few ways to do this, but we prefer to use create and manage a Python virtual environment specific to this project. -If you already have `pyenv` and `pyenv-virtualenv` configured you can -take advantage of the `setup-env` tool in this repo to automate the -entire environment configuration process. +The `setup-env` tool in this repository is our recommended method +for automating the entire environment configuration process. The +dependencies required to run this tool are +[`gnu-getopt`](https://manned.org/getopt.1), `pyenv`, and +`pyenv-virtualenv`. If these tools are already configured on +your system, you can simply run the following command. ```console ./setup-env @@ -57,13 +60,14 @@ entire environment configuration process. Otherwise, follow the steps below to manually configure your environment. -#### Installing and using `pyenv` and `pyenv-virtualenv` #### +#### Installing and using `gnu-getopt`, `pyenv`, and `pyenv-virtualenv` #### On the Mac, we recommend installing [brew](https://brew.sh/). Then -installation is as simple as `brew install pyenv pyenv-virtualenv` and +installation is as simple as `brew install gnu-getopt pyenv pyenv-virtualenv` and adding this to your profile: ```bash +export PATH="/usr/local/opt/gnu-getopt/bin:$PATH" export PYENV_ROOT="$HOME/.pyenv" export PATH="$PYENV_ROOT/bin:$PATH" eval "$(pyenv init --path)" @@ -78,6 +82,8 @@ install the necessary tools. Before running this ensure that you have installed the prerequisites for your platform according to the [`pyenv` wiki page](https://github.com/pyenv/pyenv/wiki/common-build-problems). +`Gnu-getopt` is generally included in the core utilities of most +Linux distributions. On WSL you should treat your platform as whatever Linux distribution you've chosen to install. From ba86ead6c4f757f03610c6065df46e5ba4ad4dbd Mon Sep 17 00:00:00 2001 From: Michael Saki Date: Wed, 7 Feb 2024 09:43:30 -0600 Subject: [PATCH 041/171] Fix grammar and capitalization errors Co-authored-by: Shane Frasier --- CONTRIBUTING.md | 2 +- setup-env | 12 ++++++------ 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index dbbe417..d96a0e3 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -82,7 +82,7 @@ install the necessary tools. Before running this ensure that you have installed the prerequisites for your platform according to the [`pyenv` wiki page](https://github.com/pyenv/pyenv/wiki/common-build-problems). -`Gnu-getopt` is generally included in the core utilities of most +`gnu-getopt` is generally included in the core utilities of most Linux distributions. On WSL you should treat your platform as whatever Linux distribution diff --git a/setup-env b/setup-env index f4cdf75..1054a64 100755 --- a/setup-env +++ b/setup-env @@ -9,10 +9,10 @@ USAGE=$( Configure a development environment for this repository. It does the following: - - Allows user to specify Python version. - - Allows user to choose name for their virtual environment. + - Allows the user to specify the Python version to use for the virtual environment. + - Allows the user to specify a name for the virtual environment. - Verifies pyenv and pyenv-virtualenv are installed. - - Creates a Python virtual environment. + - Creates the Python virtual environment. - Configures the activation of the virtual enviroment for the repo directory. - Installs the requirements needed for development. - Installs git pre-commit hooks. @@ -29,7 +29,7 @@ Options: -i | --install-hooks Install hook environments for all environments in the pre-commit config file. -l | --list-versions List available Python versions and select interactively. - -n | --name Choose the name of the virtual environment. + -n | --name Specify the name of the virtual environment. -v | --version Specify the Python version for the virtual environment. END_OF_LINE @@ -195,7 +195,7 @@ if [ $FORCE -ne 0 ]; then elif [[ -f .python-version ]]; then cat << 'END_OF_LINE' An existing .python-version file was found. Either remove this file yourself - or re-run with -f (force) option to have it deleted along with the associated + or re-run with the --force option to have it deleted along with the associated virtual environment. rm .python-version @@ -216,7 +216,7 @@ fi if ! pyenv virtualenv ${PYTHON_VERSION:=} "${env_name}"; then cat << END_OF_LINE An existing virtual environment named $env_name was found. Either delete this - environment yourself or re-run with -f (force) option to have it deleted. + environment yourself or re-run with the --force option to have it deleted. pyenv virtualenv-delete ${env_name} From ba0fc19811d2b9bcb2cc043660f6dcd9d6d9c64b Mon Sep 17 00:00:00 2001 From: Michael Saki Date: Wed, 7 Feb 2024 09:48:01 -0600 Subject: [PATCH 042/171] Combine PATH exports to single line Co-authored-by: Shane Frasier --- CONTRIBUTING.md | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index d96a0e3..1e6cc74 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -67,9 +67,8 @@ installation is as simple as `brew install gnu-getopt pyenv pyenv-virtualenv` an adding this to your profile: ```bash -export PATH="/usr/local/opt/gnu-getopt/bin:$PATH" export PYENV_ROOT="$HOME/.pyenv" -export PATH="$PYENV_ROOT/bin:$PATH" +export PATH="$PYENV_ROOT/bin:/usr/local/opt/gnu-getopt/bin:$PATH" eval "$(pyenv init --path)" eval "$(pyenv init -)" eval "$(pyenv virtualenv-init -)" From 1240bdd98fe3523dada222782a55df8c2b258e5c Mon Sep 17 00:00:00 2001 From: Michael Saki Date: Wed, 7 Feb 2024 09:48:39 -0600 Subject: [PATCH 043/171] Improve usage instructions Co-authored-by: Shane Frasier --- setup-env | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/setup-env b/setup-env index 1054a64..a0230ed 100755 --- a/setup-env +++ b/setup-env @@ -19,8 +19,7 @@ It does the following: - Configures git upstream remote "lineage" repositories. Usage: - setup-env (-n | --name) [virt_env_name] - setup-env (-v | --version) [python_version] + setup-env (-n | --name) [virt_env_name] (-v | --version) [python_version] setup-env (-h | --help) Options: From 297b5bd850f716372eb385c25ef29936b4bd6d7c Mon Sep 17 00:00:00 2001 From: Michael Saki Date: Wed, 7 Feb 2024 11:03:35 -0600 Subject: [PATCH 044/171] Add $(brew --prefix) to PATH for getopt Co-authored-by: dav3r --- CONTRIBUTING.md | 2 +- setup-env | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 1e6cc74..899f4ff 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -68,7 +68,7 @@ adding this to your profile: ```bash export PYENV_ROOT="$HOME/.pyenv" -export PATH="$PYENV_ROOT/bin:/usr/local/opt/gnu-getopt/bin:$PATH" +export PATH="$PYENV_ROOT/bin:$(brew --prefix)/opt/gnu-getopt/bin:$PATH" eval "$(pyenv init --path)" eval "$(pyenv init -)" eval "$(pyenv virtualenv-init -)" diff --git a/setup-env b/setup-env index a0230ed..3ddc9c9 100755 --- a/setup-env +++ b/setup-env @@ -63,7 +63,7 @@ if [[ $(getopt --version 2> /dev/null) != *"getopt from util-linux"* ]]; then is as simple as `brew install gnu-getopt` and adding this to your profile: - export PATH="/usr/local/opt/gnu-getopt/bin:$PATH" + export PATH="$(brew --prefix)/opt/gnu-getopt/bin:$PATH" END_OF_LINE exit 1 From 7af70f514fba159ed6e7c8ae85af7a0e14a667a1 Mon Sep 17 00:00:00 2001 From: Michael Saki Date: Wed, 7 Feb 2024 11:04:27 -0600 Subject: [PATCH 045/171] Fix confusing wording Co-authored-by: dav3r --- setup-env | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup-env b/setup-env index 3ddc9c9..1e2fd79 100755 --- a/setup-env +++ b/setup-env @@ -42,7 +42,7 @@ python_versions() { # Flag to force deletion and creation of virtual environment FORCE=0 -# Initialize the all other flags +# Initialize the other flags INSTALL_HOOKS=0 LIST_VERSIONS=0 PYTHON_VERSION="" From e5a2d14964a65c0f5b74e53e5f7e606778b5f003 Mon Sep 17 00:00:00 2001 From: Michael Saki Date: Wed, 7 Feb 2024 11:06:50 -0600 Subject: [PATCH 046/171] Replace virt_env_name w/ virtual_env_name for clarity Co-authored-by: dav3r --- setup-env | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup-env b/setup-env index 1e2fd79..ac20852 100755 --- a/setup-env +++ b/setup-env @@ -19,7 +19,7 @@ It does the following: - Configures git upstream remote "lineage" repositories. Usage: - setup-env (-n | --name) [virt_env_name] (-v | --version) [python_version] + setup-env (-n | --name) [virtual_env_name] (-v | --version) [python_version] setup-env (-h | --help) Options: From 82c70e089bfd43c84273a83228da38030d340c14 Mon Sep 17 00:00:00 2001 From: Michael Saki Date: Tue, 13 Feb 2024 14:31:35 -0600 Subject: [PATCH 047/171] Differentiate between GNU getopt and gnu-getopt brew formula This commit will clearly differentiate between GNU getopt the tool and gnu-getopt the Homebrew formula. Also updating the URL so that getopt points at the source repository and pyenv and pyenv-virtualenv point at their respective repositories. Updated the ending punctuation. Co-authored-by: Nick <50747025+mcdonnnj@users.noreply.github.com> --- CONTRIBUTING.md | 13 +++++++------ setup-env | 2 +- 2 files changed, 8 insertions(+), 7 deletions(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 899f4ff..afb1ae8 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -49,9 +49,10 @@ project. The `setup-env` tool in this repository is our recommended method for automating the entire environment configuration process. The dependencies required to run this tool are -[`gnu-getopt`](https://manned.org/getopt.1), `pyenv`, and -`pyenv-virtualenv`. If these tools are already configured on -your system, you can simply run the following command. +[GNU `getopt`](https://github.com/util-linux/util-linux/blob/master/misc-utils/getopt.1.adoc), +[`pyenv`](https://github.com/pyenv/pyenv), and [`pyenv-virtualenv`](https://github.com/pyenv/pyenv-virtualenv). +If these tools are already configured on your system, you can simply run the +following command: ```console ./setup-env @@ -60,7 +61,7 @@ your system, you can simply run the following command. Otherwise, follow the steps below to manually configure your environment. -#### Installing and using `gnu-getopt`, `pyenv`, and `pyenv-virtualenv` #### +#### Installing and using GNU `getopt`, `pyenv`, and `pyenv-virtualenv` #### On the Mac, we recommend installing [brew](https://brew.sh/). Then installation is as simple as `brew install gnu-getopt pyenv pyenv-virtualenv` and @@ -81,8 +82,8 @@ install the necessary tools. Before running this ensure that you have installed the prerequisites for your platform according to the [`pyenv` wiki page](https://github.com/pyenv/pyenv/wiki/common-build-problems). -`gnu-getopt` is generally included in the core utilities of most -Linux distributions. +GNU `getopt` is included in most Linux distributions as part of the +[`util-linux`](https://github.com/util-linux/util-linux) package. On WSL you should treat your platform as whatever Linux distribution you've chosen to install. diff --git a/setup-env b/setup-env index ac20852..0e9598b 100755 --- a/setup-env +++ b/setup-env @@ -58,7 +58,7 @@ SHORTOPTS="fhiln:v:" if [[ $(getopt --version 2> /dev/null) != *"getopt from util-linux"* ]]; then cat << 'END_OF_LINE' - Gnu-getopt is not detected and is a dependency to run this script. + GNU getopt is not detected and is a dependency to run this script. On the Mac, we recommend installing brew, https://brew.sh/. Then installation is as simple as `brew install gnu-getopt` and adding this to your profile: From 493a4a3e5dfe4c705998395797c75eda756c6dc4 Mon Sep 17 00:00:00 2001 From: Michael Saki Date: Tue, 13 Feb 2024 14:58:43 -0600 Subject: [PATCH 048/171] Add parenthesis over brew link Co-authored-by: Nick <50747025+mcdonnnj@users.noreply.github.com> --- setup-env | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup-env b/setup-env index 0e9598b..f0d4b36 100755 --- a/setup-env +++ b/setup-env @@ -59,7 +59,7 @@ if [[ $(getopt --version 2> /dev/null) != *"getopt from util-linux"* ]]; then cat << 'END_OF_LINE' GNU getopt is not detected and is a dependency to run this script. - On the Mac, we recommend installing brew, https://brew.sh/. Then installation + On the Mac, we recommend installing brew (https://brew.sh/). Then installation is as simple as `brew install gnu-getopt` and adding this to your profile: From 3bc9aeb12fb7a270fa6bc54fcc4deceb897906bc Mon Sep 17 00:00:00 2001 From: Michael Saki Date: Wed, 14 Feb 2024 07:37:24 -0600 Subject: [PATCH 049/171] Refactor flag names for clarity and accuracy This commit will remove the previous flags -v or --version for -p or --python-version. Also it will replace -n or --name for -v or --venv-name. The usage menu has been updated to reflect these changes as well. --- setup-env | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/setup-env b/setup-env index f0d4b36..b7b08e6 100755 --- a/setup-env +++ b/setup-env @@ -19,7 +19,7 @@ It does the following: - Configures git upstream remote "lineage" repositories. Usage: - setup-env (-n | --name) [virtual_env_name] (-v | --version) [python_version] + setup-env [--venv-name virtual_env_name] [--python-version python_version] setup-env (-h | --help) Options: @@ -28,8 +28,8 @@ Options: -i | --install-hooks Install hook environments for all environments in the pre-commit config file. -l | --list-versions List available Python versions and select interactively. - -n | --name Specify the name of the virtual environment. - -v | --version Specify the Python version for the virtual environment. + -v | --venv-name Specify the name of the virtual environment. + -p | --python-version Specify the Python version for the virtual environment. END_OF_LINE ) @@ -49,10 +49,10 @@ PYTHON_VERSION="" VENV_NAME="" # Define long options -LONGOPTS="force,help,install-hooks,list-versions,name:,version:" +LONGOPTS="force,help,install-hooks,list-versions,python-version:,venv-name:" # Define short options for getopt -SHORTOPTS="fhiln:v:" +SHORTOPTS="fhilp:v:" # Check if GNU getopt is available if [[ $(getopt --version 2> /dev/null) != *"getopt from util-linux"* ]]; then @@ -94,11 +94,7 @@ while true; do LIST_VERSIONS=1 shift ;; - -n | --name) - VENV_NAME="$2" - shift 2 - ;; - -v | --version) + -p | --python-version) PYTHON_VERSION="$2" shift 2 # Check the Python versions being passed in. @@ -113,6 +109,10 @@ while true; do fi fi ;; + -v | --venv-name) + VENV_NAME="$2" + shift 2 + ;; --) shift break From 0be1f6376dc914be4eff9ce5fde7649d085a2582 Mon Sep 17 00:00:00 2001 From: Michael Saki Date: Wed, 14 Feb 2024 08:08:07 -0600 Subject: [PATCH 050/171] Elaborate on message when checking for GNU getopt --- setup-env | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/setup-env b/setup-env index b7b08e6..6649b4d 100755 --- a/setup-env +++ b/setup-env @@ -58,7 +58,13 @@ SHORTOPTS="fhilp:v:" if [[ $(getopt --version 2> /dev/null) != *"getopt from util-linux"* ]]; then cat << 'END_OF_LINE' - GNU getopt is not detected and is a dependency to run this script. + Please note, this script requires GNU getopt due to its enhanced + functionality and compatibility with certain script features that + are not supported by the POSIX getopt found in some systems, particularly + those with a non-GNU version of getopt. This distinction is crucial + as a system might have a non-GNU version of getopt installed by default, + which could lead to unexpected behavior or script failure. + On the Mac, we recommend installing brew (https://brew.sh/). Then installation is as simple as `brew install gnu-getopt` and adding this to your profile: From c8f0b1b996958a5fa965343fa7e6295b7160cbf4 Mon Sep 17 00:00:00 2001 From: Michael Saki Date: Wed, 14 Feb 2024 08:10:21 -0600 Subject: [PATCH 051/171] Remove unnecessary nounset flipping logic --- setup-env | 2 -- 1 file changed, 2 deletions(-) diff --git a/setup-env b/setup-env index 6649b4d..05b010b 100755 --- a/setup-env +++ b/setup-env @@ -166,7 +166,6 @@ END_OF_LINE exit 1 fi -set +o nounset # Determine the virtual environment name if [ -n "$VENV_NAME" ]; then # Use the user-provided environment name @@ -175,7 +174,6 @@ else # Set the environment name to the last part of the working directory. env_name=${PWD##*/} fi -set -o nounset # List Python versions and select one interactively if [ $LIST_VERSIONS -ne 0 ]; then From 495862aeaf3430b0a69f92cf54b6415818139db2 Mon Sep 17 00:00:00 2001 From: Michael Saki Date: Wed, 14 Feb 2024 08:20:56 -0600 Subject: [PATCH 052/171] Separate pyenv PATH from GNU getopt PATH --- CONTRIBUTING.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index afb1ae8..3a5e087 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -69,7 +69,8 @@ adding this to your profile: ```bash export PYENV_ROOT="$HOME/.pyenv" -export PATH="$PYENV_ROOT/bin:$(brew --prefix)/opt/gnu-getopt/bin:$PATH" +export PATH="$PYENV_ROOT/bin:$PATH" +export PATH="$(brew --prefix)/opt/gnu-getopt/bin:$PATH" eval "$(pyenv init --path)" eval "$(pyenv init -)" eval "$(pyenv virtualenv-init -)" From 4752b37ec887a65e967f01a1f9b7633e02a02af8 Mon Sep 17 00:00:00 2001 From: Michael Saki Date: Wed, 21 Feb 2024 10:57:38 -0600 Subject: [PATCH 053/171] Improve verbiage in comments Co-authored-by: Shane Frasier --- CONTRIBUTING.md | 4 ++-- setup-env | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 3a5e087..9ff54e4 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -46,8 +46,8 @@ There are a few ways to do this, but we prefer to use create and manage a Python virtual environment specific to this project. -The `setup-env` tool in this repository is our recommended method -for automating the entire environment configuration process. The +We recommend using the `setup-env` tool located in this repository, +as it automates the entire environment configuration process. The dependencies required to run this tool are [GNU `getopt`](https://github.com/util-linux/util-linux/blob/master/misc-utils/getopt.1.adoc), [`pyenv`](https://github.com/pyenv/pyenv), and [`pyenv-virtualenv`](https://github.com/pyenv/pyenv-virtualenv). diff --git a/setup-env b/setup-env index 05b010b..6d9fec0 100755 --- a/setup-env +++ b/setup-env @@ -27,7 +27,7 @@ Options: -h | --help Show this message. -i | --install-hooks Install hook environments for all environments in the pre-commit config file. - -l | --list-versions List available Python versions and select interactively. + -l | --list-versions List available Python versions and select one interactively. -v | --venv-name Specify the name of the virtual environment. -p | --python-version Specify the Python version for the virtual environment. @@ -63,7 +63,7 @@ if [[ $(getopt --version 2> /dev/null) != *"getopt from util-linux"* ]]; then are not supported by the POSIX getopt found in some systems, particularly those with a non-GNU version of getopt. This distinction is crucial as a system might have a non-GNU version of getopt installed by default, - which could lead to unexpected behavior or script failure. + which could lead to unexpected behavior. On the Mac, we recommend installing brew (https://brew.sh/). Then installation is as simple as `brew install gnu-getopt` and adding this to your From 2e389979b0b0b34a6f32443d88f8ed0ba8c647cd Mon Sep 17 00:00:00 2001 From: Michael Saki Date: Wed, 21 Feb 2024 11:22:53 -0600 Subject: [PATCH 054/171] Clarify between pyenv and GNU getopt setup Co-authored-by: Nick <50747025+mcdonnnj@users.noreply.github.com> --- CONTRIBUTING.md | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 9ff54e4..a844f86 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -68,9 +68,12 @@ installation is as simple as `brew install gnu-getopt pyenv pyenv-virtualenv` an adding this to your profile: ```bash +# Enable GNU getopt since it is keg-only +export PATH="$(brew --prefix)/opt/gnu-getopt/bin:$PATH" + +# Setup pyenv export PYENV_ROOT="$HOME/.pyenv" export PATH="$PYENV_ROOT/bin:$PATH" -export PATH="$(brew --prefix)/opt/gnu-getopt/bin:$PATH" eval "$(pyenv init --path)" eval "$(pyenv init -)" eval "$(pyenv virtualenv-init -)" From f8824c8aa157cbee0ec95367b1754eeac67934c9 Mon Sep 17 00:00:00 2001 From: Michael Saki Date: Wed, 21 Feb 2024 12:14:23 -0600 Subject: [PATCH 055/171] Improve comment on conditional check for regex --- setup-env | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/setup-env b/setup-env index 6d9fec0..6d16958 100755 --- a/setup-env +++ b/setup-env @@ -54,7 +54,9 @@ LONGOPTS="force,help,install-hooks,list-versions,python-version:,venv-name:" # Define short options for getopt SHORTOPTS="fhilp:v:" -# Check if GNU getopt is available +# Check for GNU getopt by matching a specific pattern ("getopt from util-linux") +# in its version output. This approach presumes the output format remains stable. +# Be aware that format changes could invalidate this check. if [[ $(getopt --version 2> /dev/null) != *"getopt from util-linux"* ]]; then cat << 'END_OF_LINE' From 88724e70167a5864495eccff5532f4c59fbefc1c Mon Sep 17 00:00:00 2001 From: Michael Saki Date: Wed, 21 Feb 2024 12:25:49 -0600 Subject: [PATCH 056/171] Add comment explaining that GNU getopt is keg-only --- setup-env | 2 ++ 1 file changed, 2 insertions(+) diff --git a/setup-env b/setup-env index 6d16958..84a8305 100755 --- a/setup-env +++ b/setup-env @@ -73,6 +73,8 @@ if [[ $(getopt --version 2> /dev/null) != *"getopt from util-linux"* ]]; then export PATH="$(brew --prefix)/opt/gnu-getopt/bin:$PATH" + This will enable GNU getopt since it is keg-only. + END_OF_LINE exit 1 fi From c1870be7a16f5e88f6e6540c290f674da4861e07 Mon Sep 17 00:00:00 2001 From: Michael Saki Date: Wed, 21 Feb 2024 16:28:54 -0600 Subject: [PATCH 057/171] Improve comments to better describe `keg-only` terminology Co-authored-by: Shane Frasier --- CONTRIBUTING.md | 3 ++- setup-env | 3 ++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index a844f86..7bb8881 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -68,7 +68,8 @@ installation is as simple as `brew install gnu-getopt pyenv pyenv-virtualenv` an adding this to your profile: ```bash -# Enable GNU getopt since it is keg-only +# GNU getopt must be explicitly added to the path since it is +# keg-only export PATH="$(brew --prefix)/opt/gnu-getopt/bin:$PATH" # Setup pyenv diff --git a/setup-env b/setup-env index 84a8305..3fa8c47 100755 --- a/setup-env +++ b/setup-env @@ -73,7 +73,8 @@ if [[ $(getopt --version 2> /dev/null) != *"getopt from util-linux"* ]]; then export PATH="$(brew --prefix)/opt/gnu-getopt/bin:$PATH" - This will enable GNU getopt since it is keg-only. + GNU getopt must be explicitly added to the PATH since it + is keg-only. END_OF_LINE exit 1 From a3f69cd24e58077ee6f084bce900ed176c88cff7 Mon Sep 17 00:00:00 2001 From: Michael Saki Date: Mon, 26 Feb 2024 08:04:19 -0600 Subject: [PATCH 058/171] Change "'setup-env' tool" to "'setup-env' script" Co-authored-by: Nick <50747025+mcdonnnj@users.noreply.github.com> --- CONTRIBUTING.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 7bb8881..fca0cf7 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -46,7 +46,7 @@ There are a few ways to do this, but we prefer to use create and manage a Python virtual environment specific to this project. -We recommend using the `setup-env` tool located in this repository, +We recommend using the `setup-env` script located in this repository, as it automates the entire environment configuration process. The dependencies required to run this tool are [GNU `getopt`](https://github.com/util-linux/util-linux/blob/master/misc-utils/getopt.1.adoc), From 8ff51796ec530e68d35362fab761608e676dcd3a Mon Sep 17 00:00:00 2001 From: Michael Saki Date: Mon, 26 Feb 2024 08:07:16 -0600 Subject: [PATCH 059/171] Remove build-in error exit for generic error exit Exit code 2 usually relates to build-ins which does not fit what is failing here. Not sure what alternative would be best so we can just use the generic failure code Co-authored-by: Nick <50747025+mcdonnnj@users.noreply.github.com> --- setup-env | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup-env b/setup-env index 3fa8c47..f7ce620 100755 --- a/setup-env +++ b/setup-env @@ -83,7 +83,7 @@ fi # Use GNU getopt to parse options if ! PARSED=$(getopt --options $SHORTOPTS --longoptions $LONGOPTS --name "$0" -- "$@"); then echo "Error parsing options" - exit 2 + exit 1 fi eval set -- "$PARSED" From 1c21e2b9839980c75410584c69d0ac0a0651f097 Mon Sep 17 00:00:00 2001 From: Michael Saki Date: Mon, 26 Feb 2024 08:08:39 -0600 Subject: [PATCH 060/171] Change verbiage from 'tool' to 'script' for clarity Co-authored-by: Nick <50747025+mcdonnnj@users.noreply.github.com> --- CONTRIBUTING.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index fca0cf7..cdab589 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -48,7 +48,7 @@ project. We recommend using the `setup-env` script located in this repository, as it automates the entire environment configuration process. The -dependencies required to run this tool are +dependencies required to run this script are [GNU `getopt`](https://github.com/util-linux/util-linux/blob/master/misc-utils/getopt.1.adoc), [`pyenv`](https://github.com/pyenv/pyenv), and [`pyenv-virtualenv`](https://github.com/pyenv/pyenv-virtualenv). If these tools are already configured on your system, you can simply run the From 3acc8d6a17360221d6c59c306db1a14a98b48b8b Mon Sep 17 00:00:00 2001 From: Michael Saki Date: Mon, 26 Feb 2024 10:04:42 -0600 Subject: [PATCH 061/171] Check for pyenv earlier in the script Since the new function, python_versions expects pyenv to be installed it makes sense to have it checked earlier in the script. --- setup-env | 72 +++++++++++++++++++++++++++---------------------------- 1 file changed, 36 insertions(+), 36 deletions(-) diff --git a/setup-env b/setup-env index f7ce620..3898ca8 100755 --- a/setup-env +++ b/setup-env @@ -80,6 +80,42 @@ END_OF_LINE exit 1 fi +# Check to see if pyenv is installed +if [ -z "$(command -v pyenv)" ] || { [ -z "$(command -v pyenv-virtualenv)" ] && [ ! -f "$(pyenv root)/plugins/pyenv-virtualenv/bin/pyenv-virtualenv" ]; }; then + echo "pyenv and pyenv-virtualenv are required." + if [[ "$OSTYPE" == "darwin"* ]]; then + cat << 'END_OF_LINE' + + On the Mac, we recommend installing brew, https://brew.sh/. Then installation + is as simple as `brew install pyenv pyenv-virtualenv` and adding this to your + profile: + + eval "$(pyenv init -)" + eval "$(pyenv virtualenv-init -)" + +END_OF_LINE + + fi + cat << 'END_OF_LINE' + For Linux, Windows Subsystem for Linux (WSL), or on the Mac (if you don't want + to use "brew") you can use https://github.com/pyenv/pyenv-installer to install + the necessary tools. Before running this ensure that you have installed the + prerequisites for your platform according to the pyenv wiki page, + https://github.com/pyenv/pyenv/wiki/common-build-problems. + + On WSL you should treat your platform as whatever Linux distribution you've + chosen to install. + + Once you have installed "pyenv" you will need to add the following lines to + your ".bashrc": + + export PATH="$PATH:$HOME/.pyenv/bin" + eval "$(pyenv init -)" + eval "$(pyenv virtualenv-init -)" +END_OF_LINE + exit 1 +fi + # Use GNU getopt to parse options if ! PARSED=$(getopt --options $SHORTOPTS --longoptions $LONGOPTS --name "$0" -- "$@"); then echo "Error parsing options" @@ -135,42 +171,6 @@ while true; do esac done -# Check to see if pyenv is installed -if [ -z "$(command -v pyenv)" ] || { [ -z "$(command -v pyenv-virtualenv)" ] && [ ! -f "$(pyenv root)/plugins/pyenv-virtualenv/bin/pyenv-virtualenv" ]; }; then - echo "pyenv and pyenv-virtualenv are required." - if [[ "$OSTYPE" == "darwin"* ]]; then - cat << 'END_OF_LINE' - - On the Mac, we recommend installing brew, https://brew.sh/. Then installation - is as simple as `brew install pyenv pyenv-virtualenv` and adding this to your - profile: - - eval "$(pyenv init -)" - eval "$(pyenv virtualenv-init -)" - -END_OF_LINE - - fi - cat << 'END_OF_LINE' - For Linux, Windows Subsystem for Linux (WSL), or on the Mac (if you don't want - to use "brew") you can use https://github.com/pyenv/pyenv-installer to install - the necessary tools. Before running this ensure that you have installed the - prerequisites for your platform according to the pyenv wiki page, - https://github.com/pyenv/pyenv/wiki/common-build-problems. - - On WSL you should treat your platform as whatever Linux distribution you've - chosen to install. - - Once you have installed "pyenv" you will need to add the following lines to - your ".bashrc": - - export PATH="$PATH:$HOME/.pyenv/bin" - eval "$(pyenv init -)" - eval "$(pyenv virtualenv-init -)" -END_OF_LINE - exit 1 -fi - # Determine the virtual environment name if [ -n "$VENV_NAME" ]; then # Use the user-provided environment name From b377ce71fa3dd5f56697d415c5230f3fe18e8090 Mon Sep 17 00:00:00 2001 From: Michael Saki Date: Mon, 26 Feb 2024 11:02:39 -0600 Subject: [PATCH 062/171] Explain -r and -p in Python version prompt --- setup-env | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/setup-env b/setup-env index 3898ca8..5c2b580 100755 --- a/setup-env +++ b/setup-env @@ -180,10 +180,12 @@ else env_name=${PWD##*/} fi -# List Python versions and select one interactively +# List Python versions and select one interactively. if [ $LIST_VERSIONS -ne 0 ]; then echo Available Python versions: python_versions + # Read the user's desired Python version. + # -r: treat backslashes as literal, -p: display prompt before input. read -r -p "Enter the desired Python version: " PYTHON_VERSION # Check the Python versions being passed in. if [ -n "${PYTHON_VERSION+x}" ]; then From 74838a256e134e00208a23bb6dded7012cfb8534 Mon Sep 17 00:00:00 2001 From: Michael Saki Date: Mon, 26 Feb 2024 11:58:44 -0600 Subject: [PATCH 063/171] Refine exit code to 64 with gnu-getopt note - Set exit code to 64 per sysexits.h for clarity. - Added comment on gnu-getopt handling. Aligns with Unix standards and enhances readability. --- setup-env | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/setup-env b/setup-env index 5c2b580..549c079 100755 --- a/setup-env +++ b/setup-env @@ -165,8 +165,9 @@ while true; do break ;; *) + # Unreachable due to gnu-getopt handling all options echo "Programming error" - exit 3 + exit 64 ;; esac done From 487126e34bd85e231b7021b70d225154694ad24d Mon Sep 17 00:00:00 2001 From: Michael Saki Date: Wed, 28 Feb 2024 06:32:35 -0600 Subject: [PATCH 064/171] Rename gnu-getopt tool to GNU getopt formula Co-authored-by: Nick <50747025+mcdonnnj@users.noreply.github.com> --- setup-env | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup-env b/setup-env index 549c079..d2db6e2 100755 --- a/setup-env +++ b/setup-env @@ -165,7 +165,7 @@ while true; do break ;; *) - # Unreachable due to gnu-getopt handling all options + # Unreachable due to GNU getopt handling all options echo "Programming error" exit 64 ;; From 6c82a8d1bb86a90f0e903dd2d90efdb6198fba33 Mon Sep 17 00:00:00 2001 From: Michael Saki Date: Wed, 28 Feb 2024 07:49:16 -0600 Subject: [PATCH 065/171] Fix whitespace for usage menu Co-authored-by: Shane Frasier --- setup-env | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/setup-env b/setup-env index d2db6e2..c93eb25 100755 --- a/setup-env +++ b/setup-env @@ -28,8 +28,8 @@ Options: -i | --install-hooks Install hook environments for all environments in the pre-commit config file. -l | --list-versions List available Python versions and select one interactively. - -v | --venv-name Specify the name of the virtual environment. - -p | --python-version Specify the Python version for the virtual environment. + -v | --venv-name Specify the name of the virtual environment. + -p | --python-version Specify the Python version for the virtual environment. END_OF_LINE ) From 324f6d426a149cb2c7bfc30ffcf7c1810b8ae1fa Mon Sep 17 00:00:00 2001 From: Michael Saki Date: Wed, 28 Feb 2024 11:26:24 -0600 Subject: [PATCH 066/171] Add link to brew terminology Co-authored-by: dav3r --- CONTRIBUTING.md | 2 +- setup-env | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index cdab589..bc32a55 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -69,7 +69,7 @@ adding this to your profile: ```bash # GNU getopt must be explicitly added to the path since it is -# keg-only +# keg-only (https://docs.brew.sh/FAQ#what-does-keg-only-mean) export PATH="$(brew --prefix)/opt/gnu-getopt/bin:$PATH" # Setup pyenv diff --git a/setup-env b/setup-env index c93eb25..bb689f3 100755 --- a/setup-env +++ b/setup-env @@ -74,7 +74,7 @@ if [[ $(getopt --version 2> /dev/null) != *"getopt from util-linux"* ]]; then export PATH="$(brew --prefix)/opt/gnu-getopt/bin:$PATH" GNU getopt must be explicitly added to the PATH since it - is keg-only. + is keg-only (https://docs.brew.sh/FAQ#what-does-keg-only-mean). END_OF_LINE exit 1 From a26d0e36f0c33a7f472b91d66697e4cecb8f9557 Mon Sep 17 00:00:00 2001 From: Michael Saki Date: Wed, 28 Feb 2024 15:52:20 -0600 Subject: [PATCH 067/171] Rephrase comment to improve clarity Co-authored-by: Nick <50747025+mcdonnnj@users.noreply.github.com> --- setup-env | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup-env b/setup-env index bb689f3..3fddb36 100755 --- a/setup-env +++ b/setup-env @@ -16,7 +16,7 @@ It does the following: - Configures the activation of the virtual enviroment for the repo directory. - Installs the requirements needed for development. - Installs git pre-commit hooks. - - Configures git upstream remote "lineage" repositories. + - Configures git remotes for upstream "lineage" repositories. Usage: setup-env [--venv-name virtual_env_name] [--python-version python_version] From 05108705fa4120db51f261ad59347bb01134eafc Mon Sep 17 00:00:00 2001 From: Michael Saki Date: Wed, 28 Feb 2024 15:58:43 -0600 Subject: [PATCH 068/171] Improve comment for clarity Co-authored-by: Nick <50747025+mcdonnnj@users.noreply.github.com> --- setup-env | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup-env b/setup-env index 3fddb36..c8680ea 100755 --- a/setup-env +++ b/setup-env @@ -34,7 +34,7 @@ Options: END_OF_LINE ) -# Display installed python versions +# Display pyenv's installed Python versions python_versions() { pyenv versions --bare --skip-aliases --skip-envs } From 01abde6f580cf549d77c81af9378d4a18812fa8f Mon Sep 17 00:00:00 2001 From: Michael Saki Date: Wed, 28 Feb 2024 16:03:47 -0600 Subject: [PATCH 069/171] Improve verbiage in comment --- setup-env | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup-env b/setup-env index c8680ea..534d47f 100755 --- a/setup-env +++ b/setup-env @@ -217,7 +217,7 @@ fi # Create a new virtual environment for this project # -# If $PYTHON_VERSION is undefined then the system Python will be used. +# If $PYTHON_VERSION is undefined then the global version of Python will be used. # # We can't quote ${PYTHON_VERSION:=} below since if the variable is # undefined then we want nothing to appear; this is the reason for the From 0989d17f49a089e03e23b72ab638595d538add3f Mon Sep 17 00:00:00 2001 From: Michael Saki Date: Wed, 28 Feb 2024 16:09:15 -0600 Subject: [PATCH 070/171] Change comments for macOS and venv_name --- CONTRIBUTING.md | 4 ++-- setup-env | 8 ++++---- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index bc32a55..617db85 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -63,7 +63,7 @@ environment. #### Installing and using GNU `getopt`, `pyenv`, and `pyenv-virtualenv` #### -On the Mac, we recommend installing [brew](https://brew.sh/). Then +On macOS, we recommend installing [brew](https://brew.sh/). Then installation is as simple as `brew install gnu-getopt pyenv pyenv-virtualenv` and adding this to your profile: @@ -80,7 +80,7 @@ eval "$(pyenv init -)" eval "$(pyenv virtualenv-init -)" ``` -For Linux, Windows Subsystem for Linux (WSL), or on the Mac (if you +For Linux, Windows Subsystem for Linux (WSL), or on macOS (if you don't want to use `brew`) you can use [pyenv/pyenv-installer](https://github.com/pyenv/pyenv-installer) to install the necessary tools. Before running this ensure that you have diff --git a/setup-env b/setup-env index 534d47f..d2cd111 100755 --- a/setup-env +++ b/setup-env @@ -19,7 +19,7 @@ It does the following: - Configures git remotes for upstream "lineage" repositories. Usage: - setup-env [--venv-name virtual_env_name] [--python-version python_version] + setup-env [--venv-name venv_name] [--python-version python_version] setup-env (-h | --help) Options: @@ -67,7 +67,7 @@ if [[ $(getopt --version 2> /dev/null) != *"getopt from util-linux"* ]]; then as a system might have a non-GNU version of getopt installed by default, which could lead to unexpected behavior. - On the Mac, we recommend installing brew (https://brew.sh/). Then installation + On macOS, we recommend installing brew (https://brew.sh/). Then installation is as simple as `brew install gnu-getopt` and adding this to your profile: @@ -86,7 +86,7 @@ if [ -z "$(command -v pyenv)" ] || { [ -z "$(command -v pyenv-virtualenv)" ] && if [[ "$OSTYPE" == "darwin"* ]]; then cat << 'END_OF_LINE' - On the Mac, we recommend installing brew, https://brew.sh/. Then installation + On macOS, we recommend installing brew, https://brew.sh/. Then installation is as simple as `brew install pyenv pyenv-virtualenv` and adding this to your profile: @@ -97,7 +97,7 @@ END_OF_LINE fi cat << 'END_OF_LINE' - For Linux, Windows Subsystem for Linux (WSL), or on the Mac (if you don't want + For Linux, Windows Subsystem for Linux (WSL), or on mac OS (if you don't want to use "brew") you can use https://github.com/pyenv/pyenv-installer to install the necessary tools. Before running this ensure that you have installed the prerequisites for your platform according to the pyenv wiki page, From a9c6ed88c3c993db890a09ece5433b33e5d5d206 Mon Sep 17 00:00:00 2001 From: Michael Saki Date: Thu, 29 Feb 2024 09:19:01 -0600 Subject: [PATCH 071/171] Improve comments for clarity Co-authored-by: Nick <50747025+mcdonnnj@users.noreply.github.com> --- CONTRIBUTING.md | 2 +- setup-env | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 617db85..f9fa53a 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -80,7 +80,7 @@ eval "$(pyenv init -)" eval "$(pyenv virtualenv-init -)" ``` -For Linux, Windows Subsystem for Linux (WSL), or on macOS (if you +For Linux, Windows Subsystem for Linux (WSL), or macOS (if you don't want to use `brew`) you can use [pyenv/pyenv-installer](https://github.com/pyenv/pyenv-installer) to install the necessary tools. Before running this ensure that you have diff --git a/setup-env b/setup-env index d2cd111..3a22d43 100755 --- a/setup-env +++ b/setup-env @@ -97,7 +97,7 @@ END_OF_LINE fi cat << 'END_OF_LINE' - For Linux, Windows Subsystem for Linux (WSL), or on mac OS (if you don't want + For Linux, Windows Subsystem for Linux (WSL), or macOS (if you don't want to use "brew") you can use https://github.com/pyenv/pyenv-installer to install the necessary tools. Before running this ensure that you have installed the prerequisites for your platform according to the pyenv wiki page, @@ -217,7 +217,7 @@ fi # Create a new virtual environment for this project # -# If $PYTHON_VERSION is undefined then the global version of Python will be used. +# If $PYTHON_VERSION is undefined then the current pyenv Python version will be used. # # We can't quote ${PYTHON_VERSION:=} below since if the variable is # undefined then we want nothing to appear; this is the reason for the From b9c729f6159defe55490d2198390ba2ceab5924f Mon Sep 17 00:00:00 2001 From: Nicholas McDonnell <50747025+mcdonnnj@users.noreply.github.com> Date: Thu, 4 Jan 2024 15:03:15 -0500 Subject: [PATCH 072/171] Update pre-commit hook versions This is done automatically with the `pre-commit autoupdate` command. The pre-commit/mirrors-prettier was manually held back because the latest tags are for alpha releases of the next major version. --- .pre-commit-config.yaml | 26 +++++++++++++------------- 1 file changed, 13 insertions(+), 13 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 34f0253..e7f5d0a 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -5,7 +5,7 @@ default_language_version: repos: - repo: https://github.com/pre-commit/pre-commit-hooks - rev: v4.4.0 + rev: v4.5.0 hooks: - id: check-case-conflict - id: check-executables-have-shebangs @@ -31,7 +31,7 @@ repos: # Text file hooks - repo: https://github.com/igorshubovych/markdownlint-cli - rev: v0.36.0 + rev: v0.39.0 hooks: - id: markdownlint args: @@ -41,7 +41,7 @@ repos: hooks: - id: prettier - repo: https://github.com/adrienverge/yamllint - rev: v1.32.0 + rev: v1.35.1 hooks: - id: yamllint args: @@ -49,14 +49,14 @@ repos: # GitHub Actions hooks - repo: https://github.com/python-jsonschema/check-jsonschema - rev: 0.26.3 + rev: 0.28.0 hooks: - id: check-github-actions - id: check-github-workflows # pre-commit hooks - repo: https://github.com/pre-commit/pre-commit - rev: v3.4.0 + rev: v3.6.2 hooks: - id: validate_manifest @@ -107,44 +107,44 @@ repos: # Python hooks - repo: https://github.com/PyCQA/bandit - rev: 1.7.5 + rev: 1.7.7 hooks: - id: bandit args: - --config=.bandit.yml - repo: https://github.com/psf/black-pre-commit-mirror - rev: 23.9.1 + rev: 24.2.0 hooks: - id: black - repo: https://github.com/PyCQA/flake8 - rev: 6.1.0 + rev: 7.0.0 hooks: - id: flake8 additional_dependencies: - flake8-docstrings - repo: https://github.com/PyCQA/isort - rev: 5.12.0 + rev: 5.13.2 hooks: - id: isort - repo: https://github.com/pre-commit/mirrors-mypy - rev: v1.5.1 + rev: v1.8.0 hooks: - id: mypy - repo: https://github.com/asottile/pyupgrade - rev: v3.10.1 + rev: v3.15.1 hooks: - id: pyupgrade # Ansible hooks - repo: https://github.com/ansible/ansible-lint - rev: v6.19.0 + rev: v24.2.0 hooks: - id: ansible-lint # files: molecule/default/playbook.yml # Terraform hooks - repo: https://github.com/antonbabenko/pre-commit-terraform - rev: v1.83.2 + rev: v1.88.0 hooks: - id: terraform_fmt - id: terraform_validate From 4c933958bca75c291bcb3b28fa1ba821d1d1f6f5 Mon Sep 17 00:00:00 2001 From: Nicholas McDonnell <50747025+mcdonnnj@users.noreply.github.com> Date: Thu, 4 Jan 2024 15:08:30 -0500 Subject: [PATCH 073/171] Manually update the prettier hook The hook is manually updated to the last v3 tag available from the pre-commit/mirrors-prettier repository. We then add the latest release of v3 available from NPM as an additional depdency. --- .pre-commit-config.yaml | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index e7f5d0a..8e3941d 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -37,9 +37,16 @@ repos: args: - --config=.mdl_config.yaml - repo: https://github.com/pre-commit/mirrors-prettier - rev: v3.0.3 + # This is the last version of v3 available from the mirror. We should hold + # here until v4, which is currently in alpha, is more stable. + rev: v3.1.0 hooks: - id: prettier + # This is the latest version of v3 available from NPM. The pre-commit + # mirror does not pull tags for old major versions once a new major + # version tag is published. + additional_dependencies: + - prettier@3.2.5 - repo: https://github.com/adrienverge/yamllint rev: v1.35.1 hooks: From 7169dcf65670f1d42b9498b68624a1638de08531 Mon Sep 17 00:00:00 2001 From: Nicholas McDonnell <50747025+mcdonnnj@users.noreply.github.com> Date: Sat, 11 Nov 2023 16:20:14 -0500 Subject: [PATCH 074/171] Use Python and Go versions provided by cisagov/setup-env-github-action --- .github/workflows/build.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 4911745..de6937d 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -59,7 +59,7 @@ jobs: - id: setup-python uses: actions/setup-python@v5 with: - python-version: "3.11" + python-version: ${{ steps.setup-env.outputs.python-version }} # We need the Go version and Go cache location for the actions/cache step, # so the Go installation must happen before that. - id: setup-go @@ -68,7 +68,7 @@ jobs: # There is no expectation for actual Go code so we disable caching as # it relies on the existence of a go.sum file. cache: false - go-version: "1.20" + go-version: ${{ steps.setup-env.outputs.go-version }} - id: go-cache name: Lookup Go cache directory run: | From 035cf86cf76f40ed8b4bf67ff1b14c420701e145 Mon Sep 17 00:00:00 2001 From: Nicholas McDonnell <50747025+mcdonnnj@users.noreply.github.com> Date: Tue, 27 Feb 2024 17:17:39 -0500 Subject: [PATCH 075/171] Switch pre-commit hooks for running shellcheck This hook bundles the binaries for shellcheck with a Python package which removes the need to ensure the tool is installed for the hook to function. It also ties the version of shellcheck used to the hook which will help guarantee consistency. --- .pre-commit-config.yaml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 1d12a0d..2c5b3c8 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -115,10 +115,10 @@ repos: - --case-indent # Redirect operators are followed by a space - --space-redirects - - repo: https://github.com/detailyang/pre-commit-shell - rev: 1.0.5 + - repo: https://github.com/shellcheck-py/shellcheck-py + rev: v0.9.0.6 hooks: - - id: shell-lint + - id: shellcheck # Python hooks - repo: https://github.com/PyCQA/bandit From dd67feb6c5a01b8b9d6b4605a48bb898b4fa91ec Mon Sep 17 00:00:00 2001 From: Nicholas McDonnell <50747025+mcdonnnj@users.noreply.github.com> Date: Wed, 6 Mar 2024 13:17:07 -0500 Subject: [PATCH 076/171] Pin the version of the ansible package We should be pinning the version of Ansible we use in our Packer projects to mirror the version used in our Ansible role projects. Co-authored-by: Shane Frasier --- requirements.txt | 16 +++++++++++++++- 1 file changed, 15 insertions(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 79ae56c..c0b40d8 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,4 +1,18 @@ -ansible +# With the release of version 2.10, Ansible finally correctly +# identifies Kali Linux as being the Kali distribution of the Debian +# OS family. This simplifies a lot of things for roles that support +# Kali Linux, so it makes sense to force the installation of Ansible +# 2.10 or newer. +# +# We need at least version 6 to correctly identify Amazon Linux 2023 +# as using the dnf package manager, and version 8 is currently the +# oldest supported version. +# +# We have tested against version 9. We want to avoid automatically +# jumping to another major version without testing, since there are +# often breaking changes across major versions. This is the reason +# for the upper bound. +ansible>=8,<10 boto3 docopt semver From f7b07646cbf724963b08406c60da51a2b996f473 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 13 Sep 2023 02:01:21 +0000 Subject: [PATCH 077/171] Bump crazy-max/ghaction-github-status from 3 to 4 Bumps [crazy-max/ghaction-github-status](https://github.com/crazy-max/ghaction-github-status) from 3 to 4. - [Release notes](https://github.com/crazy-max/ghaction-github-status/releases) - [Commits](https://github.com/crazy-max/ghaction-github-status/compare/v3...v4) --- updated-dependencies: - dependency-name: crazy-max/ghaction-github-status dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] --- .github/workflows/build.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 6906d5a..32b17b1 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -28,7 +28,7 @@ jobs: egress-policy: audit - id: github-status name: Check GitHub status - uses: crazy-max/ghaction-github-status@v3 + uses: crazy-max/ghaction-github-status@v4 - id: dump-context name: Dump context uses: crazy-max/ghaction-dump-context@v2 From 5e90f7ee3584df64b057c256375d2d3603c0cacc Mon Sep 17 00:00:00 2001 From: Jeremy Frasier Date: Wed, 13 Sep 2023 12:48:59 -0400 Subject: [PATCH 078/171] Add a diagnostics job for the label syncing workflow Also add a runner hardening task to the labeler job. --- .github/workflows/sync-labels.yml | 24 ++++++++++++++++++++++++ 1 file changed, 24 insertions(+) diff --git a/.github/workflows/sync-labels.yml b/.github/workflows/sync-labels.yml index 44e8e19..5a20438 100644 --- a/.github/workflows/sync-labels.yml +++ b/.github/workflows/sync-labels.yml @@ -11,7 +11,26 @@ permissions: contents: read jobs: + diagnostics: + name: Run diagnostics + runs-on: ubuntu-latest + steps: + # Note that a duplicate of this step must be added at the top of + # each job. + - id: harden-runner + name: Harden the runner + uses: step-security/harden-runner@v2 + with: + egress-policy: audit + - id: github-status + name: Check GitHub status + uses: crazy-max/ghaction-github-status@v3 + - id: dump-context + name: Dump context + uses: crazy-max/ghaction-dump-context@v2 labeler: + needs: + - diagnostics permissions: # actions/checkout needs this to fetch code contents: read @@ -19,6 +38,11 @@ jobs: issues: write runs-on: ubuntu-latest steps: + - id: harden-runner + name: Harden the runner + uses: step-security/harden-runner@v2 + with: + egress-policy: audit - uses: actions/checkout@v4 - name: Sync repository labels if: success() From 376ed231a6f5fcf0f4d04999c1cbc9b4d36e04f4 Mon Sep 17 00:00:00 2001 From: Jeremy Frasier Date: Thu, 14 Sep 2023 15:45:17 -0400 Subject: [PATCH 079/171] Make the dev team the owners of the linter configuration files Co-authored-by: Nick <50747025+mcdonnnj@users.noreply.github.com> --- .github/CODEOWNERS | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index 17276b6..48f2077 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -14,3 +14,12 @@ /src/aws.yml @jsf9k /src/packer.pkr.hcl @jsf9k /src/python.yml @jsf9k + +# These folks own all linting configuration files. +/.*.cfg @dav3r @felddy @jasonodoom @jsf9k @mcdonnnj +/.*.yaml @dav3r @felddy @jasonodoom @jsf9k @mcdonnnj +/.*.yml @dav3r @felddy @jasonodoom @jsf9k @mcdonnnj +/.ansible-lint @dav3r @felddy @jasonodoom @jsf9k @mcdonnnj +/.flake8 @dav3r @felddy @jasonodoom @jsf9k @mcdonnnj +/.prettierignore @dav3r @felddy @jasonodoom @jsf9k @mcdonnnj +/.yamllint @dav3r @felddy @jasonodoom @jsf9k @mcdonnnj From f894a16af8c7ab940fd648e95abd52304efc21a8 Mon Sep 17 00:00:00 2001 From: Jeremy Frasier Date: Thu, 14 Sep 2023 15:59:22 -0400 Subject: [PATCH 080/171] Make dev team members the codeowners of the requirements*.txt and setup-env files Co-authored-by: Nick <50747025+mcdonnnj@users.noreply.github.com> --- .github/CODEOWNERS | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index 48f2077..2a9e98d 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -23,3 +23,5 @@ /.flake8 @dav3r @felddy @jasonodoom @jsf9k @mcdonnnj /.prettierignore @dav3r @felddy @jasonodoom @jsf9k @mcdonnnj /.yamllint @dav3r @felddy @jasonodoom @jsf9k @mcdonnnj +/requirements*.txt @dav3r @felddy @jasonodoom @jsf9k @mcdonnnj +/setup-env @dav3r @felddy @jasonodoom @jsf9k @mcdonnnj From 34471a6018a106031ec444411f1f6039e01a9c89 Mon Sep 17 00:00:00 2001 From: Jeremy Frasier Date: Fri, 15 Sep 2023 10:26:45 -0400 Subject: [PATCH 081/171] Explicitly list the linter config files the dev team should own @mcdonnnj correctly pointed out that other projects add their own configuration files that match, e.g., the /.*.yaml pattern. We want to ensure that we only own the linter configuration files from the skeleton. Co-authored-by: Nick <50747025+mcdonnnj@users.noreply.github.com> --- .github/CODEOWNERS | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index 2a9e98d..41c310f 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -16,12 +16,15 @@ /src/python.yml @jsf9k # These folks own all linting configuration files. -/.*.cfg @dav3r @felddy @jasonodoom @jsf9k @mcdonnnj -/.*.yaml @dav3r @felddy @jasonodoom @jsf9k @mcdonnnj -/.*.yml @dav3r @felddy @jasonodoom @jsf9k @mcdonnnj /.ansible-lint @dav3r @felddy @jasonodoom @jsf9k @mcdonnnj +/.bandit.yml @dav3r @felddy @jasonodoom @jsf9k @mcdonnnj /.flake8 @dav3r @felddy @jasonodoom @jsf9k @mcdonnnj +/.isort.cfg @dav3r @felddy @jasonodoom @jsf9k @mcdonnnj +/.mdl_config.yaml @dav3r @felddy @jasonodoom @jsf9k @mcdonnnj +/.pre-commit-config.yaml @dav3r @felddy @jasonodoom @jsf9k @mcdonnnj /.prettierignore @dav3r @felddy @jasonodoom @jsf9k @mcdonnnj /.yamllint @dav3r @felddy @jasonodoom @jsf9k @mcdonnnj -/requirements*.txt @dav3r @felddy @jasonodoom @jsf9k @mcdonnnj +/requirements.txt @dav3r @felddy @jasonodoom @jsf9k @mcdonnnj +/requirements-dev.txt @dav3r @felddy @jasonodoom @jsf9k @mcdonnnj +/requirements-test.txt @dav3r @felddy @jasonodoom @jsf9k @mcdonnnj /setup-env @dav3r @felddy @jasonodoom @jsf9k @mcdonnnj From a7327217c628aab92fad7b3e5005c8bcd24ba7be Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 30 Oct 2023 18:41:36 +0000 Subject: [PATCH 082/171] Bump hashicorp/setup-terraform from 2 to 3 Bumps [hashicorp/setup-terraform](https://github.com/hashicorp/setup-terraform) from 2 to 3. - [Release notes](https://github.com/hashicorp/setup-terraform/releases) - [Changelog](https://github.com/hashicorp/setup-terraform/blob/main/CHANGELOG.md) - [Commits](https://github.com/hashicorp/setup-terraform/compare/v2...v3) --- updated-dependencies: - dependency-name: hashicorp/setup-terraform dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] --- .github/workflows/build.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 32b17b1..fdb08d5 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -102,7 +102,7 @@ jobs: ${{ env.CURL_CACHE_DIR }}/"${PACKER_ZIP}" sudo mv /usr/local/bin/packer /usr/local/bin/packer-default sudo ln -s /opt/packer/packer /usr/local/bin/packer - - uses: hashicorp/setup-terraform@v2 + - uses: hashicorp/setup-terraform@v3 with: terraform_version: ${{ steps.setup-env.outputs.terraform-version }} - name: Install go-critic From c682d9a4b955b7fab2c218fd0a77ff64f1db487e Mon Sep 17 00:00:00 2001 From: Nicholas McDonnell <50747025+mcdonnnj@users.noreply.github.com> Date: Thu, 2 Nov 2023 15:49:06 -0400 Subject: [PATCH 083/171] Prefer block style to flow style We prefer block style to flow style for sequences and mappings in YAML. --- .github/workflows/build.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index fdb08d5..97774f3 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -5,7 +5,8 @@ on: push: pull_request: repository_dispatch: - types: [apb] + types: + - apb env: AWS_DEFAULT_REGION: us-east-1 From 592b26cb747b320420a300f027f3edf015a6a6bf Mon Sep 17 00:00:00 2001 From: Nicholas McDonnell <50747025+mcdonnnj@users.noreply.github.com> Date: Thu, 2 Nov 2023 15:57:37 -0400 Subject: [PATCH 084/171] Alphabetize entries in the build workflow We prefer to alphabetize mapping keys in YAML documents whenever possible. --- .github/workflows/build.yml | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 97774f3..5f1b991 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -2,8 +2,8 @@ name: build on: - push: pull_request: + push: repository_dispatch: types: - apb @@ -59,8 +59,8 @@ jobs: # it relies on the existence of a go.sum file. cache: false go-version: "1.20" - - name: Lookup Go cache directory - id: go-cache + - id: go-cache + name: Lookup Go cache directory run: | echo "dir=$(go env GOCACHE)" >> $GITHUB_OUTPUT - uses: actions/cache@v3 @@ -71,6 +71,10 @@ jobs: packer${{ steps.setup-env.outputs.packer-version }}-\ tf${{ steps.setup-env.outputs.terraform-version }}-" with: + key: "${{ env.BASE_CACHE_KEY }}\ + ${{ hashFiles('**/requirements-test.txt') }}-\ + ${{ hashFiles('**/requirements.txt') }}-\ + ${{ hashFiles('**/.pre-commit-config.yaml') }}" # Note that the .terraform directory IS NOT included in the # cache because if we were caching, then we would need to use # the `-upgrade=true` option. This option blindly pulls down the @@ -82,10 +86,6 @@ jobs: ${{ env.PRE_COMMIT_CACHE_DIR }} ${{ env.CURL_CACHE_DIR }} ${{ steps.go-cache.outputs.dir }} - key: "${{ env.BASE_CACHE_KEY }}\ - ${{ hashFiles('**/requirements-test.txt') }}-\ - ${{ hashFiles('**/requirements.txt') }}-\ - ${{ hashFiles('**/.pre-commit-config.yaml') }}" restore-keys: | ${{ env.BASE_CACHE_KEY }} - name: Setup curl cache From 780aa4d6a932b975af99226c6913c2bd22f7f158 Mon Sep 17 00:00:00 2001 From: Nicholas McDonnell <50747025+mcdonnnj@users.noreply.github.com> Date: Thu, 2 Nov 2023 16:00:00 -0400 Subject: [PATCH 085/171] Add a `merge_group` trigger to the build workflow This should improve compatibility with merge queues. We configure it to only trigger on the `checks_requested` type which is currently the only supported type for this trigger. If additional types are added in the future they should be added if appropriate. --- .github/workflows/build.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 5f1b991..1b5d51d 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -2,6 +2,9 @@ name: build on: + merge_group: + types: + - checks_requested pull_request: push: repository_dispatch: From 8a8338a72a1d2db1d2dc8bb00e4af526e261a6af Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 11 Dec 2023 18:59:21 +0000 Subject: [PATCH 086/171] Bump actions/setup-go from 4 to 5 Bumps [actions/setup-go](https://github.com/actions/setup-go) from 4 to 5. - [Release notes](https://github.com/actions/setup-go/releases) - [Commits](https://github.com/actions/setup-go/compare/v4...v5) --- updated-dependencies: - dependency-name: actions/setup-go dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] --- .github/workflows/build.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 1b5d51d..1f5617e 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -56,7 +56,7 @@ jobs: # We need the Go version and Go cache location for the actions/cache step, # so the Go installation must happen before that. - id: setup-go - uses: actions/setup-go@v4 + uses: actions/setup-go@v5 with: # There is no expectation for actual Go code so we disable caching as # it relies on the existence of a go.sum file. From 57d2993dd13fe92a99a0a025b77d739989e651d6 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 11 Dec 2023 18:59:25 +0000 Subject: [PATCH 087/171] Bump actions/setup-python from 4 to 5 Bumps [actions/setup-python](https://github.com/actions/setup-python) from 4 to 5. - [Release notes](https://github.com/actions/setup-python/releases) - [Commits](https://github.com/actions/setup-python/compare/v4...v5) --- updated-dependencies: - dependency-name: actions/setup-python dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] --- .github/workflows/build.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 1f5617e..11f63ba 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -50,7 +50,7 @@ jobs: uses: cisagov/setup-env-github-action@develop - uses: actions/checkout@v4 - id: setup-python - uses: actions/setup-python@v4 + uses: actions/setup-python@v5 with: python-version: "3.11" # We need the Go version and Go cache location for the actions/cache step, From 56e160bb8578139f9fcf7af6153029f74af68e92 Mon Sep 17 00:00:00 2001 From: Nicholas McDonnell <50747025+mcdonnnj@users.noreply.github.com> Date: Thu, 4 Jan 2024 15:03:15 -0500 Subject: [PATCH 088/171] Update pre-commit hook versions This is done automatically with the `pre-commit autoupdate` command. The pre-commit/mirrors-prettier was manually held back because the latest tags are for alpha releases of the next major version. --- .pre-commit-config.yaml | 26 +++++++++++++------------- 1 file changed, 13 insertions(+), 13 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 79894a8..90f2382 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -5,7 +5,7 @@ default_language_version: repos: - repo: https://github.com/pre-commit/pre-commit-hooks - rev: v4.4.0 + rev: v4.5.0 hooks: - id: check-case-conflict - id: check-executables-have-shebangs @@ -31,7 +31,7 @@ repos: # Text file hooks - repo: https://github.com/igorshubovych/markdownlint-cli - rev: v0.36.0 + rev: v0.39.0 hooks: - id: markdownlint args: @@ -41,7 +41,7 @@ repos: hooks: - id: prettier - repo: https://github.com/adrienverge/yamllint - rev: v1.32.0 + rev: v1.35.1 hooks: - id: yamllint args: @@ -49,14 +49,14 @@ repos: # GitHub Actions hooks - repo: https://github.com/python-jsonschema/check-jsonschema - rev: 0.26.3 + rev: 0.28.0 hooks: - id: check-github-actions - id: check-github-workflows # pre-commit hooks - repo: https://github.com/pre-commit/pre-commit - rev: v3.4.0 + rev: v3.6.2 hooks: - id: validate_manifest @@ -108,7 +108,7 @@ repos: # Python hooks # Run bandit on the "tests" tree with a configuration - repo: https://github.com/PyCQA/bandit - rev: 1.7.5 + rev: 1.7.7 hooks: - id: bandit name: bandit (tests tree) @@ -123,38 +123,38 @@ repos: name: bandit (everything else) exclude: tests - repo: https://github.com/psf/black-pre-commit-mirror - rev: 23.9.1 + rev: 24.2.0 hooks: - id: black - repo: https://github.com/PyCQA/flake8 - rev: 6.1.0 + rev: 7.0.0 hooks: - id: flake8 additional_dependencies: - flake8-docstrings - repo: https://github.com/PyCQA/isort - rev: 5.12.0 + rev: 5.13.2 hooks: - id: isort - repo: https://github.com/pre-commit/mirrors-mypy - rev: v1.5.1 + rev: v1.8.0 hooks: - id: mypy - repo: https://github.com/asottile/pyupgrade - rev: v3.10.1 + rev: v3.15.1 hooks: - id: pyupgrade # Ansible hooks - repo: https://github.com/ansible/ansible-lint - rev: v6.19.0 + rev: v24.2.0 hooks: - id: ansible-lint # files: molecule/default/playbook.yml # Terraform hooks - repo: https://github.com/antonbabenko/pre-commit-terraform - rev: v1.83.2 + rev: v1.88.0 hooks: - id: terraform_fmt - id: terraform_validate From 0d000b217f26a60c1b10e5cdc7abb235f815ad6a Mon Sep 17 00:00:00 2001 From: Nicholas McDonnell <50747025+mcdonnnj@users.noreply.github.com> Date: Thu, 4 Jan 2024 15:08:30 -0500 Subject: [PATCH 089/171] Manually update the prettier hook The hook is manually updated to the last v3 tag available from the pre-commit/mirrors-prettier repository. We then add the latest release of v3 available from NPM as an additional depdency. --- .pre-commit-config.yaml | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 90f2382..dd3f024 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -37,9 +37,16 @@ repos: args: - --config=.mdl_config.yaml - repo: https://github.com/pre-commit/mirrors-prettier - rev: v3.0.3 + # This is the last version of v3 available from the mirror. We should hold + # here until v4, which is currently in alpha, is more stable. + rev: v3.1.0 hooks: - id: prettier + # This is the latest version of v3 available from NPM. The pre-commit + # mirror does not pull tags for old major versions once a new major + # version tag is published. + additional_dependencies: + - prettier@3.2.5 - repo: https://github.com/adrienverge/yamllint rev: v1.35.1 hooks: From c23e69a4522a56ac18173220af23b659d51b6c05 Mon Sep 17 00:00:00 2001 From: Nicholas McDonnell <50747025+mcdonnnj@users.noreply.github.com> Date: Thu, 21 Sep 2023 17:49:55 -0400 Subject: [PATCH 090/171] Set the default shell for all run steps in the build workflow This sets the default shell for any run steps in the build workflow to mirror our standard shellscript writing practices. In addition to enabling our standard options it will also enable errtrace and print any commands that are run which should make debugging/troubleshooting more straightforward. --- .github/workflows/build.yml | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 11f63ba..565007a 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -11,6 +11,14 @@ on: types: - apb +# Set a default shell for any run steps. The `-Eueo pipefail` sets errtrace, +# nounset, errexit, and pipefail. The `-x` will print all commands as they are +# run. Please see the GitHub Actions documentation for more information: +# https://docs.github.com/en/actions/using-jobs/setting-default-values-for-jobs +defaults: + run: + shell: bash -Eueo pipefail -x {0} + env: AWS_DEFAULT_REGION: us-east-1 CURL_CACHE_DIR: ~/.cache/curl From 59ff98987b050c23aa06468b60dfd61637eab54a Mon Sep 17 00:00:00 2001 From: Michael Saki Date: Thu, 25 Jan 2024 12:19:41 -0600 Subject: [PATCH 091/171] Allow setup-env to specify Python version This commit is introducing 2 new flags into the setup-env script. -l or --list-versions will list available Python versions and allow the user to select a version interactively. The second flag -v or --version will allow a user to set the version if installed. (e.g. ./setup-env -v 3.9.6) --- setup-env | 42 +++++++++++++++++++++++++++++++++++++++--- 1 file changed, 39 insertions(+), 3 deletions(-) diff --git a/setup-env b/setup-env index 49f069f..9c2e31a 100755 --- a/setup-env +++ b/setup-env @@ -25,6 +25,8 @@ Options: -h --help Show this message. -i --install-hooks Install hook environments for all environments in the pre-commit config file. + -v --version Specify the Python version for the virtual environment. + -l --list-versions List available Python versions and select interactively. END_OF_LINE ) @@ -35,6 +37,13 @@ FORCE=0 # Positional parameters PARAMS="" +# Flags to allow a user to specify which version of Python they want to use +PYTHON_VERSION="" +LIST_VERSIONS=0 + +# Temp file that is used to search through available installed Python versions +TMPFILE=/tmp/versions.$$ + # Parse command line arguments while (("$#")); do case "$1" in @@ -50,6 +59,14 @@ while (("$#")); do INSTALL_HOOKS=1 shift ;; + -v | --version) + PYTHON_VERSION=$2 + shift 2 + ;; + -l | --list-versions) + LIST_VERSIONS=1 + shift + ;; -*) # unsupported flags echo "Error: Unsupported flag $1" >&2 exit 1 @@ -111,6 +128,25 @@ else fi set -o nounset +# List Python versions and select one interactively +if [ $LIST_VERSIONS -ne 0 ]; then + echo Available Python versions: + pyenv versions --bare --skip-aliases --skip-envs + read -p -r "Enter the desired Python version: " PYTHON_VERSION +fi + +# Check if PYTHON_VERSION isn't empty. If it is installed, set it locally. +pyenv versions --bare --skip-aliases --skip-envs > $TMPFILE +if [ -n "$PYTHON_VERSION" ]; then + if grep --fixed-strings --quiet "$PYTHON_VERSION" $TMPFILE; then + echo Using Python version "$PYTHON_VERSION" + pyenv local "$PYTHON_VERSION" + else + echo Error: Python version "$PYTHON_VERSION" is not installed. + fi + exit 1 +fi + # Remove any lingering local configuration. if [ $FORCE -ne 0 ]; then rm -f .python-version @@ -130,10 +166,10 @@ fi # Create a new virtual environment for this project if ! pyenv virtualenv "${env_name}"; then cat << END_OF_LINE - An existing virtual environment named $env_name was found. Either delete this - environment yourself or re-run with --force option to have it deleted. + An existing virtual environment named $env_name was found. Either delete this + environment yourself or re-run with --force option to have it deleted. - pyenv virtualenv-delete ${env_name} + pyenv virtualenv-delete ${env_name} END_OF_LINE exit 1 From 98f754e3ee808694f0c7d35faa45018039c2cd89 Mon Sep 17 00:00:00 2001 From: Michael Saki Date: Thu, 25 Jan 2024 14:15:40 -0600 Subject: [PATCH 092/171] Add /dev/null and remove TMPFILE This makes the code a bit cleaner and still accomplishes the same functionality Co-authored-by: Shane Frasier --- setup-env | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/setup-env b/setup-env index 9c2e31a..209cf89 100755 --- a/setup-env +++ b/setup-env @@ -41,9 +41,6 @@ PARAMS="" PYTHON_VERSION="" LIST_VERSIONS=0 -# Temp file that is used to search through available installed Python versions -TMPFILE=/tmp/versions.$$ - # Parse command line arguments while (("$#")); do case "$1" in @@ -136,9 +133,8 @@ if [ $LIST_VERSIONS -ne 0 ]; then fi # Check if PYTHON_VERSION isn't empty. If it is installed, set it locally. -pyenv versions --bare --skip-aliases --skip-envs > $TMPFILE if [ -n "$PYTHON_VERSION" ]; then - if grep --fixed-strings --quiet "$PYTHON_VERSION" $TMPFILE; then + if pyenv versions --bare --skip-aliases --skip-envs | grep --fixed-strings "$PYTHON_VERSION" >/dev/null; then echo Using Python version "$PYTHON_VERSION" pyenv local "$PYTHON_VERSION" else From 6f8cf91d332f3953dd3f6cd18f48b6f883cea440 Mon Sep 17 00:00:00 2001 From: Michael Saki Date: Thu, 25 Jan 2024 14:19:54 -0600 Subject: [PATCH 093/171] Place flags in the correct order for -r and -p --- setup-env | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/setup-env b/setup-env index 209cf89..28b36f0 100755 --- a/setup-env +++ b/setup-env @@ -129,12 +129,12 @@ set -o nounset if [ $LIST_VERSIONS -ne 0 ]; then echo Available Python versions: pyenv versions --bare --skip-aliases --skip-envs - read -p -r "Enter the desired Python version: " PYTHON_VERSION + read -r -p "Enter the desired Python version: " PYTHON_VERSION fi # Check if PYTHON_VERSION isn't empty. If it is installed, set it locally. if [ -n "$PYTHON_VERSION" ]; then - if pyenv versions --bare --skip-aliases --skip-envs | grep --fixed-strings "$PYTHON_VERSION" >/dev/null; then + if pyenv versions --bare --skip-aliases --skip-envs | grep --fixed-strings "$PYTHON_VERSION" > /dev/null; then echo Using Python version "$PYTHON_VERSION" pyenv local "$PYTHON_VERSION" else From 6f2adf902b1a4a248fc48e1bbb2347b50136c253 Mon Sep 17 00:00:00 2001 From: Michael Saki Date: Thu, 25 Jan 2024 14:21:49 -0600 Subject: [PATCH 094/171] Remove unneccessary spacing --- setup-env | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/setup-env b/setup-env index 28b36f0..705d44e 100755 --- a/setup-env +++ b/setup-env @@ -162,10 +162,10 @@ fi # Create a new virtual environment for this project if ! pyenv virtualenv "${env_name}"; then cat << END_OF_LINE - An existing virtual environment named $env_name was found. Either delete this - environment yourself or re-run with --force option to have it deleted. + An existing virtual environment named $env_name was found. Either delete this + environment yourself or re-run with --force option to have it deleted. - pyenv virtualenv-delete ${env_name} + pyenv virtualenv-delete ${env_name} END_OF_LINE exit 1 From ce5acf0a739f701d763b9a9de91667ce7b17ffcd Mon Sep 17 00:00:00 2001 From: Michael Saki Date: Thu, 25 Jan 2024 14:25:37 -0600 Subject: [PATCH 095/171] Alphabetize flags and descriptions Co-authored-by: Shane Frasier --- setup-env | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/setup-env b/setup-env index 705d44e..55ead36 100755 --- a/setup-env +++ b/setup-env @@ -25,8 +25,8 @@ Options: -h --help Show this message. -i --install-hooks Install hook environments for all environments in the pre-commit config file. - -v --version Specify the Python version for the virtual environment. -l --list-versions List available Python versions and select interactively. + -v --version Specify the Python version for the virtual environment. END_OF_LINE ) @@ -56,14 +56,14 @@ while (("$#")); do INSTALL_HOOKS=1 shift ;; - -v | --version) - PYTHON_VERSION=$2 - shift 2 - ;; -l | --list-versions) LIST_VERSIONS=1 shift ;; + -v | --version) + PYTHON_VERSION=$2 + shift 2 + ;; -*) # unsupported flags echo "Error: Unsupported flag $1" >&2 exit 1 From 2687bf85faa5caa21baa6273488872635bc03cee Mon Sep 17 00:00:00 2001 From: Jeremy Frasier Date: Fri, 26 Jan 2024 10:08:25 -0500 Subject: [PATCH 096/171] Move misplaced exit If the Python version exists then we want the script to continue execution. --- setup-env | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup-env b/setup-env index 55ead36..b22dd09 100755 --- a/setup-env +++ b/setup-env @@ -139,8 +139,8 @@ if [ -n "$PYTHON_VERSION" ]; then pyenv local "$PYTHON_VERSION" else echo Error: Python version "$PYTHON_VERSION" is not installed. + exit 1 fi - exit 1 fi # Remove any lingering local configuration. From 56e51dc2cdd4c6edcec114e6a777bd30e8386161 Mon Sep 17 00:00:00 2001 From: Jeremy Frasier Date: Fri, 26 Jan 2024 10:10:23 -0500 Subject: [PATCH 097/171] Remove premature pyenv local command pyenv local is run below. --- setup-env | 1 - 1 file changed, 1 deletion(-) diff --git a/setup-env b/setup-env index b22dd09..2c49ff8 100755 --- a/setup-env +++ b/setup-env @@ -136,7 +136,6 @@ fi if [ -n "$PYTHON_VERSION" ]; then if pyenv versions --bare --skip-aliases --skip-envs | grep --fixed-strings "$PYTHON_VERSION" > /dev/null; then echo Using Python version "$PYTHON_VERSION" - pyenv local "$PYTHON_VERSION" else echo Error: Python version "$PYTHON_VERSION" is not installed. exit 1 From 7c8ef13eeb8f1c716dd2992057b2a5cd9e6bea14 Mon Sep 17 00:00:00 2001 From: Jeremy Frasier Date: Fri, 26 Jan 2024 10:11:50 -0500 Subject: [PATCH 098/171] Include PYTHON_VERSION when running pyenv virtualenv If PYTHON_VERSION is an empty string then the system Python will be used. --- setup-env | 19 ++++++++++++++----- 1 file changed, 14 insertions(+), 5 deletions(-) diff --git a/setup-env b/setup-env index 2c49ff8..3bdf049 100755 --- a/setup-env +++ b/setup-env @@ -37,8 +37,8 @@ FORCE=0 # Positional parameters PARAMS="" -# Flags to allow a user to specify which version of Python they want to use -PYTHON_VERSION="" +# A flag to allow a user to specify which version of Python they want +# to use. LIST_VERSIONS=0 # Parse command line arguments @@ -132,8 +132,9 @@ if [ $LIST_VERSIONS -ne 0 ]; then read -r -p "Enter the desired Python version: " PYTHON_VERSION fi -# Check if PYTHON_VERSION isn't empty. If it is installed, set it locally. -if [ -n "$PYTHON_VERSION" ]; then +# Check if PYTHON_VERSION is defined. If it is defined then check that +# it is a valid value. +if [ -n "${PYTHON_VERSION+x}" ]; then if pyenv versions --bare --skip-aliases --skip-envs | grep --fixed-strings "$PYTHON_VERSION" > /dev/null; then echo Using Python version "$PYTHON_VERSION" else @@ -159,7 +160,15 @@ END_OF_LINE fi # Create a new virtual environment for this project -if ! pyenv virtualenv "${env_name}"; then +# +# If $PYTHON_VERSION is undefined then the system Python will be used. +# +# We can't quote ${PYTHON_VERSION:=} below since if the variable is +# undefined then we want nothing to appear; this is the reason for the +# "shellcheck disable" line below. +# +# shellcheck disable=SC2086 +if ! pyenv virtualenv ${PYTHON_VERSION:=} "${env_name}"; then cat << END_OF_LINE An existing virtual environment named $env_name was found. Either delete this environment yourself or re-run with --force option to have it deleted. From 6d74d8d15733ceae88259f846d46a2f520818459 Mon Sep 17 00:00:00 2001 From: Michael Saki Date: Tue, 30 Jan 2024 12:51:51 -0600 Subject: [PATCH 099/171] Add getopt variables and short flags --- setup-env | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/setup-env b/setup-env index 3bdf049..a1cbf12 100755 --- a/setup-env +++ b/setup-env @@ -34,9 +34,21 @@ END_OF_LINE # Flag to force deletion and creation of virtual environment FORCE=0 +# Initialize the all other flags +INSTALL_HOOKS=0 +LIST_VERSIONS=0 +PYTHON_VERSION="" +VENV_NAME="" + # Positional parameters PARAMS="" +# Define short options for getopt +OPTS="fhilv:n:" + +# Parse options using BSD getopt +OPTIND=1 + # A flag to allow a user to specify which version of Python they want # to use. LIST_VERSIONS=0 From 66f4cb843511ed04849a722677d4c17075d78935 Mon Sep 17 00:00:00 2001 From: Michael Saki Date: Tue, 30 Jan 2024 12:54:12 -0600 Subject: [PATCH 100/171] Remove redundant flag initialization --- setup-env | 4 ---- 1 file changed, 4 deletions(-) diff --git a/setup-env b/setup-env index a1cbf12..0208a2a 100755 --- a/setup-env +++ b/setup-env @@ -49,10 +49,6 @@ OPTS="fhilv:n:" # Parse options using BSD getopt OPTIND=1 -# A flag to allow a user to specify which version of Python they want -# to use. -LIST_VERSIONS=0 - # Parse command line arguments while (("$#")); do case "$1" in From fcfe83545a13139a65b08b95712aeeb7793827b1 Mon Sep 17 00:00:00 2001 From: Michael Saki Date: Tue, 30 Jan 2024 13:59:04 -0600 Subject: [PATCH 101/171] Add getopt functionality and -n flag This commit makes a couple changes. The first change is adding the BSD getopt tool to simplify the parsing of flags and arguments. Second, we are adding the -n flag so the user can specify the name of the virtual environment if they choose. --- setup-env | 57 +++++++++++++++++++++++++++++++++++-------------------- 1 file changed, 36 insertions(+), 21 deletions(-) diff --git a/setup-env b/setup-env index 0208a2a..4ddd390 100755 --- a/setup-env +++ b/setup-env @@ -49,40 +49,55 @@ OPTS="fhilv:n:" # Parse options using BSD getopt OPTIND=1 +# Display installed python versions +python_versions() { + pyenv versions --bare --skip-aliases --skip-envs +} + # Parse command line arguments -while (("$#")); do - case "$1" in - -f | --force) +while getopts :$OPTS opt; do + case $opt in + f) FORCE=1 - shift ;; - -h | --help) - echo "${USAGE}" + h) + echo "$USAGE" exit 0 ;; - -i | --install-hooks) + i) INSTALL_HOOKS=1 - shift ;; - -l | --list-versions) + l) LIST_VERSIONS=1 - shift ;; - -v | --version) - PYTHON_VERSION=$2 - shift 2 + n) + VENV_NAME="$OPTARG" + ;; + v) + PYTHON_VERSION="$OPTARG" + # Check if Python version is valid and installed + if ! python_versions | grep --fixed-strings "$PYTHON_VERSION" > /dev/null; then + echo "Error: Python version $PYTHON_VERSION is not installed. Versions available:" + python_versions + exit 1 + fi ;; - -*) # unsupported flags - echo "Error: Unsupported flag $1" >&2 + \:) + echo Error: Option -$OPTARG requires an argument. + echo "$USAGE" exit 1 ;; - *) # preserve positional arguments - PARAMS="$PARAMS $1" - shift + \?) + echo -e "Invalid option please look through usage: \n" + echo "$USAGE" + exit 1 ;; + esac done +shift $((OPTIND-1)) + # set positional arguments in their proper place eval set -- "$PARAMS" @@ -124,9 +139,9 @@ fi set +o nounset # Determine the virtual environment name -if [ "$1" ]; then +if [ -n "$VENV_NAME" ]; then # Use the user-provided environment name - env_name=$1 + env_name="$VENV_NAME" else # Set the environment name to the last part of the working directory. env_name=${PWD##*/} @@ -136,7 +151,7 @@ set -o nounset # List Python versions and select one interactively if [ $LIST_VERSIONS -ne 0 ]; then echo Available Python versions: - pyenv versions --bare --skip-aliases --skip-envs + python_versions read -r -p "Enter the desired Python version: " PYTHON_VERSION fi From f1f5b08367fca5f699e98b768b57993c1576fdfb Mon Sep 17 00:00:00 2001 From: Michael Saki Date: Tue, 30 Jan 2024 14:16:52 -0600 Subject: [PATCH 102/171] Update the usage and force documentation This commit will update the usage documentation so now it will display all the new flags, how to use them, and what they do. Since the long option isn't available for the BSD version of getopt we have to use short options only. The force documentation has been updated to reflect this change. Also removed some redundant code that wasn't necessary. --- setup-env | 47 +++++++++++++++++++---------------------------- 1 file changed, 19 insertions(+), 28 deletions(-) diff --git a/setup-env b/setup-env index 4ddd390..7a66329 100755 --- a/setup-env +++ b/setup-env @@ -17,16 +17,18 @@ It does the following: - Configures git upstream remote "lineage" repositories. Usage: - setup-env [options] [virt_env_name] - setup-env (-h | --help) + setup-env [-n] [virt_env_name] + setup-env [-v] [python_version] + setup-env (-h) Options: - -f --force Delete virtual enviroment if it already exists. - -h --help Show this message. - -i --install-hooks Install hook environments for all environments in the - pre-commit config file. - -l --list-versions List available Python versions and select interactively. - -v --version Specify the Python version for the virtual environment. + -f Delete virtual enviroment if it already exists. + -h Show this message. + -i Install hook environments for all environments in the + pre-commit config file. + -l List available Python versions and select interactively. + -n Choose the name of the virtual environment. + -v Specify the Python version for the virtual environment. END_OF_LINE ) @@ -49,9 +51,9 @@ OPTS="fhilv:n:" # Parse options using BSD getopt OPTIND=1 -# Display installed python versions +# Display installed python versions python_versions() { - pyenv versions --bare --skip-aliases --skip-envs + pyenv versions --bare --skip-aliases --skip-envs } # Parse command line arguments @@ -82,21 +84,21 @@ while getopts :$OPTS opt; do exit 1 fi ;; - \:) - echo Error: Option -$OPTARG requires an argument. + :) + echo -e "Error: Option -$OPTARG requires an argument. \n" echo "$USAGE" exit 1 ;; \?) - echo -e "Invalid option please look through usage: \n" + echo -e "Invalid option please look through usage: \n" echo "$USAGE" exit 1 ;; - + esac done -shift $((OPTIND-1)) +shift $((OPTIND - 1)) # set positional arguments in their proper place eval set -- "$PARAMS" @@ -155,17 +157,6 @@ if [ $LIST_VERSIONS -ne 0 ]; then read -r -p "Enter the desired Python version: " PYTHON_VERSION fi -# Check if PYTHON_VERSION is defined. If it is defined then check that -# it is a valid value. -if [ -n "${PYTHON_VERSION+x}" ]; then - if pyenv versions --bare --skip-aliases --skip-envs | grep --fixed-strings "$PYTHON_VERSION" > /dev/null; then - echo Using Python version "$PYTHON_VERSION" - else - echo Error: Python version "$PYTHON_VERSION" is not installed. - exit 1 - fi -fi - # Remove any lingering local configuration. if [ $FORCE -ne 0 ]; then rm -f .python-version @@ -173,7 +164,7 @@ if [ $FORCE -ne 0 ]; then elif [[ -f .python-version ]]; then cat << 'END_OF_LINE' An existing .python-version file was found. Either remove this file yourself - or re-run with --force option to have it deleted along with the associated + or re-run with -f (force) option to have it deleted along with the associated virtual environment. rm .python-version @@ -194,7 +185,7 @@ fi if ! pyenv virtualenv ${PYTHON_VERSION:=} "${env_name}"; then cat << END_OF_LINE An existing virtual environment named $env_name was found. Either delete this - environment yourself or re-run with --force option to have it deleted. + environment yourself or re-run with -f (force) option to have it deleted. pyenv virtualenv-delete ${env_name} From e953f5d04ce98b1d32924baaa7543f0856e50852 Mon Sep 17 00:00:00 2001 From: Michael Saki Date: Tue, 6 Feb 2024 19:58:49 -0600 Subject: [PATCH 103/171] Update usage with long options --- setup-env | 22 ++++++++++++---------- 1 file changed, 12 insertions(+), 10 deletions(-) diff --git a/setup-env b/setup-env index 7a66329..9adab2c 100755 --- a/setup-env +++ b/setup-env @@ -9,6 +9,8 @@ USAGE=$( Configure a development environment for this repository. It does the following: + - Allows user to specify Python version. + - Allows user to choose name for their virtual environment. - Verifies pyenv and pyenv-virtualenv are installed. - Creates a Python virtual environment. - Configures the activation of the virtual enviroment for the repo directory. @@ -17,18 +19,18 @@ It does the following: - Configures git upstream remote "lineage" repositories. Usage: - setup-env [-n] [virt_env_name] - setup-env [-v] [python_version] - setup-env (-h) + setup-env (-n | --name) [virt_env_name] + setup-env (-v | --version) [python_version] + setup-env (-h | --help) Options: - -f Delete virtual enviroment if it already exists. - -h Show this message. - -i Install hook environments for all environments in the - pre-commit config file. - -l List available Python versions and select interactively. - -n Choose the name of the virtual environment. - -v Specify the Python version for the virtual environment. + -f | --force Delete virtual enviroment if it already exists. + -h | --help Show this message. + -i | --install-hooks Install hook environments for all environments in the + pre-commit config file. + -l | --list-versions List available Python versions and select interactively. + -n | --name Choose the name of the virtual environment. + -v | --version Specify the Python version for the virtual environment. END_OF_LINE ) From 63abea54ac18f4d5075d3fe6a698f7b6c65523d2 Mon Sep 17 00:00:00 2001 From: Michael Saki Date: Tue, 6 Feb 2024 20:28:05 -0600 Subject: [PATCH 104/171] Add gnu-getopt functionality and error handling This commit will make it so long options are supported. It also includes some improved error handling. It will display a message if a Mac OS user doesn't have gnu-getopt installed on their system and suggest steps to take to get it installed via brew. --- setup-env | 107 ++++++++++++++++++++++++++++++++++-------------------- 1 file changed, 68 insertions(+), 39 deletions(-) diff --git a/setup-env b/setup-env index 9adab2c..e958148 100755 --- a/setup-env +++ b/setup-env @@ -35,6 +35,11 @@ Options: END_OF_LINE ) +# Display installed python versions +python_versions() { + pyenv versions --bare --skip-aliases --skip-envs +} + # Flag to force deletion and creation of virtual environment FORCE=0 @@ -44,67 +49,82 @@ LIST_VERSIONS=0 PYTHON_VERSION="" VENV_NAME="" -# Positional parameters -PARAMS="" +# Define long options +LONGOPTS="force,help,install-hooks,list-versions,name:,version:" # Define short options for getopt -OPTS="fhilv:n:" +SHORTOPTS="fhiln:v:" -# Parse options using BSD getopt -OPTIND=1 +# Check if GNU getopt is available +if [[ $(getopt --version 2> /dev/null) != *"getopt from util-linux"* ]]; then + cat << 'END_OF_LINE' -# Display installed python versions -python_versions() { - pyenv versions --bare --skip-aliases --skip-envs -} + Gnu-getopt is not detected and is a dependency to run this script. + On the Mac, we recommend installing brew, https://brew.sh/. Then installation + is as simple as `brew install gnu-getopt` and adding this to your + profile: + + export PATH="/usr/local/opt/gnu-getopt/bin:$PATH" + +END_OF_LINE + exit 1 +fi -# Parse command line arguments -while getopts :$OPTS opt; do - case $opt in - f) +# Use GNU getopt to parse options +if ! PARSED=$(getopt --options $SHORTOPTS --longoptions $LONGOPTS --name "$0" -- "$@"); then + echo "Error parsing options" + exit 2 +fi +eval set -- "$PARSED" + +while true; do + case "$1" in + -f | --force) FORCE=1 + shift ;; - h) + -h | --help) echo "$USAGE" exit 0 ;; - i) + -i | --install-hooks) INSTALL_HOOKS=1 + shift ;; - l) + -l | --list-versions) LIST_VERSIONS=1 + shift ;; - n) - VENV_NAME="$OPTARG" + -n | --name) + VENV_NAME="$2" + shift 2 ;; - v) - PYTHON_VERSION="$OPTARG" - # Check if Python version is valid and installed - if ! python_versions | grep --fixed-strings "$PYTHON_VERSION" > /dev/null; then - echo "Error: Python version $PYTHON_VERSION is not installed. Versions available:" - python_versions - exit 1 + -v | --version) + PYTHON_VERSION="$2" + shift 2 + # Check the Python versions being passed in. + if [ -n "${PYTHON_VERSION+x}" ]; then + if python_versions | grep -E "^${PYTHON_VERSION}$" > /dev/null; then + echo Using Python version "$PYTHON_VERSION" + else + echo Error: Python version "$PYTHON_VERSION" is not installed. + echo Installed Python versions are: + python_versions + exit 1 + fi fi ;; - :) - echo -e "Error: Option -$OPTARG requires an argument. \n" - echo "$USAGE" - exit 1 + --) + shift + break ;; - \?) - echo -e "Invalid option please look through usage: \n" - echo "$USAGE" - exit 1 + *) + echo "Programming error" + exit 3 ;; - esac done -shift $((OPTIND - 1)) - -# set positional arguments in their proper place -eval set -- "$PARAMS" - # Check to see if pyenv is installed if [ -z "$(command -v pyenv)" ] || { [ -z "$(command -v pyenv-virtualenv)" ] && [ ! -f "$(pyenv root)/plugins/pyenv-virtualenv/bin/pyenv-virtualenv" ]; }; then echo "pyenv and pyenv-virtualenv are required." @@ -157,6 +177,15 @@ if [ $LIST_VERSIONS -ne 0 ]; then echo Available Python versions: python_versions read -r -p "Enter the desired Python version: " PYTHON_VERSION + # Check the Python versions being passed in. + if [ -n "${PYTHON_VERSION+x}" ]; then + if python_versions | grep -E "^${PYTHON_VERSION}$" > /dev/null; then + echo Using Python version "$PYTHON_VERSION" + else + echo Error: Python version "$PYTHON_VERSION" is not installed. + exit 1 + fi + fi fi # Remove any lingering local configuration. From 8d469b697045b5477b0ca0440836f1a60a5179e2 Mon Sep 17 00:00:00 2001 From: Michael Saki Date: Tue, 6 Feb 2024 20:49:34 -0600 Subject: [PATCH 105/171] Add documentation in CONTRIBUTING.md for gnu-getopt --- CONTRIBUTING.md | 16 +++++++++++----- 1 file changed, 11 insertions(+), 5 deletions(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index b2077bb..5727ec5 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -46,9 +46,12 @@ There are a few ways to do this, but we prefer to use create and manage a Python virtual environment specific to this project. -If you already have `pyenv` and `pyenv-virtualenv` configured you can -take advantage of the `setup-env` tool in this repo to automate the -entire environment configuration process. +The `setup-env` tool in this repository is our recommended method +for automating the entire environment configuration process. The +dependencies required to run this tool are +[`gnu-getopt`](https://manned.org/getopt.1), `pyenv`, and +`pyenv-virtualenv`. If these tools are already configured on +your system, you can simply run the following command. ```console ./setup-env @@ -57,13 +60,14 @@ entire environment configuration process. Otherwise, follow the steps below to manually configure your environment. -#### Installing and using `pyenv` and `pyenv-virtualenv` #### +#### Installing and using `gnu-getopt`, `pyenv`, and `pyenv-virtualenv` #### On the Mac, we recommend installing [brew](https://brew.sh/). Then -installation is as simple as `brew install pyenv pyenv-virtualenv` and +installation is as simple as `brew install gnu-getopt pyenv pyenv-virtualenv` and adding this to your profile: ```bash +export PATH="/usr/local/opt/gnu-getopt/bin:$PATH" export PYENV_ROOT="$HOME/.pyenv" export PATH="$PYENV_ROOT/bin:$PATH" eval "$(pyenv init --path)" @@ -78,6 +82,8 @@ install the necessary tools. Before running this ensure that you have installed the prerequisites for your platform according to the [`pyenv` wiki page](https://github.com/pyenv/pyenv/wiki/common-build-problems). +`Gnu-getopt` is generally included in the core utilities of most +Linux distributions. On WSL you should treat your platform as whatever Linux distribution you've chosen to install. From 37f00171b93baa6da69ddf06af7b1ec78f4605f8 Mon Sep 17 00:00:00 2001 From: Michael Saki Date: Wed, 7 Feb 2024 09:43:30 -0600 Subject: [PATCH 106/171] Fix grammar and capitalization errors Co-authored-by: Shane Frasier --- CONTRIBUTING.md | 2 +- setup-env | 12 ++++++------ 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 5727ec5..bddb8c7 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -82,7 +82,7 @@ install the necessary tools. Before running this ensure that you have installed the prerequisites for your platform according to the [`pyenv` wiki page](https://github.com/pyenv/pyenv/wiki/common-build-problems). -`Gnu-getopt` is generally included in the core utilities of most +`gnu-getopt` is generally included in the core utilities of most Linux distributions. On WSL you should treat your platform as whatever Linux distribution diff --git a/setup-env b/setup-env index e958148..b33f8c3 100755 --- a/setup-env +++ b/setup-env @@ -9,10 +9,10 @@ USAGE=$( Configure a development environment for this repository. It does the following: - - Allows user to specify Python version. - - Allows user to choose name for their virtual environment. + - Allows the user to specify the Python version to use for the virtual environment. + - Allows the user to specify a name for the virtual environment. - Verifies pyenv and pyenv-virtualenv are installed. - - Creates a Python virtual environment. + - Creates the Python virtual environment. - Configures the activation of the virtual enviroment for the repo directory. - Installs the requirements needed for development. - Installs git pre-commit hooks. @@ -29,7 +29,7 @@ Options: -i | --install-hooks Install hook environments for all environments in the pre-commit config file. -l | --list-versions List available Python versions and select interactively. - -n | --name Choose the name of the virtual environment. + -n | --name Specify the name of the virtual environment. -v | --version Specify the Python version for the virtual environment. END_OF_LINE @@ -195,7 +195,7 @@ if [ $FORCE -ne 0 ]; then elif [[ -f .python-version ]]; then cat << 'END_OF_LINE' An existing .python-version file was found. Either remove this file yourself - or re-run with -f (force) option to have it deleted along with the associated + or re-run with the --force option to have it deleted along with the associated virtual environment. rm .python-version @@ -216,7 +216,7 @@ fi if ! pyenv virtualenv ${PYTHON_VERSION:=} "${env_name}"; then cat << END_OF_LINE An existing virtual environment named $env_name was found. Either delete this - environment yourself or re-run with -f (force) option to have it deleted. + environment yourself or re-run with the --force option to have it deleted. pyenv virtualenv-delete ${env_name} From 6afbee2621e48c3fcf3de415791cc8927c4ee4a2 Mon Sep 17 00:00:00 2001 From: Michael Saki Date: Wed, 7 Feb 2024 09:48:01 -0600 Subject: [PATCH 107/171] Combine PATH exports to single line Co-authored-by: Shane Frasier --- CONTRIBUTING.md | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index bddb8c7..ce64056 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -67,9 +67,8 @@ installation is as simple as `brew install gnu-getopt pyenv pyenv-virtualenv` an adding this to your profile: ```bash -export PATH="/usr/local/opt/gnu-getopt/bin:$PATH" export PYENV_ROOT="$HOME/.pyenv" -export PATH="$PYENV_ROOT/bin:$PATH" +export PATH="$PYENV_ROOT/bin:/usr/local/opt/gnu-getopt/bin:$PATH" eval "$(pyenv init --path)" eval "$(pyenv init -)" eval "$(pyenv virtualenv-init -)" From 2e0abd89256cbfef29e206e33d7404ab81e09d67 Mon Sep 17 00:00:00 2001 From: Michael Saki Date: Wed, 7 Feb 2024 09:48:39 -0600 Subject: [PATCH 108/171] Improve usage instructions Co-authored-by: Shane Frasier --- setup-env | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/setup-env b/setup-env index b33f8c3..e8173c0 100755 --- a/setup-env +++ b/setup-env @@ -19,8 +19,7 @@ It does the following: - Configures git upstream remote "lineage" repositories. Usage: - setup-env (-n | --name) [virt_env_name] - setup-env (-v | --version) [python_version] + setup-env (-n | --name) [virt_env_name] (-v | --version) [python_version] setup-env (-h | --help) Options: From 3282c1c13b765e8aaddfd3e0cb7ebc4baf1028ae Mon Sep 17 00:00:00 2001 From: Michael Saki Date: Wed, 7 Feb 2024 11:03:35 -0600 Subject: [PATCH 109/171] Add $(brew --prefix) to PATH for getopt Co-authored-by: dav3r --- CONTRIBUTING.md | 2 +- setup-env | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index ce64056..65abd87 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -68,7 +68,7 @@ adding this to your profile: ```bash export PYENV_ROOT="$HOME/.pyenv" -export PATH="$PYENV_ROOT/bin:/usr/local/opt/gnu-getopt/bin:$PATH" +export PATH="$PYENV_ROOT/bin:$(brew --prefix)/opt/gnu-getopt/bin:$PATH" eval "$(pyenv init --path)" eval "$(pyenv init -)" eval "$(pyenv virtualenv-init -)" diff --git a/setup-env b/setup-env index e8173c0..293fe3b 100755 --- a/setup-env +++ b/setup-env @@ -63,7 +63,7 @@ if [[ $(getopt --version 2> /dev/null) != *"getopt from util-linux"* ]]; then is as simple as `brew install gnu-getopt` and adding this to your profile: - export PATH="/usr/local/opt/gnu-getopt/bin:$PATH" + export PATH="$(brew --prefix)/opt/gnu-getopt/bin:$PATH" END_OF_LINE exit 1 From 769097bb7c125acfa7b01f58cd9249042b85a486 Mon Sep 17 00:00:00 2001 From: Michael Saki Date: Wed, 7 Feb 2024 11:04:27 -0600 Subject: [PATCH 110/171] Fix confusing wording Co-authored-by: dav3r --- setup-env | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup-env b/setup-env index 293fe3b..e801529 100755 --- a/setup-env +++ b/setup-env @@ -42,7 +42,7 @@ python_versions() { # Flag to force deletion and creation of virtual environment FORCE=0 -# Initialize the all other flags +# Initialize the other flags INSTALL_HOOKS=0 LIST_VERSIONS=0 PYTHON_VERSION="" From 88c95863fc1a769995d835eb36d58de20eb044d9 Mon Sep 17 00:00:00 2001 From: Michael Saki Date: Wed, 7 Feb 2024 11:06:50 -0600 Subject: [PATCH 111/171] Replace virt_env_name w/ virtual_env_name for clarity Co-authored-by: dav3r --- setup-env | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup-env b/setup-env index e801529..dd470b4 100755 --- a/setup-env +++ b/setup-env @@ -19,7 +19,7 @@ It does the following: - Configures git upstream remote "lineage" repositories. Usage: - setup-env (-n | --name) [virt_env_name] (-v | --version) [python_version] + setup-env (-n | --name) [virtual_env_name] (-v | --version) [python_version] setup-env (-h | --help) Options: From fb8bd7a78b5e3e9a34ca2d653cb011de4f4429c1 Mon Sep 17 00:00:00 2001 From: Michael Saki Date: Tue, 13 Feb 2024 14:31:35 -0600 Subject: [PATCH 112/171] Differentiate between GNU getopt and gnu-getopt brew formula This commit will clearly differentiate between GNU getopt the tool and gnu-getopt the Homebrew formula. Also updating the URL so that getopt points at the source repository and pyenv and pyenv-virtualenv point at their respective repositories. Updated the ending punctuation. Co-authored-by: Nick <50747025+mcdonnnj@users.noreply.github.com> --- CONTRIBUTING.md | 13 +++++++------ setup-env | 2 +- 2 files changed, 8 insertions(+), 7 deletions(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 65abd87..584ea4a 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -49,9 +49,10 @@ project. The `setup-env` tool in this repository is our recommended method for automating the entire environment configuration process. The dependencies required to run this tool are -[`gnu-getopt`](https://manned.org/getopt.1), `pyenv`, and -`pyenv-virtualenv`. If these tools are already configured on -your system, you can simply run the following command. +[GNU `getopt`](https://github.com/util-linux/util-linux/blob/master/misc-utils/getopt.1.adoc), +[`pyenv`](https://github.com/pyenv/pyenv), and [`pyenv-virtualenv`](https://github.com/pyenv/pyenv-virtualenv). +If these tools are already configured on your system, you can simply run the +following command: ```console ./setup-env @@ -60,7 +61,7 @@ your system, you can simply run the following command. Otherwise, follow the steps below to manually configure your environment. -#### Installing and using `gnu-getopt`, `pyenv`, and `pyenv-virtualenv` #### +#### Installing and using GNU `getopt`, `pyenv`, and `pyenv-virtualenv` #### On the Mac, we recommend installing [brew](https://brew.sh/). Then installation is as simple as `brew install gnu-getopt pyenv pyenv-virtualenv` and @@ -81,8 +82,8 @@ install the necessary tools. Before running this ensure that you have installed the prerequisites for your platform according to the [`pyenv` wiki page](https://github.com/pyenv/pyenv/wiki/common-build-problems). -`gnu-getopt` is generally included in the core utilities of most -Linux distributions. +GNU `getopt` is included in most Linux distributions as part of the +[`util-linux`](https://github.com/util-linux/util-linux) package. On WSL you should treat your platform as whatever Linux distribution you've chosen to install. diff --git a/setup-env b/setup-env index dd470b4..7bf53ec 100755 --- a/setup-env +++ b/setup-env @@ -58,7 +58,7 @@ SHORTOPTS="fhiln:v:" if [[ $(getopt --version 2> /dev/null) != *"getopt from util-linux"* ]]; then cat << 'END_OF_LINE' - Gnu-getopt is not detected and is a dependency to run this script. + GNU getopt is not detected and is a dependency to run this script. On the Mac, we recommend installing brew, https://brew.sh/. Then installation is as simple as `brew install gnu-getopt` and adding this to your profile: From 967e80de34851bf4926dfd93f14de56102160825 Mon Sep 17 00:00:00 2001 From: Michael Saki Date: Tue, 13 Feb 2024 14:58:43 -0600 Subject: [PATCH 113/171] Add parenthesis over brew link Co-authored-by: Nick <50747025+mcdonnnj@users.noreply.github.com> --- setup-env | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup-env b/setup-env index 7bf53ec..59733a4 100755 --- a/setup-env +++ b/setup-env @@ -59,7 +59,7 @@ if [[ $(getopt --version 2> /dev/null) != *"getopt from util-linux"* ]]; then cat << 'END_OF_LINE' GNU getopt is not detected and is a dependency to run this script. - On the Mac, we recommend installing brew, https://brew.sh/. Then installation + On the Mac, we recommend installing brew (https://brew.sh/). Then installation is as simple as `brew install gnu-getopt` and adding this to your profile: From 4fadb123e36460ef5753a4d7eac550f3d23dd577 Mon Sep 17 00:00:00 2001 From: Michael Saki Date: Wed, 14 Feb 2024 07:37:24 -0600 Subject: [PATCH 114/171] Refactor flag names for clarity and accuracy This commit will remove the previous flags -v or --version for -p or --python-version. Also it will replace -n or --name for -v or --venv-name. The usage menu has been updated to reflect these changes as well. --- setup-env | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/setup-env b/setup-env index 59733a4..b272898 100755 --- a/setup-env +++ b/setup-env @@ -19,7 +19,7 @@ It does the following: - Configures git upstream remote "lineage" repositories. Usage: - setup-env (-n | --name) [virtual_env_name] (-v | --version) [python_version] + setup-env [--venv-name virtual_env_name] [--python-version python_version] setup-env (-h | --help) Options: @@ -28,8 +28,8 @@ Options: -i | --install-hooks Install hook environments for all environments in the pre-commit config file. -l | --list-versions List available Python versions and select interactively. - -n | --name Specify the name of the virtual environment. - -v | --version Specify the Python version for the virtual environment. + -v | --venv-name Specify the name of the virtual environment. + -p | --python-version Specify the Python version for the virtual environment. END_OF_LINE ) @@ -49,10 +49,10 @@ PYTHON_VERSION="" VENV_NAME="" # Define long options -LONGOPTS="force,help,install-hooks,list-versions,name:,version:" +LONGOPTS="force,help,install-hooks,list-versions,python-version:,venv-name:" # Define short options for getopt -SHORTOPTS="fhiln:v:" +SHORTOPTS="fhilp:v:" # Check if GNU getopt is available if [[ $(getopt --version 2> /dev/null) != *"getopt from util-linux"* ]]; then @@ -94,11 +94,7 @@ while true; do LIST_VERSIONS=1 shift ;; - -n | --name) - VENV_NAME="$2" - shift 2 - ;; - -v | --version) + -p | --python-version) PYTHON_VERSION="$2" shift 2 # Check the Python versions being passed in. @@ -113,6 +109,10 @@ while true; do fi fi ;; + -v | --venv-name) + VENV_NAME="$2" + shift 2 + ;; --) shift break From b1079bdce42be6f50a74fe08746db48505ccac59 Mon Sep 17 00:00:00 2001 From: Michael Saki Date: Wed, 14 Feb 2024 08:08:07 -0600 Subject: [PATCH 115/171] Elaborate on message when checking for GNU getopt --- setup-env | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/setup-env b/setup-env index b272898..c7af9db 100755 --- a/setup-env +++ b/setup-env @@ -58,7 +58,13 @@ SHORTOPTS="fhilp:v:" if [[ $(getopt --version 2> /dev/null) != *"getopt from util-linux"* ]]; then cat << 'END_OF_LINE' - GNU getopt is not detected and is a dependency to run this script. + Please note, this script requires GNU getopt due to its enhanced + functionality and compatibility with certain script features that + are not supported by the POSIX getopt found in some systems, particularly + those with a non-GNU version of getopt. This distinction is crucial + as a system might have a non-GNU version of getopt installed by default, + which could lead to unexpected behavior or script failure. + On the Mac, we recommend installing brew (https://brew.sh/). Then installation is as simple as `brew install gnu-getopt` and adding this to your profile: From b7fdfb08f1cf28fb7c4ff80a8fb4637c5f87bedc Mon Sep 17 00:00:00 2001 From: Michael Saki Date: Wed, 14 Feb 2024 08:10:21 -0600 Subject: [PATCH 116/171] Remove unnecessary nounset flipping logic --- setup-env | 2 -- 1 file changed, 2 deletions(-) diff --git a/setup-env b/setup-env index c7af9db..fa3eb88 100755 --- a/setup-env +++ b/setup-env @@ -166,7 +166,6 @@ END_OF_LINE exit 1 fi -set +o nounset # Determine the virtual environment name if [ -n "$VENV_NAME" ]; then # Use the user-provided environment name @@ -175,7 +174,6 @@ else # Set the environment name to the last part of the working directory. env_name=${PWD##*/} fi -set -o nounset # List Python versions and select one interactively if [ $LIST_VERSIONS -ne 0 ]; then From 2710d5ec8a9a2f847dbb9d108a61d769cc67b9fe Mon Sep 17 00:00:00 2001 From: Michael Saki Date: Wed, 14 Feb 2024 08:20:56 -0600 Subject: [PATCH 117/171] Separate pyenv PATH from GNU getopt PATH --- CONTRIBUTING.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 584ea4a..93ce450 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -69,7 +69,8 @@ adding this to your profile: ```bash export PYENV_ROOT="$HOME/.pyenv" -export PATH="$PYENV_ROOT/bin:$(brew --prefix)/opt/gnu-getopt/bin:$PATH" +export PATH="$PYENV_ROOT/bin:$PATH" +export PATH="$(brew --prefix)/opt/gnu-getopt/bin:$PATH" eval "$(pyenv init --path)" eval "$(pyenv init -)" eval "$(pyenv virtualenv-init -)" From aacc5d0925d70ea3104df86c6df05c0fcce163e5 Mon Sep 17 00:00:00 2001 From: Michael Saki Date: Wed, 21 Feb 2024 10:57:38 -0600 Subject: [PATCH 118/171] Improve verbiage in comments Co-authored-by: Shane Frasier --- CONTRIBUTING.md | 4 ++-- setup-env | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 93ce450..f7573ad 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -46,8 +46,8 @@ There are a few ways to do this, but we prefer to use create and manage a Python virtual environment specific to this project. -The `setup-env` tool in this repository is our recommended method -for automating the entire environment configuration process. The +We recommend using the `setup-env` tool located in this repository, +as it automates the entire environment configuration process. The dependencies required to run this tool are [GNU `getopt`](https://github.com/util-linux/util-linux/blob/master/misc-utils/getopt.1.adoc), [`pyenv`](https://github.com/pyenv/pyenv), and [`pyenv-virtualenv`](https://github.com/pyenv/pyenv-virtualenv). diff --git a/setup-env b/setup-env index fa3eb88..244ef53 100755 --- a/setup-env +++ b/setup-env @@ -27,7 +27,7 @@ Options: -h | --help Show this message. -i | --install-hooks Install hook environments for all environments in the pre-commit config file. - -l | --list-versions List available Python versions and select interactively. + -l | --list-versions List available Python versions and select one interactively. -v | --venv-name Specify the name of the virtual environment. -p | --python-version Specify the Python version for the virtual environment. @@ -63,7 +63,7 @@ if [[ $(getopt --version 2> /dev/null) != *"getopt from util-linux"* ]]; then are not supported by the POSIX getopt found in some systems, particularly those with a non-GNU version of getopt. This distinction is crucial as a system might have a non-GNU version of getopt installed by default, - which could lead to unexpected behavior or script failure. + which could lead to unexpected behavior. On the Mac, we recommend installing brew (https://brew.sh/). Then installation is as simple as `brew install gnu-getopt` and adding this to your From 705a41e7b90ed3aded1608d7de342d5ba987cdfd Mon Sep 17 00:00:00 2001 From: Michael Saki Date: Wed, 21 Feb 2024 11:22:53 -0600 Subject: [PATCH 119/171] Clarify between pyenv and GNU getopt setup Co-authored-by: Nick <50747025+mcdonnnj@users.noreply.github.com> --- CONTRIBUTING.md | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index f7573ad..3f2fec4 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -68,9 +68,12 @@ installation is as simple as `brew install gnu-getopt pyenv pyenv-virtualenv` an adding this to your profile: ```bash +# Enable GNU getopt since it is keg-only +export PATH="$(brew --prefix)/opt/gnu-getopt/bin:$PATH" + +# Setup pyenv export PYENV_ROOT="$HOME/.pyenv" export PATH="$PYENV_ROOT/bin:$PATH" -export PATH="$(brew --prefix)/opt/gnu-getopt/bin:$PATH" eval "$(pyenv init --path)" eval "$(pyenv init -)" eval "$(pyenv virtualenv-init -)" From 1baa08acf5b7b113f7538db489455843632ad71e Mon Sep 17 00:00:00 2001 From: Michael Saki Date: Wed, 21 Feb 2024 12:14:23 -0600 Subject: [PATCH 120/171] Improve comment on conditional check for regex --- setup-env | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/setup-env b/setup-env index 244ef53..2b5eef8 100755 --- a/setup-env +++ b/setup-env @@ -54,7 +54,9 @@ LONGOPTS="force,help,install-hooks,list-versions,python-version:,venv-name:" # Define short options for getopt SHORTOPTS="fhilp:v:" -# Check if GNU getopt is available +# Check for GNU getopt by matching a specific pattern ("getopt from util-linux") +# in its version output. This approach presumes the output format remains stable. +# Be aware that format changes could invalidate this check. if [[ $(getopt --version 2> /dev/null) != *"getopt from util-linux"* ]]; then cat << 'END_OF_LINE' From eb40ca062ce74f92a08be340a44945a7600f7f26 Mon Sep 17 00:00:00 2001 From: Michael Saki Date: Wed, 21 Feb 2024 12:25:49 -0600 Subject: [PATCH 121/171] Add comment explaining that GNU getopt is keg-only --- setup-env | 2 ++ 1 file changed, 2 insertions(+) diff --git a/setup-env b/setup-env index 2b5eef8..2970b02 100755 --- a/setup-env +++ b/setup-env @@ -73,6 +73,8 @@ if [[ $(getopt --version 2> /dev/null) != *"getopt from util-linux"* ]]; then export PATH="$(brew --prefix)/opt/gnu-getopt/bin:$PATH" + This will enable GNU getopt since it is keg-only. + END_OF_LINE exit 1 fi From c1fe1b63103d54159f4bacd250aac5c16248cb25 Mon Sep 17 00:00:00 2001 From: Michael Saki Date: Wed, 21 Feb 2024 16:28:54 -0600 Subject: [PATCH 122/171] Improve comments to better describe `keg-only` terminology Co-authored-by: Shane Frasier --- CONTRIBUTING.md | 3 ++- setup-env | 3 ++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 3f2fec4..56c6bc5 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -68,7 +68,8 @@ installation is as simple as `brew install gnu-getopt pyenv pyenv-virtualenv` an adding this to your profile: ```bash -# Enable GNU getopt since it is keg-only +# GNU getopt must be explicitly added to the path since it is +# keg-only export PATH="$(brew --prefix)/opt/gnu-getopt/bin:$PATH" # Setup pyenv diff --git a/setup-env b/setup-env index 2970b02..34cbdd0 100755 --- a/setup-env +++ b/setup-env @@ -73,7 +73,8 @@ if [[ $(getopt --version 2> /dev/null) != *"getopt from util-linux"* ]]; then export PATH="$(brew --prefix)/opt/gnu-getopt/bin:$PATH" - This will enable GNU getopt since it is keg-only. + GNU getopt must be explicitly added to the PATH since it + is keg-only. END_OF_LINE exit 1 From cdabc4e622519b6a5c5bbc143b5e0560ed331c7b Mon Sep 17 00:00:00 2001 From: Michael Saki Date: Mon, 26 Feb 2024 08:04:19 -0600 Subject: [PATCH 123/171] Change "'setup-env' tool" to "'setup-env' script" Co-authored-by: Nick <50747025+mcdonnnj@users.noreply.github.com> --- CONTRIBUTING.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 56c6bc5..1104897 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -46,7 +46,7 @@ There are a few ways to do this, but we prefer to use create and manage a Python virtual environment specific to this project. -We recommend using the `setup-env` tool located in this repository, +We recommend using the `setup-env` script located in this repository, as it automates the entire environment configuration process. The dependencies required to run this tool are [GNU `getopt`](https://github.com/util-linux/util-linux/blob/master/misc-utils/getopt.1.adoc), From f6435d1a9fa454bead61426a63b8724df4bde9fa Mon Sep 17 00:00:00 2001 From: Michael Saki Date: Mon, 26 Feb 2024 08:07:16 -0600 Subject: [PATCH 124/171] Remove build-in error exit for generic error exit Exit code 2 usually relates to build-ins which does not fit what is failing here. Not sure what alternative would be best so we can just use the generic failure code Co-authored-by: Nick <50747025+mcdonnnj@users.noreply.github.com> --- setup-env | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup-env b/setup-env index 34cbdd0..b9e63b6 100755 --- a/setup-env +++ b/setup-env @@ -83,7 +83,7 @@ fi # Use GNU getopt to parse options if ! PARSED=$(getopt --options $SHORTOPTS --longoptions $LONGOPTS --name "$0" -- "$@"); then echo "Error parsing options" - exit 2 + exit 1 fi eval set -- "$PARSED" From 089176deda869bb87b67c5325791f2f732518f86 Mon Sep 17 00:00:00 2001 From: Michael Saki Date: Mon, 26 Feb 2024 08:08:39 -0600 Subject: [PATCH 125/171] Change verbiage from 'tool' to 'script' for clarity Co-authored-by: Nick <50747025+mcdonnnj@users.noreply.github.com> --- CONTRIBUTING.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 1104897..afa18ff 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -48,7 +48,7 @@ project. We recommend using the `setup-env` script located in this repository, as it automates the entire environment configuration process. The -dependencies required to run this tool are +dependencies required to run this script are [GNU `getopt`](https://github.com/util-linux/util-linux/blob/master/misc-utils/getopt.1.adoc), [`pyenv`](https://github.com/pyenv/pyenv), and [`pyenv-virtualenv`](https://github.com/pyenv/pyenv-virtualenv). If these tools are already configured on your system, you can simply run the From acbdc2247f951f449cdef06790f5a59eef94b0e9 Mon Sep 17 00:00:00 2001 From: Michael Saki Date: Mon, 26 Feb 2024 10:04:42 -0600 Subject: [PATCH 126/171] Check for pyenv earlier in the script Since the new function, python_versions expects pyenv to be installed it makes sense to have it checked earlier in the script. --- setup-env | 72 +++++++++++++++++++++++++++---------------------------- 1 file changed, 36 insertions(+), 36 deletions(-) diff --git a/setup-env b/setup-env index b9e63b6..2c98d07 100755 --- a/setup-env +++ b/setup-env @@ -80,6 +80,42 @@ END_OF_LINE exit 1 fi +# Check to see if pyenv is installed +if [ -z "$(command -v pyenv)" ] || { [ -z "$(command -v pyenv-virtualenv)" ] && [ ! -f "$(pyenv root)/plugins/pyenv-virtualenv/bin/pyenv-virtualenv" ]; }; then + echo "pyenv and pyenv-virtualenv are required." + if [[ "$OSTYPE" == "darwin"* ]]; then + cat << 'END_OF_LINE' + + On the Mac, we recommend installing brew, https://brew.sh/. Then installation + is as simple as `brew install pyenv pyenv-virtualenv` and adding this to your + profile: + + eval "$(pyenv init -)" + eval "$(pyenv virtualenv-init -)" + +END_OF_LINE + + fi + cat << 'END_OF_LINE' + For Linux, Windows Subsystem for Linux (WSL), or on the Mac (if you don't want + to use "brew") you can use https://github.com/pyenv/pyenv-installer to install + the necessary tools. Before running this ensure that you have installed the + prerequisites for your platform according to the pyenv wiki page, + https://github.com/pyenv/pyenv/wiki/common-build-problems. + + On WSL you should treat your platform as whatever Linux distribution you've + chosen to install. + + Once you have installed "pyenv" you will need to add the following lines to + your ".bashrc": + + export PATH="$PATH:$HOME/.pyenv/bin" + eval "$(pyenv init -)" + eval "$(pyenv virtualenv-init -)" +END_OF_LINE + exit 1 +fi + # Use GNU getopt to parse options if ! PARSED=$(getopt --options $SHORTOPTS --longoptions $LONGOPTS --name "$0" -- "$@"); then echo "Error parsing options" @@ -135,42 +171,6 @@ while true; do esac done -# Check to see if pyenv is installed -if [ -z "$(command -v pyenv)" ] || { [ -z "$(command -v pyenv-virtualenv)" ] && [ ! -f "$(pyenv root)/plugins/pyenv-virtualenv/bin/pyenv-virtualenv" ]; }; then - echo "pyenv and pyenv-virtualenv are required." - if [[ "$OSTYPE" == "darwin"* ]]; then - cat << 'END_OF_LINE' - - On the Mac, we recommend installing brew, https://brew.sh/. Then installation - is as simple as `brew install pyenv pyenv-virtualenv` and adding this to your - profile: - - eval "$(pyenv init -)" - eval "$(pyenv virtualenv-init -)" - -END_OF_LINE - - fi - cat << 'END_OF_LINE' - For Linux, Windows Subsystem for Linux (WSL), or on the Mac (if you don't want - to use "brew") you can use https://github.com/pyenv/pyenv-installer to install - the necessary tools. Before running this ensure that you have installed the - prerequisites for your platform according to the pyenv wiki page, - https://github.com/pyenv/pyenv/wiki/common-build-problems. - - On WSL you should treat your platform as whatever Linux distribution you've - chosen to install. - - Once you have installed "pyenv" you will need to add the following lines to - your ".bashrc": - - export PATH="$PATH:$HOME/.pyenv/bin" - eval "$(pyenv init -)" - eval "$(pyenv virtualenv-init -)" -END_OF_LINE - exit 1 -fi - # Determine the virtual environment name if [ -n "$VENV_NAME" ]; then # Use the user-provided environment name From 8b965927ec5bceb4285bf8517c9a91c84d5876b0 Mon Sep 17 00:00:00 2001 From: Michael Saki Date: Mon, 26 Feb 2024 11:02:39 -0600 Subject: [PATCH 127/171] Explain -r and -p in Python version prompt --- setup-env | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/setup-env b/setup-env index 2c98d07..a23fa5a 100755 --- a/setup-env +++ b/setup-env @@ -180,10 +180,12 @@ else env_name=${PWD##*/} fi -# List Python versions and select one interactively +# List Python versions and select one interactively. if [ $LIST_VERSIONS -ne 0 ]; then echo Available Python versions: python_versions + # Read the user's desired Python version. + # -r: treat backslashes as literal, -p: display prompt before input. read -r -p "Enter the desired Python version: " PYTHON_VERSION # Check the Python versions being passed in. if [ -n "${PYTHON_VERSION+x}" ]; then From e9cf4589eb9dd00a11932931dc7910b067bdd405 Mon Sep 17 00:00:00 2001 From: Michael Saki Date: Mon, 26 Feb 2024 11:58:44 -0600 Subject: [PATCH 128/171] Refine exit code to 64 with gnu-getopt note - Set exit code to 64 per sysexits.h for clarity. - Added comment on gnu-getopt handling. Aligns with Unix standards and enhances readability. --- setup-env | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/setup-env b/setup-env index a23fa5a..ae626ad 100755 --- a/setup-env +++ b/setup-env @@ -165,8 +165,9 @@ while true; do break ;; *) + # Unreachable due to gnu-getopt handling all options echo "Programming error" - exit 3 + exit 64 ;; esac done From 96ac70b60b8bf2691fbd5bd782c77b7483cd263c Mon Sep 17 00:00:00 2001 From: Michael Saki Date: Wed, 28 Feb 2024 06:32:35 -0600 Subject: [PATCH 129/171] Rename gnu-getopt tool to GNU getopt formula Co-authored-by: Nick <50747025+mcdonnnj@users.noreply.github.com> --- setup-env | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup-env b/setup-env index ae626ad..e7dfc9a 100755 --- a/setup-env +++ b/setup-env @@ -165,7 +165,7 @@ while true; do break ;; *) - # Unreachable due to gnu-getopt handling all options + # Unreachable due to GNU getopt handling all options echo "Programming error" exit 64 ;; From df6e9510cd8ac6b7ccedd5659a2ef182bfb731cd Mon Sep 17 00:00:00 2001 From: Michael Saki Date: Wed, 28 Feb 2024 07:49:16 -0600 Subject: [PATCH 130/171] Fix whitespace for usage menu Co-authored-by: Shane Frasier --- setup-env | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/setup-env b/setup-env index e7dfc9a..f141b8f 100755 --- a/setup-env +++ b/setup-env @@ -28,8 +28,8 @@ Options: -i | --install-hooks Install hook environments for all environments in the pre-commit config file. -l | --list-versions List available Python versions and select one interactively. - -v | --venv-name Specify the name of the virtual environment. - -p | --python-version Specify the Python version for the virtual environment. + -v | --venv-name Specify the name of the virtual environment. + -p | --python-version Specify the Python version for the virtual environment. END_OF_LINE ) From 60663ca0b0863a12bedad2a1fc05a2d6d33e5974 Mon Sep 17 00:00:00 2001 From: Michael Saki Date: Wed, 28 Feb 2024 11:26:24 -0600 Subject: [PATCH 131/171] Add link to brew terminology Co-authored-by: dav3r --- CONTRIBUTING.md | 2 +- setup-env | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index afa18ff..0022ab4 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -69,7 +69,7 @@ adding this to your profile: ```bash # GNU getopt must be explicitly added to the path since it is -# keg-only +# keg-only (https://docs.brew.sh/FAQ#what-does-keg-only-mean) export PATH="$(brew --prefix)/opt/gnu-getopt/bin:$PATH" # Setup pyenv diff --git a/setup-env b/setup-env index f141b8f..590ab73 100755 --- a/setup-env +++ b/setup-env @@ -74,7 +74,7 @@ if [[ $(getopt --version 2> /dev/null) != *"getopt from util-linux"* ]]; then export PATH="$(brew --prefix)/opt/gnu-getopt/bin:$PATH" GNU getopt must be explicitly added to the PATH since it - is keg-only. + is keg-only (https://docs.brew.sh/FAQ#what-does-keg-only-mean). END_OF_LINE exit 1 From 4b07dc3b558259b7c37d003bc6c23c9190f36bef Mon Sep 17 00:00:00 2001 From: Michael Saki Date: Wed, 28 Feb 2024 15:52:20 -0600 Subject: [PATCH 132/171] Rephrase comment to improve clarity Co-authored-by: Nick <50747025+mcdonnnj@users.noreply.github.com> --- setup-env | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup-env b/setup-env index 590ab73..0415bd7 100755 --- a/setup-env +++ b/setup-env @@ -16,7 +16,7 @@ It does the following: - Configures the activation of the virtual enviroment for the repo directory. - Installs the requirements needed for development. - Installs git pre-commit hooks. - - Configures git upstream remote "lineage" repositories. + - Configures git remotes for upstream "lineage" repositories. Usage: setup-env [--venv-name virtual_env_name] [--python-version python_version] From 0bcc386e356f13b19150446409e374bc560553e4 Mon Sep 17 00:00:00 2001 From: Michael Saki Date: Wed, 28 Feb 2024 15:58:43 -0600 Subject: [PATCH 133/171] Improve comment for clarity Co-authored-by: Nick <50747025+mcdonnnj@users.noreply.github.com> --- setup-env | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup-env b/setup-env index 0415bd7..ec5133e 100755 --- a/setup-env +++ b/setup-env @@ -34,7 +34,7 @@ Options: END_OF_LINE ) -# Display installed python versions +# Display pyenv's installed Python versions python_versions() { pyenv versions --bare --skip-aliases --skip-envs } From 4d4a9e3a9f970333145e529cd37222fb2105a2cb Mon Sep 17 00:00:00 2001 From: Michael Saki Date: Wed, 28 Feb 2024 16:03:47 -0600 Subject: [PATCH 134/171] Improve verbiage in comment --- setup-env | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup-env b/setup-env index ec5133e..e7cb581 100755 --- a/setup-env +++ b/setup-env @@ -217,7 +217,7 @@ fi # Create a new virtual environment for this project # -# If $PYTHON_VERSION is undefined then the system Python will be used. +# If $PYTHON_VERSION is undefined then the global version of Python will be used. # # We can't quote ${PYTHON_VERSION:=} below since if the variable is # undefined then we want nothing to appear; this is the reason for the From 352d38a9917adcedadebc4604dd4dfc1e7d41614 Mon Sep 17 00:00:00 2001 From: Michael Saki Date: Wed, 28 Feb 2024 16:09:15 -0600 Subject: [PATCH 135/171] Change comments for macOS and venv_name --- CONTRIBUTING.md | 4 ++-- setup-env | 8 ++++---- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 0022ab4..2606cc6 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -63,7 +63,7 @@ environment. #### Installing and using GNU `getopt`, `pyenv`, and `pyenv-virtualenv` #### -On the Mac, we recommend installing [brew](https://brew.sh/). Then +On macOS, we recommend installing [brew](https://brew.sh/). Then installation is as simple as `brew install gnu-getopt pyenv pyenv-virtualenv` and adding this to your profile: @@ -80,7 +80,7 @@ eval "$(pyenv init -)" eval "$(pyenv virtualenv-init -)" ``` -For Linux, Windows Subsystem for Linux (WSL), or on the Mac (if you +For Linux, Windows Subsystem for Linux (WSL), or on macOS (if you don't want to use `brew`) you can use [pyenv/pyenv-installer](https://github.com/pyenv/pyenv-installer) to install the necessary tools. Before running this ensure that you have diff --git a/setup-env b/setup-env index e7cb581..a1ef8a4 100755 --- a/setup-env +++ b/setup-env @@ -19,7 +19,7 @@ It does the following: - Configures git remotes for upstream "lineage" repositories. Usage: - setup-env [--venv-name virtual_env_name] [--python-version python_version] + setup-env [--venv-name venv_name] [--python-version python_version] setup-env (-h | --help) Options: @@ -67,7 +67,7 @@ if [[ $(getopt --version 2> /dev/null) != *"getopt from util-linux"* ]]; then as a system might have a non-GNU version of getopt installed by default, which could lead to unexpected behavior. - On the Mac, we recommend installing brew (https://brew.sh/). Then installation + On macOS, we recommend installing brew (https://brew.sh/). Then installation is as simple as `brew install gnu-getopt` and adding this to your profile: @@ -86,7 +86,7 @@ if [ -z "$(command -v pyenv)" ] || { [ -z "$(command -v pyenv-virtualenv)" ] && if [[ "$OSTYPE" == "darwin"* ]]; then cat << 'END_OF_LINE' - On the Mac, we recommend installing brew, https://brew.sh/. Then installation + On macOS, we recommend installing brew, https://brew.sh/. Then installation is as simple as `brew install pyenv pyenv-virtualenv` and adding this to your profile: @@ -97,7 +97,7 @@ END_OF_LINE fi cat << 'END_OF_LINE' - For Linux, Windows Subsystem for Linux (WSL), or on the Mac (if you don't want + For Linux, Windows Subsystem for Linux (WSL), or on mac OS (if you don't want to use "brew") you can use https://github.com/pyenv/pyenv-installer to install the necessary tools. Before running this ensure that you have installed the prerequisites for your platform according to the pyenv wiki page, From 9ad783a39afea29f1c473ecf2eff088212ce2ddf Mon Sep 17 00:00:00 2001 From: Michael Saki Date: Thu, 29 Feb 2024 09:19:01 -0600 Subject: [PATCH 136/171] Improve comments for clarity Co-authored-by: Nick <50747025+mcdonnnj@users.noreply.github.com> --- CONTRIBUTING.md | 2 +- setup-env | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 2606cc6..a7c9886 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -80,7 +80,7 @@ eval "$(pyenv init -)" eval "$(pyenv virtualenv-init -)" ``` -For Linux, Windows Subsystem for Linux (WSL), or on macOS (if you +For Linux, Windows Subsystem for Linux (WSL), or macOS (if you don't want to use `brew`) you can use [pyenv/pyenv-installer](https://github.com/pyenv/pyenv-installer) to install the necessary tools. Before running this ensure that you have diff --git a/setup-env b/setup-env index a1ef8a4..7d42d25 100755 --- a/setup-env +++ b/setup-env @@ -97,7 +97,7 @@ END_OF_LINE fi cat << 'END_OF_LINE' - For Linux, Windows Subsystem for Linux (WSL), or on mac OS (if you don't want + For Linux, Windows Subsystem for Linux (WSL), or macOS (if you don't want to use "brew") you can use https://github.com/pyenv/pyenv-installer to install the necessary tools. Before running this ensure that you have installed the prerequisites for your platform according to the pyenv wiki page, @@ -217,7 +217,7 @@ fi # Create a new virtual environment for this project # -# If $PYTHON_VERSION is undefined then the global version of Python will be used. +# If $PYTHON_VERSION is undefined then the current pyenv Python version will be used. # # We can't quote ${PYTHON_VERSION:=} below since if the variable is # undefined then we want nothing to appear; this is the reason for the From c656ce5a16a60e6ee19223d461c72e86131e91a0 Mon Sep 17 00:00:00 2001 From: Nicholas McDonnell <50747025+mcdonnnj@users.noreply.github.com> Date: Sat, 11 Nov 2023 16:20:14 -0500 Subject: [PATCH 137/171] Use Python and Go versions provided by cisagov/setup-env-github-action --- .github/workflows/build.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 565007a..55d4b4f 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -60,7 +60,7 @@ jobs: - id: setup-python uses: actions/setup-python@v5 with: - python-version: "3.11" + python-version: ${{ steps.setup-env.outputs.python-version }} # We need the Go version and Go cache location for the actions/cache step, # so the Go installation must happen before that. - id: setup-go @@ -69,7 +69,7 @@ jobs: # There is no expectation for actual Go code so we disable caching as # it relies on the existence of a go.sum file. cache: false - go-version: "1.20" + go-version: ${{ steps.setup-env.outputs.go-version }} - id: go-cache name: Lookup Go cache directory run: | From 990ba36903048694156accbe157e9d4f16245a8a Mon Sep 17 00:00:00 2001 From: Nicholas McDonnell <50747025+mcdonnnj@users.noreply.github.com> Date: Thu, 18 Jan 2024 16:25:35 -0500 Subject: [PATCH 138/171] Switch pre-commit hooks for running shfmt This hook bundles the binaries for shfmt with a Python package which removes the need to manually install the tool for the hook to function. --- .pre-commit-config.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index dd3f024..ba3ff3f 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -93,8 +93,8 @@ repos: - id: nixpkgs-fmt # Shell script hooks - - repo: https://github.com/cisagov/pre-commit-shfmt - rev: v0.0.2 + - repo: https://github.com/scop/pre-commit-shfmt + rev: v3.7.0-4 hooks: - id: shfmt args: From 2edecb5bf56dba6cae5a812177ce23e618cb9cd8 Mon Sep 17 00:00:00 2001 From: Nicholas McDonnell <50747025+mcdonnnj@users.noreply.github.com> Date: Thu, 18 Jan 2024 17:43:00 -0500 Subject: [PATCH 139/171] Remove installation of shfmt in the `build` workflow The new pre-commit hook provides `shfmt` binaries so we no longer need to ensure it is installed. --- .github/workflows/build.yml | 5 ----- 1 file changed, 5 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 55d4b4f..c5dc31d 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -127,11 +127,6 @@ jobs: PACKAGE_URL: github.com/securego/gosec/v2/cmd/gosec PACKAGE_VERSION: ${{ steps.setup-env.outputs.gosec-version }} run: go install ${PACKAGE_URL}@${PACKAGE_VERSION} - - name: Install shfmt - env: - PACKAGE_URL: mvdan.cc/sh/v3/cmd/shfmt - PACKAGE_VERSION: ${{ steps.setup-env.outputs.shfmt-version }} - run: go install ${PACKAGE_URL}@${PACKAGE_VERSION} - name: Install staticcheck env: PACKAGE_URL: honnef.co/go/tools/cmd/staticcheck From ec946f8d928db265047d6ddb4158445e33bcb499 Mon Sep 17 00:00:00 2001 From: Nicholas McDonnell <50747025+mcdonnnj@users.noreply.github.com> Date: Thu, 18 Jan 2024 16:56:43 -0500 Subject: [PATCH 140/171] Use long options for shfmt arguments Since shfmt now supports long command line options we should use them as that is our preference. The single quotes for the number of spaces to indent is changed to double quotes to align with our usual quotation style. --- .pre-commit-config.yaml | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index ba3ff3f..65a5c62 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -99,14 +99,14 @@ repos: - id: shfmt args: # Indent by two spaces - - -i - - '2' + - --indent + - "2" # Binary operators may start a line - - -bn + - --binary-next-line # Switch cases are indented - - -ci + - --case-indent # Redirect operators are followed by a space - - -sr + - --space-redirects - repo: https://github.com/detailyang/pre-commit-shell rev: 1.0.5 hooks: From a2d4790039da49a6fc3acd86bd8552c7325b1fb8 Mon Sep 17 00:00:00 2001 From: Nicholas McDonnell <50747025+mcdonnnj@users.noreply.github.com> Date: Thu, 18 Jan 2024 17:34:18 -0500 Subject: [PATCH 141/171] Add additional shfmt options These options are baked into the functionality of the old hook but must be explicitly declared for the new hook. --- .pre-commit-config.yaml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 65a5c62..4f407d9 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -98,6 +98,10 @@ repos: hooks: - id: shfmt args: + # List files that will be formatted + - --list + # Write result to file instead of stdout + - --write # Indent by two spaces - --indent - "2" From f4af1cfeb92ac6bd4bd5298351bdc4e52b304e4a Mon Sep 17 00:00:00 2001 From: Nicholas McDonnell <50747025+mcdonnnj@users.noreply.github.com> Date: Fri, 12 Jan 2024 17:55:34 -0500 Subject: [PATCH 142/171] Add linting with goimports to the pre-commit configuration This will run the Go tool `goimports` against the repository if it contains any Go files. This tool bundles the functionality of `go fmt` with the additional benefit of sorting Go imports much like the isort tool we use for Python code. --- .github/workflows/build.yml | 5 +++++ .pre-commit-config.yaml | 6 +++++- 2 files changed, 10 insertions(+), 1 deletion(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index c5dc31d..f90dfcb 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -122,6 +122,11 @@ jobs: PACKAGE_URL: github.com/go-critic/go-critic/cmd/gocritic PACKAGE_VERSION: ${{ steps.setup-env.outputs.go-critic-version }} run: go install ${PACKAGE_URL}@${PACKAGE_VERSION} + - name: Install goimports + env: + PACKAGE_URL: golang.org/x/tools/cmd/goimports + PACKAGE_VERSION: ${{ steps.setup-env.outputs.goimports-version }} + run: go install ${PACKAGE_URL}@${PACKAGE_VERSION} - name: Install gosec env: PACKAGE_URL: github.com/securego/gosec/v2/cmd/gosec diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 4f407d9..f5744ed 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -85,7 +85,11 @@ repos: - id: go-vet-repo-mod # GoSec - id: go-sec-repo-mod - + # goimports + - id: go-imports-repo + args: + # Write changes to files + - -w # Nix hooks - repo: https://github.com/nix-community/nixpkgs-fmt rev: v1.3.0 From 9c91c6a5897ae720b9d43359a2f8f7951b48d4ac Mon Sep 17 00:00:00 2001 From: Michael Saki Date: Mon, 22 Jan 2024 12:31:43 -0600 Subject: [PATCH 143/171] Add ATX Header Support for terraform-docs This is a temporary fix until @mcdonnnj has his PR approved and merged into the terraform-docs repo. This fix will perform a shallow clone of his forked branch, build the binary, and install it. --- .github/workflows/build.yml | 22 +++++++++++++++++----- 1 file changed, 17 insertions(+), 5 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index f90dfcb..132b9a3 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -21,9 +21,12 @@ defaults: env: AWS_DEFAULT_REGION: us-east-1 + BRANCH_NAME: improvement/support_atx_closed_markdown_headers CURL_CACHE_DIR: ~/.cache/curl + DEPTH: 1 PIP_CACHE_DIR: ~/.cache/pip PRE_COMMIT_CACHE_DIR: ~/.cache/pre-commit + REPO_URL: https://github.com/mcdonnnj/terraform-docs.git RUN_TMATE: ${{ secrets.RUN_TMATE }} jobs: @@ -137,11 +140,20 @@ jobs: PACKAGE_URL: honnef.co/go/tools/cmd/staticcheck PACKAGE_VERSION: ${{ steps.setup-env.outputs.staticcheck-version }} run: go install ${PACKAGE_URL}@${PACKAGE_VERSION} - - name: Install Terraform-docs - env: - PACKAGE_URL: github.com/terraform-docs/terraform-docs - PACKAGE_VERSION: ${{ steps.setup-env.outputs.terraform-docs-version }} - run: go install ${PACKAGE_URL}@${PACKAGE_VERSION} + # We are temporarily using @mcdonnnj's forked branch of terraform-docs + # until his PR: https://github.com/terraform-docs/terraform-docs/pull/745 + # is approved. This temporary fix will allow for ATX Header Support when + # terraform-docs is ran during lint. + - name: Clone ATX Headers branch from Terraform-docs Fork + run: | + git clone --branch $BRANCH_NAME --single-branch \ + --depth=$DEPTH $REPO_URL /tmp/terraform-docs + - name: Build and Install Terraform-docs Binary + run: | + cd /tmp/terraform-docs + GOBIN=$(go env GOPATH)/bin + go build -o $GOBIN/terraform-docs + echo "$GOBIN" >> $GITHUB_PATH - name: Install dependencies run: | python -m pip install --upgrade pip setuptools wheel From 70d3b9258b2a37ae8cf2070437a656ae805bf542 Mon Sep 17 00:00:00 2001 From: Michael Saki Date: Mon, 22 Jan 2024 14:03:00 -0600 Subject: [PATCH 144/171] Add prepended names to variables to describe their function Co-authored-by: Nick <50747025+mcdonnnj@users.noreply.github.com> --- .github/workflows/build.yml | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 132b9a3..487705b 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -21,13 +21,13 @@ defaults: env: AWS_DEFAULT_REGION: us-east-1 - BRANCH_NAME: improvement/support_atx_closed_markdown_headers CURL_CACHE_DIR: ~/.cache/curl - DEPTH: 1 PIP_CACHE_DIR: ~/.cache/pip PRE_COMMIT_CACHE_DIR: ~/.cache/pre-commit - REPO_URL: https://github.com/mcdonnnj/terraform-docs.git RUN_TMATE: ${{ secrets.RUN_TMATE }} + TERRAFORM_DOCS_REPO_BRANCH_NAME: improvement/support_atx_closed_markdown_headers + TERRAFORM_DOCS_REPO_DEPTH: 1 + TERRAFORM_DOCS_REPO_URL: https://github.com/mcdonnnj/terraform-docs.git jobs: diagnostics: @@ -146,8 +146,10 @@ jobs: # terraform-docs is ran during lint. - name: Clone ATX Headers branch from Terraform-docs Fork run: | - git clone --branch $BRANCH_NAME --single-branch \ - --depth=$DEPTH $REPO_URL /tmp/terraform-docs + git clone --single-branch \ + --branch $TERRAFORM_DOCS_REPO_BRANCH_NAME \ + --depth $TERRAFORM_DOCS_REPO_DEPTH \ + $TERRAFORM_DOCS_REPO_URL /tmp/terraform-docs - name: Build and Install Terraform-docs Binary run: | cd /tmp/terraform-docs From 19dea0aae665a2661bd12dc5c76bc166cca67fc2 Mon Sep 17 00:00:00 2001 From: Michael Saki Date: Mon, 22 Jan 2024 14:05:01 -0600 Subject: [PATCH 145/171] Remove unnecessary capitalizations and fix grammar Co-authored-by: Shane Frasier --- .github/workflows/build.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 487705b..80d6226 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -142,15 +142,15 @@ jobs: run: go install ${PACKAGE_URL}@${PACKAGE_VERSION} # We are temporarily using @mcdonnnj's forked branch of terraform-docs # until his PR: https://github.com/terraform-docs/terraform-docs/pull/745 - # is approved. This temporary fix will allow for ATX Header Support when - # terraform-docs is ran during lint. - - name: Clone ATX Headers branch from Terraform-docs Fork + # is approved. This temporary fix will allow for ATX header support when + # terraform-docs is run during linting. + - name: Clone ATX headers branch from terraform-docs fork run: | git clone --single-branch \ --branch $TERRAFORM_DOCS_REPO_BRANCH_NAME \ --depth $TERRAFORM_DOCS_REPO_DEPTH \ $TERRAFORM_DOCS_REPO_URL /tmp/terraform-docs - - name: Build and Install Terraform-docs Binary + - name: Build and install terraform-docs binary run: | cd /tmp/terraform-docs GOBIN=$(go env GOPATH)/bin From a3f6c30d725a9c548683881d0b7ec193abeafded Mon Sep 17 00:00:00 2001 From: Michael Saki Date: Mon, 22 Jan 2024 14:14:27 -0600 Subject: [PATCH 146/171] Simplify steps in the build/install portion of workflow PATH is handled by `setup-go` so we can refactor the code setting it. Also we are taking advantage of the -C switch to handle building from the cloned repository. Co-authored-by: Nick <50747025+mcdonnnj@users.noreply.github.com> --- .github/workflows/build.yml | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 80d6226..508d5ed 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -152,10 +152,9 @@ jobs: $TERRAFORM_DOCS_REPO_URL /tmp/terraform-docs - name: Build and install terraform-docs binary run: | - cd /tmp/terraform-docs - GOBIN=$(go env GOPATH)/bin - go build -o $GOBIN/terraform-docs - echo "$GOBIN" >> $GITHUB_PATH + go build \ + -C /tmp/terraform-docs \ + -o $(go env GOPATH)/bin/terraform-docs - name: Install dependencies run: | python -m pip install --upgrade pip setuptools wheel From 84a82019123adcc1300dc799d8d553f86febcf2f Mon Sep 17 00:00:00 2001 From: Michael Saki Date: Tue, 23 Jan 2024 08:04:07 -0600 Subject: [PATCH 147/171] Add TODO label --- .github/workflows/build.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 508d5ed..945b73e 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -64,7 +64,7 @@ jobs: uses: actions/setup-python@v5 with: python-version: ${{ steps.setup-env.outputs.python-version }} - # We need the Go version and Go cache location for the actions/cache step, + # TODO: We need the Go version and Go cache location for the actions/cache step, # so the Go installation must happen before that. - id: setup-go uses: actions/setup-go@v5 From 74ddeecb9f590474168b6f138a08da417a1765ec Mon Sep 17 00:00:00 2001 From: Michael Saki Date: Tue, 23 Jan 2024 08:24:12 -0600 Subject: [PATCH 148/171] Move TODO and add link to the issue `TODO` was placed on the wrong comment block. Also I am adding a link to the issue for the TODO. --- .github/workflows/build.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 945b73e..de4d46f 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -64,7 +64,7 @@ jobs: uses: actions/setup-python@v5 with: python-version: ${{ steps.setup-env.outputs.python-version }} - # TODO: We need the Go version and Go cache location for the actions/cache step, + # We need the Go version and Go cache location for the actions/cache step, # so the Go installation must happen before that. - id: setup-go uses: actions/setup-go@v5 @@ -140,6 +140,7 @@ jobs: PACKAGE_URL: honnef.co/go/tools/cmd/staticcheck PACKAGE_VERSION: ${{ steps.setup-env.outputs.staticcheck-version }} run: go install ${PACKAGE_URL}@${PACKAGE_VERSION} + # TODO: https://github.com/cisagov/skeleton-generic/issues/165 # We are temporarily using @mcdonnnj's forked branch of terraform-docs # until his PR: https://github.com/terraform-docs/terraform-docs/pull/745 # is approved. This temporary fix will allow for ATX header support when From 4aa54ba7382aa2cfcee499d022c779925233d0e5 Mon Sep 17 00:00:00 2001 From: Michael Saki Date: Tue, 23 Jan 2024 08:41:41 -0600 Subject: [PATCH 149/171] Alphabetize switches Co-authored-by: dav3r --- .github/workflows/build.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index de4d46f..389e2f1 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -147,9 +147,10 @@ jobs: # terraform-docs is run during linting. - name: Clone ATX headers branch from terraform-docs fork run: | - git clone --single-branch \ + git clone \ --branch $TERRAFORM_DOCS_REPO_BRANCH_NAME \ --depth $TERRAFORM_DOCS_REPO_DEPTH \ + --single-branch \ $TERRAFORM_DOCS_REPO_URL /tmp/terraform-docs - name: Build and install terraform-docs binary run: | From 19789e5f031bc4983c235c3e43dfe1c5b0cf28de Mon Sep 17 00:00:00 2001 From: Nicholas McDonnell <50747025+mcdonnnj@users.noreply.github.com> Date: Tue, 27 Feb 2024 17:17:39 -0500 Subject: [PATCH 150/171] Switch pre-commit hooks for running shellcheck This hook bundles the binaries for shellcheck with a Python package which removes the need to ensure the tool is installed for the hook to function. It also ties the version of shellcheck used to the hook which will help guarantee consistency. --- .pre-commit-config.yaml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index f5744ed..0ac4d82 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -115,10 +115,10 @@ repos: - --case-indent # Redirect operators are followed by a space - --space-redirects - - repo: https://github.com/detailyang/pre-commit-shell - rev: 1.0.5 + - repo: https://github.com/shellcheck-py/shellcheck-py + rev: v0.9.0.6 hooks: - - id: shell-lint + - id: shellcheck # Python hooks # Run bandit on the "tests" tree with a configuration From 7d12ea79e2ff73d8a76a85a92bb7891354957a36 Mon Sep 17 00:00:00 2001 From: Jeremy Frasier Date: Wed, 6 Mar 2024 10:04:24 -0500 Subject: [PATCH 151/171] Reorder YAML keys to placate ansible-lint --- src/aws.yml | 4 ++-- src/base.yml | 4 ++-- src/example.yml | 4 ++-- src/python.yml | 4 ++-- src/upgrade.yml | 4 ++-- 5 files changed, 10 insertions(+), 10 deletions(-) diff --git a/src/aws.yml b/src/aws.yml index f05d07f..7b7604b 100644 --- a/src/aws.yml +++ b/src/aws.yml @@ -1,6 +1,6 @@ --- -- hosts: all - name: AWS-specific roles +- name: AWS-specific roles + hosts: all become: true become_method: ansible.builtin.sudo tasks: diff --git a/src/base.yml b/src/base.yml index 5064841..fed4116 100644 --- a/src/base.yml +++ b/src/base.yml @@ -1,6 +1,6 @@ --- -- hosts: all - name: Setup base image +- name: Setup base image + hosts: all become: true become_method: ansible.builtin.sudo tasks: diff --git a/src/example.yml b/src/example.yml index ca2e6f5..7ada732 100644 --- a/src/example.yml +++ b/src/example.yml @@ -1,6 +1,6 @@ --- -- hosts: all - name: Project-specific roles +- name: Project-specific roles + hosts: all become: true become_method: ansible.builtin.sudo tasks: diff --git a/src/python.yml b/src/python.yml index c1d369f..6a645bd 100644 --- a/src/python.yml +++ b/src/python.yml @@ -1,6 +1,6 @@ --- -- hosts: all - name: Install pip3/python3 and remove pip2/python2 +- name: Install pip3/python3 and remove pip2/python2 + hosts: all become: true become_method: ansible.builtin.sudo tasks: diff --git a/src/upgrade.yml b/src/upgrade.yml index e8dfe7d..c065512 100644 --- a/src/upgrade.yml +++ b/src/upgrade.yml @@ -1,6 +1,6 @@ --- -- hosts: all - name: Upgrade base image +- name: Upgrade base image + hosts: all become: true become_method: ansible.builtin.sudo tasks: From c5841179f1d8cc4bc309791f3d04fe6207bcc2b2 Mon Sep 17 00:00:00 2001 From: Jeremy Frasier Date: Wed, 6 Mar 2024 10:06:57 -0500 Subject: [PATCH 152/171] Use Python version output from cisagov/setup-env-github-action --- .github/workflows/build.yml | 4 ++-- .github/workflows/prerelease.yml | 2 +- .github/workflows/release.yml | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 389e2f1..39fe8d2 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -190,7 +190,7 @@ jobs: - id: setup-python uses: actions/setup-python@v4 with: - python-version: "3.10" + python-version: ${{ steps.setup-env.outputs.python-version }} - uses: actions/cache@v3 env: BASE_CACHE_KEY: "${{ github.job }}-${{ runner.os }}-\ @@ -250,7 +250,7 @@ jobs: - id: setup-python uses: actions/setup-python@v4 with: - python-version: "3.10" + python-version: ${{ steps.setup-env.outputs.python-version }} - uses: actions/cache@v3 env: BASE_CACHE_KEY: "${{ github.job }}-${{ runner.os }}-\ diff --git a/.github/workflows/prerelease.yml b/.github/workflows/prerelease.yml index 03eb8d3..424b8af 100644 --- a/.github/workflows/prerelease.yml +++ b/.github/workflows/prerelease.yml @@ -45,7 +45,7 @@ jobs: - id: setup-python uses: actions/setup-python@v4 with: - python-version: "3.10" + python-version: ${{ steps.setup-env.outputs.python-version }} - uses: actions/cache@v3 env: BASE_CACHE_KEY: "${{ github.job }}-${{ runner.os }}-\ diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 2ea1223..e2773a2 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -52,7 +52,7 @@ jobs: - id: setup-python uses: actions/setup-python@v4 with: - python-version: "3.10" + python-version: ${{ steps.setup-env.outputs.python-version }} - uses: actions/cache@v3 env: BASE_CACHE_KEY: "${{ github.job }}-${{ runner.os }}-\ From 656c5efc41b3308b4dd718d3d271cc1242a0ae7f Mon Sep 17 00:00:00 2001 From: Jeremy Frasier Date: Wed, 6 Mar 2024 10:15:03 -0500 Subject: [PATCH 153/171] Add whitespace change for Black linter/formatter --- tests/conftest.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/conftest.py b/tests/conftest.py index 4189122..cb1cac7 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -2,6 +2,7 @@ https://docs.pytest.org/en/latest/writing_plugins.html#conftest-py-plugins """ + # Third-Party Libraries import pytest From 907f68a2cc1b57593d142469957b78a4a74648d5 Mon Sep 17 00:00:00 2001 From: Jeremy Frasier Date: Wed, 14 Jun 2023 09:21:19 -0400 Subject: [PATCH 154/171] Update base AMI to Debian Bookworm Debian Bookworm was officially released on June 10,2023. --- src/packer.pkr.hcl | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/src/packer.pkr.hcl b/src/packer.pkr.hcl index cce1c94..849c29d 100644 --- a/src/packer.pkr.hcl +++ b/src/packer.pkr.hcl @@ -63,9 +63,9 @@ variable "skip_create_ami" { type = bool } -data "amazon-ami" "debian_bullseye" { +data "amazon-ami" "debian_bookworm" { filters = { - name = "debian-11-amd64-*" + name = "debian-12-amd64-*" root-device-type = "ebs" virtualization-type = "hvm" } @@ -93,7 +93,7 @@ source "amazon-ebs" "example" { region = var.build_region region_kms_key_ids = var.region_kms_keys skip_create_ami = var.skip_create_ami - source_ami = data.amazon-ami.debian_bullseye.id + source_ami = data.amazon-ami.debian_bookworm.id ssh_username = "admin" subnet_filter { filters = { @@ -102,9 +102,9 @@ source "amazon-ebs" "example" { } tags = { Application = "Example" - Base_AMI_Name = data.amazon-ami.debian_bullseye.name + Base_AMI_Name = data.amazon-ami.debian_bookworm.name GitHub_Release_URL = var.release_url - OS_Version = "Debian Bullseye" + OS_Version = "Debian Bookworm" Pre_Release = var.is_prerelease Release = var.release_tag Team = "VM Fusion - Development" From 39e3d3102531e932c02d45d68b2b8dc8895f343e Mon Sep 17 00:00:00 2001 From: Jeremy Frasier Date: Fri, 12 Jan 2024 15:07:24 -0500 Subject: [PATCH 155/171] Add a playbook to install prerequisites for fixing a broken Netplan configuration See these issues for more information: - cisagov/skeleton-packer#300 - canonical/cloud-init#4764 --- src/fix-netplan-configuration.yml | 18 ++++++++++++++++++ src/playbook.yml | 3 +++ 2 files changed, 21 insertions(+) create mode 100644 src/fix-netplan-configuration.yml diff --git a/src/fix-netplan-configuration.yml b/src/fix-netplan-configuration.yml new file mode 100644 index 0000000..1461335 --- /dev/null +++ b/src/fix-netplan-configuration.yml @@ -0,0 +1,18 @@ +--- +# Install the prerequisites for the user script that fixes the Netplan +# configuration generated by cloud-init from the instance metadata. +# +# See these issues for more details: +# - cisagov/skeleton-packer#300 +# - canonical/cloud-init#4764 +- hosts: all + name: >- + Install prerequisites for the script that fixes the Netplan + configuration generated by cloud-init from the instance metadata + become: true + become_method: ansible.builtin.sudo + tasks: + - name: Install python3-pyyaml + ansible.builtin.package: + name: + - python3-yaml diff --git a/src/playbook.yml b/src/playbook.yml index b1e5b7c..5197ff0 100644 --- a/src/playbook.yml +++ b/src/playbook.yml @@ -2,6 +2,9 @@ - name: Import base image playbook ansible.builtin.import_playbook: base.yml +- name: Import playbook to fix Netplan configuration + ansible.builtin.import_playbook: fix-netplan-configuration.yml + - name: Import AWS playbook ansible.builtin.import_playbook: aws.yml From 90277d71693eee1cd6f51a4eeb792007418ba908 Mon Sep 17 00:00:00 2001 From: Jeremy Frasier Date: Fri, 12 Jan 2024 16:17:55 -0500 Subject: [PATCH 156/171] Rename playbook to better describe what it does Co-authored-by: Nick <50747025+mcdonnnj@users.noreply.github.com> Co-authored-by: dav3r --- ...install-prerequisites-for-netplan-configuration-fix.yml} | 0 src/playbook.yml | 6 ++++-- 2 files changed, 4 insertions(+), 2 deletions(-) rename src/{fix-netplan-configuration.yml => install-prerequisites-for-netplan-configuration-fix.yml} (100%) diff --git a/src/fix-netplan-configuration.yml b/src/install-prerequisites-for-netplan-configuration-fix.yml similarity index 100% rename from src/fix-netplan-configuration.yml rename to src/install-prerequisites-for-netplan-configuration-fix.yml diff --git a/src/playbook.yml b/src/playbook.yml index 5197ff0..4f0d18f 100644 --- a/src/playbook.yml +++ b/src/playbook.yml @@ -2,8 +2,10 @@ - name: Import base image playbook ansible.builtin.import_playbook: base.yml -- name: Import playbook to fix Netplan configuration - ansible.builtin.import_playbook: fix-netplan-configuration.yml +- name: >- + Import playbook to install prerequisites for the Netplan + configuration fix + ansible.builtin.import_playbook: install-prerequisites-for-netplan-configuration-fix.yml - name: Import AWS playbook ansible.builtin.import_playbook: aws.yml From cf34d76fe28b7df6b5b12ae630854d71d6039786 Mon Sep 17 00:00:00 2001 From: Jeremy Frasier Date: Sat, 13 Jan 2024 09:24:31 -0500 Subject: [PATCH 157/171] Add comments mentioning cisagov/skeleton-packer#301 These comments remind us to remove the code to fix up the Netplan configuration generated by cloud-init when and if that becomes possible. --- src/install-prerequisites-for-netplan-configuration-fix.yml | 3 +++ src/playbook.yml | 2 ++ 2 files changed, 5 insertions(+) diff --git a/src/install-prerequisites-for-netplan-configuration-fix.yml b/src/install-prerequisites-for-netplan-configuration-fix.yml index 1461335..667e6f1 100644 --- a/src/install-prerequisites-for-netplan-configuration-fix.yml +++ b/src/install-prerequisites-for-netplan-configuration-fix.yml @@ -5,6 +5,9 @@ # See these issues for more details: # - cisagov/skeleton-packer#300 # - canonical/cloud-init#4764 +# +# TODO: Remove this playbook when and if that becomes possible. See +# cisagov/skeleton-packer#301 for more details. - hosts: all name: >- Install prerequisites for the script that fixes the Netplan diff --git a/src/playbook.yml b/src/playbook.yml index 4f0d18f..703a7de 100644 --- a/src/playbook.yml +++ b/src/playbook.yml @@ -2,6 +2,8 @@ - name: Import base image playbook ansible.builtin.import_playbook: base.yml +# TODO: Remove this when and if that becomes possible. See +# cisagov/skeleton-packer#301. - name: >- Import playbook to install prerequisites for the Netplan configuration fix From 341e7aff3880556f1bfb227767dcd7e31c019b18 Mon Sep 17 00:00:00 2001 From: Nicholas McDonnell <50747025+mcdonnnj@users.noreply.github.com> Date: Thu, 7 Mar 2024 14:39:46 -0500 Subject: [PATCH 158/171] Finalize version from 1.0.0-rc.2 to 1.0.0 --- src/version.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/version.txt b/src/version.txt index 1acb7e3..5becc17 100644 --- a/src/version.txt +++ b/src/version.txt @@ -1 +1 @@ -__version__ = "1.0.0-rc.2" +__version__ = "1.0.0" From 41cc1f96822ecf0cf708eace62ec3b314e76f72e Mon Sep 17 00:00:00 2001 From: Nicholas McDonnell <50747025+mcdonnnj@users.noreply.github.com> Date: Thu, 7 Mar 2024 16:18:00 -0500 Subject: [PATCH 159/171] Reorder YAML keys to placate ansible-lint --- src/install-prerequisites-for-netplan-configuration-fix.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/install-prerequisites-for-netplan-configuration-fix.yml b/src/install-prerequisites-for-netplan-configuration-fix.yml index 667e6f1..1c86cbc 100644 --- a/src/install-prerequisites-for-netplan-configuration-fix.yml +++ b/src/install-prerequisites-for-netplan-configuration-fix.yml @@ -8,10 +8,10 @@ # # TODO: Remove this playbook when and if that becomes possible. See # cisagov/skeleton-packer#301 for more details. -- hosts: all - name: >- +- name: >- Install prerequisites for the script that fixes the Netplan configuration generated by cloud-init from the instance metadata + hosts: all become: true become_method: ansible.builtin.sudo tasks: From c0465a3c9687b9e584c7ac2aa82ac55425c32ded Mon Sep 17 00:00:00 2001 From: Nicholas McDonnell <50747025+mcdonnnj@users.noreply.github.com> Date: Fri, 8 Mar 2024 01:51:40 -0500 Subject: [PATCH 160/171] Bump hashicorp/setup-terraform from 2 to 3 This covers the prerelease.yml and release.yml workflows. --- .github/workflows/prerelease.yml | 2 +- .github/workflows/release.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/prerelease.yml b/.github/workflows/prerelease.yml index 424b8af..9379e20 100644 --- a/.github/workflows/prerelease.yml +++ b/.github/workflows/prerelease.yml @@ -75,7 +75,7 @@ jobs: ${{ env.CURL_CACHE_DIR }}/"${PACKER_ZIP}" sudo mv /usr/local/bin/packer /usr/local/bin/packer-default sudo ln -s /opt/packer/packer /usr/local/bin/packer - - uses: hashicorp/setup-terraform@v2 + - uses: hashicorp/setup-terraform@v3 with: terraform_version: ${{ steps.setup-env.outputs.terraform-version }} - name: Install dependencies diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index e2773a2..a41f122 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -82,7 +82,7 @@ jobs: ${{ env.CURL_CACHE_DIR }}/"${PACKER_ZIP}" sudo mv /usr/local/bin/packer /usr/local/bin/packer-default sudo ln -s /opt/packer/packer /usr/local/bin/packer - - uses: hashicorp/setup-terraform@v2 + - uses: hashicorp/setup-terraform@v3 with: terraform_version: ${{ steps.setup-env.outputs.terraform-version }} - name: Install dependencies From 47f746495d6124208a3a86cd97c7ca582874cbed Mon Sep 17 00:00:00 2001 From: Nicholas McDonnell <50747025+mcdonnnj@users.noreply.github.com> Date: Fri, 8 Mar 2024 01:53:07 -0500 Subject: [PATCH 161/171] Bump actions/setup-python from 4 to 5 This covers all missed bumps in the build.yml, prerelease.yml, and release.yml workflows. --- .github/workflows/build.yml | 4 ++-- .github/workflows/prerelease.yml | 2 +- .github/workflows/release.yml | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 821d432..2cc2474 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -188,7 +188,7 @@ jobs: uses: cisagov/setup-env-github-action@develop - uses: actions/checkout@v4 - id: setup-python - uses: actions/setup-python@v4 + uses: actions/setup-python@v5 with: python-version: ${{ steps.setup-env.outputs.python-version }} - uses: actions/cache@v3 @@ -248,7 +248,7 @@ jobs: uses: cisagov/setup-env-github-action@develop - uses: actions/checkout@v4 - id: setup-python - uses: actions/setup-python@v4 + uses: actions/setup-python@v5 with: python-version: ${{ steps.setup-env.outputs.python-version }} - uses: actions/cache@v3 diff --git a/.github/workflows/prerelease.yml b/.github/workflows/prerelease.yml index 9379e20..1277d6f 100644 --- a/.github/workflows/prerelease.yml +++ b/.github/workflows/prerelease.yml @@ -43,7 +43,7 @@ jobs: uses: cisagov/setup-env-github-action@develop - uses: actions/checkout@v4 - id: setup-python - uses: actions/setup-python@v4 + uses: actions/setup-python@v5 with: python-version: ${{ steps.setup-env.outputs.python-version }} - uses: actions/cache@v3 diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index a41f122..e666902 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -50,7 +50,7 @@ jobs: uses: cisagov/setup-env-github-action@develop - uses: actions/checkout@v4 - id: setup-python - uses: actions/setup-python@v4 + uses: actions/setup-python@v5 with: python-version: ${{ steps.setup-env.outputs.python-version }} - uses: actions/cache@v3 From b9dd3c44ebc5152c521e2a21cda2c31d793a1fc8 Mon Sep 17 00:00:00 2001 From: Nicholas McDonnell <50747025+mcdonnnj@users.noreply.github.com> Date: Fri, 8 Mar 2024 01:56:28 -0500 Subject: [PATCH 162/171] Bump crazy-max/ghaction-github-status from 3 to 4 This covers the prerelease.yml and release.yml workflows. --- .github/workflows/prerelease.yml | 2 +- .github/workflows/release.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/prerelease.yml b/.github/workflows/prerelease.yml index 1277d6f..f322580 100644 --- a/.github/workflows/prerelease.yml +++ b/.github/workflows/prerelease.yml @@ -25,7 +25,7 @@ jobs: egress-policy: audit - id: github-status name: Check GitHub status - uses: crazy-max/ghaction-github-status@v3 + uses: crazy-max/ghaction-github-status@v4 - id: dump-context name: Dump context uses: crazy-max/ghaction-dump-context@v2 diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index e666902..b50f9b6 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -32,7 +32,7 @@ jobs: egress-policy: audit - id: github-status name: Check GitHub status - uses: crazy-max/ghaction-github-status@v3 + uses: crazy-max/ghaction-github-status@v4 - id: dump-context name: Dump context uses: crazy-max/ghaction-dump-context@v2 From e603f655f358f33c66d96e69f382e850f47a15a8 Mon Sep 17 00:00:00 2001 From: Jeremy Frasier Date: Fri, 8 Mar 2024 15:20:48 -0500 Subject: [PATCH 163/171] Pin ansible-core to earlier than 2.16.3 ansible-core 2.16.3 and later suffer from the bug discussed in ansible/ansible#82702, which breaks any symlinked files in vars, tasks, etc. for any Ansible role installed via ansible-galaxy. --- requirements.txt | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/requirements.txt b/requirements.txt index c0b40d8..0a43002 100644 --- a/requirements.txt +++ b/requirements.txt @@ -13,6 +13,10 @@ # often breaking changes across major versions. This is the reason # for the upper bound. ansible>=8,<10 +# ansible-core 2.16.3 and later suffer from the bug discussed in +# ansible/ansible#82702, which breaks any symlinked files in vars, +# tasks, etc. for any Ansible role installed via ansible-galaxy. +ansible-core<2.16.3 boto3 docopt semver From 21c2ca4ab3cf1b8b4ba3552027265d533007d415 Mon Sep 17 00:00:00 2001 From: Jeremy Frasier Date: Fri, 8 Mar 2024 16:26:08 -0500 Subject: [PATCH 164/171] Include a reference to cisagov/skeleton-packer#312 This issue reminds us to remove the pin against ansible-core when that becomes possible. --- requirements.txt | 3 +++ 1 file changed, 3 insertions(+) diff --git a/requirements.txt b/requirements.txt index 0a43002..0b23b07 100644 --- a/requirements.txt +++ b/requirements.txt @@ -13,6 +13,9 @@ # often breaking changes across major versions. This is the reason # for the upper bound. ansible>=8,<10 +# TODO: Remove this pin when possible. See +# cisagov/skeleton-packer##312 for more details. +# # ansible-core 2.16.3 and later suffer from the bug discussed in # ansible/ansible#82702, which breaks any symlinked files in vars, # tasks, etc. for any Ansible role installed via ansible-galaxy. From 842e16512f6581128ccf8cc62b5d641186a377b7 Mon Sep 17 00:00:00 2001 From: Jeremy Frasier Date: Sun, 10 Mar 2024 12:20:32 -0400 Subject: [PATCH 165/171] Mention the related issue cisagov/skeleton-ansible-role#178 --- requirements.txt | 2 ++ 1 file changed, 2 insertions(+) diff --git a/requirements.txt b/requirements.txt index 0b23b07..699e9c6 100644 --- a/requirements.txt +++ b/requirements.txt @@ -19,6 +19,8 @@ ansible>=8,<10 # ansible-core 2.16.3 and later suffer from the bug discussed in # ansible/ansible#82702, which breaks any symlinked files in vars, # tasks, etc. for any Ansible role installed via ansible-galaxy. +# +# See also cisagov/skeleton-ansible-role#178. ansible-core<2.16.3 boto3 docopt From aaa538dfa9bb66c7e465aee519d9da4fee34b933 Mon Sep 17 00:00:00 2001 From: Shane Frasier Date: Sun, 10 Mar 2024 16:03:02 -0400 Subject: [PATCH 166/171] Remove duplicate octothorpe Co-authored-by: Nick <50747025+mcdonnnj@users.noreply.github.com> --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 699e9c6..272a7a3 100644 --- a/requirements.txt +++ b/requirements.txt @@ -14,7 +14,7 @@ # for the upper bound. ansible>=8,<10 # TODO: Remove this pin when possible. See -# cisagov/skeleton-packer##312 for more details. +# cisagov/skeleton-packer#312 for more details. # # ansible-core 2.16.3 and later suffer from the bug discussed in # ansible/ansible#82702, which breaks any symlinked files in vars, From f5282301b6baf9d67eb7b514f2fde61611605442 Mon Sep 17 00:00:00 2001 From: Jeremy Frasier Date: Wed, 13 Mar 2024 15:54:35 -0400 Subject: [PATCH 167/171] Reorder blocks to match skeleton Co-authored-by: Nick <50747025+mcdonnnj@users.noreply.github.com> --- .github/CODEOWNERS | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index 12308bb..040c5ea 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -5,16 +5,16 @@ # requested for review when someone opens a pull request. * @dav3r @jasonodoom @jsf9k @mcdonnnj +# These folks own any files in the .github directory at the root of +# the repository and any of its subdirectories. +/.github/ @dav3r @felddy @jasonodoom @jsf9k @mcdonnnj + # Let jsf9k own the sometimes-touchy AWS and Python playbooks, as well # as the packer.pkr.hcl file. /src/aws.yml @jsf9k /src/packer.pkr.hcl @jsf9k /src/python.yml @jsf9k -# These folks own any files in the .github directory at the root of -# the repository and any of its subdirectories. -/.github/ @dav3r @felddy @jasonodoom @jsf9k @mcdonnnj - # These folks own all linting configuration files. /.ansible-lint @dav3r @felddy @jasonodoom @jsf9k @mcdonnnj /.bandit.yml @dav3r @felddy @jasonodoom @jsf9k @mcdonnnj From f045c905381908ea528ae66884812382ec03a06c Mon Sep 17 00:00:00 2001 From: Jeremy Frasier Date: Wed, 13 Mar 2024 15:55:19 -0400 Subject: [PATCH 168/171] Reorder YAML keys to placate ansible-lint --- src/cloud_init.yml | 4 ++-- src/cobalt_strike.yml | 4 ++-- src/ufw.yml | 4 ++-- src/vnc.yml | 4 ++-- src/xfce.yml | 4 ++-- 5 files changed, 10 insertions(+), 10 deletions(-) diff --git a/src/cloud_init.yml b/src/cloud_init.yml index 2fb82f8..fdd2138 100644 --- a/src/cloud_init.yml +++ b/src/cloud_init.yml @@ -2,8 +2,8 @@ # For some reason cloud-init-local, cloud-config, and cloud-final are # not enabled in the base Kali AMI. We need cloud-config enabled, and # we may as well enable the others while we're at it. -- hosts: all - name: Enable all cloud-init services +- name: Enable all cloud-init services + hosts: all become: true become_method: ansible.builtin.sudo tasks: diff --git a/src/cobalt_strike.yml b/src/cobalt_strike.yml index 6225d46..a6eb4a7 100644 --- a/src/cobalt_strike.yml +++ b/src/cobalt_strike.yml @@ -1,6 +1,6 @@ --- -- hosts: all - name: Install Cobalt Strike +- name: Install Cobalt Strike + hosts: all become: true become_method: ansible.builtin.sudo tasks: diff --git a/src/ufw.yml b/src/ufw.yml index d5cae05..2d39864 100644 --- a/src/ufw.yml +++ b/src/ufw.yml @@ -1,6 +1,6 @@ --- -- hosts: all - name: Install and configure UFW +- name: Install and configure UFW + hosts: all become: true become_method: ansible.builtin.sudo tasks: diff --git a/src/vnc.yml b/src/vnc.yml index 77664e1..9d6b9b8 100644 --- a/src/vnc.yml +++ b/src/vnc.yml @@ -1,6 +1,6 @@ --- -- hosts: all - name: Install VNC and configure VNC user +- name: Install VNC and configure VNC user + hosts: all become: true become_method: ansible.builtin.sudo tasks: diff --git a/src/xfce.yml b/src/xfce.yml index a6148e4..8ab6735 100644 --- a/src/xfce.yml +++ b/src/xfce.yml @@ -1,6 +1,6 @@ --- -- hosts: all - name: Install Xfce +- name: Install Xfce + hosts: all become: true become_method: ansible.builtin.sudo tasks: From 645f616ac72e625106cb154971f14d19d8d10244 Mon Sep 17 00:00:00 2001 From: Jeremy Frasier Date: Wed, 13 Mar 2024 16:46:23 -0400 Subject: [PATCH 169/171] Bump version from 0.3.7+build.2 to 0.4.0 --- src/version.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/version.txt b/src/version.txt index c324745..6a9beea 100644 --- a/src/version.txt +++ b/src/version.txt @@ -1 +1 @@ -__version__ = "0.3.7+build.2" +__version__ = "0.4.0" From 46aad4bbd0a30edf93bead13083af49360f2b9cd Mon Sep 17 00:00:00 2001 From: Jeremy Frasier Date: Wed, 13 Mar 2024 16:46:35 -0400 Subject: [PATCH 170/171] Bump version from 0.4.0 to 0.4.0-rc.1 --- src/version.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/version.txt b/src/version.txt index 6a9beea..0ced42e 100644 --- a/src/version.txt +++ b/src/version.txt @@ -1 +1 @@ -__version__ = "0.4.0" +__version__ = "0.4.0-rc.1" From 0eb5e228009761552045e0a4c37bab92987a6e64 Mon Sep 17 00:00:00 2001 From: Jeremy Frasier Date: Wed, 20 Mar 2024 15:09:35 -0400 Subject: [PATCH 171/171] Finalize version from 0.4.0-rc.1 to 0.4.0 --- src/version.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/version.txt b/src/version.txt index 0ced42e..6a9beea 100644 --- a/src/version.txt +++ b/src/version.txt @@ -1 +1 @@ -__version__ = "0.4.0-rc.1" +__version__ = "0.4.0"