update pe_source scripts to save to mdl #82
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| --- | |
| name: build | |
| on: | |
| push: | |
| pull_request: | |
| repository_dispatch: | |
| types: [apb] | |
| env: | |
| CURL_CACHE_DIR: ~/.cache/curl | |
| PIP_CACHE_DIR: ~/.cache/pip | |
| PRE_COMMIT_CACHE_DIR: ~/.cache/pre-commit | |
| RUN_TMATE: ${{ secrets.RUN_TMATE }} | |
| jobs: | |
| diagnostics: | |
| name: Run diagnostics | |
| runs-on: ubuntu-latest | |
| steps: | |
| # Note that a duplicate of this step must be added at the top of | |
| # each job. | |
| - id: harden-runner | |
| name: Harden the runner | |
| uses: step-security/harden-runner@v2 | |
| with: | |
| egress-policy: audit | |
| - id: github-status | |
| name: Check GitHub status | |
| uses: crazy-max/ghaction-github-status@v3 | |
| - id: dump-context | |
| name: Dump context | |
| uses: crazy-max/ghaction-dump-context@v2 | |
| lint: | |
| needs: | |
| - diagnostics | |
| runs-on: ubuntu-latest | |
| steps: | |
| - id: harden-runner | |
| name: Harden the runner | |
| uses: step-security/harden-runner@v2 | |
| with: | |
| egress-policy: audit | |
| - id: setup-env | |
| uses: cisagov/setup-env-github-action@develop | |
| - uses: actions/checkout@v3 | |
| - id: setup-python | |
| uses: actions/setup-python@v4 | |
| with: | |
| # Temporarily hold back support for Python 3.11 due to the | |
| # version pin for the PyMuPDF package. Pre-built wheels for | |
| # Python 3.11 were not made available until the 1.21.0 | |
| # release and building the PyMuPDF package from source | |
| # proves problematic due to dependencies. Please see the | |
| # following issue for more information: | |
| # https://github.com/cisagov/pe-reports/issues/441 | |
| # | |
| # It's also the case that we cannot build the wheel for | |
| # reportlab under Python 3.11 because its C code accesses | |
| # the C struct members of PyFrameObject directly, and these | |
| # struct members are no longer public as of Python 3.11. | |
| # See here for more details: | |
| # https://docs.python.org/3/whatsnew/3.11.html#pyframeobject-3-11-hiding | |
| python-version: "3.10" | |
| # We need the Go version and Go cache location for the actions/cache step, | |
| # so the Go installation must happen before that. | |
| - id: setup-go | |
| uses: actions/setup-go@v4 | |
| with: | |
| go-version: "1.19" | |
| - name: Lookup Go cache directory | |
| id: go-cache | |
| run: | | |
| echo "dir=$(go env GOCACHE)" >> $GITHUB_OUTPUT | |
| - uses: actions/cache@v3 | |
| env: | |
| BASE_CACHE_KEY: "${{ github.job }}-${{ runner.os }}-\ | |
| py${{ steps.setup-python.outputs.python-version }}-\ | |
| go${{ steps.setup-go.outputs.go-version }}-\ | |
| packer${{ steps.setup-env.outputs.packer-version }}-\ | |
| tf${{ steps.setup-env.outputs.terraform-version }}-" | |
| with: | |
| # Note that the .terraform directory IS NOT included in the | |
| # cache because if we were caching, then we would need to use | |
| # the `-upgrade=true` option. This option blindly pulls down the | |
| # latest modules and providers instead of checking to see if an | |
| # update is required. That behavior defeats the benefits of caching. | |
| # so there is no point in doing it for the .terraform directory. | |
| path: | | |
| ${{ env.PIP_CACHE_DIR }} | |
| ${{ env.PRE_COMMIT_CACHE_DIR }} | |
| ${{ env.CURL_CACHE_DIR }} | |
| ${{ steps.go-cache.outputs.dir }} | |
| # We do not use '**/setup.py' in the cache key so only the 'setup.py' | |
| # file in the root of the repository is used. This is in case a Python | |
| # package were to have a 'setup.py' as part of its internal codebase. | |
| key: "${{ env.BASE_CACHE_KEY }}\ | |
| ${{ hashFiles('**/requirements-test.txt') }}-\ | |
| ${{ hashFiles('**/requirements.txt') }}-\ | |
| ${{ hashFiles('**/.pre-commit-config.yaml') }}-\ | |
| ${{ hashFiles('setup.py') }}" | |
| restore-keys: | | |
| ${{ env.BASE_CACHE_KEY }} | |
| - name: Setup curl cache | |
| run: mkdir -p ${{ env.CURL_CACHE_DIR }} | |
| - name: Install Packer | |
| env: | |
| PACKER_VERSION: ${{ steps.setup-env.outputs.packer-version }} | |
| run: | | |
| PACKER_ZIP="packer_${PACKER_VERSION}_linux_amd64.zip" | |
| curl --output ${{ env.CURL_CACHE_DIR }}/"${PACKER_ZIP}" \ | |
| --time-cond ${{ env.CURL_CACHE_DIR }}/"${PACKER_ZIP}" \ | |
| --location \ | |
| "https://releases.hashicorp.com/packer/${PACKER_VERSION}/${PACKER_ZIP}" | |
| sudo unzip -d /opt/packer \ | |
| ${{ env.CURL_CACHE_DIR }}/"${PACKER_ZIP}" | |
| sudo mv /usr/local/bin/packer /usr/local/bin/packer-default | |
| sudo ln -s /opt/packer/packer /usr/local/bin/packer | |
| - uses: hashicorp/setup-terraform@v2 | |
| with: | |
| terraform_version: ${{ steps.setup-env.outputs.terraform-version }} | |
| - name: Install go-critic | |
| env: | |
| PACKAGE_URL: github.com/go-critic/go-critic/cmd/gocritic | |
| PACKAGE_VERSION: ${{ steps.setup-env.outputs.go-critic-version }} | |
| run: go install ${PACKAGE_URL}@${PACKAGE_VERSION} | |
| - name: Install gosec | |
| env: | |
| PACKAGE_URL: github.com/securego/gosec/v2/cmd/gosec | |
| PACKAGE_VERSION: ${{ steps.setup-env.outputs.gosec-version }} | |
| run: go install ${PACKAGE_URL}@${PACKAGE_VERSION} | |
| - name: Install shfmt | |
| env: | |
| PACKAGE_URL: mvdan.cc/sh/v3/cmd/shfmt | |
| PACKAGE_VERSION: ${{ steps.setup-env.outputs.shfmt-version }} | |
| run: go install ${PACKAGE_URL}@${PACKAGE_VERSION} | |
| - name: Install staticcheck | |
| env: | |
| PACKAGE_URL: honnef.co/go/tools/cmd/staticcheck | |
| PACKAGE_VERSION: ${{ steps.setup-env.outputs.staticcheck-version }} | |
| run: go install ${PACKAGE_URL}@${PACKAGE_VERSION} | |
| - name: Install Terraform-docs | |
| env: | |
| PACKAGE_URL: github.com/terraform-docs/terraform-docs | |
| PACKAGE_VERSION: ${{ steps.setup-env.outputs.terraform-docs-version }} | |
| run: go install ${PACKAGE_URL}@${PACKAGE_VERSION} | |
| # Install some necessary packages for building wheels | |
| - uses: awalsh128/cache-apt-pkgs-action@latest | |
| with: | |
| packages: libmupdf-dev | |
| version: 1.3.0 | |
| - name: Install dependencies | |
| run: | | |
| python -m pip install --upgrade pip setuptools wheel | |
| pip install --upgrade --requirement requirements-test.txt | |
| - name: Set up pre-commit hook environments | |
| run: pre-commit install-hooks | |
| - name: Run pre-commit on all files | |
| run: pre-commit run --all-files | |
| - name: Setup tmate debug session | |
| uses: mxschmitt/action-tmate@v3 | |
| if: env.RUN_TMATE | |
| test: | |
| name: test source - py${{ matrix.python-version }} | |
| runs-on: ${{ matrix.os }} | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| os: | |
| - ubuntu-latest | |
| python-version: | |
| - "3.7" | |
| - "3.8" | |
| - "3.9" | |
| - "3.10" | |
| # Temporarily hold back support for Python 3.11 due to the version pin | |
| # for the PyMuPDF package. Pre-built wheels for Python 3.11 were not made | |
| # available until the 1.21.0 release and building the PyMuPDF package from | |
| # source proves problematic due to dependencies. Please see the following | |
| # issue for more information: https://github.com/cisagov/pe-reports/issues/441 | |
| # - "3.11" | |
| include: | |
| - os: ubuntu-20.04 | |
| python-version: "3.6" | |
| steps: | |
| - uses: actions/checkout@v3 | |
| - id: setup-python | |
| uses: actions/setup-python@v4 | |
| with: | |
| python-version: ${{ matrix.python-version }} | |
| - uses: actions/cache@v3 | |
| env: | |
| BASE_CACHE_KEY: "${{ github.job }}-${{ runner.os }}-\ | |
| py${{ steps.setup-python.outputs.python-version }}-" | |
| with: | |
| path: ${{ env.PIP_CACHE_DIR }} | |
| # We do not use '**/setup.py' in the cache key so only the 'setup.py' | |
| # file in the root of the repository is used. This is in case a Python | |
| # package were to have a 'setup.py' as part of its internal codebase. | |
| key: "${{ env.BASE_CACHE_KEY }}\ | |
| ${{ hashFiles('**/requirements-test.txt') }}-\ | |
| ${{ hashFiles('**/requirements.txt') }}-\ | |
| ${{ hashFiles('setup.py') }}" | |
| restore-keys: | | |
| ${{ env.BASE_CACHE_KEY }} | |
| # Install some necessary packages for building wheels | |
| - uses: awalsh128/cache-apt-pkgs-action@latest | |
| with: | |
| packages: libmupdf-dev | |
| version: 1.3.0 | |
| - name: Install dependencies | |
| run: | | |
| python -m pip install --upgrade pip | |
| pip install --upgrade --requirement requirements-test.txt | |
| - name: Run tests | |
| env: | |
| RELEASE_TAG: ${{ github.event.release.tag_name }} | |
| run: pytest | |
| - name: Upload coverage report | |
| run: coveralls | |
| env: | |
| COVERALLS_FLAG_NAME: "py${{ matrix.python-version }}" | |
| COVERALLS_PARALLEL: true | |
| COVERALLS_SERVICE_NAME: github | |
| GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} | |
| if: success() | |
| - name: Setup tmate debug session | |
| uses: mxschmitt/action-tmate@v3 | |
| if: env.RUN_TMATE | |
| coveralls-finish: | |
| runs-on: ubuntu-latest | |
| needs: | |
| - test | |
| steps: | |
| - uses: actions/checkout@v3 | |
| - id: setup-python | |
| uses: actions/setup-python@v4 | |
| with: | |
| python-version: "3.10" | |
| - uses: actions/cache@v3 | |
| env: | |
| BASE_CACHE_KEY: "${{ github.job }}-${{ runner.os }}-\ | |
| py${{ steps.setup-python.outputs.python-version }}-" | |
| with: | |
| path: ${{ env.PIP_CACHE_DIR }} | |
| # We do not use '**/setup.py' in the cache key so only the 'setup.py' | |
| # file in the root of the repository is used. This is in case a Python | |
| # package were to have a 'setup.py' as part of its internal codebase. | |
| key: "${{ env.BASE_CACHE_KEY }}\ | |
| ${{ hashFiles('**/requirements-test.txt') }}-\ | |
| ${{ hashFiles('**/requirements.txt') }}-\ | |
| ${{ hashFiles('setup.py') }}" | |
| restore-keys: | | |
| ${{ env.BASE_CACHE_KEY }} | |
| # Install some necessary packages for building wheels | |
| - uses: awalsh128/cache-apt-pkgs-action@latest | |
| with: | |
| packages: libmupdf-dev | |
| version: 1.3.0 | |
| - name: Install dependencies | |
| run: | | |
| python -m pip install --upgrade pip | |
| pip install --upgrade --requirement requirements-test.txt | |
| - name: Finished coveralls reports | |
| run: coveralls --finish | |
| env: | |
| GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} | |
| - name: Setup tmate debug session | |
| uses: mxschmitt/action-tmate@v3 | |
| if: env.RUN_TMATE | |
| build: | |
| name: build wheel - py${{ matrix.python-version }} | |
| needs: | |
| - lint | |
| - test | |
| runs-on: ${{ matrix.os }} | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| os: | |
| - ubuntu-latest | |
| python-version: | |
| - "3.7" | |
| - "3.8" | |
| - "3.9" | |
| - "3.10" | |
| # Temporarily hold back support for Python 3.11 due to the version pin | |
| # for the PyMuPDF package. Pre-built wheels for Python 3.11 were not made | |
| # available until the 1.21.0 release and building the PyMuPDF package from | |
| # source proves problematic due to dependencies. Please see the following | |
| # issue for more information: https://github.com/cisagov/pe-reports/issues/441 | |
| # - "3.11" | |
| include: | |
| - os: ubuntu-20.04 | |
| python-version: "3.6" | |
| steps: | |
| - uses: actions/checkout@v3 | |
| - id: setup-python | |
| uses: actions/setup-python@v4 | |
| with: | |
| python-version: ${{ matrix.python-version }} | |
| - uses: actions/cache@v3 | |
| env: | |
| BASE_CACHE_KEY: "${{ github.job }}-${{ runner.os }}-\ | |
| py${{ steps.setup-python.outputs.python-version }}-" | |
| with: | |
| path: ${{ env.PIP_CACHE_DIR }} | |
| # We do not use '**/setup.py' in the cache key so only the 'setup.py' | |
| # file in the root of the repository is used. This is in case a Python | |
| # package were to have a 'setup.py' as part of its internal codebase. | |
| key: "${{ env.BASE_CACHE_KEY }}\ | |
| ${{ hashFiles('**/requirements.txt') }}-\ | |
| ${{ hashFiles('setup.py') }}" | |
| restore-keys: | | |
| ${{ env.BASE_CACHE_KEY }} | |
| # Install some necessary packages for building wheels | |
| - uses: awalsh128/cache-apt-pkgs-action@latest | |
| with: | |
| packages: libmupdf-dev | |
| version: 1.3.0 | |
| - name: Install build dependencies | |
| run: | | |
| python -m pip install --upgrade pip setuptools wheel | |
| python -m pip install --upgrade build | |
| - name: Build artifacts | |
| run: python -m build | |
| - name: Upload artifacts | |
| uses: actions/upload-artifact@v3 | |
| with: | |
| name: dist-${{ matrix.python-version }} | |
| path: dist | |
| - name: Setup tmate debug session | |
| uses: mxschmitt/action-tmate@v3 | |
| if: env.RUN_TMATE | |
| test-build: | |
| name: test built wheel - py${{ matrix.python-version }} | |
| needs: | |
| - build | |
| runs-on: ${{ matrix.os }} | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| os: | |
| - ubuntu-latest | |
| python-version: | |
| - "3.7" | |
| - "3.8" | |
| - "3.9" | |
| - "3.10" | |
| # Temporarily hold back support for Python 3.11 due to the version pin | |
| # for the PyMuPDF package. Pre-built wheels for Python 3.11 were not made | |
| # available until the 1.21.0 release and building the PyMuPDF package from | |
| # source proves problematic due to dependencies. Please see the following | |
| # issue for more information: https://github.com/cisagov/pe-reports/issues/441 | |
| # - "3.11" | |
| include: | |
| - os: ubuntu-20.04 | |
| python-version: "3.6" | |
| steps: | |
| - uses: actions/checkout@v3 | |
| - id: setup-python | |
| uses: actions/setup-python@v4 | |
| with: | |
| python-version: ${{ matrix.python-version }} | |
| - uses: actions/cache@v3 | |
| env: | |
| BASE_CACHE_KEY: "${{ github.job }}-${{ runner.os }}-\ | |
| py${{ steps.setup-python.outputs.python-version }}-" | |
| with: | |
| path: ${{ env.PIP_CACHE_DIR }} | |
| # We do not use '**/setup.py' in the cache key so only the 'setup.py' | |
| # file in the root of the repository is used. This is in case a Python | |
| # package were to have a 'setup.py' as part of its internal codebase. | |
| key: "${{ env.BASE_CACHE_KEY }}\ | |
| ${{ hashFiles('**/requirements.txt') }}-\ | |
| ${{ hashFiles('setup.py') }}" | |
| restore-keys: | | |
| ${{ env.BASE_CACHE_KEY }} | |
| - name: Retrieve the built wheel | |
| uses: actions/download-artifact@v3 | |
| with: | |
| name: dist-${{ matrix.python-version }} | |
| path: dist | |
| - id: find-wheel | |
| name: Get the name of the retrieved wheel (there should only be one) | |
| run: echo "wheel=$(ls dist/*whl)" >> $GITHUB_OUTPUT | |
| - name: Update core Python packages | |
| run: python -m pip install --upgrade pip setuptools wheel | |
| - name: Install the built wheel (along with testing dependencies) | |
| run: python -m pip install ${{ steps.find-wheel.outputs.wheel }}[test] | |
| - name: Run tests | |
| env: | |
| RELEASE_TAG: ${{ github.event.release.tag_name }} | |
| run: pytest | |
| - name: Setup tmate debug session | |
| uses: mxschmitt/action-tmate@v3 | |
| if: env.RUN_TMATE |