diff --git a/.github/workflows/branch_ci.yml b/.github/workflows/branch_ci.yml index b14ac899..12ce9c47 100644 --- a/.github/workflows/branch_ci.yml +++ b/.github/workflows/branch_ci.yml @@ -1,10 +1,11 @@ # Workflow that runs on pushes to non-default branches -name: Non-Default Branch CI (Python) +name: Non-Default Branch Push CI (Python) on: push: - branches-ignore: ["main"] + branches-ignore: ['main'] + paths-ignore: ['README.md'] # Specify concurrency such that only one workflow can run at a time # * Different workflow files are not affected @@ -24,6 +25,7 @@ permissions: jobs: + # Job to run a linter and typechecker against the codebase lint-typecheck: runs-on: ubuntu-latest @@ -51,6 +53,8 @@ jobs: - name: Typecheck package run: uv run mypy . + # Job to run unittests + # * Produces a JUnit XML report that can be displayed in the GitHub UI test-unit: runs-on: ubuntu-latest needs: lint-typecheck @@ -119,8 +123,9 @@ jobs: name: docs path: docs + # Job for building container image # * Builds and pushes an OCI Container image to the registry defined in the environment variables - # * Only runs if test job passes + # * Only runs if test and lint jobs pass build-container: runs-on: ubuntu-latest permissions: @@ -153,8 +158,7 @@ jobs: uses: docker/metadata-action@v5 with: images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }} - tags: | - type=ref,event=branch + tags: type=ref,event=branch # Build and push the Container image to the registry # * Creates a multiplatform-aware image diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml deleted file mode 100644 index eb7e2d05..00000000 --- a/.github/workflows/ci.yaml +++ /dev/null @@ -1,274 +0,0 @@ -name: Python CI - -on: - push: - branches: [] - paths-ignore: - - 'README.md' - tags: - - 'v*' - pull_request: - branches: [] - paths-ignore: - - 'README.md' - workflow_dispatch: - -# Specify concurrency such that only one workflow can run at a time -# * Different workflow files are not affected -concurrency: - group: ${{ github.workflow }}-${{ github.ref }} - cancel-in-progress: true - -# Registry for storing Container images -env: - REGISTRY: ghcr.io - IMAGE_NAME: ${{ github.repository }} - -# Ensure the GitHub token can remove packages -permissions: - packages: write - - -jobs: - - # Define a job that builds the documentation - # * Surfaces the documentation as an artifact - build-docs: - runs-on: ubuntu-latest - - steps: - - name: Checkout repository - uses: actions/checkout@v4 - - - name: Set up Python - uses: actions/setup-python@v5 - with: - python-version: '3.12' - - - name: Install dependencies - run: | - python -m pip install --upgrade pip - pip install pydoctor - - - name: Build documentation - run: | - python -m pydoctor - - - name: Upload documentation - uses: actions/upload-artifact@v4 - with: - name: docs - path: docs - - - # Define a dependencies job that runs on all branches and PRs - # * Installs dependencies and caches them - build-venv: - runs-on: ubuntu-latest - container: quay.io/condaforge/miniforge3:latest - - steps: - - name: Checkout repository - uses: actions/checkout@v4 - - # Restore cached virtualenv, if available - # * The pyproject.toml hash is part of the cache key, invalidating - # the cache if the file changes - - name: Restore cached virtualenv - id: restore-cache - uses: actions/cache/restore@v4 - with: - path: ./venv - key: ${{ runner.os }}-venv-${{ hashFiles('**/pyproject.toml') }} - - # Should mirror the build-venv stage in the Containerfile - - name: Build venv - run: | - apt -qq update && apt -qq install -y build-essential - conda create -p ./venv python=3.12 - ./venv/bin/python -m pip install --upgrade -q pip wheel setuptools - if: steps.restore-cache.outputs.cache-hit != 'true' - - # Should mirror the build-reqs stage in the Containerfile - # * Except this installs the dev dependencies as well - - name: Install all dependencies - run: | - conda install -p ./venv -q -y eccodes zarr - ./venv/bin/python -m pip install -q .[dev] --no-binary=nwp-consumer - if: steps.restore-cache.outputs.cache-hit != 'true' - - # Cache the virtualenv for future runs - - name: Cache virtualenv - uses: actions/cache/save@v4 - with: - path: ./venv - key: ${{ steps.restore-cache.outputs.cache-primary-key }} - if: steps.restore-cache.outputs.cache-hit != 'true' - - # Define a unittest job that runs on all branches and PRs - test-unit: - runs-on: ubuntu-latest - container: quay.io/condaforge/miniforge3:latest - needs: build-venv - if: github.event.name == ' - - steps: - - name: Checkout repository - uses: actions/checkout@v4 - - # Restore cached virtualenv - - name: Restore cached virtualenv - uses: actions/cache/restore@v4 - with: - path: ./venv - key: ${{ runner.os }}-venv-${{ hashFiles('**/pyproject.toml') }} - - - name: Install package - run: ./venv/bin/python -m pip install -q . - - # Run unittests - # * Produce JUnit XML report - - name: Run unit tests - env: - ECCODES_DEFINITION_PATH: ${{ github.workspace }}/venv/share/eccodes/definitions - run: ./venv/bin/python -m xmlrunner discover -s src/nwp_consumer -p "test_*.py" --output-file ut-report.xml - - # Create test summary to be visualised on the job summary screen on GitHub - # * Runs even if previous steps fail - - name: Create test summary - uses: test-summary/action@v2 - with: - paths: "*t-report.xml" - show: "fail, skip" - if: always() - - # Define an autotagger job that runs on merge requests - tag: - runs-on: ubuntu-latest - needs: test-unit - if: | - github.event_name == 'pull_request' && - github.event.action == 'closed' && - github.event.pull_request.merged == true - - steps: - - name: Checkout repository - uses: actions/checkout@v4 - with: - fetch-depth: 0 - - name: Bump version and push tag - uses: RueLaLa/auto-tagger@master - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - GITHUB_PR_NUMBER: ${{ github.event.number }} - - # Define a "build-container" job that runs on branch commits only - # * Builds and pushes an OCI Container image to the registry defined in the environment variables - # * Only runs if test job passes - build-container: - runs-on: ubuntu-latest - permissions: - contents: read - packages: write - needs: test-unit - if: github.event_name != 'pull_request' - - steps: - # Do a non-shallow clone of the repo to ensure tags are present - # * This allows setuptools-git-versioning to automatically set the version - - name: Checkout repository - uses: actions/checkout@v4 - with: - fetch-depth: 0 - - - name: Set up Buildx - uses: docker/setup-buildx-action@v2 - - - name: Log in to the Container registry - uses: docker/login-action@65b78e6e13532edd9afa3aa52ac7964289d1a9c1 - with: - registry: ${{ env.REGISTRY }} - username: ${{ github.actor }} - password: ${{ secrets.GITHUB_TOKEN }} - - # Tag the built image according to the event type - # * If the event is a valid version tag, use the tag name - # * If the event is a branch commit, use the commit sha - - name: Extract metadata (tags, labels) for Container - id: meta - uses: docker/metadata-action@9ec57ed1fcdbf14dcef7dfbe97b2010124a938b7 - with: - images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }} - tags: | - type=ref,event=branch - type=semver,pattern={{version}} - - # Build and push the Container image to the registry - # * Creates a multiplatform-aware image - # * Semantic versioning is handled via the metadata action - # * The image layers are cached between action runs with the following strategy - # * - On push to main, also push build cache - # * - On push to other branches, only pull build cache - - name: Build and push Container image and cache - uses: docker/build-push-action@v4 - if: github.event_name == 'push' && github.ref == 'refs/heads/main' - with: - context: . - file: Containerfile - push: true - tags: ${{ steps.meta.outputs.tags }} - labels: ${{ steps.meta.outputs.labels }} - platforms: linux/amd64,linux/arm64 - cache-from: type=registry,ref=${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:buildcache - cache-to: type=registry,ref=${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:buildcache,mode=max - - name: Build and push container image - uses: docker/build-push-action@v4 - if: github.event_name != 'push' || github.ref != 'refs/heads/main' - with: - context: . - file: Containerfile - push: true - tags: ${{ steps.meta.outputs.tags }} - labels: ${{ steps.meta.outputs.labels }} - platforms: linux/amd64,linux/arm64 - cache-from: type=registry,ref=${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:buildcache - - # Define a "build-wheel" job that runs on version tags - # * Only runs if integration test job passes - build-wheel: - runs-on: ubuntu-latest - if: contains(github.ref, 'refs/tags/v') && github.event_name == 'push' - - steps: - # Do a non-shallow clone of the repo to ensure tags are present - # * This allows setuptools-git-versioning to automatically set the version - - name: Checkout repository - uses: actions/checkout@v4 - with: - fetch-depth: 0 - - # Restore cached virtualenv - - name: Restore cached virtualenv - uses: actions/cache/restore@v4 - with: - path: ./venv - key: ${{ runner.os }}-venv-${{ hashFiles('**/pyproject.toml') }} - - # Building the wheel dynamically assigns the version according to git - # * The setuptools_git_versioning package reads the git tags and assigns the version - # * The version is then used in the wheel filename and made available in the package - # * setuptools_git_versioning is configured in pyproject.toml - - name: Build wheel - run: ./venv/bin/python -m pip wheel . --no-deps --wheel-dir dist - - - name: Upload wheel - uses: actions/upload-artifact@v4 - with: - name: wheel - path: dist/*.whl - - - name: Publish wheel - uses: pypa/gh-action-pypi-publish@v1.8.10 - with: - user: __token__ - password: ${{ secrets.PYPI_API_TOKEN }} diff --git a/.github/workflows/main_ci.yml b/.github/workflows/main_ci.yml new file mode 100644 index 00000000..2d0201b6 --- /dev/null +++ b/.github/workflows/main_ci.yml @@ -0,0 +1,36 @@ +# Workflow that runs on closed PRs to the default branch + +name: Default Branch PR Merged CI (Python) + +on: + pull_request: + types: ["closed"] + branches: ["main"] + +# Specify concurrency such that only one workflow can run at a time +# * Different workflow files are not affected +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: false + + +jobs: + + # Define an autotagger job that creates tags on changes to master + tag: + runs-on: ubuntu-latest + if: | + github.event_name == 'pull_request' && + github.event.action == 'closed' && + github.event.pull_request.merged == true + + steps: + - name: Checkout repository + uses: actions/checkout@v4 + with: + fetch-depth: 0 + - name: Bump version and push tag + uses: RueLaLa/auto-tagger@v2 + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + GITHUB_PR_NUMBER: ${{ github.event.number }} diff --git a/.github/workflows/tagged_ci.yml b/.github/workflows/tagged_ci.yml new file mode 100644 index 00000000..42fcc688 --- /dev/null +++ b/.github/workflows/tagged_ci.yml @@ -0,0 +1,121 @@ +# Workflow that runs on new SemVer tags on the default branch + +name: Default Branch SemVer Tagged CI (Python) + +on: + push: + branches: ['main'] + tags: ['v[0-9]+.[0-9]+.[0-9]+'] + paths-ignore: ['README.md'] + +# Specify concurrency such that only one workflow can run at a time +# * Different workflow files are not affected +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: false + +# Registry for storing Container images +env: + REGISTRY: ghcr.io + IMAGE_NAME: ${{ github.repository }} + + +jobs: + + # Job to create a container + # Job for building container image + # * Builds and pushes an OCI Container image to the registry defined in the environment variables + build-container: + runs-on: ubuntu-latest + permissions: + contents: read + packages: write + needs: ["lint-typecheck", "test-unit"] + + steps: + # Do a non-shallow clone of the repo to ensure tags are present + # * This allows setuptools-git-versioning to automatically set the version + - name: Checkout repository + uses: actions/checkout@v4 + with: + fetch-depth: 0 + + - name: Set up Buildx + uses: docker/setup-buildx-action@v2 + + - name: Log in to the Container registry + uses: docker/login-action@65b78e6e13532edd9afa3aa52ac7964289d1a9c1 + with: + registry: ${{ env.REGISTRY }} + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + + # Tag the built image according to the event type + # The event is a semver release, so use the version + - name: Extract metadata (tags, labels) for Container + id: meta + uses: docker/metadata-action@v5 + with: + images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }} + tags: type=semver,pattern={{version}} + + # Build and push the Container image to the registry + # * Creates a multiplatform-aware image + # * Pulls build cache from the registry and pushes new cache back + - name: Build and push container image + uses: docker/build-push-action@v4 + with: + context: . + file: Containerfile + push: true + tags: ${{ steps.meta.outputs.tags }} + labels: ${{ steps.meta.outputs.labels }} + platforms: linux/amd64,linux/arm64 + cache-from: type=registry,ref=${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:buildcache + cache-to: type=registry,ref=${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:buildcache,mode=max + + + # Job to build and publish the package on PyPi as a wheel + build-wheel: + runs-on: ubuntu-latest + + steps: + # Do a non-shallow clone of the repo to ensure tags are present + # * This allows setuptools-git-versioning to automatically set the version + - name: Checkout repository + uses: actions/checkout@v4 + with: + fetch-depth: 0 + + - name: Setup uv + uses: astral-sh/setup-uv@v3 + with: + enable-cache: true + cache-dependency-glob: "pyproject.toml" + + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version-file: "pyproject.toml" + + - name: Install editable package and required dependencies + run: uv sync + + # Building the wheel dynamically assigns the version according to git + # * The setuptools_git_versioning package reads the git tags and assigns the version + # * The version is then used in the wheel filename and made available in the package + # * setuptools_git_versioning is configured in pyproject.toml + - name: Build wheel + run: uv pip wheel . --no-deps --wheel-dir dist + + - name: Upload wheel + uses: actions/upload-artifact@v4 + with: + name: wheel + path: dist/*.whl + + - name: Publish wheel + uses: pypa/gh-action-pypi-publish@v1.10 + with: + user: __token__ + password: ${{ secrets.PYPI_API_TOKEN }} \ No newline at end of file