ci-workflow.yaml

name: Gazette Continuous Integration

# We build on any push to a branch, or when a release is created.
on:
  pull_request:
    branches:
      - "arize"
    paths-ignore:
      - "docs/**"
  push:
    branches:
      - "arize"
      - "arize-dev/*"
    # Ignore pushes to tags, since those ought to be handled by the release created event.
    tags-ignore:
      - "*"
    paths-ignore:
      - "docs/**"
  release:
    # Without this additional restriction, GH actions will trigger multiple runs for a single
    # release, because it fires off separate events creating vs publishing the release.
    types: [created]
  workflow_dispatch:

env:
  # This is only used as the cache key to prevent rebuilding rocksdb every time. Eventually
  # we'll need to figure out a solution that doesn't duplicate this version everywhere.
  # For now, ensure that it's changed both here and in mk/common-config.mk.
  ROCKSDB_VERSION: "6.22.1"

permissions:
  id-token: write
  contents: read

jobs:
  build:
    name: "Build"
    runs-on: ubuntu-20.04
    steps:
      - name: "Checkout"
        uses: actions/checkout@v2

        # Sets outputs for version, docker tag, and whether we should push images and upload release
        # artifacts. These outputs are used by later steps.
      - name: "Release Info"
        id: release_info
        env:
          # just having this in the env is enough to make it visible in the "raw" logs attached to the run
          GITHUB_EVENT_JSON: "${{ toJson(github.event) }}"
        run: |
          is_release=${{ github.event_name == 'release' }}
          if [[ "$is_release" == "true" ]]; then
            push_images=true
            tag_name=${{ github.event.release.tag_name }}
            # The version regex doesn't need to be super sophisticated, since the goal is not to
            # validate that this is a semantic version number. Rather the goal is just to see if
            # matches the typical pattern we use for release tags (e.g. v0.86.1). If it does, then
            # we'll remove the 'v' prefix and use the remainder as the docker tag
            # If a release tag does not match that format, then we'll just use the tag value as is.
            if echo "$tag_name" | grep -qE '^v[0-9]+\.[0-9]+\.[0-9]+'; then
              version="${tag_name#v}"
            else
              version="${tag_name}"
            fi
          else
            # This is not a release, so we'll use 'dev-<sha>' for the version number
            # and just 'latest' for the docker tag.
            sha=${{ github.sha }}
            version="dev-${sha:0:7}"
            # If this is a master build, then we'll treat this as a release and just use the
            # hard-coded tag as the docker image tag.
            if [[ '${{ github.ref }}' == 'refs/heads/master' ]]; then
                # We don't want to put the git sha in the docker tag because otherwise they'll
                # accumulate forever and just clutter up the page on docker hub. So 'latest'
                # just always gets you the most recent master build, and if you want a specific master
                # build, then you can use the '@sha256:...' syntax.
                docker_tag="${{ secrets.REGISTRY_PATH }}/gazette/broker:latest"
                push_images='true'
            elif [[ '${{ github.ref }}' == 'refs/heads/arize' ]]; then
                version="0.99.0-arize-${sha:0:7}"
                docker_tag="${{ secrets.REGISTRY_PATH }}/gazette/broker:arize-${sha:0:7}"
                push_images='true'
            elif [[ '${{ github.ref }}' == *'arize'* ]]; then
                version="0.99.0-dev-${sha:0:7}"
                docker_tag="${{ secrets.REGISTRY_PATH }}/gazette/broker:dev-${sha:0:7}"
                push_images='true'
            else
                docker_tag="latest"
                push_images='false'
            fi
          fi
          echo ::set-output name=VERSION::${version}
          echo ::set-output name=DOCKER_TAG::${docker_tag}
          echo ::set-output name=PUSH_IMAGES::${push_images}
          echo ::set-output name=IS_RELEASE::${is_release}

        # Try to load the ci-builder docker image from the cache. If this misses, then it will be
        # built automatically by a make rule. A later step will then run 'docker save' to export the
        # image as a tar archive so we can cache it.
      - name: "CI Builder Docker Image Cache"
        id: "ci_builder_cache"
        uses: actions/cache@v2
        with:
          key: ci-builder-${{ hashFiles('mk/ci-builder.Dockerfile') }}
          path: ".build/ci-builder-image.tar"

        # The 'c<n>' in these Cache steps is just for changing the cache key so that
        # we can manually invalidate the cache if we need.
      - name: "RocksDB Cache"
        uses: actions/cache@v2
        with:
          key: rocksdb-c4-${{ env.ROCKSDB_VERSION }}
          path: ".build-ci/rocksdb-v${{ env.ROCKSDB_VERSION }}"

      - name: "Go Module Cache"
        uses: actions/cache@v2
        with:
          key: go-mod-c4-${{ hashFiles('go.sum') }}
          path: ".build-ci/go-path/pkg"
          # If we don't have a cached directory that matches the hash exactly,
          # then this will allow a non-matching directory to be pulled in. This is safe
          # because go will use its own finer-grained cache invalidation logic.
          restore-keys: "go-mod-c4-"


      - uses: 'google-github-actions/auth@v1'
        with:
          token_format: "access_token"          
          project_id: ${{ secrets.PROJECT_ID }}
          workload_identity_provider: projects/${{ secrets.PROJECT_NUMBER }}/locations/global/workloadIdentityPools/github/providers/github-actions
          service_account: ${{ secrets.SERVICE_ACCOUNT }}

      - name: 'Set up Cloud SDK'
        uses: 'google-github-actions/setup-gcloud@v1'

      - name: 'Use gcloud CLI'
        run: gcloud info

      - name: "Build Binaries"
        run: "make as-ci target=release-linux-binaries VERSION=${{ steps.release_info.outputs.VERSION }}"

      - name: "Test"
        run: "make as-ci target=go-test-ci VERSION=${{ steps.release_info.outputs.VERSION }}"

        # We upload this to the artifacts that are attached to the action just to make it easy for
        # someone to pull down a build from another branch.
      - name: "Upload Binaries"
        uses: actions/upload-artifact@v4
        with:
          name: "gazette-x86_64-linux-gnu.zip"
          path: ".build-ci/gazette-x86_64-linux-gnu.zip"

      - name: "Upload Release Binaries"
        if: steps.release_info.outputs.IS_RELEASE == 'true'
        uses: actions/upload-release-asset@v1
        env:
          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
        with:
          asset_name: gazette-x86_64-linux-gnu.zip
          asset_path: ".build-ci/gazette-x86_64-linux-gnu.zip"
          upload_url: "${{ github.event.release.upload_url }}"
          asset_content_type: application/zip


      - name: "Build and Push Docker Images"
        if: steps.release_info.outputs.PUSH_IMAGES == 'true'
        run: |
          make as-ci target=ci-release-gazette-broker VERSION=${{ steps.release_info.outputs.VERSION }}
          docker tag gazette/broker:latest ${{ steps.release_info.outputs.DOCKER_TAG }}
          gcloud auth configure-docker ${{ secrets.REGISTRY }}
          docker push ${{ steps.release_info.outputs.DOCKER_TAG }}