continuous.yaml

name: Continuous
on:
  pull_request:
    types:
      - opened
      - reopened
      - synchronize
      - ready_for_review
  merge_group:
  push:
    branches:
      - master

concurrency:
  group: ${{ github.ref }}
  cancel-in-progress: true

jobs:
  build-generic:
    if: ${{ github.event_name == 'pull_request' || github.event_name == 'push' }}
    name: "Continuous Image Build"
    permissions:
      contents: 'read'
      id-token: 'write'
    runs-on: ubuntu-latest
    strategy:
      matrix:
        app: [ web, node ]
    steps:
      - uses: actions/checkout@v4
      - name: Set up QEMU
        uses: docker/setup-qemu-action@v3
      - name: Set up Docker Buildx
        uses: docker/setup-buildx-action@v3
      - id: auth
        name: Authenticate to Google Cloud
        uses: google-github-actions/auth@v2
        with:
          token_format: 'access_token'
          workload_identity_provider: 'projects/${{ secrets.DEV_GKE_PROJECT_ID}}/locations/global/workloadIdentityPools/github/providers/github'
          service_account: '${{ secrets.DEV_GKE_SA }}'
      - name: Login to GAR
        uses: docker/login-action@v3
        with:
          registry: us-east1-docker.pkg.dev
          username: oauth2accesstoken
          password: '${{ steps.auth.outputs.access_token }}'
      - name: Get branch name
        id: branch-raw
        uses: tj-actions/branch-names@v5.1
      - name: Format branch name
        id: branch-name
        run: >-
          echo "current_branch="$(echo ${{ steps.branch-raw.outputs.current_branch }}
          | awk '{print tolower($0)}'
          | sed 's|.*/\([^/]*\)/.*|\1|; t; s|.*|\0|'
          | sed 's/[^a-z0-9\.\-]//g')
          >> $GITHUB_OUTPUT
      - name: Get current date
        id: date
        run: echo "date=$(date +'%Y%m%d%H%M')" >> $GITHUB_OUTPUT
      - name: Generate image metadata
        id: meta
        uses: docker/metadata-action@v3
        with:
          images: |
            us-east1-docker.pkg.dev/${{ secrets.DEV_PROJECT }}/containers/sefaria-${{ matrix.app }}-${{ steps.branch-name.outputs.current_branch }}
          # generate Docker tags based on the following events/attributes
          tags: |
            type=ref,event=branch
            type=sha,enable=true,priority=100,prefix=sha-,suffix=-${{ steps.date.outputs.date }},format=short
            type=sha
          flavor: |
            latest=true
      - name: build and push
        uses: docker/build-push-action@v3
        with:
#         cache-from: type=registry,ref=gcr.io/${{ secrets.DEV_PROJECT }}/sefaria-${{ matrix.app }}/cache
#         cache-to: type=registry,ref=gcr.io/${{ secrets.DEV_PROJECT }}/sefaria-${{ matrix.app }}/cache, mode=max
          context: .
          push: true
          build-args: |
            TYPE=build
          file: ./build/${{ matrix.app }}/Dockerfile
          tags: ${{ steps.meta.outputs.tags }}
          labels: ${{ steps.meta.outputs.labels }}
  build-derived:
    if: ${{ github.event_name == 'pull_request' || github.event_name == 'push' }}
    name: "Continuous Image Build Stage 2"
    runs-on: ubuntu-latest
    permissions:
      contents: 'read'
      id-token: 'write'
    needs:
      - build-generic
    strategy:
      matrix:
        app: [ asset, linker ]
    steps:
      - uses: actions/checkout@v4
      - name: Set up QEMU
        uses: docker/setup-qemu-action@v3
      - name: Set up Docker Buildx
        uses: docker/setup-buildx-action@v3
      - id: auth
        name: Authenticate to Google Cloud
        uses: google-github-actions/auth@v2
        with:
          token_format: 'access_token'
          workload_identity_provider: 'projects/${{ secrets.DEV_GKE_PROJECT_ID}}/locations/global/workloadIdentityPools/github/providers/github'
          service_account: '${{ secrets.DEV_GKE_SA }}'
      - name: Login to GAR
        uses: docker/login-action@v3
        with:
          registry: us-east1-docker.pkg.dev
          username: oauth2accesstoken
          password: '${{ steps.auth.outputs.access_token }}'
      - name: Get branch name
        id: branch-raw
        uses: tj-actions/branch-names@v5.1
      - name: Format branch name
        id: branch-name
        run: >-
          echo "current_branch="$(echo ${{ steps.branch-raw.outputs.current_branch }}
          | awk '{print tolower($0)}'
          | sed 's|.*/\([^/]*\)/.*|\1|; t; s|.*|\0|'
          | sed 's/[^a-z0-9\.\-]//g')
          >> $GITHUB_OUTPUT
      - name: Get current date
        id: date
        run: echo "date=$(date +'%Y%m%d%H%M')" >> $GITHUB_OUTPUT
      - name: Generate image metadata
        id: meta
        uses: docker/metadata-action@v3
        with:
          images: |
            us-east1-docker.pkg.dev/${{ secrets.DEV_PROJECT }}/containers/sefaria-${{ matrix.app }}-${{ steps.branch-name.outputs.current_branch }}
          # generate Docker tags based on the following events/attributes
          tags: |
            type=ref,event=branch
            type=sha,enable=true,priority=100,prefix=sha-,suffix=-${{ steps.date.outputs.date }},format=short
            type=sha
          flavor: |
            latest=true
      - name: Set outputs
        id: get-sha
        run: echo "sha_short=$(git rev-parse --short HEAD)" >> $GITHUB_OUTPUT
      - name: build and push
        uses: docker/build-push-action@v3
        with:
#         cache-from: type=registry,ref=gcr.io/${{ secrets.DEV_PROJECT }}/sefaria-${{ matrix.app }}/cache
#         cache-to: type=registry,ref=gcr.io/${{ secrets.DEV_PROJECT }}/sefaria-${{ matrix.app }}/cache,mode=max
          context: .
          push: true
          build-args: |
            SRC_IMG=us-east1-docker.pkg.dev/${{ secrets.DEV_PROJECT }}/containers/sefaria-web-${{ steps.branch-name.outputs.current_branch }}:sha-${{ steps.get-sha.outputs.sha_short }}
          file: ./build/${{ matrix.app }}/Dockerfile
          tags: ${{ steps.meta.outputs.tags }}
          labels: ${{ steps.meta.outputs.labels }}
#######
# Below Tests only run if PR is NOT draft
#######
  jest-tests:
    name: "Continuous Testing: Jest"   # This name is referenced when slacking status
    runs-on: ubuntu-latest
    if: github.event.pull_request.draft == false
    steps:
      - name: Checkout Code
        uses: actions/checkout@v4
      - name: Install dependencies
        run: |
          npm install
          npm run build-prod
      - run: ls
      - run: pwd
      - name: Run Jest Tests
        run: npm run jest-gha
      - name: Handle Jest Test Results
        run: cat /home/runner/jestResults.json; STATUS=`jq ".numFailedTestSuites" /home/runner/jestResults.json`; exit $STATUS
        if: ${{ always() }}
  check-python-files:
    runs-on: ubuntu-latest
    outputs:
      python_files_changed: ${{ steps.check-python-files.outputs.python_files_changed }}
    steps:
      - name: Checkout
        uses: actions/checkout@v4
      - name: Check for Python file changes
        id: check-python-files
        uses: dorny/paths-filter@v2
        with:
          filters: |
            python_files_changed:
              - added|modified:
                - '**/*.py'
                - 'requirements.txt'
                - 'setup.py'
                - 'pyproject.toml'
  sandbox-deploy:
    name: "Continuous Testing: Sandbox Deploy"
    concurrency:
      group: dev-mongo
      cancel-in-progress: false
    needs: [ build-derived, check-python-files ]
    if: >
      github.event.pull_request.draft == false &&
      needs.check-python-files.outputs.python_files_changed == 'true'
    runs-on: ubuntu-latest
    permissions:
      contents: 'read'
      id-token: 'write'
    steps:
      - name: Checkout Code
        uses: actions/checkout@v4
      - id: auth
        name: Authenticate to Google Cloud
        uses: google-github-actions/auth@v2
        with:
          token_format: 'access_token'
          workload_identity_provider: 'projects/${{ secrets.DEV_GKE_PROJECT_ID}}/locations/global/workloadIdentityPools/github/providers/github'
          service_account: '${{ secrets.DEV_GKE_SA }}'
      - name: Setup GCloud
        uses: google-github-actions/setup-gcloud@v0
        with:
          project_id: ${{ secrets.DEV_PROJECT }}
          install_components: 'gke-gcloud-auth-plugin'
      - name: Get branch name
        id: branch-raw
        uses: tj-actions/branch-names@v5.1
      - name: Format branch name
        id: branch-name
        run: >-
          echo "current_branch="$(echo ${{ steps.branch-raw.outputs.current_branch }}
          | awk '{print tolower($0)}'
          | sed 's|.*/\([^/]*\)/.*|\1|; t; s|.*|\0|'
          | sed 's/[^a-z0-9\.\-]//g')
          >> $GITHUB_OUTPUT
      - name: Set outputs
        id: get-sha
        run: echo "sha_short=$(git rev-parse --short HEAD)" >> $GITHUB_OUTPUT
      - name: Set up yq
        uses: frenck/action-setup-yq@v1
          #      - name: setup helm
          #        uses: azure/setup-helm@v3
      - name: Authenticate GHA Runner To Target Cluster
        uses: google-github-actions/get-gke-credentials@v0
        with:
          cluster_name: ${{secrets.DEV_GKE_CLUSTER}}
          location: ${{secrets.DEV_GKE_REGION}}
          project_id: ${{secrets.DEV_GCP_PROJECT}}
      - name: Deploy Sandbox
        run: ./build/ci/sandbox-helm-deploy.sh build/ci/sandbox-values.yaml
        env:
          GIT_COMMIT: "${{ steps.get-sha.outputs.sha_short }}"
          BRANCH: "${{ steps.branch-name.outputs.current_branch }}"
          PROJECT_ID: "${{ secrets.DEV_PROJECT }}"
          NAMESPACE: "${{secrets.DEV_SANDBOX_NAMESPACE}}"
  sandbox-ready:
    if: >
      github.event.pull_request.draft == false &&
      needs.check-python-files.outputs.python_files_changed == 'true'
    needs: sandbox-deploy
    runs-on: ubuntu-latest
    steps:
      - name: Checkout
        uses: actions/checkout@v2
      - name: Set outputs
        id: get-sha
        run: echo "sha_short=$(git rev-parse --short HEAD)" >> $GITHUB_OUTPUT
      - name: Wait for test sandbox to become available
        # https://gist.github.com/rgl/f90ff293d56dbb0a1e0f7e7e89a81f42
        run: ./build/ci/waitForSandbox.bash
        env:
          WAIT_DURATION: "3000"
          GIT_COMMIT: "${{ steps.get-sha.outputs.sha_short }}"
  pytest-job:
    name: "Continuous Testing: PyTest"
    concurrency:
      group: dev-mongo
      cancel-in-progress: false
    needs: [ sandbox-ready, check-python-files ]
    if: >
      github.event.pull_request.draft == false &&
      needs.check-python-files.outputs.python_files_changed == 'true'
    permissions:
      contents: 'read'
      id-token: 'write'
    runs-on: ubuntu-latest
    steps:
      - name: Checkout
        uses: actions/checkout@v4
      - id: auth
        name: Authenticate to Google Cloud
        uses: google-github-actions/auth@v2
        with:
          token_format: 'access_token'
          workload_identity_provider: 'projects/${{ secrets.DEV_GKE_PROJECT_ID}}/locations/global/workloadIdentityPools/github/providers/github'
          service_account: '${{ secrets.DEV_GKE_SA }}'
      - name: Setup GCloud
        uses: google-github-actions/setup-gcloud@v0
        with:
          project_id: ${{ secrets.DEV_PROJECT }}
          install_components: 'gke-gcloud-auth-plugin'
      - name: Authenticate GHA Runner To Target Cluster
        uses: google-github-actions/get-gke-credentials@v0
        with:
          cluster_name: ${{secrets.DEV_GKE_CLUSTER}}
          location: ${{secrets.DEV_GKE_REGION}}
          project_id: ${{secrets.DEV_GCP_PROJECT}}
      - name: Set outputs
        id: get-sha
        run: echo "sha_short=$(git rev-parse --short HEAD)" >> $GITHUB_OUTPUT
      - name: Get branch name
        id: branch-raw
        uses: tj-actions/branch-names@v5.1
      - name: Format branch name
        id: branch-name
        run: >-
          echo "current_branch="$(echo ${{ steps.branch-raw.outputs.current_branch }}
          | awk '{print tolower($0)}'
          | sed 's|.*/\([^/]*\)/.*|\1|; t; s|.*|\0|'
          | sed 's/[^a-z0-9\.\-]//g')
          >> $GITHUB_OUTPUT
      - name: Start Job
        run: ./build/ci/createJobFromRollout.sh $GITHUB_RUN_ID $DEPLOY_ENV
        env:
          # dependent on GITHUB_RUN_ID, which is implicitly passed in
          DEPLOY_ENV: sandbox-${{ steps.get-sha.outputs.sha_short }}
      - name: Wait For Job To Finish
        run: ./build/ci/waitForCIJob.bash
        timeout-minutes: 60
        env:
          # dependent on GITHUB_RUN_ID, which is implicitly passed in
          TEST_NAME: pytest
      - name: Get Logs From Cluster and propogate test result
        run: "kubectl logs --tail=-1 -l ci-run=$GITHUB_RUN_ID,test-name=pytest; LASTLINE=`kubectl logs --tail=1 -l ci-run=$GITHUB_RUN_ID,test-name=pytest`; STAT=${LASTLINE: -1}; exit $STAT"
      - name: Cleanup pyTest Pod
        run: kubectl delete jobs -l ci-run=$GITHUB_RUN_ID,test-name=pytest
        if: always()
  ending-notification:
    name: "Continuous Testing: Notifications"
    runs-on: ubuntu-latest
    if: ${{ always() && (github.event.pull_request.draft == false) }}
    needs:
      - pytest-job
    steps:
      - uses: actions/checkout@v4
      - uses: actions/setup-node@v2
        with:
          node-version: '14'
      - run: npm ci
        working-directory: ./build/notify
      - run: node notifyEnd.js
        working-directory: ./build/notify
        env:
          SLACK_TEST_SUCCESS_WEBHOOK_URL: ${{secrets.SLACK_TEST_SUCCESS_WEBHOOK_URL}}
          SLACK_TEST_FAILURE_WEBHOOK_URL: ${{secrets.SLACK_TEST_FAILURE_WEBHOOK_URL}}
          GITUSER_SLACK_MAP: ${{secrets.GITUSER_SLACK_MAP}}
          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
  sandbox-cleanup:
    name: "Continuous Testing: Clean up"
    if: ${{ always() && (github.event.pull_request.draft == false) }}
    needs:
      - pytest-job
    runs-on: ubuntu-latest
    permissions:
      contents: 'read'
      id-token: 'write'
    steps:
      - name: Checkout
        uses: actions/checkout@v4
      - id: auth
        name: Authenticate to Google Cloud
        uses: google-github-actions/auth@v2
        with:
          token_format: 'access_token'
          workload_identity_provider: 'projects/${{ secrets.DEV_GKE_PROJECT_ID}}/locations/global/workloadIdentityPools/github/providers/github'
          service_account: '${{ secrets.DEV_GKE_SA }}'
      - name: Setup GCloud
        uses: google-github-actions/setup-gcloud@v0
        with:
          project_id: ${{ secrets.DEV_PROJECT }}
          install_components: 'gke-gcloud-auth-plugin'
      - name: Authenticate GHA Runner To Target Cluster
        uses: google-github-actions/get-gke-credentials@v0
        with:
          cluster_name: ${{secrets.DEV_GKE_CLUSTER}}
          location: ${{secrets.DEV_GKE_REGION}}
          project_id: ${{secrets.DEV_GCP_PROJECT}}
      - name: Set outputs
        id: get-sha
        run: echo "sha_short=$(git rev-parse --short HEAD)" >> $GITHUB_OUTPUT
      - name: check helm
        id: get-helm
        run: echo "count=$(helm list -n $NAMESPACE | grep $NAME | wc -l)" >> $GITHUB_OUTPUT
        env:
          NAMESPACE: ${{ secrets.DEV_SANDBOX_NAMESPACE }}
          NAME: sandbox-${{ steps.get-sha.outputs.sha_short }}
      - name: Uninstall
        run: helm delete sandbox-${{ steps.get-sha.outputs.sha_short }} -n ${{ secrets.DEV_SANDBOX_NAMESPACE }} --debug --timeout 10m0s
        if: steps.get-helm.outputs.count > 0
  continuous-branch-protection:
    needs:
      - build-generic
      - build-derived
      - check-python-files
      - sandbox-deploy
      - sandbox-ready
      - pytest-job
    runs-on: ubuntu-latest
    if: always()
    steps:
      - name:
        run: |
          if [ "${{ github.event_name }}" == "merge_group" ]; then
            exit 0
          fi
          if [ "${{ needs.build-generic.result }}" != "success" ] || \
             [ "${{ needs.build-derived.result }}" != "success" ]; then
            echo "One or more build jobs failed"
            exit 1
          fi
          if [ "${{ github.event.pull_request.draft }}" == "false" ]; then
            if [ "${{ needs.check-python-files.outputs.python_files_changed }}" == "true" ]; then
              if [ "${{ needs.sandbox-deploy.result }}" != "success" ] || \
                 [ "${{ needs.sandbox-ready.result }}" != "success" ] || \
                 [ "${{ needs.pytest-job.result }}" != "success" ]; then
                echo "One or more Python-related test jobs failed"
                exit 1
              fi
            fi
          fi