build.yml

name: Build
permissions:
  id-token: write # This is required for requesting the JWT
  contents: write # This is required for actions/checkout and create release
  pull-requests: write
  actions: write  # This is required for trigger another action which is used by JetBrains integrateion tests
on:
  pull_request:
    types: [ opened, edited ]
  push:
  workflow_dispatch:
    inputs:
      recreate_vm:
        required: true
        type: string
        description: "Whether to recreate the VM"
        default: "false"

jobs:
  create-runner:
    uses: gitpod-io/gce-github-runner/.github/workflows/create-vm.yml@main
    secrets:
      runner_token: ${{ secrets.SELF_HOSTED_GITHUB_RUNNER_TOKEN }}
      gcp_credentials: ${{ secrets.SELF_HOSTED_GITHUB_RUNNER_GCP_CREDENTIALS }}
    concurrency:
      group: ${{ github.ref == 'refs/heads/main' && github.run_id || github.sha }}-create-runner
      cancel-in-progress: false

  configuration:
    name: Configure job parameters
    runs-on: ${{ needs.create-runner.outputs.label }}
    needs: [ create-runner ]
    concurrency:
      group: ${{ github.ref == 'refs/heads/main' && github.run_id || github.sha }}-configuration
      cancel-in-progress: true
    outputs:
      is_main_branch: ${{ (github.head_ref || github.ref) == 'refs/heads/main' }}
      version: ${{ steps.branches.outputs.sanitized-branch-name }}-gha.${{github.run_number}}
      preview_enable: ${{ contains( steps.pr-details.outputs.pr_body, '[x] /werft with-preview') || (steps.output.outputs.with_integration_tests != '') }}
      preview_name: ${{ github.head_ref || github.ref_name }}
      build_no_cache: ${{ contains( steps.pr-details.outputs.pr_body, '[x] leeway-no-cache') }}
      build_no_test: ${{ contains( steps.pr-details.outputs.pr_body, '[x] /werft no-test') }}
      with_large_vm: ${{ contains( steps.pr-details.outputs.pr_body, '[X] /werft with-large-vm') || (steps.output.outputs.with_integration_tests != '') }}
      publish_to_npm: ${{ contains( steps.pr-details.outputs.pr_body, '[X] /werft publish-to-npm') }}
      publish_to_jbmp: ${{ contains( steps.pr-details.outputs.pr_body, '[X] /werft publish-to-jb-marketplace') }}
      with_preemptible: ${{ contains( steps.pr-details.outputs.pr_body, '[X] /werft preemptible') }}
      with_dedicated_emulation: ${{ contains( steps.pr-details.outputs.pr_body, '[X] with-dedicated-emulation') }}
      analytics: ${{ steps.output.outputs.analytics }}
      workspace_feature_flags: ${{ steps.output.outputs.workspace_feature_flags }}
      pr_no_diff_skip: ${{ steps.pr-diff.outputs.pr_no_diff_skip }}
      with_werft: ${{ steps.output.outputs.with-werft }}
      with_integration_tests: ${{ steps.output.outputs.with_integration_tests }}
      with_monitoring: ${{ contains( steps.pr-details.outputs.pr_body, '[x] with-monitoring') }}
      latest_ide_version: ${{ contains( steps.pr-details.outputs.pr_body, '[x] latest-ide-version=true') }}
      leeway_cache_bucket: ${{ steps.output.outputs.leeway_cache_bucket }}
      image_repo_base: ${{ steps.output.outputs.image_repo_base }}
      pr_number: ${{ steps.pr-details.outputs.number }}
      pr_body: ${{ steps.pr-details.outputs.pr_body }}
    steps:
      - name: "Determine Branch"
        id: branches
        uses: transferwise/sanitize-branch-name@v1
        # Since we trigger this worklow on other event types, besides pull_request
        # We use this action to help us get the pr body, as it's not included in push/workflow_dispatch events
      - uses: 8BitJonny/gh-get-current-pr@2.2.0
        id: pr-details
        with:
          sha: ${{ github.event.pull_request.head.sha }}
        # If the PR got edited, but no checkbox changes occurred, we don't want to run the whole build
        # Therefore we set a flag and use it to skip the next jobs
      - name: "Check PR diff"
        id: pr-diff
        if: (github.event_name == 'pull_request' && github.event.action == 'edited')
        env:
          PR_DESC: "${{  steps.pr-details.outputs.pr_body }}"
          OLD_BODY: "${{ github.event.changes.body.from }}"
        run: |
          if ! diff <(echo "$OLD_BODY") <(echo "$PR_DESC") | grep -e '\[x\]' -e '\[X\]'; then
             echo "pr_no_diff_skip=true" >> $GITHUB_OUTPUT
          fi
      - name: "Set outputs"
        id: output
        env:
          PR_DESC: "${{ steps.pr-details.outputs.pr_body }}"
          MAIN_BRANCH: ${{ (github.head_ref || github.ref) == 'refs/heads/main' }}
        shell: bash
        run: |
          {
            echo "workspace_feature_flags=$(echo "$PR_DESC" | grep -oiP '(?<=\[x\] workspace-feature-flags=).*?(?=\s*$)')"
            echo "with_integration_tests=$(echo "$PR_DESC" | grep -oiP '(?<=\[x\] with-integration-tests=).*?(?=\s*$)')"
            echo "analytics=$(echo "$PR_DESC" | grep -oiP '(?<=\[x\] analytics=).*?(?=\s*$)')"
            echo "leeway_cache_bucket=$([[ "$MAIN_BRANCH"  = "true" ]] && echo "leeway-cache-main-c514a01" || echo "leeway-cache-dev-3ac8ef5")"
            echo "image_repo_base=$([[ "$MAIN_BRANCH"  = "true" ]] && echo "eu.gcr.io/gitpod-core-dev" || echo "eu.gcr.io/gitpod-dev-artifact")"
          } >> $GITHUB_OUTPUT

  build-previewctl:
    name: Build previewctl
    if: |
      (needs.configuration.outputs.pr_no_diff_skip != 'true') &&
      (needs.configuration.outputs.preview_enable == 'true')
    needs: [ configuration, create-runner ]
    concurrency:
      group: ${{ github.ref == 'refs/heads/main' && github.run_id || github.sha }}-build-previewctl
      cancel-in-progress: ${{ needs.configuration.outputs.is_main_branch == 'false' }}
    runs-on: ${{ needs.create-runner.outputs.label }}
    container:
      image: eu.gcr.io/gitpod-core-dev/dev/dev-environment:main-gha.30393
    steps:
      - uses: actions/checkout@v4
      - name: Setup Environment
        uses: ./.github/actions/setup-environment
        with:
          identity_provider: ${{ github.ref == 'refs/heads/main' && secrets.CORE_DEV_PROVIDER || secrets.DEV_PREVIEW_PROVIDER }}
          service_account: ${{ github.ref == 'refs/heads/main' && secrets.CORE_DEV_SA || secrets.DEV_PREVIEW_SA }}
          leeway_segment_key: ${{ secrets.LEEWAY_SEGMENT_KEY }}
      - name: Build previewctl
        id: build
        shell: bash
        env:
          LEEWAY_REMOTE_CACHE_BUCKET: ${{needs.configuration.outputs.leeway_cache_bucket}}
        run: |
          leeway build dev/preview/previewctl:cli --cache remote
  infrastructure:
    needs: [ configuration, build-previewctl, create-runner ]
    if: |
      (needs.configuration.outputs.pr_no_diff_skip != 'true') &&
      (needs.configuration.outputs.preview_enable == 'true') &&
      (needs.configuration.outputs.is_main_branch != 'true')
    runs-on: ${{ needs.create-runner.outputs.label }}
    concurrency:
      group: ${{ github.ref == 'refs/heads/main' && github.run_id || github.sha }}-infrastructure
      cancel-in-progress: true
    steps:
      - uses: actions/checkout@v4
      - name: Setup Environment
        uses: ./.github/actions/setup-environment
        with:
          identity_provider: ${{ secrets.DEV_PREVIEW_PROVIDER }}
          service_account: ${{ secrets.DEV_PREVIEW_SA }}
          leeway_segment_key: ${{ secrets.LEEWAY_SEGMENT_KEY }}
      - name: Create preview environment infrastructure
        id: create
        uses: ./.github/actions/preview-create
        with:
          name: ${{ needs.configuration.outputs.preview_name }}
          previewctl_hash: ${{ needs.build-previewctl.outputs.previewctl_hash }}
          large_vm: ${{ needs.configuration.outputs.with_large_vm }}
          preemptible: ${{ needs.configuration.outputs.with_preemptible }}
          recreate_vm: ${{ inputs.recreate_vm }}
          image_repo_base: ${{needs.configuration.outputs.image_repo_base}}/build

  build-gitpod:
    name: Build Gitpod
    needs: [ configuration, create-runner ]
    runs-on: ${{ needs.create-runner.outputs.label }}
    concurrency:
      group: ${{ github.ref == 'refs/heads/main' && github.run_id || github.sha }}-build-gitpod
      cancel-in-progress: ${{ needs.configuration.outputs.is_main_branch == 'false' }}
    services:
      mysql:
        image: bitnami/mysql:8.0.33-debian-11-r24
        env:
          MYSQL_ROOT_PASSWORD: test
          #MYSQL_TCP_PORT: 23306 bitnami/mysql does not honor this, but has it's own:
          MYSQL_PORT_NUMBER: 23306
          MYSQL_AUTHENTICATION_PLUGIN: mysql_native_password
        ports:
          - 23306:23306
      redis:
        image: redis
        ports:
          - 6379:6379
    container:
      image: eu.gcr.io/gitpod-core-dev/dev/dev-environment:main-gha.30393
      env:
        DB_HOST: "mysql"
        DB_PORT: "23306"
        REDIS_HOST: "redis"
        # GitHub action + MySQL 8.0 need longer to initialize
        DB_RETRIES: 5
    steps:
      - uses: actions/checkout@v4
      - uses: ./.github/actions/setup-environment
        with:
          identity_provider: ${{ github.ref == 'refs/heads/main' && secrets.CORE_DEV_PROVIDER || secrets.DEV_PREVIEW_PROVIDER }}
          service_account: ${{ github.ref == 'refs/heads/main' && secrets.CORE_DEV_SA || secrets.DEV_PREVIEW_SA }}
          leeway_segment_key: ${{ secrets.LEEWAY_SEGMENT_KEY }}
      - name: Leeway Vet
        shell: bash
        run: |
          leeway vet --ignore-warnings
      - name: Pre-Commit Checks
        shell: bash
        run: |
          RESULT=0
          pre-commit run --show-diff-on-failure || RESULT=$?
          if [ $RESULT -ne 0 ]; then
            echo "::error title=The Pre-Commit Checks Failed.::Please run 'pre-commit run --show-diff-on-failure'"
          fi
          exit "$RESULT"
      - name: Dev Build
        id: dev-build
        env:
          JAVA_HOME: /home/gitpod/.sdkman/candidates/java/current
          VERSION: ${{needs.configuration.outputs.version}}
          LEEWAY_REMOTE_CACHE_BUCKET: ${{needs.configuration.outputs.leeway_cache_bucket}}
          IMAGE_REPO_BASE: ${{needs.configuration.outputs.image_repo_base}}/dev
        shell: bash
        run: |
          RESULT=0
          set -x

          leeway build dev:all \
            --docker-build-options network=host \
            --cache remote \
            -Dversion=$VERSION \
            -DlocalAppVersion=$VERSION \
            -DimageRepoBase=$IMAGE_REPO_BASE \
            --report large-report.html || RESULT=$?

          set +x

          # capture logs for the dev/image build, they're too big for GITHUB_STEP_SUMMARY
          pup -p -f large-report.html '[id="dev/image:docker"] + details + details' > remove.txt

          # remove logs for the dev/image build output from the report file
          grep -vxFf  remove.txt large-report.html > report.html

          cat report.html >> $GITHUB_STEP_SUMMARY

          exit $RESULT
      - name: Leeway Build
        id: leeway
        shell: bash
        env:
          NODE_OPTIONS: "--max_old_space_size=4096"
          JAVA_HOME: /home/gitpod/.sdkman/candidates/java/current
          VERSION: ${{needs.configuration.outputs.version}}
          PR_NO_CACHE: ${{needs.configuration.outputs.build_no_cache}}
          PR_NO_TEST: ${{needs.configuration.outputs.build_no_test}}
          NPM_AUTH_TOKEN: "${{ secrets.NPM_AUTH_TOKEN }}"
          PUBLISH_TO_NPM: ${{ needs.configuration.outputs.publish_to_npm == 'true'  || needs.configuration.outputs.is_main_branch == 'true' }}
          JB_MARKETPLACE_PUBLISH_TOKEN: "${{ secrets.JB_MARKETPLACE_PUBLISH_TOKEN }}"
          PUBLISH_TO_JBPM: ${{ needs.configuration.outputs.publish_to_jbmp == 'true'  || needs.configuration.outputs.is_main_branch == 'true' }}
          CODECOV_TOKEN: "${{ secrets.CODECOV_TOKEN }}"
          LEEWAY_REMOTE_CACHE_BUCKET: ${{needs.configuration.outputs.leeway_cache_bucket}}
          IMAGE_REPO_BASE: ${{needs.configuration.outputs.image_repo_base}}/build

          # SCM tokens for integration tests
          GITPOD_TEST_TOKEN_BITBUCKET: "${{ secrets.GITPOD_TEST_TOKEN_BITBUCKET }}"
          GITPOD_TEST_TOKEN_BITBUCKET_SERVER: "${{ secrets.GITPOD_TEST_TOKEN_BITBUCKET_SERVER }}"
          GITPOD_TEST_TOKEN_BITBUCKET_SERVER_WRITE: "${{ secrets.GITPOD_TEST_TOKEN_BITBUCKET_SERVER_WRITE }}"
          GITPOD_TEST_TOKEN_BITBUCKET_SERVER_READ: "${{ secrets.GITPOD_TEST_TOKEN_BITBUCKET_SERVER_READ }}"
          GITPOD_TEST_TOKEN_GITHUB: "${{ secrets.GITPOD_TEST_TOKEN_GITHUB }}"
          GITPOD_TEST_TOKEN_GITLAB: "${{ secrets.GITPOD_TEST_TOKEN_GITLAB }}"
          GITPOD_TEST_TOKEN_AZURE_DEVOPS: "${{ secrets.GITPOD_TEST_TOKEN_AZURE_DEVOPS }}"
        run: |
          [[ "$PR_NO_CACHE" = "true" ]] && CACHE="none"       || CACHE="remote"
          [[ "$PR_NO_TEST"  = "true" ]] && TEST="--dont-test" || TEST=""
          [[ "${PUBLISH_TO_NPM}" = 'true' ]] && NPM_PUBLISH_TRIGGER=$(date +%s%3N) || NPM_PUBLISH_TRIGGER="false"

          RESULT=0
          set -x
          # CI=true is a var set by GHA. Unsetting it for the build, as yarn builds treat warnings as errors if that var is set to true
          CI= leeway build \
            --cache $CACHE \
            $TEST \
            -Dversion=$VERSION \
            --docker-build-options network=host \
            --max-concurrent-tasks 1 \
            -DlocalAppVersion=$VERSION \
            -DpublishToNPM="${PUBLISH_TO_NPM}" \
            -DnpmPublishTrigger="${NPM_PUBLISH_TRIGGER}" \
            -DpublishToJBMarketplace="${PUBLISH_TO_JBPM}" \
            -DimageRepoBase=$IMAGE_REPO_BASE \
            --report report.html || RESULT=$?
          set +x

          cat report.html >> $GITHUB_STEP_SUMMARY

          SUMMARY_SIZE=$(wc -c < $GITHUB_STEP_SUMMARY)
          if [[ $SUMMARY_SIZE -gt 1000000 ]]; then
            head -c 1000000 $GITHUB_STEP_SUMMARY > $GITHUB_STEP_SUMMARY.tmp
            mv $GITHUB_STEP_SUMMARY.tmp $GITHUB_STEP_SUMMARY
            (
              echo '.....'
              echo ''
              echo ':x: **WARNING: Summary is too large and has been truncated.**'
              echo ''
            )  >> $GITHUB_STEP_SUMMARY
          fi

          exit $RESULT
      - name: Tag the release
        if: github.ref == 'refs/heads/main'
        run: |
          git config --global user.name $GITHUB_USER
          git config --global user.email $GITHUB_EMAIL
          git tag -a ${VERSION} -m "Release ${VERSION}"
          git push origin ${VERSION}
        env:
          GITHUB_USER: roboquat
          GITHUB_EMAIL: roboquat@gitpod.io
          VERSION: ${{ needs.configuration.outputs.version }}

  install-app:
    runs-on: ${{ needs.create-runner.outputs.label }}
    needs: [ configuration, build-gitpod, create-runner ]
    if: ${{ needs.configuration.outputs.is_main_branch == 'true' }}
    strategy:
      fail-fast: false
      matrix:
        env: [ staging, production ]
    steps:
      - uses: gitpod-io/gh-app-auth@v0.1
        id: auth
        with:
          private-key: ${{ secrets.ACTIONS_APP_PKEY }}
          app-id: 308947
          installation-id: 35574470
      - name: trigger installation
        uses: actions/github-script@v6
        with:
          github-token: ${{ steps.auth.outputs.token }}
          script: |
            const result = await github.rest.actions.createWorkflowDispatch({
              owner: context.repo.owner,
              repo: 'gitpod-dedicated',
              workflow_id: 'install-app.yaml',
              ref: 'main',
              inputs: {
                "version": '${{ needs.configuration.outputs.version }}',
                "repo": "https://github.com/gitpod-io/gitpod",
                "commit": "${{github.sha}}",
                "release": "${{ matrix.env }}",
                "env": "${{ matrix.env }}"
              }
            })

  install:
    name: "Install Gitpod"
    needs:
      - configuration
      - build-previewctl
      - build-gitpod
      - infrastructure
      - create-runner
    runs-on: ${{ needs.create-runner.outputs.label }}
    concurrency:
      group: ${{ github.ref == 'refs/heads/main' && github.run_id || github.sha }}-install
      cancel-in-progress: ${{ needs.configuration.outputs.is_main_branch == 'false' }}
    steps:
      - uses: actions/checkout@v4
      - name: Setup Environment
        uses: ./.github/actions/setup-environment
        with:
          identity_provider: ${{ secrets.DEV_PREVIEW_PROVIDER }}
          service_account: ${{ secrets.DEV_PREVIEW_SA }}
          leeway_segment_key: ${{ secrets.LEEWAY_SEGMENT_KEY }}
      - name: Deploy Gitpod to the preview environment
        id: deploy-gitpod
        uses: ./.github/actions/deploy-gitpod
        with:
          name: ${{ needs.configuration.outputs.preview_name }}
          version: ${{needs.configuration.outputs.version}}
          with_dedicated_emu: ${{needs.configuration.outputs.with_dedicated_emulation}}
          analytics: ${{needs.configuration.outputs.analytics}}
          workspace_feature_flags: ${{needs.configuration.outputs.workspace_feature_flags}}
          image_repo_base: ${{needs.configuration.outputs.image_repo_base}}/build
      - uses: actions/github-script@v6
        if: needs.configuration.outputs.pr_number != '' && contains(needs.configuration.outputs.pr_body, 'gitpod:summary')
        with:
          script: |
            const prNumber = ${{ needs.configuration.outputs.pr_number }};

            const summaryEncoded = `${{ steps.deploy-gitpod.outputs.report }}`;
            const summary = Buffer.from(summaryEncoded, 'base64').toString('utf8');

            const { data: pr } = await github.rest.pulls.get({
              ...context.repo,
              pull_number: prNumber,
            });

            const prBody = pr.body;

            const newBody = prBody.replace('gitpod:summary', summary);

            await github.rest.pulls.update({
              ...context.repo,
              pull_number: prNumber,
              body: newBody,
            });

  monitoring:
    name: "Install Monitoring Satellite"
    needs: [ infrastructure, build-previewctl, create-runner ]
    runs-on: ${{ needs.create-runner.outputs.label }}
    if: needs.configuration.outputs.with_monitoring == 'true'
    concurrency:
      group: ${{ github.ref == 'refs/heads/main' && github.run_id || github.sha }}-monitoring
      cancel-in-progress: true
    steps:
      - uses: actions/checkout@v4
      - name: Setup Environment
        uses: ./.github/actions/setup-environment
        with:
          identity_provider: ${{ secrets.DEV_PREVIEW_PROVIDER }}
          service_account: ${{ secrets.DEV_PREVIEW_SA }}
          leeway_segment_key: ${{ secrets.LEEWAY_SEGMENT_KEY }}
      - name: Deploy monitoring satellite to the preview environment
        id: deploy-monitoring-satellite
        uses: ./.github/actions/deploy-monitoring-satellite
        with:
          previewctl_hash: ${{ needs.build-previewctl.outputs.previewctl_hash }}
          image_repo_base: ${{needs.configuration.outputs.image_repo_base}}/build

  integration-test:
    name: "Run integration test"
    needs:
      - configuration
      - build-previewctl
      - build-gitpod
      - infrastructure
      - install
      - create-runner
    runs-on: ${{ needs.create-runner.outputs.label }}
    container:
      image: eu.gcr.io/gitpod-core-dev/dev/dev-environment:main-gha.30393
    if: needs.configuration.outputs.with_integration_tests != ''
    concurrency:
      group: ${{ needs.configuration.outputs.preview_name }}-integration-test
      cancel-in-progress: true
    steps:
      - uses: actions/checkout@v4
      - name: Run integration test
        id: integration-test
        uses: ./.github/actions/integration-tests
        with:
          preview_name: ${{ needs.configuration.outputs.preview_name }}
          test_suite: ${{ needs.configuration.outputs.with_integration_tests }}
          notify_slack_webhook: ''
          github_token: ${{ secrets.GITHUB_TOKEN }}
          latest_ide_version: ${{ needs.configuration.outputs.latest_ide_version }}
          test_build_id: ${{ github.run_id }}
          test_build_url: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}
          test_build_ref: ${{ github.head_ref || github.ref }}
          identity_provider: ${{ secrets.DEV_PREVIEW_PROVIDER }}
          service_account: ${{ secrets.DEV_PREVIEW_SA }}
          leeway_segment_key: ${{ secrets.LEEWAY_SEGMENT_KEY }}
          integration_test_username: ${{ secrets.WORKSPACE_INTEGRATION_TEST_USERNAME }}
          integration_test_usertoken: ${{ secrets.WORKSPACE_INTEGRATION_TEST_USER_TOKEN }}

  workspace-integration-tests-main:
    name: "Run workspace integration tests on main branch"
    needs:
      - configuration
      - build-gitpod
      - create-runner
    if: needs.configuration.outputs.is_main_branch == 'true'
    uses: ./.github/workflows/workspace-integration-tests.yml
    with:
      version: ${{ needs.configuration.outputs.version }}
    secrets: inherit

  ide-code-updates:
    name: "Run VS Code update jobs on main branch"
    needs:
      - configuration
      - build-gitpod
      - create-runner
    if: needs.configuration.outputs.is_main_branch == 'true'
    uses: ./.github/workflows/code-updates.yml
    secrets: inherit

  ide-jb-updates:
    name: "Run JetBrains update jobs on main branch"
    needs:
      - configuration
      - build-gitpod
      - create-runner
    if: needs.configuration.outputs.is_main_branch == 'true'
    uses: ./.github/workflows/jetbrains-updates.yml
    secrets: inherit

  delete-runner:
    if: always()
    needs:
      - create-runner
      - configuration
      - build-previewctl
      - infrastructure
      - build-gitpod
      - install-app
      - install
      - monitoring
      - integration-test
    uses: gitpod-io/gce-github-runner/.github/workflows/delete-vm.yml@main
    secrets:
      gcp_credentials: ${{ secrets.SELF_HOSTED_GITHUB_RUNNER_GCP_CREDENTIALS }}
    with:
      runner-label: ${{ needs.create-runner.outputs.label }}
      machine-zone: ${{ needs.create-runner.outputs.machine-zone }}