ide-integration-tests.yml

name: "IDE integration tests"
permissions:
  id-token: write # This is required for requesting the JWT
  contents: read  # This is required for actions/checkout
  actions: write  # This is required for trigger another action which is used by JetBrains integrateion tests
on:
  workflow_dispatch:
    inputs:
      name:
        required: true
        description: "The name of the preview environment"
        default: "latest"
      version:
        required: true
        description: "The version of Gitpod to install"
        default: "latest"
      image_repo_base:
        type: choice
        required: false
        description: "The base repo of image"
        options:
          - "eu.gcr.io/gitpod-core-dev/build"
          - "eu.gcr.io/gitpod-dev-artifact/build"
        default: "eu.gcr.io/gitpod-core-dev/build"
      skip_deploy:
        required: false
        description: "Skip deploy preview environment (debug only)"
      skip_delete:
        required: false
        description: "Skip delete preview environment (debug only)"
  schedule:
    - cron: "0 3 * * *"

jobs:
  create-runner:
    uses: gitpod-io/gce-github-runner/.github/workflows/create-vm.yml@main
    secrets:
      runner_token: ${{ secrets.SELF_HOSTED_GITHUB_RUNNER_TOKEN }}
      gcp_credentials: ${{ secrets.SELF_HOSTED_GITHUB_RUNNER_GCP_CREDENTIALS }}

  configuration:
    name: Configuration
    runs-on: ${{ needs.create-runner.outputs.label }}
    needs: [create-runner]
    container:
      image: eu.gcr.io/gitpod-core-dev/dev/dev-environment:main-gha.30393
    outputs:
      name: ${{ steps.configuration.outputs.name }}
      version: ${{ steps.configuration.outputs.version }}
      image_repo_base: ${{ steps.configuration.outputs.image_repo_base }}
    steps:
      - name: "Set outputs"
        id: configuration
        shell: bash
        env:
          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
        run: |
          if [[ '${{ github.event.inputs.name }}' != '' && '${{ github.event.inputs.name }}' != 'latest' ]]; then
              # The workflow was triggered by workflow_dispatch
              {
                  echo "version=${{ github.event.inputs.version }}"
                  echo "name=${{ github.event.inputs.name }}"
              } >> $GITHUB_OUTPUT
          else
              # others
              RUNID=$(gh run list -R gitpod-io/gitpod -b main -w build.yml --limit 1 --json databaseId --jq .[0].databaseId)
              gh run watch "$RUNID" -R gitpod-io/gitpod
              CONCLUSION=$(gh run view "$RUNID" -R gitpod-io/gitpod --json jobs --jq '.jobs[] | select(.name == "Build Gitpod") | {name, status, conclusion} | .conclusion')
              if [ "$CONCLUSION" != "success" ]; then
                echo main branch build is failed, see https://github.com/gitpod-io/gitpod/actions/runs/"$RUNID" for detail | tee -a $GITHUB_STEP_SUMMARY
                exit 1
              fi

              {
                  echo "version=$(gh run view "$RUNID" --log -R gitpod-io/gitpod | grep 'main-gha.[0-9]*' -o | head -n 1)"
                  echo "name=ide-integration-test-${{ github.run_id }}-${{ github.run_attempt }}"
              } >> $GITHUB_OUTPUT
          fi
          if [[ '${{ inputs.image_repo_base }}' != '' ]]; then
              {
                  echo "image_repo_base=${{ inputs.image_repo_base }}"
              } >> $GITHUB_OUTPUT
          else
              {
                  echo "image_repo_base=eu.gcr.io/gitpod-core-dev/build"
              } >> $GITHUB_OUTPUT
          fi
      - name: Slack Notification
        uses: rtCamp/action-slack-notify@v2
        if: failure()
        env:
          SLACK_WEBHOOK: ${{ secrets.IDE_SLACK_WEBHOOK }}
          SLACK_COLOR: ${{ job.status }}
          SLACK_MESSAGE: main branch build failed
          SLACK_FOOTER: "<https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }}|Workflow logs>"

  infrastructure:
    name: Create preview environment infrastructure
    needs: [configuration, create-runner]
    runs-on: ${{ needs.create-runner.outputs.label }}
    concurrency:
      group: ${{ needs.configuration.outputs.name }}-infrastructure
    steps:
      - uses: actions/checkout@v4
      - name: Setup Environment
        uses: ./.github/actions/setup-environment
        with:
          identity_provider: ${{ secrets.DEV_PREVIEW_PROVIDER }}
          service_account: ${{ secrets.DEV_PREVIEW_SA }}
          leeway_segment_key: ${{ secrets.LEEWAY_SEGMENT_KEY }}
      - name: Create preview environment infrastructure
        id: create
        uses: ./.github/actions/preview-create
        with:
          name: ${{ needs.configuration.outputs.name }}
          infrastructure_provider: gce
          large_vm: true
          preemptible: true
          image_repo_base: ${{ needs.configuration.outputs.image_repo_base }}
      - name: Deploy Gitpod to the preview environment
        id: deploy-gitpod
        if: github.event.inputs.skip_deploy != 'true'
        uses: ./.github/actions/deploy-gitpod
        with:
          name: ${{ needs.configuration.outputs.name }}
          version: ${{ needs.configuration.outputs.version}}
          image_repo_base: ${{ needs.configuration.outputs.image_repo_base }}

  check:
    name: Check for regressions
    needs: [configuration, infrastructure, create-runner]
    runs-on: ${{ needs.create-runner.outputs.label }}
    container:
      image: eu.gcr.io/gitpod-core-dev/dev/dev-environment:main-gha.30393
      volumes:
        - /var/tmp:/var/tmp
        - /tmp:/tmp
    steps:
      - uses: actions/checkout@v4
      - name: Setup Environment
        env:
          HOME: /home/gitpod
        uses: ./.github/actions/setup-environment
        with:
          identity_provider: ${{ secrets.DEV_PREVIEW_PROVIDER }}
          service_account: ${{ secrets.DEV_PREVIEW_SA }}
          leeway_segment_key: ${{ secrets.LEEWAY_SEGMENT_KEY }}
      - name: Integration Test
        shell: bash
        env:
          ROBOQUAT_TOKEN: ${{ secrets.GITHUB_TOKEN }}
          USERNAME: ${{ secrets.IDE_INTEGRATION_TEST_USERNAME }}
          USER_TOKEN: ${{ secrets.IDE_INTEGRATION_TEST_USER_TOKEN }}
          PREVIEW_NAME: ${{ needs.configuration.outputs.name }}
          TEST_BUILD_ID: ${{ github.run_id }}
          TEST_BUILD_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}
          TEST_BUILD_REF: ${{ github.head_ref || github.ref }}
          HOME: /home/gitpod
        run: |
          set -euo pipefail

          export LEEWAY_WORKSPACE_ROOT="$(pwd)"

          leeway run dev/preview/previewctl:install

          previewctl install-context --branch "${PREVIEW_NAME}" --log-level debug --timeout 1m

          # start integration test
          args=()
          args+=( "-kubeconfig=$HOME/.kube/config" )
          args+=( "-namespace=default" )
          [[ "$USERNAME" != "" ]] && args+=( "-username=$USERNAME" )
          args+=( "-timeout=60m" )

          IDE_TESTS_DIR="$GITHUB_WORKSPACE/test/tests/ide"
          JETBRAINS_TESTS="$IDE_TESTS_DIR/jetbrains"
          VSCODE_TESTS="$IDE_TESTS_DIR/vscode"
          SSH_TESTS="$IDE_TESTS_DIR/ssh"

          go install github.com/jstemmer/go-junit-report/v2@latest

          FAILURE_COUNT=0

          IDE_TEST_LIST=("$SSH_TESTS" "$VSCODE_TESTS" "$JETBRAINS_TESTS")
          for TEST_PATH in "${IDE_TEST_LIST[@]}"
          do
              TEST_NAME=$(basename "${TEST_PATH}")
              echo "running integration for ${TEST_NAME}"

              cd "${TEST_PATH}"
              set +e
              go test -parallel=3 -v ./... "${args[@]}" 2>&1 | go-junit-report -subtest-mode=exclude-parents -set-exit-code -out "TEST-${TEST_NAME}.xml" -iocopy
              RC=${PIPESTATUS[0]}
              set -e

              if [ "${RC}" -ne "0" ]; then
                FAILURE_COUNT=$((FAILURE_COUNT+1))
              fi
          done

          exit $FAILURE_COUNT
      - name: Test Summary
        id: test_summary
        uses: test-summary/action@v2
        with:
          paths: "test/tests/**/TEST-*.xml"
        if: always()
      - name: Slack Notification
        uses: rtCamp/action-slack-notify@v2
        if: success() || failure()
        env:
          SLACK_WEBHOOK: ${{ secrets.IDE_SLACK_WEBHOOK }}
          SLACK_COLOR: ${{ job.status }}
          SLACK_MESSAGE: ${{ steps.test_summary.outputs.passed }}/${{ steps.test_summary.outputs.total }} tests passed
          SLACK_FOOTER: "<https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }}|Workflow logs>"

  delete:
    name: Delete preview environment
    needs: [configuration, infrastructure, check, create-runner]
    if: github.event.inputs.skip_delete != 'true' && always()
    runs-on: ${{ needs.create-runner.outputs.label }}
    steps:
      - uses: actions/checkout@v4
      - name: Setup Environment
        uses: ./.github/actions/setup-environment
        with:
          identity_provider: ${{ secrets.DEV_PREVIEW_PROVIDER }}
          service_account: ${{ secrets.DEV_PREVIEW_SA }}
          leeway_segment_key: ${{ secrets.LEEWAY_SEGMENT_KEY }}
      - name: Delete preview environment
        uses: ./.github/actions/delete-preview
        with:
          name: ${{ needs.configuration.outputs.name }}

  delete-runner:
    if: always()
    needs:
      - create-runner
      - configuration
      - infrastructure
      - check
      - delete
    uses: gitpod-io/gce-github-runner/.github/workflows/delete-vm.yml@main
    secrets:
      gcp_credentials: ${{ secrets.SELF_HOSTED_GITHUB_RUNNER_GCP_CREDENTIALS }}
    with:
      runner-label: ${{ needs.create-runner.outputs.label }}
      machine-zone: ${{ needs.create-runner.outputs.machine-zone }}