Run integration tests in parallel to speed up testing #2933
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Go Tests | |
| on: | |
| workflow_dispatch: | |
| pull_request: | |
| schedule: | |
| - cron: '0 0 * * *' | |
| push: | |
| branches: [ master ] | |
| jobs: | |
| test: | |
| runs-on: ${{ matrix.os }} | |
| strategy: | |
| matrix: | |
| os: [ubuntu-latest, macos-latest, macos-14] | |
| test_shard: ["0", "1", "2", "3", "4", "5", "6"] | |
| fail-fast: false | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - name: Set up Go | |
| uses: actions/setup-go@v4 | |
| with: | |
| go-version: 1.21 | |
| - name: Linux Setup | |
| if: ${{ matrix.os == 'ubuntu-latest'}} | |
| run: | | |
| # Install our dependencies | |
| sudo apt-get update | |
| sudo apt-get install -y zsh tmux fish | |
| # Work around a weird bug where zsh on ubuntu actions gives that directory 0777 which makes zsh refuse to start | |
| sudo chmod 0755 -R /usr/share/zsh/ | |
| # Set a consistent hostname so we can run tests that depend on it | |
| sudo hostname ghaction-runner-hostname | |
| - name: MacOS Setup | |
| if: ${{ matrix.os == 'macos-latest' || matrix.os == 'macos-14' }} | |
| run: | | |
| # Install our dependencies | |
| brew install fish tmux bash | |
| # Set a consistent hostname so we can run tests that depend on it | |
| sudo scutil --set HostName ghaction-runner-hostname | |
| - name: MacOS Docker Setup | |
| if: ${{ matrix.os == 'macos-latest' || matrix.os == 'macos-14 '}} | |
| continue-on-error: true | |
| run: | | |
| # Install docker so it can be used for datadog | |
| brew install docker | |
| colima start | |
| sudo ln -sf $HOME/.colima/default/docker.sock /var/run/docker.sock | |
| - name: Set up Datadog | |
| if: ${{ github.ref == 'refs/heads/master' && matrix.os != 'macos-14' }} | |
| uses: datadog/[email protected] | |
| with: | |
| api_key: ${{ secrets.DD_API_KEY }} | |
| - name: Extra Delay | |
| if: ${{ startsWith(github.event.head_commit.message, 'Release') }} | |
| run: | | |
| # If this is a release, then sleep for before starting the tests so that the newest version is released | |
| # and pushed to the updated server before we run the tests | |
| sleep 1200 # 20 minutes | |
| - name: Go test | |
| env: | |
| DD_API_KEY: ${{ secrets.DD_API_KEY }} | |
| OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }} | |
| run: | | |
| go install gotest.tools/gotestsum@bc98120 | |
| NUM_TEST_SHARDS=6 CURRENT_SHARD_NUM=${{ matrix.test_shard }} make test | |
| - name: Extra Delay | |
| run: | | |
| # Add an extra short delay to allow datadog to flush metrics | |
| sleep 300 # 5 minutes | |
| - name: Upload test results json | |
| uses: actions/upload-artifact@v3 | |
| if: success() || failure() | |
| with: | |
| name: test-results-${{ matrix.os }}-${{ matrix.test_shard }}.json | |
| path: /tmp/testrun.json | |
| - name: Upload failed test goldens | |
| uses: actions/upload-artifact@v3 | |
| if: success() || failure() | |
| with: | |
| name: test-goldens-${{ matrix.os }}-${{ matrix.test_shard }}.zip | |
| path: /tmp/test-goldens/ | |
| - name: Upload test log | |
| uses: actions/upload-artifact@v3 | |
| if: success() || failure() | |
| with: | |
| name: testlog-${{ matrix.os }}-${{ matrix.test_shard }}.txt | |
| path: /tmp/test.log | |
| - name: Upload test log | |
| uses: actions/upload-artifact@v4 | |
| if: success() || failure() | |
| with: | |
| name: goldens-used-${{ matrix.os }}-${{ matrix.test_shard }} | |
| path: /tmp/goldens-used.txt | |
| # - name: Setup tmate session | |
| # if: ${{ failure() }} | |
| # uses: mxschmitt/action-tmate@v3 | |
| # with: | |
| # limit-access-to-actor: true | |
| check-goldens: | |
| runs-on: ubuntu-latest | |
| needs: test | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - name: Set up Go | |
| uses: actions/setup-go@v4 | |
| with: | |
| go-version: 1.21 | |
| - name: Download artifact | |
| uses: actions/download-artifact@v4 | |
| - name: Check all goldens were used | |
| run: | | |
| go run client/posttest/main.go check-goldens |