reuse.benchmark.yml

name: Benchmark

on:
  workflow_call:
    inputs:
      sha:
        description: Git sha of benchmark
        required: true
        type: string
      run_id:
        description: The run id of benchmark
        required: true
        type: string
      source:
        description: The source of benchmark, pr/release
        required: true
        type: string
      source_id:
        description: The source id of benchmark, pr number/release tag
        required: true
        type: string
      version:
        description: The version of databend to run
        required: true
        type: string
      runner_provider:
        description: "Self-hosted runner provider, aws or gcp"
        type: string
        required: true
      target:
        description: "The target of benchmark, local or cloud or all"
        type: string
        required: true

permissions:
  id-token: write
  pull-requests: write
  contents: read

env:
  BUILD_PROFILE: release
  RUNNER_PROVIDER: ${{ inputs.runner_provider }}

jobs:
  local:
    timeout-minutes: 60
    runs-on: [self-hosted, X64, Linux, 16c32g, "${{ inputs.runner_provider }}"]
    strategy:
      matrix:
        dataset:
          - internal
          # - hits
      fail-fast: true
      max-parallel: 1
    steps:
      - uses: actions/checkout@v4
        if: inputs.source == 'release'
        with:
          ref: ${{ inputs.sha }}
      - uses: actions/checkout@v4
        if: inputs.source == 'pr'
        with:
          ref: "refs/pull/${{ inputs.source_id }}/merge"
      - uses: ./.github/actions/setup_bendsql
      - uses: actions/setup-python@v5
        with:
          python-version: "3.12"
      - name: Download artifact
        uses: ./.github/actions/artifact_download
        with:
          sha: ${{ inputs.sha }}
          target: x86_64-unknown-linux-gnu
          category: udf
          artifacts: meta,query
      - name: Setup Databend Binary
        shell: bash
        run: |
          sudo cp ./target/release/databend-* /usr/local/bin/
          databend-query --version
          databend-meta --version
      - uses: ./.github/actions/benchmark_local
        if: inputs.target == 'local' || inputs.target == 'all'
        timeout-minutes: 30
        id: benchmark
        with:
          sha: ${{ inputs.sha }}
          run_id: ${{ inputs.run_id }}
          dataset: ${{ matrix.dataset }}
          source: ${{ inputs.source }}
          source_id: ${{ inputs.source_id }}

  load:
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v4
        if: inputs.source == 'release'
        with:
          ref: ${{ inputs.sha }}
      - uses: actions/checkout@v4
        if: inputs.source == 'pr'
        with:
          ref: "refs/pull/${{ inputs.source_id }}/merge"
      - uses: ./.github/actions/setup_bendsql
      - uses: ./.github/actions/benchmark_cloud
        if: inputs.target == 'load' || inputs.target == 'all'
        timeout-minutes: 120
        id: benchmark
        with:
          sha: ${{ inputs.sha }}
          run_id: ${{ inputs.run_id }}
          dataset: load
          source: ${{ inputs.source }}
          source_id: ${{ inputs.source_id }}
          size: Small
          version: ${{ inputs.version }}
          cloud_user: ${{ secrets.BENCHMARK_CLOUD_USER }}
          cloud_password: ${{ secrets.BENCHMARK_CLOUD_PASSWORD }}
          cloud_gateway: ${{ secrets.BENCHMARK_CLOUD_GATEWAY }}
      - name: clean
        if: always()
        continue-on-error: true
        env:
          BENDSQL_DSN: "databend://${{ secrets.BENCHMARK_CLOUD_USER }}:${{ secrets.BENCHMARK_CLOUD_PASSWORD }}@${{ secrets.BENCHMARK_CLOUD_GATEWAY }}:443/?warehouse=default"
        run: |
          echo "DROP DATABASE IF EXISTS load_test_${{ inputs.run_id }};" | bendsql
          echo "DROP WAREHOUSE IF EXISTS 'benchmark-${{ inputs.run_id }}';" | bendsql

  cloud:
    runs-on: ubuntu-latest
    needs: load
    strategy:
      matrix:
        include:
          - { dataset: hits, size: Small }
          - { dataset: hits, size: Large }
          - { dataset: tpch, size: Small }
          - { dataset: tpch, size: Large }
      fail-fast: true
      max-parallel: 1
    steps:
      - uses: actions/checkout@v4
        if: inputs.source == 'release'
        with:
          ref: ${{ inputs.sha }}
      - uses: actions/checkout@v4
        if: inputs.source == 'pr'
        with:
          ref: "refs/pull/${{ inputs.source_id }}/merge"
      - uses: ./.github/actions/setup_bendsql
      - uses: ./.github/actions/benchmark_cloud
        if: inputs.target == 'cloud' || inputs.target == 'all'
        timeout-minutes: 20
        id: benchmark
        with:
          sha: ${{ inputs.sha }}
          run_id: ${{ inputs.run_id }}
          dataset: ${{ matrix.dataset }}
          source: ${{ inputs.source }}
          source_id: ${{ inputs.source_id }}
          size: ${{ matrix.size }}
          version: ${{ inputs.version }}
          cloud_user: ${{ secrets.BENCHMARK_CLOUD_USER }}
          cloud_password: ${{ secrets.BENCHMARK_CLOUD_PASSWORD }}
          cloud_gateway: ${{ secrets.BENCHMARK_CLOUD_GATEWAY }}
      - name: clean
        if: always()
        continue-on-error: true
        env:
          BENDSQL_DSN: "databend://${{ secrets.BENCHMARK_CLOUD_USER }}:${{ secrets.BENCHMARK_CLOUD_PASSWORD }}@${{ secrets.BENCHMARK_CLOUD_GATEWAY }}:443/?warehouse=default"
        run: |
          echo "DROP WAREHOUSE IF EXISTS 'benchmark-${{ inputs.run_id }}';" | bendsql

  comment:
    needs: [local, cloud]
    if: inputs.source == 'pr'
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v4
      - name: Install Dependencies
        run: |
          sudo apt-get update -yq
          sudo apt-get install -yq python3-jinja2
      - uses: actions/download-artifact@v4
        with:
          path: benchmark/clickbench/results
          pattern: benchmark-*
          merge-multiple: true
      - name: Get Report Prefix
        run: |
          for result in benchmark/clickbench/results/*.json; do
            dataset=$(echo $result | sed -E 's/.*result-(\w+)-.*\.json/\1/')
            mkdir -p benchmark/clickbench/results/${dataset}/
            mv $result benchmark/clickbench/results/${dataset}/$(basename $result)
          done
          echo "REPORT_S3_PREFIX=s3://benchmark/clickbench/pr/${{ inputs.source_id }}/${{ inputs.run_id }}" >> $GITHUB_ENV
      - name: Upload PR clickbench result to R2
        id: result
        env:
          AWS_ACCESS_KEY_ID: ${{ secrets.R2_ACCESS_KEY_ID }}
          AWS_SECRET_ACCESS_KEY: ${{ secrets.R2_SECRET_ACCESS_KEY }}
          AWS_DEFAULT_REGION: auto
          AWS_ENDPOINT_URL: ${{ secrets.R2_ENDPOINT_URL }}
        working-directory: benchmark/clickbench
        run: |
          echo -e "## ClickBench Report\n" > /tmp/body
          for p in results/*; do
            dataset=$(basename $p)
            aws s3 sync results/$dataset/ ${REPORT_S3_PREFIX}/ --include "*.json"
            aws s3 sync "s3://benchmark/clickbench/release/${dataset}/latest/" ./results/${dataset}/ --exclude "*" --include "*.json"
            ./update_results.py --dataset $dataset --pr ${{ inputs.source_id }}
            aws s3 cp ./results/${dataset}.html ${REPORT_S3_PREFIX}/${dataset}.html
            echo "* **${dataset}**: https://benchmark.databend.com/clickbench/pr/${{ inputs.source_id }}/${{ inputs.run_id }}/${dataset}.html" >> /tmp/body
          done
      - name: Comment on PR
        uses: everpcpc/comment-on-pr-action@v1
        with:
          token: ${{ github.token }}
          files: /tmp/body
          number: ${{ inputs.source_id }}

  archive:
    needs: [local, cloud]
    if: inputs.source == 'release'
    runs-on: ubuntu-latest
    strategy:
      matrix:
        dataset:
          - "tpch"
          - "hits"
          - "load"
          - "internal"
    steps:
      - uses: actions/checkout@v4
      - name: Install Dependencies
        run: |
          sudo apt-get update -yq
          sudo apt-get install -yq python3-jinja2
      - uses: actions/download-artifact@v4
        with:
          path: benchmark/clickbench/results
          pattern: benchmark-${{ matrix.dataset }}-*
          merge-multiple: true
      - name: Generate report and upload to R2
        working-directory: benchmark/clickbench
        env:
          AWS_ACCESS_KEY_ID: ${{ secrets.R2_ACCESS_KEY_ID }}
          AWS_SECRET_ACCESS_KEY: ${{ secrets.R2_SECRET_ACCESS_KEY }}
          AWS_DEFAULT_REGION: auto
          AWS_ENDPOINT_URL: ${{ secrets.R2_ENDPOINT_URL }}
        run: |
          for result in results/*.json; do
            dataset=$(echo $result | sed -E 's/.*result-(\w+)-.*\.json/\1/')
            mkdir -p results/${dataset}/
            mv $result results/${dataset}/$(basename $result)
          done

          aws s3 sync s3://benchmark/clickbench/release/${{ matrix.dataset }}/$(date --date='-1 month' -u +%Y)/$(date --date='-1 month' -u +%m)/ ./results/${{ matrix.dataset }}/ --no-progress
          aws s3 sync s3://benchmark/clickbench/release/${{ matrix.dataset }}/$(date -u +%Y)/$(date -u +%m)/ ./results/${{ matrix.dataset }}/ --no-progress
          ./update_results.py --dataset ${{ matrix.dataset }} --release ${{ inputs.source_id }}

          RESULT_PREFIX="s3://benchmark/clickbench/release/${{ matrix.dataset }}/$(date -u +%Y)/$(date -u +%m)/$(date -u +%Y-%m-%d)/${{ inputs.source_id }}"
          LATEST_PREFIX="s3://benchmark/clickbench/release/${{ matrix.dataset }}/latest/latest"
          for file in ./results/${{ matrix.dataset }}/*.json; do
            aws s3 cp $file "${RESULT_PREFIX}-$(basename $file)"
            aws s3 cp $file "${LATEST_PREFIX}-$(basename $file)"
          done

          aws s3 cp ./results/${{ matrix.dataset }}.html s3://benchmark/clickbench/release/${{ matrix.dataset }}.html