develop.yml

on:
  merge_group:
  pull_request:
    types: [opened, synchronize, reopened, ready_for_review]
    paths-ignore:
      - 'docs/**'
      - 'config/**'
      - '**.md'
      - '.dockerignore'
      - 'docker/**'
      - '.gitignore'
      - 'grafana/**'
  push:
    branches:
      - main
    paths-ignore:
      - 'docs/**'
      - 'config/**'
      - '**.md'
      - '.dockerignore'
      - 'docker/**'
      - '.gitignore'
      - 'grafana/**'
  workflow_dispatch:

name: CI

concurrency:
  group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }}
  cancel-in-progress: true

env:
  RUST_TOOLCHAIN: nightly-2023-12-19

jobs:
  typos:
    name: Spell Check with Typos
    runs-on: ubuntu-20.04
    steps:
      - uses: actions/checkout@v4
      - uses: crate-ci/typos@v1.13.10

  check:
    name: Check
    runs-on: ${{ matrix.os }}
    strategy:
      matrix:
        os: [ windows-latest, ubuntu-20.04 ]
    timeout-minutes: 60
    steps:
      - uses: actions/checkout@v4
      - uses: arduino/setup-protoc@v3
        with:
          repo-token: ${{ secrets.GITHUB_TOKEN }}
      - uses: dtolnay/rust-toolchain@master
        with:
          toolchain: ${{ env.RUST_TOOLCHAIN }}
      - name: Rust Cache
        uses: Swatinem/rust-cache@v2
        with: 
          # Shares across multiple jobs
          # Shares with `Clippy` job
          shared-key: "check-lint"
      - name: Run cargo check
        run: cargo check --locked --workspace --all-targets

  toml:
    name: Toml Check
    runs-on: ubuntu-20.04
    timeout-minutes: 60
    steps:
      - uses: actions/checkout@v4
      - uses: dtolnay/rust-toolchain@master
        with:
          toolchain: stable
      - name: Rust Cache
        uses: Swatinem/rust-cache@v2
        with: 
          # Shares across multiple jobs
          shared-key: "check-toml"
      - name: Install taplo
        run: cargo +stable install taplo-cli --version ^0.9 --locked
      - name: Run taplo
        run: taplo format --check

  build:
    name: Build GreptimeDB binaries
    runs-on: ${{ matrix.os }}
    strategy:
      matrix:
        os: [ ubuntu-20.04 ]
    timeout-minutes: 60
    steps:
      - uses: actions/checkout@v4
      - uses: arduino/setup-protoc@v3
      - uses: dtolnay/rust-toolchain@master
        with:
          toolchain: ${{ env.RUST_TOOLCHAIN }}
      - uses: Swatinem/rust-cache@v2
        with:
          # Shares across multiple jobs
          shared-key: "build-binaries"
      - name: Build greptime binaries
        shell: bash
        run: cargo build
      - name: Pack greptime binaries
        shell: bash
        run: |
          mkdir bins && \
          mv ./target/debug/greptime bins && \
          mv ./target/debug/sqlness-runner bins
      - name: Print greptime binaries info
        run: ls -lh bins
      - name: Upload artifacts
        uses: ./.github/actions/upload-artifacts
        with:
          artifacts-dir: bins
          version: current

  sqlness:
    name: Sqlness Test
    needs: build
    runs-on: ${{ matrix.os }}
    strategy:
      matrix:
        os: [ ubuntu-20.04 ]
    timeout-minutes: 60
    steps:
      - uses: actions/checkout@v4
      - name: Download pre-built binaries
        uses: actions/download-artifact@v4
        with:
          name: bins
          path: .
      - name: Unzip binaries
        run: tar -xvf ./bins.tar.gz
      - name: Run sqlness
        run: RUST_BACKTRACE=1 ./bins/sqlness-runner -c ./tests/cases --bins-dir ./bins
      # FIXME: Logs cannot found be on failure (or even success). Need to figure out the cause.
      - name: Upload sqlness logs
        if: always()
        uses: actions/upload-artifact@v3
        with:
          name: sqlness-logs
          path: ${{ runner.temp }}/greptime-*.log
          retention-days: 3

  sqlness-kafka-wal:
    name: Sqlness Test with Kafka Wal
    needs: build
    runs-on: ${{ matrix.os }}
    strategy:
      matrix:
        os: [ ubuntu-20.04 ]
    timeout-minutes: 60
    steps:
      - uses: actions/checkout@v4
      - name: Download pre-built binaries
        uses: actions/download-artifact@v4
        with:
          name: bins
          path: .
      - name: Unzip binaries
        run: tar -xvf ./bins.tar.gz
      - name: Setup kafka server
        working-directory: tests-integration/fixtures/kafka
        run: docker compose -f docker-compose-standalone.yml up -d --wait
      - name: Run sqlness
        run: RUST_BACKTRACE=1 ./bins/sqlness-runner -w kafka -k 127.0.0.1:9092 -c ./tests/cases --bins-dir ./bins
      # FIXME: Logs cannot be found on failure (or even success). Need to figure out the cause.
      - name: Upload sqlness logs
        if: always()
        uses: actions/upload-artifact@v3
        with:
          name: sqlness-logs
          path: ${{ runner.temp }}/greptime-*.log
          retention-days: 3

  fmt:
    name: Rustfmt
    runs-on: ubuntu-20.04
    timeout-minutes: 60
    steps:
      - uses: actions/checkout@v4
      - uses: arduino/setup-protoc@v3
        with:
          repo-token: ${{ secrets.GITHUB_TOKEN }}
      - uses: dtolnay/rust-toolchain@master
        with:
          toolchain: ${{ env.RUST_TOOLCHAIN }}
          components: rustfmt
      - name: Rust Cache
        uses: Swatinem/rust-cache@v2
        with: 
          # Shares across multiple jobs
          shared-key: "check-rust-fmt"
      - name: Run cargo fmt
        run: cargo fmt --all -- --check

  clippy:
    name: Clippy
    runs-on: ubuntu-20.04
    timeout-minutes: 60
    steps:
      - uses: actions/checkout@v4
      - uses: arduino/setup-protoc@v3
        with:
          repo-token: ${{ secrets.GITHUB_TOKEN }}
      - uses: dtolnay/rust-toolchain@master
        with:
          toolchain: ${{ env.RUST_TOOLCHAIN }}
          components: clippy
      - name: Rust Cache
        uses: Swatinem/rust-cache@v2
        with: 
          # Shares across multiple jobs
          # Shares with `Check` job
          shared-key: "check-lint"
      - name: Run cargo clippy
        run: cargo clippy --workspace --all-targets -- -D warnings

  coverage:
    if: github.event.pull_request.draft == false
    runs-on: ubuntu-20.04-8-cores
    timeout-minutes: 60
    steps:
      - uses: actions/checkout@v4
      - uses: arduino/setup-protoc@v3
        with:
          repo-token: ${{ secrets.GITHUB_TOKEN }}
      - uses: KyleMayes/install-llvm-action@v1
        with:
          version: "14.0"
      - name: Install toolchain
        uses: dtolnay/rust-toolchain@master
        with:
          toolchain: ${{ env.RUST_TOOLCHAIN }}
          components: llvm-tools-preview
      - name: Rust Cache
        uses: Swatinem/rust-cache@v2
        with:
          # Shares cross multiple jobs
          shared-key: "coverage-test"
      - name: Install latest nextest release
        uses: taiki-e/install-action@nextest
      - name: Install cargo-llvm-cov
        uses: taiki-e/install-action@cargo-llvm-cov
      - name: Install Python
        uses: actions/setup-python@v5
        with:
          python-version: '3.10'
      - name: Install PyArrow Package
        run: pip install pyarrow
      - name: Setup etcd server
        working-directory: tests-integration/fixtures/etcd
        run: docker compose -f docker-compose-standalone.yml up -d --wait
      - name: Setup kafka server
        working-directory: tests-integration/fixtures/kafka
        run: docker compose -f docker-compose-standalone.yml up -d --wait
      - name: Run nextest cases
        run: cargo llvm-cov nextest --workspace --lcov --output-path lcov.info -F pyo3_backend -F dashboard
        env:
          CARGO_BUILD_RUSTFLAGS: "-C link-arg=-fuse-ld=lld"
          RUST_BACKTRACE: 1
          CARGO_INCREMENTAL: 0
          GT_S3_BUCKET: ${{ secrets.S3_BUCKET }}
          GT_S3_ACCESS_KEY_ID: ${{ secrets.S3_ACCESS_KEY_ID }}
          GT_S3_ACCESS_KEY: ${{ secrets.S3_ACCESS_KEY }}
          GT_S3_REGION: ${{ secrets.S3_REGION }}
          GT_ETCD_ENDPOINTS: http://127.0.0.1:2379
          GT_KAFKA_ENDPOINTS: 127.0.0.1:9092
          UNITTEST_LOG_DIR: "__unittest_logs"
      - name: Codecov upload
        uses: codecov/codecov-action@v2
        with:
          token: ${{ secrets.CODECOV_TOKEN }}
          files: ./lcov.info
          flags: rust
          fail_ci_if_error: false
          verbose: true

  compat:
    name: Compatibility Test
    needs: build
    runs-on: ubuntu-20.04
    timeout-minutes: 60
    steps:
      - uses: actions/checkout@v3
      - name: Download pre-built binaries
        uses: actions/download-artifact@v4
        with:
          name: bins
          path: .
      - name: Unzip binaries
        run: |
          mkdir -p ./bins/current
          tar -xvf ./bins.tar.gz --strip-components=1 -C ./bins/current
      - run: ./tests/compat/test-compat.sh 0.6.0