release.yml

name: Release

# There are two kinds of formal release:
# 1. The tag('v*.*.*') push release: the release workflow will be triggered by the tag push event.
# 2. The scheduled release(the version will be '${{ env.NEXT_RELEASE_VERSION }}-nightly-YYYYMMDD'): the release workflow will be triggered by the schedule event.
on:
  push:
    tags:
      - "v*.*.*"
  schedule:
    # At 00:00 on Monday.
    - cron: '0 0 * * 1'
  workflow_dispatch: # Allows you to run this workflow manually.
    # Notes: The GitHub Actions ONLY support 10 inputs, and it's already used up.
    inputs:
      linux_amd64_runner:
        type: choice
        description: The runner uses to build linux-amd64 artifacts
        default: ec2-c6i.4xlarge-amd64
        options:
          - ubuntu-latest
          - ubuntu-latest-8-cores
          - ubuntu-latest-16-cores
          - ubuntu-latest-32-cores
          - ubuntu-latest-64-cores
          - ec2-c6i.xlarge-amd64 # 4C8G
          - ec2-c6i.2xlarge-amd64 # 8C16G
          - ec2-c6i.4xlarge-amd64 # 16C32G
          - ec2-c6i.8xlarge-amd64 # 32C64G
          - ec2-c6i.16xlarge-amd64 # 64C128G
      linux_arm64_runner:
        type: choice
        description: The runner uses to build linux-arm64 artifacts
        default: ec2-c6g.4xlarge-arm64
        options:
          - ec2-c6g.xlarge-arm64 # 4C8G
          - ec2-c6g.2xlarge-arm64 # 8C16G
          - ec2-c6g.4xlarge-arm64 # 16C32G
          - ec2-c6g.8xlarge-arm64 # 32C64G
          - ec2-c6g.16xlarge-arm64 # 64C128G
      macos_runner:
        type: choice
        description: The runner uses to build macOS artifacts
        default: macos-latest
        options:
          - macos-latest
      skip_test:
        description: Do not run integration tests during the build
        type: boolean
        default: true
      build_linux_amd64_artifacts:
        type: boolean
        description: Build linux-amd64 artifacts
        required: false
        default: false
      build_linux_arm64_artifacts:
        type: boolean
        description: Build linux-arm64 artifacts
        required: false
        default: false
      build_macos_artifacts:
        type: boolean
        description: Build macos artifacts
        required: false
        default: false
      release_artifacts:
        type: boolean
        description: Create GitHub release and upload artifacts
        required: false
        default: false
      release_images:
        type: boolean
        description: Build and push images to DockerHub and ACR
        required: false
        default: false
      release_dev_builder_image:
        type: boolean
        description: Release dev-builder image
        required: false
        default: false

# Use env variables to control all the release process.
env:
  # The arguments of building greptime.
  RUST_TOOLCHAIN: nightly-2023-08-07
  CARGO_PROFILE: nightly

  # Controls whether to run tests, include unit-test, integration-test and sqlness.
  DISABLE_RUN_TESTS: ${{ inputs.skip_test || vars.DEFAULT_SKIP_TEST }}

  # The scheduled version is '${{ env.NEXT_RELEASE_VERSION }}-nightly-YYYYMMDD', like v0.2.0-nigthly-20230313;
  NIGHTLY_RELEASE_PREFIX: nightly
  # Note: The NEXT_RELEASE_VERSION should be modified manually by every formal release.
  NEXT_RELEASE_VERSION: v0.4.0

jobs:
  allocate-runners:
    name: Allocate runners
    if: ${{ github.repository == 'GreptimeTeam/greptimedb' }}
    runs-on: ubuntu-latest
    outputs:
      linux-amd64-runner: ${{ steps.start-linux-amd64-runner.outputs.label }}
      linux-arm64-runner: ${{ steps.start-linux-arm64-runner.outputs.label }}
      macos-runner: ${{ inputs.macos_runner || vars.DEFAULT_MACOS_RUNNER }}

      # The following EC2 resource id will be used for resource releasing.
      linux-amd64-ec2-runner-label: ${{ steps.start-linux-amd64-runner.outputs.label }}
      linux-amd64-ec2-runner-instance-id: ${{ steps.start-linux-amd64-runner.outputs.ec2-instance-id }}
      linux-arm64-ec2-runner-label: ${{ steps.start-linux-arm64-runner.outputs.label }}
      linux-arm64-ec2-runner-instance-id: ${{ steps.start-linux-arm64-runner.outputs.ec2-instance-id }}

      # The 'version' use as the global tag name of the release workflow.
      version: ${{ steps.create-version.outputs.version }}
    steps:
      - name: Checkout
        uses: actions/checkout@v3
        with:
          fetch-depth: 0

      # The create-version will create a global variable named 'version' in the global workflows.
      # - If it's a tag push release, the version is the tag name(${{ github.ref_name }});
      # - If it's a scheduled release, the version is '${{ env.NEXT_RELEASE_VERSION }}-nightly-$buildTime', like v0.2.0-nigthly-20230313;
      # - If it's a manual release, the version is '${{ env.NEXT_RELEASE_VERSION }}-<short-git-sha>-YYYYMMDDSS', like v0.2.0-e5b243c-2023071245;
      - name: Create version
        id: create-version
        run: |
          echo "version=$(./.github/scripts/create-version.sh)" >> $GITHUB_OUTPUT
        env:
          GITHUB_EVENT_NAME: ${{ github.event_name }}
          GITHUB_REF_NAME: ${{ github.ref_name }}
          NEXT_RELEASE_VERSION: ${{ env.NEXT_RELEASE_VERSION }}
          NIGHTLY_RELEASE_PREFIX: ${{ env.NIGHTLY_RELEASE_PREFIX }}

      - name: Allocate linux-amd64 runner
        if: ${{ inputs.build_linux_amd64_artifacts || github.event_name == 'push' || github.event_name == 'schedule' }}
        uses: ./.github/actions/start-runner
        id: start-linux-amd64-runner
        with:
          runner: ${{ inputs.linux_amd64_runner || vars.DEFAULT_AMD64_RUNNER }}
          aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }}
          aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
          aws-region: ${{ vars.EC2_RUNNER_REGION }}
          github-token: ${{ secrets.GH_PERSONAL_ACCESS_TOKEN }}
          image-id: ${{ vars.EC2_RUNNER_LINUX_AMD64_IMAGE_ID }}
          security-group-id: ${{ vars.EC2_RUNNER_SECURITY_GROUP_ID }}
          subnet-id: ${{ vars.EC2_RUNNER_SUBNET_ID }}

      - name: Allocate linux-arm64 runner
        if: ${{ inputs.build_linux_arm64_artifacts || github.event_name == 'push' || github.event_name == 'schedule' }}
        uses: ./.github/actions/start-runner
        id: start-linux-arm64-runner
        with:
          runner: ${{ inputs.linux_arm64_runner || vars.DEFAULT_ARM64_RUNNER }}
          aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }}
          aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
          aws-region: ${{ vars.EC2_RUNNER_REGION }}
          github-token: ${{ secrets.GH_PERSONAL_ACCESS_TOKEN }}
          image-id: ${{ vars.EC2_RUNNER_LINUX_ARM64_IMAGE_ID }}
          security-group-id: ${{ vars.EC2_RUNNER_SECURITY_GROUP_ID }}
          subnet-id: ${{ vars.EC2_RUNNER_SUBNET_ID }}

  build-linux-amd64-artifacts:
    name: Build linux-amd64 artifacts
    if: ${{ inputs.build_linux_amd64_artifacts || github.event_name == 'push' || github.event_name == 'schedule' }}
    needs: [
      allocate-runners,
    ]
    runs-on: ${{ needs.allocate-runners.outputs.linux-amd64-runner }}
    steps:
      - uses: actions/checkout@v3
        with:
          fetch-depth: 0

      - uses: ./.github/actions/build-linux-artifacts
        with:
          arch: amd64
          cargo-profile: ${{ env.CARGO_PROFILE }}
          version: ${{ needs.allocate-runners.outputs.version }}
          disable-run-tests: ${{ env.DISABLE_RUN_TESTS }}
          release-to-s3-bucket: ${{ vars.AWS_RELEASE_BUCKET }}
          aws-access-key-id: ${{ secrets.AWS_CN_ACCESS_KEY_ID }}
          aws-secret-access-key: ${{ secrets.AWS_CN_SECRET_ACCESS_KEY }}
          aws-region: ${{ vars.AWS_RELEASE_BUCKET_REGION }}
          upload-to-s3: ${{ vars.UPLOAD_TO_S3 }}

  build-linux-arm64-artifacts:
    name: Build linux-arm64 artifacts
    if: ${{ inputs.build_linux_arm64_artifacts || github.event_name == 'push' || github.event_name == 'schedule' }}
    needs: [
      allocate-runners,
    ]
    runs-on: ${{ needs.allocate-runners.outputs.linux-arm64-runner }}
    steps:
      - uses: actions/checkout@v3
        with:
          fetch-depth: 0

      - uses: ./.github/actions/build-linux-artifacts
        with:
          arch: arm64
          cargo-profile: ${{ env.CARGO_PROFILE }}
          version: ${{ needs.allocate-runners.outputs.version }}
          disable-run-tests: ${{ env.DISABLE_RUN_TESTS }}
          release-to-s3-bucket: ${{ vars.AWS_RELEASE_BUCKET }}
          aws-access-key-id: ${{ secrets.AWS_CN_ACCESS_KEY_ID }}
          aws-secret-access-key: ${{ secrets.AWS_CN_SECRET_ACCESS_KEY }}
          aws-region: ${{ vars.AWS_RELEASE_BUCKET_REGION }}
          upload-to-s3: ${{ vars.UPLOAD_TO_S3 }}

  build-macos-artifacts:
    name: Build macOS artifacts
    strategy:
      fail-fast: false
      matrix:
        include:
          - os: ${{ needs.allocate-runners.outputs.macos-runner }}
            arch: aarch64-apple-darwin
            features: servers/dashboard
            artifacts-dir-prefix: greptime-darwin-arm64
          - os: ${{ needs.allocate-runners.outputs.macos-runner }}
            arch: aarch64-apple-darwin
            features: pyo3_backend,servers/dashboard
            artifacts-dir-prefix: greptime-darwin-arm64-pyo3
          - os: ${{ needs.allocate-runners.outputs.macos-runner }}
            features: servers/dashboard
            arch: x86_64-apple-darwin
            artifacts-dir-prefix: greptime-darwin-amd64
          - os: ${{ needs.allocate-runners.outputs.macos-runner }}
            features: pyo3_backend,servers/dashboard
            arch: x86_64-apple-darwin
            artifacts-dir-prefix: greptime-darwin-amd64-pyo3
    runs-on: ${{ matrix.os }}
    needs: [
      allocate-runners,
    ]
    if: ${{ inputs.build_macos_artifacts || github.event_name == 'push' || github.event_name == 'schedule' }}
    steps:
      - uses: actions/checkout@v3
        with:
          fetch-depth: 0

      - uses: ./.github/actions/build-macos-artifacts
        with:
          arch: ${{ matrix.arch }}
          rust-toolchain: ${{ env.RUST_TOOLCHAIN }}
          cargo-profile: ${{ env.CARGO_PROFILE }}
          features: ${{ matrix.features }}
          version: ${{ needs.allocate-runners.outputs.version }}
          disable-run-tests: ${{ env.DISABLE_RUN_TESTS }}
          release-to-s3-bucket: ${{ vars.AWS_RELEASE_BUCKET }}
          artifacts-dir: ${{ matrix.artifacts-dir-prefix }}-${{ needs.allocate-runners.outputs.version }}
          aws-access-key-id: ${{ secrets.AWS_CN_ACCESS_KEY_ID }}
          aws-secret-access-key: ${{ secrets.AWS_CN_SECRET_ACCESS_KEY }}
          aws-region: ${{ vars.AWS_RELEASE_BUCKET_REGION }}
          upload-to-s3: ${{ vars.UPLOAD_TO_S3 }}

  release-images-to-dockerhub:
    name: Build and push images to DockerHub
    if: ${{ inputs.release_images || github.event_name == 'push' || github.event_name == 'schedule' }}
    needs: [
      allocate-runners,
      build-linux-amd64-artifacts,
      build-linux-arm64-artifacts,
    ]
    runs-on: ubuntu-2004-16-cores
    steps:
      - uses: actions/checkout@v3
        with:
          fetch-depth: 0

      - name: Build and push images to dockerhub
        uses: ./.github/actions/build-images
        with:
          image-registry: docker.io
          image-namespace: ${{ vars.IMAGE_NAMESPACE }}
          image-registry-username: ${{ secrets.DOCKERHUB_USERNAME }}
          image-registry-password: ${{ secrets.DOCKERHUB_TOKEN }}
          version: ${{ needs.allocate-runners.outputs.version }}

  release-images-to-acr:
    name: Build and push images to ACR
    if: ${{ inputs.release_images || github.event_name == 'push' || github.event_name == 'schedule' }}
    needs: [
      allocate-runners,
      build-linux-amd64-artifacts,
      build-linux-arm64-artifacts,
    ]
    runs-on: ubuntu-2004-16-cores
    # When we push to ACR, it's easy to fail due to some unknown network issues.
    # However, we don't want to fail the whole workflow because of this.
    # The ACR have daily sync with DockerHub, so don't worry about the image not being updated.
    continue-on-error: true
    steps:
      - uses: actions/checkout@v3
        with:
          fetch-depth: 0

      - name: Build and push images to ACR
        uses: ./.github/actions/build-images
        with:
          image-registry: ${{ vars.ACR_IMAGE_REGISTRY }}
          image-namespace: ${{ vars.IMAGE_NAMESPACE }}
          image-registry-username: ${{ secrets.ALICLOUD_USERNAME }}
          image-registry-password: ${{ secrets.ALICLOUD_PASSWORD }}
          version: ${{ needs.allocate-runners.outputs.version }}

  release-artifacts:
    name: Create GitHub release and upload artifacts
    if: ${{ inputs.release_artifacts || github.event_name == 'push' || github.event_name == 'schedule' }}
    needs: [
      allocate-runners,
      build-linux-amd64-artifacts,
      build-linux-arm64-artifacts,
      build-macos-artifacts,
      release-images-to-dockerhub,
    ]
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v3
        with:
          fetch-depth: 0

      - name: Release artifacts
        uses: ./.github/actions/release-artifacts
        with:
          version: ${{ needs.allocate-runners.outputs.version }}

  release-dev-builder-image:
    name: Release dev builder image
    if: ${{ inputs.release_dev_builder_image }} # Only manually trigger this job.
    runs-on: ubuntu-latest-16-cores
    steps:
      - name: Checkout
        uses: actions/checkout@v3
        with:
          fetch-depth: 0

      - name: Build and push dev builder image
        uses: ./.github/actions/build-dev-builder-image
        with:
          dockerhub-image-registry-username: ${{ secrets.DOCKERHUB_USERNAME }}
          dockerhub-image-registry-token: ${{ secrets.DOCKERHUB_TOKEN }}
          acr-image-registry: ${{ vars.ACR_IMAGE_REGISTRY }}
          acr-image-registry-username: ${{ secrets.ALICLOUD_USERNAME }}
          acr-image-registry-password: ${{ secrets.ALICLOUD_PASSWORD }}

  ### Stop runners ###
  # It's very necessary to split the job of releasing runners into 'stop-linux-amd64-runner' and 'stop-linux-arm64-runner'.
  # Because we can terminate the specified EC2 instance immediately after the job is finished without uncessary waiting.
  stop-linux-amd64-runner: # It's always run as the last job in the workflow to make sure that the runner is released.
    name: Stop linux-amd64 runner
    # Only run this job when the runner is allocated.
    if: ${{ always() }}
    runs-on: ubuntu-latest
    needs: [
      allocate-runners,
      build-linux-amd64-artifacts,
    ]
    steps:
      - name: Checkout
        uses: actions/checkout@v3
        with:
          fetch-depth: 0

      - name: Stop EC2 runner
        uses: ./.github/actions/stop-runner
        with:
          label: ${{ needs.allocate-runners.outputs.linux-amd64-ec2-runner-label }}
          ec2-instance-id: ${{ needs.allocate-runners.outputs.linux-amd64-ec2-runner-instance-id }}
          aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }}
          aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
          aws-region: ${{ vars.EC2_RUNNER_REGION }}
          github-token: ${{ secrets.GH_PERSONAL_ACCESS_TOKEN }}

  stop-linux-arm64-runner: # It's always run as the last job in the workflow to make sure that the runner is released.
    name: Stop linux-arm64 runner
    # Only run this job when the runner is allocated.
    if: ${{ always() }}
    runs-on: ubuntu-latest
    needs: [
      allocate-runners,
      build-linux-arm64-artifacts,
    ]
    steps:
      - name: Checkout
        uses: actions/checkout@v3
        with:
          fetch-depth: 0

      - name: Stop EC2 runner
        uses: ./.github/actions/stop-runner
        with:
          label: ${{ needs.allocate-runners.outputs.linux-arm64-ec2-runner-label }}
          ec2-instance-id: ${{ needs.allocate-runners.outputs.linux-arm64-ec2-runner-instance-id }}
          aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }}
          aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
          aws-region: ${{ vars.EC2_RUNNER_REGION }}
          github-token: ${{ secrets.GH_PERSONAL_ACCESS_TOKEN }}