From 5e2fdec1b6150a0eec5e04a3dc3a3c0fb181f5e1 Mon Sep 17 00:00:00 2001 From: zyy17 Date: Tue, 8 Aug 2023 15:58:59 +0800 Subject: [PATCH] ci: add dev-build (#2114) --- .github/actions/build-images/action.yml | 5 + .../actions/build-linux-artifacts/action.yml | 7 +- .github/scripts/create-version.sh | 17 +- .github/workflows/dev-build.yml | 343 ++++++++++++++++++ .github/workflows/nightly-build.yml | 4 +- 5 files changed, 371 insertions(+), 5 deletions(-) create mode 100644 .github/workflows/dev-build.yml diff --git a/.github/actions/build-images/action.yml b/.github/actions/build-images/action.yml index dc63d15eba97..9045fd562925 100644 --- a/.github/actions/build-images/action.yml +++ b/.github/actions/build-images/action.yml @@ -20,6 +20,10 @@ inputs: description: Whether to push the latest tag required: false default: 'true' + dev-mode: + description: Enable dev mode, only build standard greptime + required: false + default: 'false' runs: using: composite steps: @@ -39,6 +43,7 @@ runs: push-latest-tag: ${{ inputs.push-latest-tag }} - name: Build and push centos images to dockerhub + if: ${{ inputs.dev-mode == 'false' }} uses: ./.github/actions/build-greptime-images with: image-registry: ${{ inputs.image-registry }} diff --git a/.github/actions/build-linux-artifacts/action.yml b/.github/actions/build-linux-artifacts/action.yml index 1e82098455bc..ec1b565c47f1 100644 --- a/.github/actions/build-linux-artifacts/action.yml +++ b/.github/actions/build-linux-artifacts/action.yml @@ -25,6 +25,10 @@ inputs: aws-region: description: AWS region required: true + dev-mode: + description: Enable dev mode, only build standard greptime + required: false + default: 'false' runs: using: composite steps: @@ -57,6 +61,7 @@ runs: aws-region: ${{ inputs.aws-region }} - name: Build greptime without pyo3 + if: ${{ inputs.dev-mode == 'false' }} uses: ./.github/actions/build-greptime-binary with: base-image: ubuntu @@ -71,7 +76,7 @@ runs: - name: Build greptime on centos base image uses: ./.github/actions/build-greptime-binary - if: ${{ inputs.arch == 'amd64' }} # Only build centos7 base image for amd64. + if: ${{ inputs.arch == 'amd64' && inputs.dev-mode = 'false' }} # Only build centos7 base image for amd64. with: base-image: centos features: servers/dashboard diff --git a/.github/scripts/create-version.sh b/.github/scripts/create-version.sh index 9047b0bdb940..458fa93ca3c7 100755 --- a/.github/scripts/create-version.sh +++ b/.github/scripts/create-version.sh @@ -25,11 +25,23 @@ function create_version() { fi # Reuse $NEXT_RELEASE_VERSION to identify whether it's a nightly build. - if [ "$NEXT_RELEASE_VERSION" = dev ]; then + # It will be like 'nigtly-20230808-7d0d8dc6'. + if [ "$NEXT_RELEASE_VERSION" = nightly ]; then echo "$NIGHTLY_RELEASE_PREFIX-$(date "+%Y%m%d")-$(git rev-parse --short HEAD)" exit 0 fi + # Reuse $NEXT_RELEASE_VERSION to identify whether it's a dev build. + # It will be like 'dev-2023080819-f0e7216c'. + if [ "$NEXT_RELEASE_VERSION" = dev ]; then + if [ -z "$COMMIT_SHA" ]; then + echo "COMMIT_SHA is empty in dev build" + exit 1 + fi + echo "dev-$(date "+%Y%m%d%S")-$(echo "$COMMIT_SHA" | cut -c1-8)" + exit 0 + fi + # Note: Only output 'version=xxx' to stdout when everything is ok, so that it can be used in GitHub Actions Outputs. if [ "$GITHUB_EVENT_NAME" = push ]; then if [ -z "$GITHUB_REF_NAME" ]; then @@ -51,5 +63,6 @@ function create_version() { # GITHUB_EVENT_NAME=push NEXT_RELEASE_VERSION=v0.4.0 NIGHTLY_RELEASE_PREFIX=nigtly GITHUB_REF_NAME=v0.3.0 ./create-version.sh # GITHUB_EVENT_NAME=workflow_dispatch NEXT_RELEASE_VERSION=v0.4.0 NIGHTLY_RELEASE_PREFIX=nigtly ./create-version.sh # GITHUB_EVENT_NAME=schedule NEXT_RELEASE_VERSION=v0.4.0 NIGHTLY_RELEASE_PREFIX=nigtly ./create-version.sh -# GITHUB_EVENT_NAME=schedule NEXT_RELEASE_VERSION=dev NIGHTLY_RELEASE_PREFIX=nigtly ./create-version.sh +# GITHUB_EVENT_NAME=schedule NEXT_RELEASE_VERSION=nightly NIGHTLY_RELEASE_PREFIX=nigtly ./create-version.sh +# GITHUB_EVENT_NAME=workflow_dispatch COMMIT_SHA=f0e7216c4bb6acce9b29a21ec2d683be2e3f984a NEXT_RELEASE_VERSION=dev NIGHTLY_RELEASE_PREFIX=nigtly ./create-version.sh create_version diff --git a/.github/workflows/dev-build.yml b/.github/workflows/dev-build.yml new file mode 100644 index 000000000000..29a68266cb34 --- /dev/null +++ b/.github/workflows/dev-build.yml @@ -0,0 +1,343 @@ +# Development build only build the debug version of the artifacts manually. +name: GreptimeDB Development Build + +on: + workflow_dispatch: # Allows you to run this workflow manually. + inputs: + repository: + description: The public repository to build + required: false + default: GreptimeTeam/greptimedb + commit: # Note: We only pull the source code and use the current workflow to build the artifacts. + description: The commit to build + required: true + linux_amd64_runner: + type: choice + description: The runner uses to build linux-amd64 artifacts + default: ec2-c6i.4xlarge-amd64 + options: + - ubuntu-latest + - ubuntu-latest-8-cores + - ubuntu-latest-16-cores + - ubuntu-latest-32-cores + - ubuntu-latest-64-cores + - ec2-c6i.xlarge-amd64 # 4C8G + - ec2-c6i.2xlarge-amd64 # 8C16G + - ec2-c6i.4xlarge-amd64 # 16C32G + - ec2-c6i.8xlarge-amd64 # 32C64G + - ec2-c6i.16xlarge-amd64 # 64C128G + linux_arm64_runner: + type: choice + description: The runner uses to build linux-arm64 artifacts + default: ec2-c6g.4xlarge-arm64 + options: + - ec2-c6g.xlarge-arm64 # 4C8G + - ec2-c6g.2xlarge-arm64 # 8C16G + - ec2-c6g.4xlarge-arm64 # 16C32G + - ec2-c6g.8xlarge-arm64 # 32C64G + - ec2-c6g.16xlarge-arm64 # 64C128G + skip_test: + description: Do not run integration tests during the build + type: boolean + default: true + build_linux_amd64_artifacts: + type: boolean + description: Build linux-amd64 artifacts + required: false + default: true + build_linux_arm64_artifacts: + type: boolean + description: Build linux-arm64 artifacts + required: false + default: true + release_images: + type: boolean + description: Build and push images to DockerHub and ACR + required: false + default: true + +# Use env variables to control all the release process. +env: + CARGO_PROFILE: nightly + + # Controls whether to run tests, include unit-test, integration-test and sqlness. + DISABLE_RUN_TESTS: ${{ inputs.skip_test || vars.DEFAULT_SKIP_TEST }} + + # Always use 'dev' to indicate it's the dev build. + NEXT_RELEASE_VERSION: dev + + NIGHTLY_RELEASE_PREFIX: nightly + + # Use the different image namespace to avoid conflict with the release images. + IMAGE_NAMESPACE: greptime-dev + +jobs: + allocate-runners: + name: Allocate runners + if: ${{ github.repository == 'GreptimeTeam/greptimedb' }} + runs-on: ubuntu-latest + outputs: + linux-amd64-runner: ${{ steps.start-linux-amd64-runner.outputs.label }} + linux-arm64-runner: ${{ steps.start-linux-arm64-runner.outputs.label }} + + # The following EC2 resource id will be used for resource releasing. + linux-amd64-ec2-runner-label: ${{ steps.start-linux-amd64-runner.outputs.label }} + linux-amd64-ec2-runner-instance-id: ${{ steps.start-linux-amd64-runner.outputs.ec2-instance-id }} + linux-arm64-ec2-runner-label: ${{ steps.start-linux-arm64-runner.outputs.label }} + linux-arm64-ec2-runner-instance-id: ${{ steps.start-linux-arm64-runner.outputs.ec2-instance-id }} + + # The 'version' use as the global tag name of the release workflow. + version: ${{ steps.create-version.outputs.version }} + steps: + - name: Checkout + uses: actions/checkout@v3 + with: + fetch-depth: 0 + + - name: Create version + id: create-version + run: | + version=$(./.github/scripts/create-version.sh) && \ + echo $version && \ + echo "version=$version" >> $GITHUB_OUTPUT + env: + GITHUB_EVENT_NAME: ${{ github.event_name }} + GITHUB_REF_NAME: ${{ github.ref_name }} + COMMIT_SHA: ${{ inputs.commit }} + NEXT_RELEASE_VERSION: ${{ env.NEXT_RELEASE_VERSION }} + NIGHTLY_RELEASE_PREFIX: ${{ env.NIGHTLY_RELEASE_PREFIX }} + + - name: Allocate linux-amd64 runner + if: ${{ inputs.build_linux_amd64_artifacts || github.event_name == 'schedule' }} + uses: ./.github/actions/start-runner + id: start-linux-amd64-runner + with: + runner: ${{ inputs.linux_amd64_runner || vars.DEFAULT_AMD64_RUNNER }} + aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }} + aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + aws-region: ${{ vars.EC2_RUNNER_REGION }} + github-token: ${{ secrets.GH_PERSONAL_ACCESS_TOKEN }} + image-id: ${{ vars.EC2_RUNNER_LINUX_AMD64_IMAGE_ID }} + security-group-id: ${{ vars.EC2_RUNNER_SECURITY_GROUP_ID }} + subnet-id: ${{ vars.EC2_RUNNER_SUBNET_ID }} + + - name: Allocate linux-arm64 runner + if: ${{ inputs.build_linux_arm64_artifacts || github.event_name == 'schedule' }} + uses: ./.github/actions/start-runner + id: start-linux-arm64-runner + with: + runner: ${{ inputs.linux_arm64_runner || vars.DEFAULT_ARM64_RUNNER }} + aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }} + aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + aws-region: ${{ vars.EC2_RUNNER_REGION }} + github-token: ${{ secrets.GH_PERSONAL_ACCESS_TOKEN }} + image-id: ${{ vars.EC2_RUNNER_LINUX_ARM64_IMAGE_ID }} + security-group-id: ${{ vars.EC2_RUNNER_SECURITY_GROUP_ID }} + subnet-id: ${{ vars.EC2_RUNNER_SUBNET_ID }} + + build-linux-amd64-artifacts: + name: Build linux-amd64 artifacts + if: ${{ inputs.build_linux_amd64_artifacts || github.event_name == 'schedule' }} + needs: [ + allocate-runners, + ] + runs-on: ${{ needs.allocate-runners.outputs.linux-amd64-runner }} + steps: + - name: Checkout + uses: actions/checkout@v3 + with: + fetch-depth: 0 + + - name: Checkout greptimedb + uses: actions/checkout@v3 + with: + repository: ${{ inputs.repository }} + ref: ${{ inputs.commit }} + path: dev/greptimedb + + - name: Enter greptimedb + run: | + cd dev/greptimedb + + - uses: ../.github/actions/build-linux-artifacts + with: + arch: amd64 + cargo-profile: ${{ env.CARGO_PROFILE }} + version: ${{ needs.allocate-runners.outputs.version }} + disable-run-tests: ${{ env.DISABLE_RUN_TESTS }} + release-to-s3-bucket: ${{ vars.AWS_RELEASE_BUCKET }} + aws-access-key-id: ${{ secrets.AWS_CN_ACCESS_KEY_ID }} + aws-secret-access-key: ${{ secrets.AWS_CN_SECRET_ACCESS_KEY }} + aws-region: ${{ vars.AWS_RELEASE_BUCKET_REGION }} + dev-mode: true # Only build the standard greptime binary. + + build-linux-arm64-artifacts: + name: Build linux-arm64 artifacts + if: ${{ inputs.build_linux_arm64_artifacts || github.event_name == 'schedule' }} + needs: [ + allocate-runners, + ] + runs-on: ${{ needs.allocate-runners.outputs.linux-arm64-runner }} + steps: + - name: Checkout + uses: actions/checkout@v3 + with: + fetch-depth: 0 + + - name: Checkout greptimedb + uses: actions/checkout@v3 + with: + repository: ${{ inputs.repository }} + ref: ${{ inputs.commit }} + path: dev/greptimedb + + - name: Enter greptimedb + run: | + cd dev/greptimedb + + - uses: ../.github/actions/build-linux-artifacts + with: + arch: arm64 + cargo-profile: ${{ env.CARGO_PROFILE }} + version: ${{ needs.allocate-runners.outputs.version }} + disable-run-tests: ${{ env.DISABLE_RUN_TESTS }} + release-to-s3-bucket: ${{ vars.AWS_RELEASE_BUCKET }} + aws-access-key-id: ${{ secrets.AWS_CN_ACCESS_KEY_ID }} + aws-secret-access-key: ${{ secrets.AWS_CN_SECRET_ACCESS_KEY }} + aws-region: ${{ vars.AWS_RELEASE_BUCKET_REGION }} + dev-mode: true # Only build the standard greptime binary. + + release-images-to-dockerhub: + name: Build and push images to DockerHub + if: ${{ inputs.release_images || github.event_name == 'schedule' }} + needs: [ + allocate-runners, + build-linux-amd64-artifacts, + build-linux-arm64-artifacts, + ] + runs-on: ubuntu-latest + outputs: + build-result: ${{ steps.set-build-result.outputs.build-result }} + steps: + - uses: actions/checkout@v3 + with: + fetch-depth: 0 + + - name: Build and push images to dockerhub + uses: ./.github/actions/build-images + with: + image-registry: docker.io + image-namespace: ${{ env.IMAGE_NAMESPACE }} + image-registry-username: ${{ secrets.DOCKERHUB_USERNAME }} + image-registry-password: ${{ secrets.DOCKERHUB_TOKEN }} + version: ${{ needs.allocate-runners.outputs.version }} + push-latest-tag: false # Don't push the latest tag to registry. + dev-mode: true # Only build the standard images. + + - name: Set build result + id: set-build-result + run: | + echo "build-result=success" >> $GITHUB_OUTPUT + + release-images-to-acr: + name: Build and push images to ACR + if: ${{ inputs.release_images || github.event_name == 'schedule' }} + needs: [ + allocate-runners, + build-linux-amd64-artifacts, + build-linux-arm64-artifacts, + ] + runs-on: ubuntu-latest + # When we push to ACR, it's easy to fail due to some unknown network issues. + # However, we don't want to fail the whole workflow because of this. + # The ACR have daily sync with DockerHub, so don't worry about the image not being updated. + continue-on-error: true + steps: + - uses: actions/checkout@v3 + with: + fetch-depth: 0 + + - name: Build and push images to ACR + uses: ./.github/actions/build-images + with: + image-registry: ${{ vars.ACR_IMAGE_REGISTRY }} + image-namespace: ${{ env.IMAGE_NAMESPACE }} + image-registry-username: ${{ secrets.ALICLOUD_USERNAME }} + image-registry-password: ${{ secrets.ALICLOUD_PASSWORD }} + version: ${{ needs.allocate-runners.outputs.version }} + push-latest-tag: false # Don't push the latest tag to registry. + dev-mode: true # Only build the standard images. + + stop-linux-amd64-runner: # It's always run as the last job in the workflow to make sure that the runner is released. + name: Stop linux-amd64 runner + # Only run this job when the runner is allocated. + if: ${{ always() }} + runs-on: ubuntu-latest + needs: [ + allocate-runners, + build-linux-amd64-artifacts, + ] + steps: + - name: Checkout + uses: actions/checkout@v3 + with: + fetch-depth: 0 + + - name: Stop EC2 runner + uses: ./.github/actions/stop-runner + with: + label: ${{ needs.allocate-runners.outputs.linux-amd64-ec2-runner-label }} + ec2-instance-id: ${{ needs.allocate-runners.outputs.linux-amd64-ec2-runner-instance-id }} + aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }} + aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + aws-region: ${{ vars.EC2_RUNNER_REGION }} + github-token: ${{ secrets.GH_PERSONAL_ACCESS_TOKEN }} + + stop-linux-arm64-runner: # It's always run as the last job in the workflow to make sure that the runner is released. + name: Stop linux-arm64 runner + # Only run this job when the runner is allocated. + if: ${{ always() }} + runs-on: ubuntu-latest + needs: [ + allocate-runners, + build-linux-arm64-artifacts, + ] + steps: + - name: Checkout + uses: actions/checkout@v3 + with: + fetch-depth: 0 + + - name: Stop EC2 runner + uses: ./.github/actions/stop-runner + with: + label: ${{ needs.allocate-runners.outputs.linux-arm64-ec2-runner-label }} + ec2-instance-id: ${{ needs.allocate-runners.outputs.linux-arm64-ec2-runner-instance-id }} + aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }} + aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + aws-region: ${{ vars.EC2_RUNNER_REGION }} + github-token: ${{ secrets.GH_PERSONAL_ACCESS_TOKEN }} + + notification: + if: ${{ always() }} # Not requiring successful dependent jobs, always run. + name: Send notification to Greptime team + needs: [ + release-images-to-dockerhub + ] + runs-on: ubuntu-latest + env: + SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_URL_DEVELOP_CHANNEL }} + steps: + - name: Notifiy nightly build successful result + uses: slackapi/slack-github-action@v1.23.0 + if: ${{ needs.release-images-to-dockerhub.outputs.build-result == 'success' }} + with: + payload: | + {"text": "GreptimeDB ${{ NEXT_RELEASE_VERSION }} build successful"} + + - name: Notifiy nightly build failed result + uses: slackapi/slack-github-action@v1.23.0 + if: ${{ needs.release-images-to-dockerhub.outputs.build-result != 'success' }} + with: + payload: | + {"text": "GreptimeDB ${{ NEXT_RELEASE_VERSION }} build failed, please check 'https://github.com/GreptimeTeam/greptimedb/actions/workflows/${{ NEXT_RELEASE_VERSION }}-build.yml'"} diff --git a/.github/workflows/nightly-build.yml b/.github/workflows/nightly-build.yml index 36adef76821a..234c200158b6 100644 --- a/.github/workflows/nightly-build.yml +++ b/.github/workflows/nightly-build.yml @@ -61,8 +61,8 @@ env: # Controls whether to run tests, include unit-test, integration-test and sqlness. DISABLE_RUN_TESTS: ${{ inputs.skip_test || vars.DEFAULT_SKIP_TEST }} - # Always use 'dev' to indicate it's the nightly build. - NEXT_RELEASE_VERSION: dev + # Always use 'nightly' to indicate it's the nightly build. + NEXT_RELEASE_VERSION: nightly NIGHTLY_RELEASE_PREFIX: nightly