From 7078959cd0de5c6f52a7867d0efd85c7eb2dda15 Mon Sep 17 00:00:00 2001 From: Huangjw <1223644280@qq.com> Date: Tue, 12 Sep 2023 18:10:13 +0800 Subject: [PATCH] chore: remove ci main workflow (#12232) --- ci/workflows/docker.yml | 7 +- ci/workflows/main-cron.yml | 111 ++++++++++ ci/workflows/main.yml | 429 ------------------------------------- 3 files changed, 114 insertions(+), 433 deletions(-) delete mode 100644 ci/workflows/main.yml diff --git a/ci/workflows/docker.yml b/ci/workflows/docker.yml index 3fe1cb5db67e5..91b3dadfcf28a 100644 --- a/ci/workflows/docker.yml +++ b/ci/workflows/docker.yml @@ -44,7 +44,7 @@ steps: DOCKER_TOKEN: docker-token retry: *auto-retry - - label: "pre build binary" + - label: "release" command: "ci/scripts/release.sh" plugins: - seek-oss/aws-sm#v2.3.1: @@ -53,10 +53,9 @@ steps: - docker-compose#v4.9.0: run: release-env config: ci/docker-compose.yml + mount-buildkite-agent: true + propagate-environment: true environment: - BINARY_NAME - - BUILDKITE_SOURCE - GITHUB_TOKEN - - BUILDKITE_COMMIT - - BUILDKITE_TAG retry: *auto-retry diff --git a/ci/workflows/main-cron.yml b/ci/workflows/main-cron.yml index adb2291d23dc8..ca4da72794b14 100644 --- a/ci/workflows/main-cron.yml +++ b/ci/workflows/main-cron.yml @@ -68,6 +68,40 @@ steps: timeout_in_minutes: 60 retry: *auto-retry + - label: "end-to-end test (parallel) (release)" + command: "ci/scripts/e2e-test-parallel.sh -p ci-release" + depends_on: + - "build" + - "docslt" + plugins: + - seek-oss/aws-sm#v2.3.1: + env: + BUILDKITE_ANALYTICS_TOKEN: buildkite-build-analytics-sqllogictest-token + - docker-compose#v4.9.0: + run: rw-build-env + config: ci/docker-compose.yml + mount-buildkite-agent: true + - test-collector#v1.0.0: + files: "*-junit.xml" + format: "junit" + - ./ci/plugins/upload-failure-logs + timeout_in_minutes: 10 + retry: *auto-retry + + - label: "end-to-end test (parallel, in-memory) (release)" + command: "ci/scripts/e2e-test-parallel-in-memory.sh -p ci-release" + depends_on: + - "build" + - "docslt" + plugins: + - docker-compose#v4.9.0: + run: rw-build-env + config: ci/docker-compose.yml + mount-buildkite-agent: true + - ./ci/plugins/upload-failure-logs + timeout_in_minutes: 10 + retry: *auto-retry + - label: "end-to-end source test (release)" command: "ci/scripts/e2e-source-test.sh -p ci-release" depends_on: @@ -82,6 +116,20 @@ steps: timeout_in_minutes: 15 retry: *auto-retry + - label: "end-to-end sink test (release)" + command: "ci/scripts/e2e-sink-test.sh -p ci-release" + depends_on: + - "build" + - "build-other" + plugins: + - docker-compose#v4.9.0: + run: sink-test-env + config: ci/docker-compose.yml + mount-buildkite-agent: true + - ./ci/plugins/upload-failure-logs + timeout_in_minutes: 35 + retry: *auto-retry + - label: "fuzz test" command: "ci/scripts/cron-fuzz-test.sh -p ci-release" depends_on: @@ -197,6 +245,69 @@ steps: timeout_in_minutes: 5 retry: *auto-retry + - label: "connector node integration test Java {{matrix.java_version}}" + command: "ci/scripts/connector-node-integration-test.sh -p ci-release -v {{matrix.java_version}}" + depends_on: + - "build" + - "build-other" + plugins: + - docker-compose#v4.9.0: + run: rw-build-env + config: ci/docker-compose.yml + mount-buildkite-agent: true + - ./ci/plugins/upload-failure-logs + matrix: + setup: + java_version: + - "11" + - "17" + timeout_in_minutes: 10 + retry: *auto-retry + + - label: "end-to-end iceberg sink test (release)" + command: "ci/scripts/e2e-iceberg-sink-test.sh -p ci-release" + depends_on: + - "build" + - "build-other" + plugins: + - docker-compose#v4.9.0: + run: rw-build-env + config: ci/docker-compose.yml + mount-buildkite-agent: true + - ./ci/plugins/upload-failure-logs + timeout_in_minutes: 5 + retry: *auto-retry + + - label: "end-to-end iceberg sink v2 test (release)" + command: "ci/scripts/e2e-iceberg-sink-v2-test.sh -p ci-release" + depends_on: + - "build" + - "build-other" + plugins: + - docker-compose#v4.9.0: + run: rw-build-env + config: ci/docker-compose.yml + mount-buildkite-agent: true + - ./ci/plugins/upload-failure-logs + timeout_in_minutes: 5 + retry: *auto-retry + + - label: "e2e java-binding test (release)" + command: "ci/scripts/java-binding-test.sh -p ci-release" + depends_on: + - "build" + - "build-other" + plugins: + - docker-compose#v4.9.0: + run: rw-build-env + config: ci/docker-compose.yml + mount-buildkite-agent: true + - ./ci/plugins/upload-failure-logs + # Extra 2 minutes to account for docker-compose latency. + # See: https://github.com/risingwavelabs/risingwave/issues/9423#issuecomment-1521222169 + timeout_in_minutes: 10 + retry: *auto-retry + - label: "S3 source check on AWS (json parser)" command: "ci/scripts/s3-source-test.sh -p ci-release -s run" depends_on: build diff --git a/ci/workflows/main.yml b/ci/workflows/main.yml deleted file mode 100644 index b02e4ba147ffd..0000000000000 --- a/ci/workflows/main.yml +++ /dev/null @@ -1,429 +0,0 @@ -auto-retry: &auto-retry - automatic: - # Agent terminated because the AWS EC2 spot instance killed by AWS. - - signal_reason: agent_stop - limit: 3 - -steps: - - label: "build (dev mode)" - command: "ci/scripts/build.sh -p ci-dev" - key: "build-dev" - plugins: - - docker-compose#v4.9.0: - run: rw-build-env - config: ci/docker-compose.yml - mount-buildkite-agent: true - timeout_in_minutes: 15 - retry: *auto-retry - - - label: "build (release mode)" - command: "ci/scripts/build.sh -p ci-release" - key: "build-release" - plugins: - - docker-compose#v4.9.0: - run: rw-build-env - config: ci/docker-compose.yml - mount-buildkite-agent: true - env: - - BUILDKITE_COMMIT - timeout_in_minutes: 20 - retry: *auto-retry - - - label: "build other components" - command: "ci/scripts/build-other.sh" - key: "build-other" - plugins: - - seek-oss/aws-sm#v2.3.1: - env: - GITHUB_TOKEN: github-token - - docker-compose#v4.9.0: - run: rw-build-env - config: ci/docker-compose.yml - mount-buildkite-agent: true - environment: - - GITHUB_TOKEN - timeout_in_minutes: 12 - retry: *auto-retry - - - label: "build (deterministic simulation)" - command: "ci/scripts/build-simulation.sh" - key: "build-simulation" - plugins: - - docker-compose#v4.9.0: - run: rw-build-env - config: ci/docker-compose.yml - mount-buildkite-agent: true - timeout_in_minutes: 15 - retry: *auto-retry - - - label: "docslt" - command: "ci/scripts/docslt.sh" - key: "docslt" - plugins: - - docker-compose#v4.9.0: - run: rw-build-env - config: ci/docker-compose.yml - mount-buildkite-agent: true - timeout_in_minutes: 10 - retry: *auto-retry - - - label: "end-to-end test (dev mode)" - command: "ci/scripts/e2e-test.sh -p ci-dev -m ci-3streaming-2serving-3fe" - depends_on: - - "build-dev" - - "build-other" - - "docslt" - plugins: - - seek-oss/aws-sm#v2.3.1: - env: - BUILDKITE_ANALYTICS_TOKEN: buildkite-build-analytics-sqllogictest-token - - docker-compose#v4.9.0: - run: rw-build-env - config: ci/docker-compose.yml - mount-buildkite-agent: true - - test-collector#v1.0.0: - files: "*-junit.xml" - format: "junit" - - ./ci/plugins/upload-failure-logs - timeout_in_minutes: 15 - retry: *auto-retry - - - label: "end-to-end test (release mode)" - command: "ci/scripts/e2e-test.sh -p ci-release -m ci-3streaming-2serving-3fe" - depends_on: - - "build-release" - - "build-other" - - "docslt" - plugins: - - seek-oss/aws-sm#v2.3.1: - env: - BUILDKITE_ANALYTICS_TOKEN: buildkite-build-analytics-sqllogictest-token - - docker-compose#v4.9.0: - run: rw-build-env - config: ci/docker-compose.yml - mount-buildkite-agent: true - - test-collector#v1.0.0: - files: "*-junit.xml" - format: "junit" - - ./ci/plugins/upload-failure-logs - timeout_in_minutes: 10 - retry: *auto-retry - - - label: "end-to-end test (parallel) (dev mode)" - command: "ci/scripts/e2e-test-parallel.sh -p ci-dev" - depends_on: - - "build-dev" - - "docslt" - plugins: - - seek-oss/aws-sm#v2.3.1: - env: - BUILDKITE_ANALYTICS_TOKEN: buildkite-build-analytics-sqllogictest-token - - docker-compose#v4.9.0: - run: rw-build-env - config: ci/docker-compose.yml - mount-buildkite-agent: true - - test-collector#v1.0.0: - files: "*-junit.xml" - format: "junit" - - ./ci/plugins/upload-failure-logs - timeout_in_minutes: 15 - retry: *auto-retry - - - label: "end-to-end test (parallel) (release mode)" - command: "ci/scripts/e2e-test-parallel.sh -p ci-release" - depends_on: - - "build-release" - - "docslt" - plugins: - - seek-oss/aws-sm#v2.3.1: - env: - BUILDKITE_ANALYTICS_TOKEN: buildkite-build-analytics-sqllogictest-token - - docker-compose#v4.9.0: - run: rw-build-env - config: ci/docker-compose.yml - mount-buildkite-agent: true - - test-collector#v1.0.0: - files: "*-junit.xml" - format: "junit" - - ./ci/plugins/upload-failure-logs - timeout_in_minutes: 10 - retry: *auto-retry - - - label: "end-to-end test (parallel, in-memory) (release mode)" - command: "ci/scripts/e2e-test-parallel-in-memory.sh -p ci-release" - depends_on: - - "build-release" - - "docslt" - plugins: - - docker-compose#v4.9.0: - run: rw-build-env - config: ci/docker-compose.yml - mount-buildkite-agent: true - - ./ci/plugins/upload-failure-logs - timeout_in_minutes: 10 - retry: *auto-retry - - - label: "end-to-end source test (release mode)" - command: "ci/scripts/e2e-source-test.sh -p ci-release" - depends_on: - - "build-release" - - "build-other" - plugins: - - docker-compose#v4.9.0: - run: source-test-env - config: ci/docker-compose.yml - mount-buildkite-agent: true - - ./ci/plugins/upload-failure-logs - timeout_in_minutes: 15 - retry: *auto-retry - - - label: "unit test" - command: "ci/scripts/pr-unit-test.sh" - plugins: - - ./ci/plugins/swapfile - - seek-oss/aws-sm#v2.3.1: - env: - CODECOV_TOKEN: my-codecov-token - - docker-compose#v4.9.0: - run: rw-build-env - config: ci/docker-compose.yml - environment: - - CODECOV_TOKEN - timeout_in_minutes: 15 - retry: *auto-retry - - - label: "unit test (deterministic simulation)" - command: "MADSIM_TEST_NUM=50 ci/scripts/deterministic-unit-test.sh" - plugins: - - docker-compose#v4.9.0: - run: rw-build-env - config: ci/docker-compose.yml - mount-buildkite-agent: true - timeout_in_minutes: 15 - retry: *auto-retry - - - label: "integration test (deterministic simulation) - scale" - command: "TEST_NUM=30 ci/scripts/deterministic-it-test.sh scale::" - depends_on: "build-simulation" - plugins: - - docker-compose#v4.9.0: - run: rw-build-env - config: ci/docker-compose.yml - mount-buildkite-agent: true - timeout_in_minutes: 40 - retry: *auto-retry - - - label: "integration test (deterministic simulation) - recovery" - command: "TEST_NUM=30 ci/scripts/deterministic-it-test.sh recovery::" - depends_on: "build-simulation" - plugins: - - docker-compose#v4.9.0: - run: rw-build-env - config: ci/docker-compose.yml - mount-buildkite-agent: true - timeout_in_minutes: 40 - retry: *auto-retry - - - label: "integration test (deterministic simulation) - others" - command: "TEST_NUM=10 ci/scripts/deterministic-it-test.sh backfill_tests:: storage:: sink::" - depends_on: "build-simulation" - plugins: - - docker-compose#v4.9.0: - run: rw-build-env - config: ci/docker-compose.yml - mount-buildkite-agent: true - timeout_in_minutes: 40 - retry: *auto-retry - - - label: "end-to-end test (deterministic simulation)" - command: "TEST_NUM=32 ci/scripts/deterministic-e2e-test.sh" - depends_on: "build-simulation" - plugins: - - seek-oss/aws-sm#v2.3.1: - env: - GITHUB_TOKEN: github-token - - docker-compose#v4.9.0: - run: rw-build-env - config: ci/docker-compose.yml - mount-buildkite-agent: true - environment: - - GITHUB_TOKEN - - ./ci/plugins/upload-failure-logs - timeout_in_minutes: 30 - retry: *auto-retry - - - label: "recovery test (deterministic simulation)" - command: "TEST_NUM=16 KILL_RATE=0.5 ci/scripts/deterministic-recovery-test.sh" - depends_on: "build-simulation" - plugins: - # - seek-oss/aws-sm#v2.3.1: - # env: - # BUILDKITE_ANALYTICS_TOKEN: buildkite-build-analytics-deterministic-token - - docker-compose#v4.9.0: - run: rw-build-env - config: ci/docker-compose.yml - mount-buildkite-agent: true - # - test-collector#v1.0.0: - # files: "*-junit.xml" - # format: "junit" - - ./ci/plugins/upload-failure-logs - timeout_in_minutes: 30 - retry: *auto-retry - - - label: "end-to-end sink test (release mode)" - command: "ci/scripts/e2e-sink-test.sh -p ci-release" - depends_on: - - "build-release" - - "build-other" - plugins: - - docker-compose#v4.9.0: - run: sink-test-env - config: ci/docker-compose.yml - mount-buildkite-agent: true - - ./ci/plugins/upload-failure-logs - timeout_in_minutes: 35 - retry: *auto-retry - - - label: "connector node integration test Java {{matrix.java_version}}" - command: "ci/scripts/connector-node-integration-test.sh -p ci-release -v {{matrix.java_version}}" - depends_on: - - "build-release" - - "build-other" - plugins: - - docker-compose#v4.9.0: - run: rw-build-env - config: ci/docker-compose.yml - mount-buildkite-agent: true - - ./ci/plugins/upload-failure-logs - matrix: - setup: - java_version: - - "11" - - "17" - timeout_in_minutes: 10 - retry: *auto-retry - - - label: "end-to-end iceberg sink test (release mode)" - command: "ci/scripts/e2e-iceberg-sink-test.sh -p ci-release" - depends_on: - - "build-release" - - "build-other" - plugins: - - docker-compose#v4.9.0: - run: rw-build-env - config: ci/docker-compose.yml - mount-buildkite-agent: true - - ./ci/plugins/upload-failure-logs - timeout_in_minutes: 5 - retry: *auto-retry - - - label: "end-to-end iceberg sink v2 test (release mode)" - command: "ci/scripts/e2e-iceberg-sink-v2-test.sh -p ci-release" - depends_on: - - "build-release" - - "build-other" - plugins: - - docker-compose#v4.9.0: - run: rw-build-env - config: ci/docker-compose.yml - mount-buildkite-agent: true - - ./ci/plugins/upload-failure-logs - timeout_in_minutes: 5 - retry: *auto-retry - - - label: "end-to-end clickhouse sink test (release mode)" - command: "ci/scripts/e2e-clickhouse-sink-test.sh -p ci-release" - depends_on: - - "build-release" - - "build-other" - plugins: - - docker-compose#v4.9.0: - run: sink-test-env - config: ci/docker-compose.yml - mount-buildkite-agent: true - - ./ci/plugins/upload-failure-logs - timeout_in_minutes: 5 - retry: *auto-retry - - - label: "e2e java-binding test (at release)" - command: "ci/scripts/java-binding-test.sh -p ci-release" - depends_on: - - "build-release" - - "build-other" - plugins: - - docker-compose#v4.9.0: - run: rw-build-env - config: ci/docker-compose.yml - mount-buildkite-agent: true - - ./ci/plugins/upload-failure-logs - # Extra 2 minutes to account for docker-compose latency. - # See: https://github.com/risingwavelabs/risingwave/issues/9423#issuecomment-1521222169 - timeout_in_minutes: 10 - retry: *auto-retry - - - label: "release" - command: "ci/scripts/release.sh" - if: build.tag != null - plugins: - - seek-oss/aws-sm#v2.3.1: - env: - GITHUB_TOKEN: github-token - - docker-compose#v4.9.0: - run: release-env - config: ci/docker-compose.yml - mount-buildkite-agent: true - environment: - - GITHUB_TOKEN - - BUILDKITE_TAG - - BUILDKITE_SOURCE - timeout_in_minutes: 60 - retry: *auto-retry - - - label: "release docker image: amd64" - command: "ci/scripts/docker.sh" - key: "build-amd64" - if: build.tag != null - env: - PUSH: true - plugins: - - seek-oss/aws-sm#v2.3.1: - env: - GHCR_USERNAME: ghcr-username - GHCR_TOKEN: ghcr-token - DOCKER_TOKEN: docker-token - GITHUB_TOKEN: github-token - timeout_in_minutes: 60 - retry: *auto-retry - - - label: "docker-build-push: aarch64" - command: "ci/scripts/docker.sh" - key: "build-aarch64" - if: build.tag != null - env: - PUSH: true - plugins: - - seek-oss/aws-sm#v2.3.1: - env: - GHCR_USERNAME: ghcr-username - GHCR_TOKEN: ghcr-token - DOCKER_TOKEN: docker-token - GITHUB_TOKEN: github-token - timeout_in_minutes: 60 - agents: - queue: "linux-arm64" - retry: *auto-retry - - - label: "multi arch image create push" - command: "ci/scripts/multi-arch-docker.sh" - if: build.tag != null - depends_on: - - "build-amd64" - - "build-aarch64" - plugins: - - seek-oss/aws-sm#v2.3.1: - env: - GHCR_USERNAME: ghcr-username - GHCR_TOKEN: ghcr-token - DOCKER_TOKEN: docker-token - timeout_in_minutes: 10 - retry: *auto-retry