Skip to content

Commit

Permalink
Merge branch 'main' into kwannoel/support-rate-limit-2
Browse files Browse the repository at this point in the history
  • Loading branch information
@kwannoel
kwannoel authored Oct 31, 2023
2 parents cebe5b7 + dc31d8b commit e60c070
Show file tree
Hide file tree
Showing 126 changed files with 2,118 additions and 693 deletions.
5 changes: 4 additions & 1 deletion .git-blame-ignore-revs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -35,4 +35,7 @@ f8266748dcb70541da944664552c1944ff8362e4
f2a3fd021059e680b35b24c63cff5f8dbe9f9d5f

# chore(rustfmt): format let-chains and let-else #9409
d70dba827c303373f3220c9733f7c7443e5c2d37
d70dba827c303373f3220c9733f7c7443e5c2d37

# chore: cargo +nightly fmt (#13162) (format let-chains)
c583e2c6c054764249acf484438c7bf7197765f4
1 change: 1 addition & 0 deletions Cargo.lock
View file

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

8 changes: 4 additions & 4 deletions README.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -47,16 +47,16 @@
<img alt="Docker" src="https://img.shields.io/docker/v/risingwavelabs/risingwave" />
</a>
<a
href="https://buildkite.com/risingwavelabs/main"
href="https://twitter.com/risingwavelabs"
target="_blank"
>
<img alt="Build status" src="https://badge.buildkite.com/9394d2bca0f87e2e97aa78b25f765c92d4207c0b65e7f6648f.svg" />
<img alt="X" src="https://img.shields.io/twitter/follow/risingwavelabs" />
</a>
<a
href="https://codecov.io/gh/risingwavelabs/risingwave"
href="https://www.youtube.com/@risingwave-labs"
target="_blank"
>
<img alt="codecov" src="https://codecov.io/gh/risingwavelabs/risingwave/branch/main/graph/badge.svg?token=EB44K9K38B" />
<img alt="YouTube" src="https://img.shields.io/youtube/channel/views/UCsHwdyBRxBpmkA5RRd0YNEA" />
</a>
</div>
Expand Down
13 changes: 10 additions & 3 deletions ci/scripts/release.sh
View file
Original file line number Diff line number Diff line change
Expand Up @@ -66,9 +66,15 @@ fi

echo "--- Build connector node"
cd ${REPO_ROOT}/java && mvn -B package -Dmaven.test.skip=true -Dno-build-rust
cd ${REPO_ROOT} && mv ${REPO_ROOT}/java/connector-node/assembly/target/risingwave-connector-1.0.0.tar.gz risingwave-connector-"${BUILDKITE_TAG}".tar.gz

if [[ -n "${BUILDKITE_TAG}" ]]; then
echo "--- Collect all release assets"
cd ${REPO_ROOT} && mkdir release-assets && cd release-assets
cp -r ${REPO_ROOT}/target/release/* .
mv ${REPO_ROOT}/java/connector-node/assembly/target/risingwave-connector-1.0.0.tar.gz risingwave-connector-"${BUILDKITE_TAG}".tar.gz
tar -zxvf risingwave-connector-"${BUILDKITE_TAG}".tar.gz libs
ls -l

echo "--- Install gh cli"
yum install -y dnf
dnf install -y 'dnf-command(config-manager)'
Expand All @@ -90,8 +96,9 @@ if [[ -n "${BUILDKITE_TAG}" ]]; then
tar -czvf risectl-"${BUILDKITE_TAG}"-x86_64-unknown-linux.tar.gz risectl
gh release upload "${BUILDKITE_TAG}" risectl-"${BUILDKITE_TAG}"-x86_64-unknown-linux.tar.gz

echo "--- Release build and upload risingwave connector node jar asset"
gh release upload "${BUILDKITE_TAG}" risingwave-connector-"${BUILDKITE_TAG}".tar.gz
echo "--- Release upload risingwave-all-in-one asset"
tar -czvf risingwave-"${BUILDKITE_TAG}"-x86_64-unknown-linux-all-in-one.tar.gz risingwave libs
gh release upload "${BUILDKITE_TAG}" risingwave-"${BUILDKITE_TAG}"-x86_64-unknown-linux-all-in-one.tar.gz
fi


Expand Down
39 changes: 38 additions & 1 deletion ci/workflows/main-cron.yml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,7 @@ auto-retry: &auto-retry
steps:
- label: "build"
command: "ci/scripts/build.sh -p ci-release"
if: (!build.pull_request.labels includes "ci/main-cron/skip-ci" || build.pull_request.labels includes "ci/run-build")
key: "build"
plugins:
- docker-compose#v4.9.0:
Expand All @@ -18,6 +19,7 @@ steps:

- label: "build other components"
command: "ci/scripts/build-other.sh"
if: (!build.pull_request.labels includes "ci/main-cron/skip-ci" || build.pull_request.labels includes "ci/run-build-other")
key: "build-other"
plugins:
- seek-oss/aws-sm#v2.3.1:
Expand All @@ -34,6 +36,7 @@ steps:

- label: "build (deterministic simulation)"
command: "ci/scripts/build-simulation.sh"
if: (!build.pull_request.labels includes "ci/main-cron/skip-ci" || build.pull_request.labels includes "ci/run-build-simulation")
key: "build-simulation"
plugins:
- docker-compose#v4.9.0:
Expand All @@ -45,6 +48,7 @@ steps:

- label: "docslt"
command: "ci/scripts/docslt.sh"
if: (!build.pull_request.labels includes "ci/main-cron/skip-ci" || build.pull_request.labels includes "ci/run-docslt")
key: "docslt"
plugins:
- docker-compose#v4.9.0:
Expand All @@ -56,6 +60,7 @@ steps:

- label: "end-to-end test (release)"
command: "ci/scripts/cron-e2e-test.sh -p ci-release -m ci-3streaming-2serving-3fe"
if: (!build.pull_request.labels includes "ci/main-cron/skip-ci" || build.pull_request.labels includes "ci/run-e2e-test")
depends_on:
- "build"
- "docslt"
Expand All @@ -70,6 +75,7 @@ steps:

- label: "end-to-end test (parallel) (release)"
command: "ci/scripts/e2e-test-parallel.sh -p ci-release"
if: (!build.pull_request.labels includes "ci/main-cron/skip-ci" || build.pull_request.labels includes "ci/run-e2e-parallel-tests")
depends_on:
- "build"
- "docslt"
Expand All @@ -90,6 +96,7 @@ steps:

- label: "end-to-end test (parallel, in-memory) (release)"
command: "ci/scripts/e2e-test-parallel-in-memory.sh -p ci-release"
if: (!build.pull_request.labels includes "ci/main-cron/skip-ci" || build.pull_request.labels includes "ci/run-e2e-parallel-in-memory-tests")
depends_on:
- "build"
- "docslt"
Expand All @@ -104,6 +111,7 @@ steps:

- label: "end-to-end source test (release)"
command: "ci/scripts/e2e-source-test.sh -p ci-release"
if: (!build.pull_request.labels includes "ci/main-cron/skip-ci" || build.pull_request.labels includes "ci/run-e2e-source-tests")
depends_on:
- "build"
- "build-other"
Expand All @@ -118,6 +126,7 @@ steps:

- label: "end-to-end sink test (release)"
command: "ci/scripts/e2e-sink-test.sh -p ci-release"
if: (!build.pull_request.labels includes "ci/main-cron/skip-ci" || build.pull_request.labels includes "ci/run-e2e-sink-tests")
depends_on:
- "build"
- "build-other"
Expand All @@ -132,6 +141,7 @@ steps:

- label: "fuzz test"
command: "ci/scripts/cron-fuzz-test.sh -p ci-release"
if: (!build.pull_request.labels includes "ci/main-cron/skip-ci" || build.pull_request.labels includes "ci/run-sqlsmith-fuzzing-tests")
depends_on:
- "build"
- "build-simulation"
Expand All @@ -149,6 +159,7 @@ steps:
# This ensures our `main-cron` workflow will be stable.
- label: "unit test"
command: "ci/scripts/unit-test.sh"
if: (!build.pull_request.labels includes "ci/main-cron/skip-ci" || build.pull_request.labels includes "ci/run-unit-test")
plugins:
- ./ci/plugins/swapfile
- seek-oss/aws-sm#v2.3.1:
Expand All @@ -164,6 +175,7 @@ steps:

- label: "unit test (deterministic simulation)"
command: "MADSIM_TEST_NUM=100 timeout 15m ci/scripts/deterministic-unit-test.sh"
if: (!build.pull_request.labels includes "ci/main-cron/skip-ci" || build.pull_request.labels includes "ci/run-unit-test-deterministic-simulation")
plugins:
- docker-compose#v4.9.0:
run: rw-build-env
Expand All @@ -174,6 +186,7 @@ steps:

- label: "integration test (deterministic simulation) - scale"
command: "TEST_NUM=60 ci/scripts/deterministic-it-test.sh scale::"
if: (!build.pull_request.labels includes "ci/main-cron/skip-ci" || build.pull_request.labels includes "ci/run-integration-test-deterministic-simulation")
depends_on: "build-simulation"
plugins:
- docker-compose#v4.9.0:
Expand All @@ -185,6 +198,7 @@ steps:

- label: "integration test (deterministic simulation) - recovery"
command: "TEST_NUM=60 ci/scripts/deterministic-it-test.sh recovery::"
if: (!build.pull_request.labels includes "ci/main-cron/skip-ci" || build.pull_request.labels includes "ci/run-integration-test-deterministic-simulation")
depends_on: "build-simulation"
plugins:
- docker-compose#v4.9.0:
Expand All @@ -196,6 +210,7 @@ steps:

- label: "integration test (deterministic simulation) - others"
command: "TEST_NUM=30 ci/scripts/deterministic-it-test.sh backfill_tests:: storage:: sink::"
if: (!build.pull_request.labels includes "ci/main-cron/skip-ci" || build.pull_request.labels includes "ci/run-integration-test-deterministic-simulation")
depends_on: "build-simulation"
plugins:
- docker-compose#v4.9.0:
Expand All @@ -207,6 +222,7 @@ steps:

- label: "end-to-end test (deterministic simulation)"
command: "TEST_NUM=64 timeout 55m ci/scripts/deterministic-e2e-test.sh"
if: (!build.pull_request.labels includes "ci/main-cron/skip-ci" || build.pull_request.labels includes "ci/run-e2e-test-deterministic-simulation")
depends_on: "build-simulation"
plugins:
- seek-oss/aws-sm#v2.3.1:
Expand All @@ -224,6 +240,7 @@ steps:

- label: "recovery test (deterministic simulation)"
command: "TEST_NUM=12 KILL_RATE=1.0 timeout 55m ci/scripts/deterministic-recovery-test.sh"
if: (!build.pull_request.labels includes "ci/main-cron/skip-ci" || build.pull_request.labels includes "ci/run-recovery-test-deterministic-simulation")
depends_on: "build-simulation"
plugins:
- docker-compose#v4.9.0:
Expand All @@ -236,6 +253,7 @@ steps:

- label: "misc check"
command: "ci/scripts/misc-check.sh"
if: (!build.pull_request.labels includes "ci/main-cron/skip-ci" || build.pull_request.labels includes "ci/run-misc-check")
plugins:
- docker-compose#v4.9.0:
run: rw-build-env
Expand All @@ -247,6 +265,7 @@ steps:

- label: "end-to-end iceberg sink test (release)"
command: "ci/scripts/e2e-iceberg-sink-test.sh -p ci-release"
if: (!build.pull_request.labels includes "ci/main-cron/skip-ci" || build.pull_request.labels includes "ci/run-e2e-iceberg-sink-tests")
depends_on:
- "build"
- "build-other"
Expand All @@ -261,6 +280,7 @@ steps:

- label: "end-to-end iceberg sink v2 test (release)"
command: "ci/scripts/e2e-iceberg-sink-v2-test.sh -p ci-release"
if: (!build.pull_request.labels includes "ci/main-cron/skip-ci" || build.pull_request.labels includes "ci/run-e2e-iceberg-sink-v2-tests")
depends_on:
- "build"
- "build-other"
Expand All @@ -275,6 +295,7 @@ steps:

- label: "e2e java-binding test (release)"
command: "ci/scripts/java-binding-test.sh -p ci-release"
if: (!build.pull_request.labels includes "ci/main-cron/skip-ci" || build.pull_request.labels includes "ci/run-e2e-java-binding-tests")
depends_on:
- "build"
- "build-other"
Expand All @@ -291,6 +312,7 @@ steps:

- label: "S3 source check on AWS (json parser)"
command: "ci/scripts/s3-source-test.sh -p ci-release -s run.py"
if: (!build.pull_request.labels includes "ci/main-cron/skip-ci" || build.pull_request.labels includes "ci/run-s3-source-tests")
depends_on: build
plugins:
- seek-oss/aws-sm#v2.3.1:
Expand All @@ -308,6 +330,7 @@ steps:

- label: "S3 source check on AWS (json parser)"
command: "ci/scripts/s3-source-test.sh -p ci-release -s json_file.py"
if: (!build.pull_request.labels includes "ci/main-cron/skip-ci" || build.pull_request.labels includes "ci/run-s3-source-tests")
depends_on: build
plugins:
- seek-oss/aws-sm#v2.3.1:
Expand All @@ -325,6 +348,7 @@ steps:

- label: "S3 source check on AWS (csv parser)"
command: "ci/scripts/s3-source-test.sh -p ci-release -s run_csv.py"
if: (!build.pull_request.labels includes "ci/main-cron/skip-ci" || build.pull_request.labels includes "ci/run-s3-source-tests")
depends_on: build
plugins:
- seek-oss/aws-sm#v2.3.1:
Expand All @@ -342,6 +366,7 @@ steps:

- label: "S3_v2 source check on AWS (json parser)"
command: "ci/scripts/s3-source-test.sh -p ci-release -s 'fs_source_v2.py json'"
if: (!build.pull_request.labels includes "ci/main-cron/skip-ci" || build.pull_request.labels includes "ci/run-s3-source-tests")
depends_on: build
plugins:
- seek-oss/aws-sm#v2.3.1:
Expand All @@ -359,6 +384,7 @@ steps:

- label: "S3_v2 source check on AWS (csv parser)"
command: "ci/scripts/s3-source-test.sh -p ci-release -s 'fs_source_v2.py csv_without_header'"
if: (!build.pull_request.labels includes "ci/main-cron/skip-ci" || build.pull_request.labels includes "ci/run-s3-source-tests")
depends_on: build
plugins:
- seek-oss/aws-sm#v2.3.1:
Expand All @@ -376,6 +402,7 @@ steps:

- label: "S3 source on OpenDAL fs engine"
command: "ci/scripts/s3-source-test-for-opendal-fs-engine.sh -p ci-release -s run"
if: (!build.pull_request.labels includes "ci/main-cron/skip-ci" || build.pull_request.labels includes "ci/run-s3-source-tests")
depends_on: build
plugins:
- seek-oss/aws-sm#v2.3.1:
Expand All @@ -393,6 +420,7 @@ steps:

- label: "pulsar source check"
command: "ci/scripts/pulsar-source-test.sh -p ci-release"
if: (!build.pull_request.labels includes "ci/main-cron/skip-ci" || build.pull_request.labels includes "ci/run-pulsar-source-tests")
depends_on:
- build
- build-other
Expand All @@ -413,6 +441,7 @@ steps:

- label: "micro benchmark"
command: "ci/scripts/run-micro-benchmarks.sh"
if: (!build.pull_request.labels includes "ci/main-cron/skip-ci" || build.pull_request.labels includes "ci/run-micro-benchmarks")
key: "run-micro-benchmarks"
plugins:
- docker-compose#v4.9.0:
Expand All @@ -423,7 +452,7 @@ steps:
retry: *auto-retry

- label: "upload micro-benchmark"
if: build.branch == "main" || build.pull_request.labels includes "ci/upload-micro-benchmark"
if: build.branch == "main" || (!build.pull_request.labels includes "ci/main-cron/skip-ci" || build.pull_request.labels includes "ci/run-micro-benchmarks")
command:
- "BUILDKITE_BUILD_NUMBER=$BUILDKITE_BUILD_NUMBER ci/scripts/upload-micro-bench-results.sh"
depends_on: "run-micro-benchmarks"
Expand All @@ -444,6 +473,7 @@ steps:
# Backwards compatibility tests
- label: "Backwards compatibility tests"
command: "RW_COMMIT=$BUILDKITE_COMMIT ci/scripts/backwards-compat-test.sh -p ci-release"
if: (!build.pull_request.labels includes "ci/main-cron/skip-ci" || build.pull_request.labels includes "ci/run-backwards-compat-tests")
depends_on:
- "build"
plugins:
Expand All @@ -458,6 +488,7 @@ steps:
# Sqlsmith differential testing
- label: "Sqlsmith Differential Testing"
command: "ci/scripts/sqlsmith-differential-test.sh -p ci-release"
if: (!build.pull_request.labels includes "ci/main-cron/skip-ci" || build.pull_request.labels includes "ci/run-sqlsmith-differential-tests")
depends_on:
- "build"
plugins:
Expand All @@ -470,6 +501,7 @@ steps:

- label: "Backfill tests"
command: "ci/scripts/backfill-test.sh -p ci-release"
if: (!build.pull_request.labels includes "ci/main-cron/skip-ci" || build.pull_request.labels includes "ci/run-backfill-tests")
depends_on:
- "build"
plugins:
Expand All @@ -483,6 +515,7 @@ steps:

- label: "e2e standalone binary test"
command: "ci/scripts/e2e-test.sh -p ci-release -m standalone"
if: (!build.pull_request.labels includes "ci/main-cron/skip-ci" || build.pull_request.labels includes "ci/run-e2e-standalone-tests")
depends_on:
- "build"
- "build-other"
Expand All @@ -498,6 +531,7 @@ steps:

- label: "end-to-end test for opendal (parallel)"
command: "ci/scripts/e2e-test-parallel-for-opendal.sh -p ci-release"
if: (!build.pull_request.labels includes "ci/main-cron/skip-ci" || build.pull_request.labels includes "ci/run-e2e-parallel-tests-for-opendal")
depends_on:
- "build"
- "docslt"
Expand All @@ -512,6 +546,7 @@ steps:

- label: "end-to-end clickhouse sink test"
command: "ci/scripts/e2e-clickhouse-sink-test.sh -p ci-release"
if: (!build.pull_request.labels includes "ci/main-cron/skip-ci" || build.pull_request.labels includes "ci/run-e2e-clickhouse-sink-tests")
depends_on:
- "build"
- "build-other"
Expand All @@ -526,6 +561,7 @@ steps:

- label: "end-to-end pulsar sink test"
command: "ci/scripts/e2e-pulsar-sink-test.sh -p ci-release"
if: (!build.pull_request.labels includes "ci/main-cron/skip-ci" || build.pull_request.labels includes "ci/run-e2e-pulsar-sink-tests")
depends_on:
- "build"
- "build-other"
Expand All @@ -540,6 +576,7 @@ steps:

- label: "connector node integration test Java {{matrix.java_version}}"
command: "ci/scripts/connector-node-integration-test.sh -p ci-release -v {{matrix.java_version}}"
if: (!build.pull_request.labels includes "ci/main-cron/skip-ci" || build.pull_request.labels includes "ci/run-connector-node-integration-tests")
depends_on:
- "build"
- "build-other"
Expand Down
Loading

0 comments on commit e60c070

Please sign in to comment.