Merge branch 'main' into wrj/detach-java-udf

.cargo/config.toml

@@ -37,23 +37,8 @@ rustflags = [
     # uncomment the following two lines to enable `TaskLocalAlloc`
     # "--cfg",
     # "enable_task_local_alloc",
-    # lints
-    # TODO: use lint configuration in cargo https://github.com/rust-lang/cargo/issues/5034
-    "-Funused_must_use",
-    "-Aclippy::uninlined_format_args",
-    "-Wclippy::dbg_macro",
-    "-Wclippy::disallowed_methods",
-    "-Wclippy::disallowed_types",
-    "-Wclippy::doc_markdown",
-    "-Wclippy::explicit_into_iter_loop",
-    "-Wclippy::explicit_iter_loop",
-    "-Wclippy::inconsistent_struct_constructor",
-    "-Wclippy::unused_async",
-    "-Wclippy::map_flatten",
-    "-Wclippy::no_effect_underscore_binding",
-    "-Wclippy::await_holding_lock",
-    "-Wrustdoc::broken_intra_doc_links",
-    "-Wfuture_incompatible",
-    "-Wnonstandard_style",
-    "-Wrust_2018_idioms",
+
 ]
+
+[unstable]
+lints = true

.github/workflows/cherry-pick-to-release-branch.yml

@@ -3,7 +3,7 @@ on:
   pull_request:
     branches:
       - main
-    types: ["closed"]
+    types: ["closed", "labeled"]
 
 jobs:
   release_pull_request_1_1:

.github/workflows/nightly-rust.yml

@@ -11,13 +11,23 @@ jobs:
   build:
     runs-on: ubuntu-latest
     steps:
+      -  name: Maximize build space
+         uses: easimon/maximize-build-space@master
+         with:
+           remove-dotnet: 'true'
+           remove-android: 'true'
+           remove-haskell: 'true'
+           remove-codeql: 'true'
+           remove-docker-images: 'true'
       - uses: actions/checkout@v3
       - name: Setup Rust toolchain
         run: |
           rustup override set nightly
           rustup update nightly
       - name: Install dependencies
         run: sudo apt-get update && sudo apt-get install -y make build-essential cmake protobuf-compiler curl openssl libssl-dev libsasl2-dev libcurl4-openssl-dev pkg-config postgresql-client tmux lld
-      - name: cargo build
+      - name: cargo check
         run: |
-          cargo build
+          export CARGO_INCREMENTAL=0
+          export CARGO_PROFILE_DEV_DEBUG=false
+          cargo check

Cargo.toml

@@ -61,22 +61,48 @@ repository = "https://github.com/risingwavelabs/risingwave"
 
 [workspace.dependencies]
 await-tree = "0.1.1"
-aws-config = { version = "0.55", default-features = false, features = ["rt-tokio", "native-tls"] }
-aws-credential-types = { version = "0.55", default-features = false, features = ["hardcoded-credentials"] }
-aws-sdk-kinesis = { version = "0.28", default-features = false, features = ["rt-tokio", "native-tls"] }
-aws-sdk-s3 = { version = "0.28", default-features = false, features = ["rt-tokio","native-tls"] }
-aws-sdk-ec2 = { version = "0.28", default-features = false, features = ["rt-tokio","native-tls"] }
-aws-sdk-sqs = { version = "0.28", default-features = false, features = ["rt-tokio", "native-tls"] }
-aws-smithy-client = { version = "0.55", default-features = false, features = ["rt-tokio", "native-tls"] }
+aws-config = { version = "0.55", default-features = false, features = [
+  "rt-tokio",
+  "native-tls",
+] }
+aws-credential-types = { version = "0.55", default-features = false, features = [
+  "hardcoded-credentials",
+] }
+aws-sdk-kinesis = { version = "0.28", default-features = false, features = [
+  "rt-tokio",
+  "native-tls",
+] }
+aws-sdk-s3 = { version = "0.28", default-features = false, features = [
+  "rt-tokio",
+  "native-tls",
+] }
+aws-sdk-ec2 = { version = "0.28", default-features = false, features = [
+  "rt-tokio",
+  "native-tls",
+] }
+aws-sdk-sqs = { version = "0.28", default-features = false, features = [
+  "rt-tokio",
+  "native-tls",
+] }
+aws-smithy-client = { version = "0.55", default-features = false, features = [
+  "rt-tokio",
+  "native-tls",
+] }
 aws-smithy-http = "0.55"
 aws-smithy-types = "0.55"
 aws-endpoint = "0.55"
 aws-types = "0.55"
 etcd-client = { package = "madsim-etcd-client", version = "0.3" }
 futures-async-stream = "0.2"
 hytra = "0.1"
-rdkafka = { package = "madsim-rdkafka", git = "https://github.com/madsim-rs/madsim.git", rev = "bb8f063", features = ["cmake-build"] }
-hashbrown = { version = "0.14.0", features = ["ahash", "inline-more", "nightly"] }
+rdkafka = { package = "madsim-rdkafka", git = "https://github.com/madsim-rs/madsim.git", rev = "bb8f063", features = [
+  "cmake-build",
+] }
+hashbrown = { version = "0.14.0", features = [
+  "ahash",
+  "inline-more",
+  "nightly",
+] }
 criterion = { version = "0.5", features = ["async_futures"] }
 tonic = { package = "madsim-tonic", version = "0.3.1" }
 tonic-build = { package = "madsim-tonic-build", version = "0.3.1" }
@@ -117,6 +143,32 @@ risingwave_variables = { path = "./src/utils/variables" }
 risingwave_java_binding = { path = "./src/java_binding" }
 risingwave_jni_core = { path = "src/jni_core" }
 
+[workspace.lints.rust]
+# `forbid` will also prevent the misuse of `#[allow(unused)]`
+unused_must_use = "forbid"
+future_incompatible = "warn"
+nonstandard_style = "warn"
+rust_2018_idioms = "warn"
+
+[workspace.lints.clippy]
+uninlined_format_args = "allow"
+dbg_macro = "warn"
+disallowed_methods = "warn"
+disallowed_types = "warn"
+doc_markdown = "warn"
+explicit_into_iter_loop = "warn"
+explicit_iter_loop = "warn"
+inconsistent_struct_constructor = "warn"
+unused_async = "warn"
+map_flatten = "warn"
+no_effect_underscore_binding = "warn"
+await_holding_lock = "warn"
+
+[workspace.lints.rustdoc]
+private_intra_doc_links = "allow"
+# Explicit lints don't hurt, and sometimes rust-analyzer works better with explicit links.
+redundant_explicit_links = "allow"
+
 [profile.dev]
 lto = 'off'
 

ci/build-ci-image.sh

@@ -13,7 +13,7 @@ cat ../rust-toolchain
 # !!! CHANGE THIS WHEN YOU WANT TO BUMP CI IMAGE !!! #
 #          AND ALSO docker-compose.yml               #
 ######################################################
-export BUILD_ENV_VERSION=v20230821
+export BUILD_ENV_VERSION=v20230909
 
 export BUILD_TAG="public.ecr.aws/x5u3w5h6/rw-build-env:${BUILD_ENV_VERSION}"
 

ci/docker-compose.yml

@@ -71,7 +71,7 @@ services:
       retries: 5
 
   source-test-env:
-    image: public.ecr.aws/x5u3w5h6/rw-build-env:v20230821
+    image: public.ecr.aws/x5u3w5h6/rw-build-env:v20230909
     depends_on:
       - mysql
       - db
@@ -81,7 +81,7 @@ services:
       - ..:/risingwave
 
   sink-test-env:
-    image: public.ecr.aws/x5u3w5h6/rw-build-env:v20230821
+    image: public.ecr.aws/x5u3w5h6/rw-build-env:v20230909
     depends_on:
       - mysql
       - db
@@ -91,12 +91,12 @@ services:
       - ..:/risingwave
 
   rw-build-env:
-    image: public.ecr.aws/x5u3w5h6/rw-build-env:v20230821
+    image: public.ecr.aws/x5u3w5h6/rw-build-env:v20230909
     volumes:
       - ..:/risingwave
 
   ci-flamegraph-env:
-    image: public.ecr.aws/x5u3w5h6/rw-build-env:v20230821
+    image: public.ecr.aws/x5u3w5h6/rw-build-env:v20230909
     # NOTE(kwannoel): This is used in order to permit
     # syscalls for `nperf` (perf_event_open),
     # so it can do CPU profiling.
@@ -107,7 +107,7 @@ services:
       - ..:/risingwave
 
   regress-test-env:
-    image: public.ecr.aws/x5u3w5h6/rw-build-env:v20230821
+    image: public.ecr.aws/x5u3w5h6/rw-build-env:v20230909
     depends_on:
       db:
         condition: service_healthy

ci/rust-toolchain

@@ -1,2 +1,2 @@
 [toolchain]
-channel = "nightly-2023-05-31"
+channel = "nightly-2023-09-09"

ci/scripts/check.sh

@@ -35,7 +35,7 @@ sccache --show-stats
 sccache --zero-stats
 
 echo "--- Build documentation"
-RUSTDOCFLAGS="-Dwarnings -Arustdoc::private_intra_doc_links" cargo doc --document-private-items --no-deps
+RUSTDOCFLAGS="-Dwarnings" cargo doc --document-private-items --no-deps
 
 echo "--- Show sccache stats"
 sccache --show-stats

ci/scripts/common.sh

@@ -13,6 +13,7 @@ export CARGO_MAKE_PRINT_TIME_SUMMARY=true
 export MINIO_DOWNLOAD_BIN=https://ci-deps-dist.s3.amazonaws.com/minio
 export MCLI_DOWNLOAD_BIN=https://ci-deps-dist.s3.amazonaws.com/mc
 export GCLOUD_DOWNLOAD_TGZ=https://ci-deps-dist.s3.amazonaws.com/google-cloud-cli-406.0.0-linux-x86_64.tar.gz
+export NEXTEST_HIDE_PROGRESS_BAR=true
 unset LANG
 if [ -n "${BUILDKITE_COMMIT:-}" ]; then
   export GIT_SHA=$BUILDKITE_COMMIT

ci/scripts/gen-flamegraph.sh

@@ -6,6 +6,8 @@ set -euo pipefail
 
 source ci/scripts/common.sh
 
+RUST_TOOLCHAIN=$(cat rust-toolchain)
+
 QUERY_DIR="/risingwave/ci/scripts/sql/nexmark"
 
 # TODO(kwannoel): This is a workaround since workdir is `/risingwave` in the docker container.
@@ -79,7 +81,7 @@ install_all() {
   git clone https://github.com/gimli-rs/addr2line
   pushd addr2line
   git checkout 0.20.0
-  echo "nightly-2023-04-07" > rust-toolchain
+  echo "$RUST_TOOLCHAIN" > rust-toolchain
   cargo b --examples -r
   mv ./target/release/examples/addr2line $(which addr2line)
   popd

ci/scripts/run-fuzz-test.sh

@@ -8,7 +8,7 @@ export RUST_LOG=info
 
 if [[ $RUN_SQLSMITH_FRONTEND -eq "1" ]]; then
     echo "--- Run sqlsmith frontend tests"
-     NEXTEST_PROFILE=ci cargo nextest run --package risingwave_sqlsmith --features "enable_sqlsmith_unit_test" 2> >(tee);
+    NEXTEST_PROFILE=ci cargo nextest run --package risingwave_sqlsmith --features "enable_sqlsmith_unit_test"
 fi
 
 extract_error_sql() {

ci/scripts/run-unit-test.sh

@@ -12,7 +12,7 @@ cd ${REPO_ROOT}
 
 echo "+++ Run unit tests with coverage"
 # use tee to disable progress bar
-NEXTEST_PROFILE=ci cargo llvm-cov nextest --lcov --output-path lcov.info --features failpoints,sync_point --workspace --exclude risingwave_simulation 2> >(tee);
+NEXTEST_PROFILE=ci cargo llvm-cov nextest --lcov --output-path lcov.info --features failpoints,sync_point --workspace --exclude risingwave_simulation
 
 echo "--- Codecov upload coverage reports"
 curl -Os https://uploader.codecov.io/latest/linux/codecov && chmod +x codecov

grafana/common.py

@@ -56,6 +56,7 @@
         ]
     )
 
+
 class Layout:
 
     def __init__(self):
@@ -198,10 +199,10 @@ def timeseries_latency(self,
         )
 
     def timeseries_latency_ms(self,
-                           title,
-                           description,
-                           targets,
-                           legendCols=["mean"]):
+                              title,
+                              description,
+                              targets,
+                              legendCols=["mean"]):
         gridPos = self.layout.next_half_width_graph()
         return TimeSeries(
             title=title,
@@ -525,7 +526,8 @@ def metric(name, filter=None, node_filter_enabled=True, table_id_filter_enabled=
     if risingwave_name_filter_enabled:
         filters.append("risingwave_name=~\"$instance\"")
     if table_id_filter_enabled:
-        filters.append("table_id=~\"$table\"")
+        # We use "%table|" instead of "%table" here to match empty string for the table_id filter
+        filters.append("table_id=~\"$table|\"")
     if node_filter_enabled:
         filters.append("job=~\"$job\"")
         filters.append("instance=~\"$node\"")
@@ -534,6 +536,7 @@ def metric(name, filter=None, node_filter_enabled=True, table_id_filter_enabled=
     else:
         return name
 
+
 def table_metric(name, filter=None):
     return metric(name, filter, True, True)