Merge branch 'main' into sts/change_watermark_filter_to_wait_committe…

…d_epoch

.github/workflows/typo.yml

@@ -10,4 +10,4 @@ jobs:
       uses: actions/checkout@v3
 
     - name: Check spelling of the entire repository
-      uses: crate-ci/typos@v1.20.4
+      uses: crate-ci/typos@v1.23.2

.pre-commit-config.yaml

@@ -7,17 +7,17 @@ repos:
     hooks:
     -   id: end-of-file-fixer
     -   id: trailing-whitespace
+-   repo: https://github.com/crate-ci/typos
+    rev: v1.23.1
+    hooks:
+      - id: typos
 -   repo: local
     hooks:
     -   id: rustfmt
         name: rustfmt
         entry: rustfmt --edition 2021
         language: system
         types: [rust]
-    -   id: typos
-        name: typos
-        entry: typos -w
-        language: system
     -   id: cargo sort
         name: cargo sort
         entry: cargo sort -g -w

.typos.toml

@@ -30,4 +30,8 @@ extend-exclude = [
     "**/Cargo.toml",
     "**/go.mod",
     "**/go.sum",
+    # https://github.com/risingwavelabs/risingwave/blob/0ce6228df6a4da183ae91146f2cdfff1ca9cc6a7/src/common/src/cast/mod.rs#L30
+    # We don't want to fix "fals" here, but may want in other places.
+    # Ideally, we should just ignore that line: https://github.com/crate-ci/typos/issues/316
+    "src/common/src/cast/mod.rs",
 ]

Cargo.toml

@@ -94,6 +94,7 @@ aws-config = { version = "1", default-features = false, features = [
 aws-credential-types = { version = "1", default-features = false, features = [
     "hardcoded-credentials",
 ] }
+aws-sdk-glue = "1"
 aws-sdk-kinesis = { version = "1", default-features = false, features = [
     "rt-tokio",
     "rustls",
@@ -143,6 +144,7 @@ arrow-cast-iceberg = { package = "arrow-cast", version = "52" }
 # After apache/iceberg-rust#411 is merged, we move to the upstream version.
 iceberg = { git = "https://github.com/risingwavelabs/iceberg-rust.git", rev = "0c6e133e6f4655ff9ce4ad57b577dc7f692dd902" }
 iceberg-catalog-rest = { git = "https://github.com/risingwavelabs/iceberg-rust.git", rev = "0c6e133e6f4655ff9ce4ad57b577dc7f692dd902" }
+opendal = "0.47"
 arrow-array = "50"
 arrow-arith = "50"
 arrow-cast = "50"
@@ -333,8 +335,6 @@ tokio-stream = { git = "https://github.com/madsim-rs/tokio.git", rev = "fe39bb8e
 tokio-retry = { git = "https://github.com/madsim-rs/rust-tokio-retry.git", rev = "95e2fd3" }
 tokio-postgres = { git = "https://github.com/madsim-rs/rust-postgres.git", rev = "ac00d88" }
 futures-timer = { git = "https://github.com/madsim-rs/futures-timer.git", rev = "05b33b4" }
-# patch: unlimit 4MB message size for grpc client
-etcd-client = { git = "https://github.com/risingwavelabs/etcd-client.git", rev = "4e84d40" }
 # patch to remove preserve_order from serde_json
 deno_core = { git = "https://github.com/bakjos/deno_core", rev = "9b241c6" }
 # patch to user reqwest 0.12.2

Makefile.toml

@@ -28,6 +28,8 @@ env_scripts = [
 
   set_env ENABLE_TELEMETRY "false"
   set_env RW_TELEMETRY_TYPE "test"
+  set_env RW_SECRET_STORE_PRIVATE_KEY_HEX "0123456789abcdef"
+  set_env RW_TEMP_SECRET_FILE_DIR "${PREFIX_SECRET}"
 
   is_sanitizer_enabled = get_env ENABLE_SANITIZER
   is_hdfs_backend = get_env ENABLE_HDFS
@@ -144,6 +146,7 @@ rm -rf "${PREFIX_DATA}"
 rm -rf "${PREFIX_LOG}"
 rm -rf "${PREFIX_CONFIG}"
 rm -rf "${PREFIX_PROFILING}"
+rm -rf "${PREFIX_SECRET}"
 '''
 
 [tasks.reset-rw]
@@ -578,7 +581,7 @@ set -euo pipefail
 RC_ENV_FILE="${PREFIX_CONFIG}/risedev-env"
 
 if [ ! -f "${RC_ENV_FILE}" ]; then
-  echo "risedev-env file not found. Did you start cluster using $(tput setaf 4)\`./risedev d\`$(tput sgr0) or $(tput setaf 4)\`./risedev p\`$(tput sgr0)?"
+  echo "risedev-env file not found at ${RC_ENV_FILE}. Did you start cluster using $(tput setaf 4)\`./risedev d\`$(tput sgr0) or $(tput setaf 4)\`./risedev p\`$(tput sgr0)?"
   exit 1
 fi
 '''
@@ -710,10 +713,10 @@ if ! ${TMUX} ls &>/dev/null ; then
   exit 0
 fi
 
-# Kill other components with Ctrl+C/Ctrl+D
+# Kill tmux components with Ctrl+C
 ${TMUX} list-windows -F "#{window_name} #{pane_id}" \
 | awk '{ print $2 }' \
-| xargs -I {} ${TMUX} send-keys -t {} C-c C-d
+| xargs -I {} ${TMUX} send-keys -t {} C-c
 
 # Stop docker components
 containers=$(docker ps -a -q -f name=risedev- 2>/dev/null) || true
@@ -1135,7 +1138,7 @@ fi
 private = true
 category = "RiseDev - Check"
 description = "Run cargo typos-cli check"
-install_crate = { min_version = "1.20.4", crate_name = "typos-cli", binary = "typos", test_arg = [
+install_crate = { min_version = "1.23.2", crate_name = "typos-cli", binary = "typos", test_arg = [
     "--help",
 ], install_command = "binstall" }
 script = """
@@ -1258,7 +1261,7 @@ echo If you still feel this is not enough, you may copy $(tput setaf 4)risedev$(
 [tasks.ci-start]
 category = "RiseDev - CI"
 dependencies = ["clean-data", "pre-start-dev"]
-command = "target/${BUILD_MODE_DIR}/risedev-dev"
+command = "target/debug/risedev-dev" # `risedev-dev` is always built in dev profile
 args = ["${@}"]
 description = "Clean data and start a full RisingWave dev cluster using risedev-dev"
 

README.md

@@ -150,4 +150,4 @@ RisingWave is distributed under the Apache License (Version 2.0). Please refer t
 
 ## Contributing
 
-Thanks for your interest in contributing to the project! Please refer to [contribution guidelines](CONTRIBUTING.md) for more information.
+Thanks for your interest in contributing to the project! Please refer to [RisingWave Developer Guide](https://risingwavelabs.github.io/risingwave/)for more information.

ci/scripts/common.sh

@@ -15,6 +15,8 @@ export MCLI_DOWNLOAD_BIN=https://rw-ci-deps-dist.s3.amazonaws.com/mc
 export GCLOUD_DOWNLOAD_TGZ=https://rw-ci-deps-dist.s3.amazonaws.com/google-cloud-cli-475.0.0-linux-x86_64.tar.gz
 export NEXTEST_HIDE_PROGRESS_BAR=true
 export RW_TELEMETRY_TYPE=test
+export RW_SECRET_STORE_PRIVATE_KEY_HEX="0123456789abcdef"
+
 unset LANG
 if [ -n "${BUILDKITE_COMMIT:-}" ]; then
   export GIT_SHA=$BUILDKITE_COMMIT

ci/scripts/deterministic-recovery-test.sh

@@ -13,7 +13,9 @@ export RUST_LOG="risingwave_meta::barrier::recovery=debug,\
 risingwave_meta::manager::catalog=debug,\
 risingwave_meta::rpc::ddl_controller=debug,\
 risingwave_meta::barrier::mod=debug,\
-risingwave_simulation=debug"
+risingwave_simulation=debug,\
+risingwave_meta::stream::stream_manager=debug,\
+risingwave_meta::barrier::progress=debug"
 
 # Extra logs you can enable if the existing trace does not give enough info.
 #risingwave_stream::executor::backfill=trace,

ci/scripts/e2e-sqlserver-sink-test.sh

@@ -74,6 +74,7 @@ if [[ ${#actual[@]} -eq ${#expected[@]} && ${actual[@]} == ${expected[@]} ]]; th
 else
   cat ./query_result.txt
   echo "The output is not as expected."
+  exit 1
 fi
 
 echo "--- Kill cluster"