Merge branch 'main' into tab/decrypt-func

Cargo.toml

@@ -133,7 +133,7 @@ arrow-flight = "50"
 arrow-select = "50"
 arrow-ord = "50"
 arrow-row = "50"
-arrow-udf-js = { git = "https://github.com/risingwavelabs/arrow-udf.git", rev = "7ba1c22" }
+arrow-udf-js = { git = "https://github.com/risingwavelabs/arrow-udf.git", rev = "ea7664d" }
 arrow-udf-wasm = "0.1"
 arrow-array-deltalake = { package = "arrow-array", version = "48.0.1" }
 arrow-buffer-deltalake = { package = "arrow-buffer", version = "48.0.1" }

ci/Dockerfile

@@ -13,7 +13,7 @@ RUN sed -i 's|http://archive.ubuntu.com/ubuntu|http://us-east-2.ec2.archive.ubun
 RUN apt-get update -yy && \
     DEBIAN_FRONTEND=noninteractive apt-get -y install make build-essential cmake protobuf-compiler curl parallel python3 python3-pip \
     openssl libssl-dev libsasl2-dev libcurl4-openssl-dev pkg-config bash openjdk-11-jdk wget unzip git tmux lld postgresql-client kafkacat netcat mysql-client \
-    maven zstd libzstd-dev locales -yy \
+    maven zstd libzstd-dev locales clang -yy \
     && rm -rf /var/lib/{apt,dpkg,cache,log}/
 
 SHELL ["/bin/bash", "-c"]
@@ -48,11 +48,11 @@ ENV CARGO_REGISTRIES_CRATES_IO_PROTOCOL=sparse
 RUN curl -L --proto '=https' --tlsv1.2 -sSf https://raw.githubusercontent.com/cargo-bins/cargo-binstall/main/install-from-binstall-release.sh | bash
 RUN cargo binstall -y --no-symlinks cargo-llvm-cov cargo-nextest cargo-hakari cargo-sort cargo-cache cargo-audit \
     [email protected] \
-    [email protected] [email protected] \
     [email protected] \
     [email protected] \
     && cargo cache -a \
     && rm -rf "/root/.cargo/registry/index" \
     && rm -rf "/root/.cargo/registry/cache" \
     && rm -rf "/root/.cargo/git/db"
+RUN cargo install [email protected] [email protected]
 RUN cargo uninstall cargo-binstall cargo-cache

ci/build-ci-image.sh

@@ -10,7 +10,7 @@ cat ../rust-toolchain
 # shellcheck disable=SC2155
 
 # REMEMBER TO ALSO UPDATE ci/docker-compose.yml
-export BUILD_ENV_VERSION=v20240122_1
+export BUILD_ENV_VERSION=v20240124_1
 
 export BUILD_TAG="public.ecr.aws/x5u3w5h6/rw-build-env:${BUILD_ENV_VERSION}"
 

ci/docker-compose.yml

@@ -71,7 +71,7 @@ services:
       retries: 5
 
   source-test-env:
-    image: public.ecr.aws/x5u3w5h6/rw-build-env:v20240122_1
+    image: public.ecr.aws/x5u3w5h6/rw-build-env:v20240124_1
     depends_on:
       - mysql
       - db
@@ -81,7 +81,7 @@ services:
       - ..:/risingwave
 
   sink-test-env:
-    image: public.ecr.aws/x5u3w5h6/rw-build-env:v20240122_1
+    image: public.ecr.aws/x5u3w5h6/rw-build-env:v20240124_1
     depends_on:
       - mysql
       - db
@@ -93,12 +93,12 @@ services:
       - ..:/risingwave
 
   rw-build-env:
-    image: public.ecr.aws/x5u3w5h6/rw-build-env:v20240122_1
+    image: public.ecr.aws/x5u3w5h6/rw-build-env:v20240124_1
     volumes:
       - ..:/risingwave
 
   ci-flamegraph-env:
-    image: public.ecr.aws/x5u3w5h6/rw-build-env:v20240122_1
+    image: public.ecr.aws/x5u3w5h6/rw-build-env:v20240124_1
     # NOTE(kwannoel): This is used in order to permit
     # syscalls for `nperf` (perf_event_open),
     # so it can do CPU profiling.
@@ -109,7 +109,7 @@ services:
       - ..:/risingwave
 
   regress-test-env:
-    image: public.ecr.aws/x5u3w5h6/rw-build-env:v20240122_1
+    image: public.ecr.aws/x5u3w5h6/rw-build-env:v20240124_1
     depends_on:
       db:
         condition: service_healthy

ci/plugins/upload-failure-logs-zipped/hooks/post-command

@@ -0,0 +1,18 @@
+#!/usr/bin/env bash
+
+set -euo pipefail
+
+if [ $BUILDKITE_COMMAND_EXIT_STATUS -ne 0 ]; then
+  mv .risingwave/log risedev-logs
+  zip -q -r risedev-logs.zip risedev-logs/
+  buildkite-agent artifact upload risedev-logs.zip
+  REGRESS_TEST_DIR="$PWD/src/tests/regress/output/results/"
+  if [ -d "$REGRESS_TEST_DIR" ]; then
+    mkdir regress-test-logs && cp src/tests/regress/output/results/* regress-test-logs/
+    zip -q -r regress-test.zip regress-test-logs/
+    buildkite-agent artifact upload regress-test-logs.zip
+  fi
+  if [ -e "$PWD/connector-node.log" ]; then
+    buildkite-agent artifact upload "$PWD/connector-node.log"
+  fi
+fi

ci/scripts/common.sh

@@ -87,3 +87,16 @@ function download_and_prepare_rw() {
   cargo make pre-start-dev
   cargo make --allow-private link-all-in-one-binaries
 }
+
+function filter_stack_trace() {
+  # Only keep first 3 lines of backtrace: 0-2.
+  echo "filtering stack trace for $1"
+  touch tmp
+  cat "$1" \
+  | sed -E '/  [1-9][0-9]+:/d' \
+  | sed -E '/  [3-9]+:/d' \
+  | sed -E '/  at .rustc/d' \
+  | sed -E '/  at ...cargo/d' > tmp
+  cp tmp "$1"
+  rm tmp
+}

ci/scripts/deterministic-recovery-test.sh

@@ -14,21 +14,70 @@ risingwave_meta::manager::catalog=debug,\
 risingwave_meta::rpc::ddl_controller=debug,\
 risingwave_meta::barrier::mod=debug,\
 risingwave_simulation=debug"
+
+# Extra logs you can enable if the existing trace does not give enough info.
+#risingwave_stream::executor::backfill=trace,
+#risingwave_meta::barrier::progress=debug,
+
+# ========= Some tips for debugging recovery tests =========
+# 1. If materialized view failed to create after multiple retries
+#    - Check logs to see where the materialized view creation was stuck.
+#      1. Is it stuck at waiting for backfill executor response?
+#         In that case perhaps some backfill logic is flawed, add more trace in backfill to debug.
+#      2. Is it stuck at waiting for backfill executor to finish?
+#         In that case perhaps some part of the backfill loop is slow / stuck.
+#      3. Is it stuck at waiting for some executor to report progress?
+#         In that case perhaps the tracking of backfill's progress in meta is flawed.
+
 export LOGDIR=.risingwave/log
 
 mkdir -p $LOGDIR
 
+filter_stack_trace_for_all_logs() {
+  # Defined in `common.sh`
+  for log in "${LOGDIR}"/*.log; do
+    filter_stack_trace $log
+  done
+}
+
+trap filter_stack_trace_for_all_logs ERR
+
 echo "--- deterministic simulation e2e, ci-3cn-2fe-3meta, recovery, background_ddl"
-seq $TEST_NUM | parallel MADSIM_TEST_SEED={} './risingwave_simulation --kill --kill-rate=${KILL_RATE} ./e2e_test/background_ddl/sim/basic.slt 2> $LOGDIR/recovery-ddl-{}.log && rm $LOGDIR/recovery-ddl-{}.log'
+seq $TEST_NUM | parallel MADSIM_TEST_SEED={} './risingwave_simulation \
+--kill \
+--kill-rate=${KILL_RATE} \
+${USE_ARRANGEMENT_BACKFILL:-} \
+./e2e_test/background_ddl/sim/basic.slt \
+2> $LOGDIR/recovery-background-ddl-{}.log && rm $LOGDIR/recovery-background-ddl-{}.log'
 
 echo "--- deterministic simulation e2e, ci-3cn-2fe-3meta, recovery, ddl"
-seq $TEST_NUM | parallel MADSIM_TEST_SEED={} './risingwave_simulation --kill --kill-rate=${KILL_RATE} --background-ddl-rate=${BACKGROUND_DDL_RATE} ./e2e_test/ddl/\*\*/\*.slt 2> $LOGDIR/recovery-ddl-{}.log && rm $LOGDIR/recovery-ddl-{}.log'
+seq $TEST_NUM | parallel MADSIM_TEST_SEED={} './risingwave_simulation \
+--kill \
+--kill-rate=${KILL_RATE} \
+--background-ddl-rate=${BACKGROUND_DDL_RATE} \
+${USE_ARRANGEMENT_BACKFILL:-} \
+./e2e_test/ddl/\*\*/\*.slt 2> $LOGDIR/recovery-ddl-{}.log && rm $LOGDIR/recovery-ddl-{}.log'
 
 echo "--- deterministic simulation e2e, ci-3cn-2fe-3meta, recovery, streaming"
-seq $TEST_NUM | parallel MADSIM_TEST_SEED={} './risingwave_simulation --kill --kill-rate=${KILL_RATE} --background-ddl-rate=${BACKGROUND_DDL_RATE} ./e2e_test/streaming/\*\*/\*.slt 2> $LOGDIR/recovery-streaming-{}.log && rm $LOGDIR/recovery-streaming-{}.log'
+seq $TEST_NUM | parallel MADSIM_TEST_SEED={} './risingwave_simulation \
+--kill \
+--kill-rate=${KILL_RATE} \
+--background-ddl-rate=${BACKGROUND_DDL_RATE} \
+${USE_ARRANGEMENT_BACKFILL:-} \
+./e2e_test/streaming/\*\*/\*.slt 2> $LOGDIR/recovery-streaming-{}.log && rm $LOGDIR/recovery-streaming-{}.log'
 
 echo "--- deterministic simulation e2e, ci-3cn-2fe-3meta, recovery, batch"
-seq $TEST_NUM | parallel MADSIM_TEST_SEED={} './risingwave_simulation --kill --kill-rate=${KILL_RATE} --background-ddl-rate=${BACKGROUND_DDL_RATE} ./e2e_test/batch/\*\*/\*.slt 2> $LOGDIR/recovery-batch-{}.log && rm $LOGDIR/recovery-batch-{}.log'
+seq $TEST_NUM | parallel MADSIM_TEST_SEED={} './risingwave_simulation \
+--kill \
+--kill-rate=${KILL_RATE} \
+--background-ddl-rate=${BACKGROUND_DDL_RATE} \
+${USE_ARRANGEMENT_BACKFILL:-} \
+./e2e_test/batch/\*\*/\*.slt 2> $LOGDIR/recovery-batch-{}.log && rm $LOGDIR/recovery-batch-{}.log'
 
 echo "--- deterministic simulation e2e, ci-3cn-2fe-3meta, recovery, kafka source,sink"
-seq $TEST_NUM | parallel MADSIM_TEST_SEED={} './risingwave_simulation --kill --kill-rate=${KILL_RATE} --kafka-datadir=./scripts/source/test_data ./e2e_test/source/basic/kafka\*.slt 2> $LOGDIR/recovery-source-{}.log && rm $LOGDIR/recovery-source-{}.log'
+seq $TEST_NUM | parallel MADSIM_TEST_SEED={} './risingwave_simulation \
+--kill \
+--kill-rate=${KILL_RATE} \
+--kafka-datadir=./scripts/source/test_data \
+${USE_ARRANGEMENT_BACKFILL:-} \
+./e2e_test/source/basic/kafka\*.slt 2> $LOGDIR/recovery-source-{}.log && rm $LOGDIR/recovery-source-{}.log'