Merge remote-tracking branch 'origin/main' into xx/sequence

risingwavelabs · Mar 28, 2024 · 68fadf9 · 68fadf9
2 parents 18bd40b + e0b71ba
commit 68fadf9
Show file tree

Hide file tree

Showing 185 changed files with 3,884 additions and 2,751 deletions.
diff --git a/Cargo.lock b/Cargo.lock
diff --git a/Makefile.toml b/Makefile.toml
@@ -91,6 +91,8 @@ if ${is_not_ci}
     set_env RUST_LOG "pgwire_query_log=info,${rust_log}"
   end
 end
+
+set_env TMUX "tmux -L risedev"
 ''',
 ]
 
@@ -155,15 +157,15 @@ category = "Misc"
 description = "List all nodes in the cluster"
 script = '''
 #!/usr/bin/env bash
-tmux list-windows -t risedev | grep -v active | cut -d'(' -f1
+${TMUX} list-windows -t risedev | grep -v active | cut -d'(' -f1
 '''
 
 [tasks.lsw]
 category = "Misc"
 description = "List --watch all nodes in the cluster"
 script = '''
 #!/usr/bin/env bash
-watch -n 1 "tmux list-windows -t risedev | grep -v active | cut -d'(' -f1"
+watch -n 1 "${TMUX} list-windows -t risedev | grep -v active | cut -d'(' -f1"
 '''
 
 [tasks.del]
@@ -176,7 +178,7 @@ script = '''
 #!/usr/bin/env bash
 
 risedev_ls () {
-  tmux list-windows -t risedev | grep -v active | cut -d'(' -f1
+  ${TMUX} list-windows -t risedev | grep -v active | cut -d'(' -f1
 }
 
 err () {
@@ -195,7 +197,7 @@ if [[ -z $(risedev_ls | grep "$1" ) ]]; then
   err
 fi
 
-tmux kill-window -t $1
+${TMUX} kill-window -t $1
 '''
 
 [tasks.f]
@@ -695,26 +697,31 @@ kill_zookeeper() {
   wait_zookeeper_exit
 }
 
+if ! ${TMUX} ls &>/dev/null ; then
+  echo "No risedev cluster to kill. Exiting..."
+  exit 0
+fi
+
 # Kill other components
-tmux list-windows -t risedev -F "#{window_name} #{pane_id}" \
+${TMUX} list-windows -F "#{window_name} #{pane_id}" \
 | grep -v 'kafka' \
 | grep -v 'zookeeper' \
 | awk '{ print $2 }' \
-| xargs -I {} tmux send-keys -t {} C-c C-d
+| xargs -I {} ${TMUX} send-keys -t {} C-c C-d
 
-if [[ -n $(tmux list-windows -t risedev | grep kafka) ]];
+if [[ -n $(${TMUX} list-windows | grep kafka) ]];
 then
   echo "kill kafka"
   kill_kafka || true
 
   echo "kill zookeeper"
   kill_zookeeper || true
 
-  # Kill their tmux sessions
-  tmux list-windows -t risedev -F "#{pane_id}" | xargs -I {} tmux send-keys -t {} C-c C-d
+  # Kill their ${TMUX} sessions
+  ${TMUX} list-windows -t risedev -F "#{pane_id}" | xargs -I {} ${TMUX} send-keys -t {} C-c C-d
 fi
 
-tmux kill-session -t risedev
+${TMUX} kill-server
 test $? -eq 0 || { echo "Failed to stop all RiseDev components."; exit 1; }
 '''
 
@@ -768,8 +775,8 @@ do
   echo
 done
 
-echo "check: $(tput setaf 4)tmux >= v3.2a$(tput sgr0)"
-tmux -V || echo "$(tput setaf 3)tmux$(tput sgr0) not found."
+echo "check: $(tput setaf 4)${TMUX} >= v3.2a$(tput sgr0)"
+${TMUX} -V || echo "$(tput setaf 3)tmux$(tput sgr0) not found."
 echo
 
 echo "check: $(tput setaf 4)psql >= 14$(tput sgr0)"

diff --git a/backwards-compat-tests/scripts/utils.sh b/backwards-compat-tests/scripts/utils.sh
@@ -50,12 +50,12 @@ kill_zookeeper() {
 }
 
 wait_for_process() {
-    process_name="$1"
+  process_name="$1"
 
-    while pgrep -x "$process_name" > /dev/null; do
-        echo "Process $process_name is still running... Wait for 1 sec"
-        sleep 1
-    done
+  while pgrep -x "$process_name" >/dev/null; do
+    echo "Process $process_name is still running... Wait for 1 sec"
+    sleep 1
+  done
 }
 
 wait_all_process_exit() {
@@ -69,33 +69,41 @@ wait_all_process_exit() {
 # Older versions of RW may not gracefully kill kafka.
 # So we duplicate the definition here.
 kill_cluster() {
+  if tmux -L risedev ls &>/dev/null; then
+    TMUX="tmux -L risedev"
+  else
+    TMUX="tmux"
+  fi
+
   # Kill other components
-  tmux list-windows -t risedev -F "#{window_name} #{pane_id}" \
-  | grep -v 'kafka' \
-  | grep -v 'zookeeper' \
-  | awk '{ print $2 }' \
-  | xargs -I {} tmux send-keys -t {} C-c C-d
+  $TMUX list-windows -t risedev -F "#{window_name} #{pane_id}" |
+    grep -v 'kafka' |
+    grep -v 'zookeeper' |
+    awk '{ print $2 }' |
+    xargs -I {} $TMUX send-keys -t {} C-c C-d
 
   set +e
-  if [[ -n $(tmux list-windows -t risedev | grep kafka) ]];
-  then
+  if [[ -n $($TMUX list-windows -t risedev | grep kafka) ]]; then
     echo "kill kafka"
     kill_kafka
 
     echo "kill zookeeper"
     kill_zookeeper
 
     # Kill their tmux sessions
-    tmux list-windows -t risedev -F "#{pane_id}" | xargs -I {} tmux send-keys -t {} C-c C-d
+    $TMUX list-windows -t risedev -F "#{pane_id}" | xargs -I {} $TMUX send-keys -t {} C-c C-d
   fi
   set -e
 
-  tmux kill-session -t risedev
-  test $? -eq 0 || { echo "Failed to stop all RiseDev components."; exit 1; }
+  $TMUX kill-server
+  test $? -eq 0 || {
+    echo "Failed to stop all RiseDev components."
+    exit 1
+  }
   wait_all_process_exit
 }
 
-run_sql () {
+run_sql() {
   psql -h localhost -p 4566 -d dev -U root -c "$@"
 }
 
@@ -140,42 +148,41 @@ seed_json_kafka() {
 
 # https://stackoverflow.com/a/4024263
 version_le() {
-    printf '%s\n' "$1" "$2" | sort -C -V
+  printf '%s\n' "$1" "$2" | sort -C -V
 }
 
 version_lt() {
-    ! version_le "$2" "$1"
+  ! version_le "$2" "$1"
 }
 
 ################################### Entry Points
 
 get_old_version() {
-   # For backwards compat test we assume we are testing the latest version of RW (i.e. latest main commit)
-   # against the Nth latest release candidate, where N > 1. N can be larger,
-   # in case some old cluster did not upgrade.
-   if [[ -z $VERSION_OFFSET ]]
-   then
-       local VERSION_OFFSET=1
-   fi
-
-   # First we obtain a list of versions from git branch names.
-   # Then we normalize them to semver format (MAJOR.MINOR.PATCH).
-   echo "--- git branch origin output"
-   git branch -r | grep origin
-
-   # Extract X.Y.Z tags
-   echo "--- VERSION BRANCHES"
-   local tags=$(git tag | grep -E "^v[0-9]+\.[0-9]+\.[0-9]+$" | tr -d 'v' | tr -d ' ')
-   echo "$tags"
-
-   # Then we sort them in descending order.
-   echo "--- VERSIONS"
-   local sorted_versions=$(echo -e "$tags" | sort -t '.' -n)
-   echo "$sorted_versions"
-
-   # Then we take the Nth latest version.
-   # We set $OLD_VERSION to this.
-   OLD_VERSION=$(echo -e "$sorted_versions" | tail -n $VERSION_OFFSET | head -1)
+  # For backwards compat test we assume we are testing the latest version of RW (i.e. latest main commit)
+  # against the Nth latest release candidate, where N > 1. N can be larger,
+  # in case some old cluster did not upgrade.
+  if [[ -z $VERSION_OFFSET ]]; then
+    local VERSION_OFFSET=1
+  fi
+
+  # First we obtain a list of versions from git branch names.
+  # Then we normalize them to semver format (MAJOR.MINOR.PATCH).
+  echo "--- git branch origin output"
+  git branch -r | grep origin
+
+  # Extract X.Y.Z tags
+  echo "--- VERSION BRANCHES"
+  local tags=$(git tag | grep -E "^v[0-9]+\.[0-9]+\.[0-9]+$" | tr -d 'v' | tr -d ' ')
+  echo "$tags"
+
+  # Then we sort them in descending order.
+  echo "--- VERSIONS"
+  local sorted_versions=$(echo -e "$tags" | sort -t '.' -n)
+  echo "$sorted_versions"
+
+  # Then we take the Nth latest version.
+  # We set $OLD_VERSION to this.
+  OLD_VERSION=$(echo -e "$sorted_versions" | tail -n $VERSION_OFFSET | head -1)
 }
 
 get_new_version() {
@@ -246,7 +253,7 @@ seed_old_cluster() {
   seed_json_kafka
   sqllogictest -d dev -h localhost -p 4566 "$TEST_DIR/kafka/seed.slt"
   # use the old syntax for version at most 1.5.4
-  if version_le "$OLD_VERSION" "1.5.4" ; then
+  if version_le "$OLD_VERSION" "1.5.4"; then
     sqllogictest -d dev -h localhost -p 4566 "$TEST_DIR/kafka/upsert/deprecate_upsert.slt"
   else
     sqllogictest -d dev -h localhost -p 4566 "$TEST_DIR/kafka/upsert/include_key_as.slt"
@@ -259,7 +266,7 @@ seed_old_cluster() {
   sqllogictest -d dev -h localhost -p 4566 "$TEST_DIR/kafka/validate_original.slt"
 
   # Test invalid WITH options, if OLD_VERSION <= 1.5.0
-  if version_le "$OLD_VERSION" "1.5.0" ; then
+  if version_le "$OLD_VERSION" "1.5.0"; then
     echo "--- KAFKA TEST (invalid options): Seeding old cluster with data"
     sqllogictest -d dev -h localhost -p 4566 "$TEST_DIR/kafka/invalid_options/seed.slt"
 
@@ -300,10 +307,10 @@ validate_new_cluster() {
   sqllogictest -d dev -h localhost -p 4566 "$TEST_DIR/kafka/validate_restart.slt"
 
   # Test invalid WITH options, if OLD_VERSION <= 1.5.0
-  if version_le "$OLD_VERSION" "1.5.0" ; then
+  if version_le "$OLD_VERSION" "1.5.0"; then
     echo "--- KAFKA TEST (invalid options): Validating new cluster"
     sqllogictest -d dev -h localhost -p 4566 "$TEST_DIR/kafka/invalid_options/validate_restart.slt"
   fi
 
   kill_cluster
-}
+}
diff --git a/ci/scripts/e2e-test-parallel.sh b/ci/scripts/e2e-test-parallel.sh
@@ -26,29 +26,33 @@ download_and_prepare_rw "$profile" common
 echo "--- Download artifacts"
 download-and-decompress-artifact e2e_test_generated ./
 
+kill_cluster() {
+  echo "--- Kill cluster"
+  cargo make ci-kill
+}
+
 host_args="-h localhost -p 4565 -h localhost -p 4566 -h localhost -p 4567"
 
+RUST_LOG="info,risingwave_stream=info,risingwave_batch=info,risingwave_storage=info,risingwave_storage::hummock::compactor::compactor_runner=warn"
+
 echo "--- e2e, ci-3streaming-2serving-3fe, streaming"
-RUST_LOG="info,risingwave_stream=info,risingwave_batch=info,risingwave_storage=info" \
+RUST_LOG=$RUST_LOG \
 cargo make ci-start ci-3streaming-2serving-3fe
 sqllogictest ${host_args} -d dev './e2e_test/streaming/**/*.slt' -j 16 --junit "parallel-streaming-${profile}"
 
-echo "--- Kill cluster"
-cargo make ci-kill
+kill_cluster
 
 echo "--- e2e, ci-3streaming-2serving-3fe, batch"
-RUST_LOG="info,risingwave_stream=info,risingwave_batch=info,risingwave_storage=info" \
+RUST_LOG=$RUST_LOG \
 cargo make ci-start ci-3streaming-2serving-3fe
 sqllogictest ${host_args} -d dev './e2e_test/ddl/**/*.slt' --junit "parallel-batch-ddl-${profile}"
 sqllogictest ${host_args} -d dev './e2e_test/visibility_mode/*.slt' -j 16 --junit "parallel-batch-${profile}"
 
-echo "--- Kill cluster"
-cargo make ci-kill
+kill_cluster
 
 echo "--- e2e, ci-3streaming-2serving-3fe, generated"
-RUST_LOG="info,risingwave_stream=info,risingwave_batch=info,risingwave_storage=info" \
+RUST_LOG=$RUST_LOG \
 cargo make ci-start ci-3streaming-2serving-3fe
 sqllogictest ${host_args} -d dev './e2e_test/generated/**/*.slt' -j 16 --junit "parallel-generated-${profile}"
 
-echo "--- Kill cluster"
-cargo make ci-kill
+kill_cluster
diff --git a/ci/scripts/release.sh b/ci/scripts/release.sh
@@ -66,6 +66,11 @@ fi
 
 echo "--- Build risingwave release binary"
 export ENABLE_BUILD_DASHBOARD=1
+if [ ${ARCH} == "aarch64" ]; then
+  # enable large page size support for jemalloc
+  # see https://github.com/tikv/jemallocator/blob/802969384ae0c581255f3375ee2ba774c8d2a754/jemalloc-sys/build.rs#L218
+  export JEMALLOC_SYS_WITH_LG_PAGE=16
+fi
 cargo build -p risingwave_cmd_all --features "rw-static-link" --profile release
 cargo build -p risingwave_cmd --bin risectl --features "rw-static-link" --profile release
 cd target/release && chmod +x risingwave risectl
@@ -75,8 +80,8 @@ if [ "${BUILDKITE_SOURCE}" == "schedule" ]; then
   tar -czvf risingwave-"$(date '+%Y%m%d')"-${ARCH}-unknown-linux.tar.gz risingwave
   aws s3 cp risingwave-"$(date '+%Y%m%d')"-${ARCH}-unknown-linux.tar.gz s3://rw-nightly-pre-built-binary
 elif [[ -n "${BINARY_NAME+x}" ]]; then
-    tar -czvf risingwave-${BINARY_NAME}-${ARCH}-unknown-linux.tar.gz risingwave
-    aws s3 cp risingwave-${BINARY_NAME}-${ARCH}-unknown-linux.tar.gz s3://rw-nightly-pre-built-binary
+  tar -czvf risingwave-${BINARY_NAME}-${ARCH}-unknown-linux.tar.gz risingwave
+  aws s3 cp risingwave-${BINARY_NAME}-${ARCH}-unknown-linux.tar.gz s3://rw-nightly-pre-built-binary
 fi
 
 echo "--- Build connector node"

diff --git a/ci/workflows/main-cron.yml b/ci/workflows/main-cron.yml
@@ -336,7 +336,7 @@ steps:
 
   - label: "end-to-end test (madsim)"
     key: "e2e-test-deterministic"
-    command: "TEST_NUM=64 timeout 55m ci/scripts/deterministic-e2e-test.sh"
+    command: "TEST_NUM=64 timeout 75m ci/scripts/deterministic-e2e-test.sh"
     if: |
       !(build.pull_request.labels includes "ci/main-cron/skip-ci") && build.env("CI_STEPS") == null
       || build.pull_request.labels includes "ci/run-e2e-test-deterministic-simulation"
@@ -353,7 +353,7 @@ steps:
           environment:
             - GITHUB_TOKEN
       - ./ci/plugins/upload-failure-logs
-    timeout_in_minutes: 60
+    timeout_in_minutes: 80
     retry: *auto-retry
 
   - label: "recovery test (madsim)"
@@ -396,7 +396,7 @@ steps:
   # Ddl statements will randomly run with background_ddl.
   - label: "background_ddl recovery test (madsim)"
     key: "background-ddl-recovery-test-deterministic"
-    command: "TEST_NUM=12 KILL_RATE=1.0 BACKGROUND_DDL_RATE=0.8 timeout 55m ci/scripts/deterministic-recovery-test.sh"
+    command: "TEST_NUM=12 KILL_RATE=1.0 BACKGROUND_DDL_RATE=0.8 timeout 60m ci/scripts/deterministic-recovery-test.sh"
     if: |
       !(build.pull_request.labels includes "ci/main-cron/skip-ci") && build.env("CI_STEPS") == null
       || build.pull_request.labels includes "ci/run-recovery-test-deterministic-simulation"
@@ -409,7 +409,7 @@ steps:
           mount-buildkite-agent: true
       # Only upload zipped files, otherwise the logs is too much.
       - ./ci/plugins/upload-failure-logs-zipped
-    timeout_in_minutes: 60
+    timeout_in_minutes: 65
     retry: *auto-retry
 
   - label: "end-to-end iceberg sink test (release)"