Merge branch 'main' of https://github.com/risingwavelabs/risingwave i…

…nto li0k/storage_max_sub_compaction_config

Cargo.toml

@@ -317,7 +317,7 @@ futures-timer = { git = "https://github.com/madsim-rs/futures-timer.git", rev =
 # patch: unlimit 4MB message size for grpc client
 etcd-client = { git = "https://github.com/risingwavelabs/etcd-client.git", rev = "4e84d40" }
 # todo(wcy-fdu): remove this patch fork after opendal release a new version to apply azure workload identity change.
-reqsign = { git = "https://github.com/wcy-fdu/reqsign.git", rev = "002ee2a" }
+reqsign = { git = "https://github.com/wcy-fdu/reqsign.git", rev = "c7dd668" }
 # patch to remove preserve_order from serde_json
 deno_core = { git = "https://github.com/bakjos/deno_core", rev = "9b241c6" }
 # patch to user reqwest 0.12.2

ci/scripts/run-e2e-test.sh

@@ -80,7 +80,6 @@ RUST_LOG="info,risingwave_stream=info,risingwave_batch=info,risingwave_storage=i
 cluster_start
 # Please make sure the regression is expected before increasing the timeout.
 sqllogictest -p 4566 -d dev './e2e_test/streaming/**/*.slt' --junit "streaming-${profile}"
-sqllogictest -p 4566 -d dev './e2e_test/backfill/rate_limit/*.slt'
 sqllogictest -p 4566 -d dev './e2e_test/backfill/sink/different_pk_and_dist_key.slt'
 
 echo "--- Kill cluster"
@@ -115,7 +114,6 @@ pkill python3
 
 sqllogictest -p 4566 -d dev './e2e_test/udf/alter_function.slt'
 sqllogictest -p 4566 -d dev './e2e_test/udf/graceful_shutdown_python.slt'
-sqllogictest -p 4566 -d dev './e2e_test/udf/always_retry_python.slt'
 # FIXME: flaky test
 # sqllogictest -p 4566 -d dev './e2e_test/udf/retry_python.slt'
 

ci/scripts/slow-e2e-test.sh

@@ -0,0 +1,40 @@
+#!/usr/bin/env bash
+
+# Exits as soon as any line fails.
+set -euo pipefail
+
+while getopts 'p:m:' opt; do
+    case ${opt} in
+        p )
+            profile=$OPTARG
+            ;;
+        m )
+            mode=$OPTARG
+            ;;
+        \? )
+            echo "Invalid Option: -$OPTARG" 1>&2
+            exit 1
+            ;;
+        : )
+            echo "Invalid option: $OPTARG requires an argument" 1>&2
+            ;;
+    esac
+done
+shift $((OPTIND -1))
+
+source ci/scripts/common.sh
+
+download_and_prepare_rw "$profile" common
+
+
+echo "--- Download artifacts"
+mkdir -p e2e_test/udf/wasm/target/wasm32-wasi/release/
+buildkite-agent artifact download udf.wasm e2e_test/udf/wasm/target/wasm32-wasi/release/
+buildkite-agent artifact download udf.jar ./
+
+echo "--- e2e, $mode, slow-udf-tests"
+python3 -m pip install --break-system-packages arrow-udf==0.2.1
+RUST_LOG="info" \
+risedev ci-start "$mode"
+sqllogictest -p 4566 -d dev './e2e_test_slow/udf/always_retry_python.slt'
+sqllogictest -p 4566 -d dev './e2e_test_slow/backfill/rate_limit/slow-udf.slt'

ci/workflows/main-cron.yml

@@ -90,6 +90,25 @@ steps:
     timeout_in_minutes: 15
     retry: *auto-retry
 
+  - label: "slow end-to-end test (release)"
+    key: "slow-e2e-test-release"
+    command: "ci/scripts/slow-e2e-test.sh -p ci-release -m ci-3streaming-2serving-3fe"
+    if: |
+        !(build.pull_request.labels includes "ci/main-cron/skip-ci") && build.env("CI_STEPS") == null
+        || build.pull_request.labels includes "ci/run-slow-e2e-tests"
+        || build.env("CI_STEPS") =~ /(^|,)slow-e2e-tests?(,|$$)/
+    depends_on:
+      - "build"
+      - "build-other"
+    plugins:
+      - docker-compose#v5.1.0:
+          run: rw-build-env
+          config: ci/docker-compose.yml
+          mount-buildkite-agent: true
+      - ./ci/plugins/upload-failure-logs
+    timeout_in_minutes: 8
+    retry: *auto-retry
+
   - label: "meta backup test (release)"
     key: "e2e-meta-backup-test-release"
     command: "ci/scripts/run-meta-backup-test.sh -p ci-release -m ci-3streaming-2serving-3fe"

ci/workflows/pull-request.yml

@@ -118,6 +118,25 @@ steps:
     timeout_in_minutes: 23
     retry: *auto-retry
 
+  - label: "slow end-to-end test"
+    key: "slow-e2e-test"
+    command: "ci/scripts/slow-e2e-test.sh -p ci-dev -m ci-3streaming-2serving-3fe"
+    if: |
+      !(build.pull_request.labels includes "ci/skip-ci") && build.env("CI_STEPS") == null
+      || build.pull_request.labels includes "ci/run-slow-e2e-tests"
+      || build.env("CI_STEPS") =~ /(^|,)slow-e2e-tests?(,|$$)/
+    depends_on:
+      - "build"
+      - "build-other"
+    plugins:
+      - docker-compose#v5.1.0:
+          run: rw-build-env
+          config: ci/docker-compose.yml
+          mount-buildkite-agent: true
+      - ./ci/plugins/upload-failure-logs
+    timeout_in_minutes: 8
+    retry: *auto-retry
+
   - label: "end-to-end test (parallel)"
     command: "ci/scripts/e2e-test-parallel.sh -p ci-dev"
     if: |

e2e_test/ddl/secret.slt

@@ -0,0 +1,23 @@
+statement error secret backend "fake-backend" is not supported
+create secret secret_1 with (
+  backend = 'fake-backend'
+) as 'demo_secret';
+
+statement ok
+create secret secret_1 with (
+  backend = 'meta'
+) as 'demo_secret';
+
+# wait for support for hashicorp_vault backend
+# statement ok
+# create secret secret_2 with (
+# backend = 'hashicorp_vault'
+# );
+
+query T
+show secrets;
+----
+secret_1
+
+statement ok
+drop secret secret_1;