feat(ci): introduce slow e2e test step (#16953) (#16978)

ci/scripts/run-e2e-test.sh

@@ -80,7 +80,6 @@ RUST_LOG="info,risingwave_stream=info,risingwave_batch=info,risingwave_storage=i
 cluster_start
 # Please make sure the regression is expected before increasing the timeout.
 sqllogictest -p 4566 -d dev './e2e_test/streaming/**/*.slt' --junit "streaming-${profile}"
-sqllogictest -p 4566 -d dev './e2e_test/backfill/rate_limit/*.slt'
 sqllogictest -p 4566 -d dev './e2e_test/backfill/sink/different_pk_and_dist_key.slt'
 
 echo "--- Kill cluster"
@@ -114,7 +113,6 @@ pkill python3
 
 sqllogictest -p 4566 -d dev './e2e_test/udf/alter_function.slt'
 sqllogictest -p 4566 -d dev './e2e_test/udf/graceful_shutdown_python.slt'
-sqllogictest -p 4566 -d dev './e2e_test/udf/always_retry_python.slt'
 # FIXME: flaky test
 # sqllogictest -p 4566 -d dev './e2e_test/udf/retry_python.slt'
 

ci/scripts/slow-e2e-test.sh

@@ -0,0 +1,40 @@
+#!/usr/bin/env bash
+
+# Exits as soon as any line fails.
+set -euo pipefail
+
+while getopts 'p:m:' opt; do
+    case ${opt} in
+        p )
+            profile=$OPTARG
+            ;;
+        m )
+            mode=$OPTARG
+            ;;
+        \? )
+            echo "Invalid Option: -$OPTARG" 1>&2
+            exit 1
+            ;;
+        : )
+            echo "Invalid option: $OPTARG requires an argument" 1>&2
+            ;;
+    esac
+done
+shift $((OPTIND -1))
+
+source ci/scripts/common.sh
+
+download_and_prepare_rw "$profile" common
+
+
+echo "--- Download artifacts"
+mkdir -p e2e_test/udf/wasm/target/wasm32-wasi/release/
+buildkite-agent artifact download udf.wasm e2e_test/udf/wasm/target/wasm32-wasi/release/
+buildkite-agent artifact download risingwave-udf-example.jar ./
+
+echo "--- e2e, $mode, slow-udf-tests"
+python3 -m pip install --break-system-packages arrow-udf==0.2.1
+RUST_LOG="info" \
+risedev ci-start "$mode"
+sqllogictest -p 4566 -d dev './e2e_test_slow/udf/always_retry_python.slt'
+sqllogictest -p 4566 -d dev './e2e_test_slow/backfill/rate_limit/slow-udf.slt'

ci/workflows/main-cron.yml

@@ -90,6 +90,25 @@ steps:
     timeout_in_minutes: 15
     retry: *auto-retry
 
+  - label: "slow end-to-end test (release)"
+    key: "slow-e2e-test-release"
+    command: "ci/scripts/slow-e2e-test.sh -p ci-release -m ci-3streaming-2serving-3fe"
+    if: |
+        !(build.pull_request.labels includes "ci/main-cron/skip-ci") && build.env("CI_STEPS") == null
+        || build.pull_request.labels includes "ci/run-slow-e2e-tests"
+        || build.env("CI_STEPS") =~ /(^|,)slow-e2e-tests?(,|$$)/
+    depends_on:
+      - "build"
+      - "build-other"
+    plugins:
+      - docker-compose#v5.1.0:
+          run: rw-build-env
+          config: ci/docker-compose.yml
+          mount-buildkite-agent: true
+      - ./ci/plugins/upload-failure-logs
+    timeout_in_minutes: 4
+    retry: *auto-retry
+
   - label: "meta backup test (release)"
     key: "e2e-meta-backup-test-release"
     command: "ci/scripts/run-meta-backup-test.sh -p ci-release -m ci-3streaming-2serving-3fe"

ci/workflows/pull-request.yml

@@ -100,6 +100,25 @@ steps:
     timeout_in_minutes: 23
     retry: *auto-retry
 
+  - label: "slow end-to-end test"
+    key: "slow-e2e-test"
+    command: "ci/scripts/slow-e2e-test.sh -p ci-dev -m ci-3streaming-2serving-3fe"
+    if: |
+      !(build.pull_request.labels includes "ci/skip-ci") && build.env("CI_STEPS") == null
+      || build.pull_request.labels includes "ci/run-slow-e2e-tests"
+      || build.env("CI_STEPS") =~ /(^|,)slow-e2e-tests?(,|$$)/
+    depends_on:
+      - "build"
+      - "build-other"
+    plugins:
+      - docker-compose#v5.1.0:
+          run: rw-build-env
+          config: ci/docker-compose.yml
+          mount-buildkite-agent: true
+      - ./ci/plugins/upload-failure-logs
+    timeout_in_minutes: 6
+    retry: *auto-retry
+
   - label: "end-to-end test (parallel)"
     command: "ci/scripts/e2e-test-parallel.sh -p ci-dev"
     if: |

e2e_test/udf/always_retry_python.slt → e2e_test_slow/udf/always_retry_python.slt

@@ -33,7 +33,7 @@ CREATE MATERIALIZED VIEW mv_no_retry AS SELECT sleep_no_retry(v1) as s1 from t;
 statement ok
 CREATE MATERIALIZED VIEW mv_always_retry AS SELECT sleep_always_retry(v1) as s1 from t;
 
-# Immediately kill the server, sleep for 1minute.
+# Immediately kill the server, sleep
 system ok
 pkill -9 -i python && sleep 60