Merge branch 'release-2.1' into auto-release-2.1-d6277fb78f287e433ccf…

…8a473b3a30393b57cc9e-1729233132

.gitignore

@@ -83,7 +83,7 @@ e2e_test/iceberg/metastore_db
 **/*.sqlite
 **/*.sqlite-journal
 
-*.slt.temp
+*.slt*.temp
 
 .direnv/
 

ci/scripts/e2e-source-test.sh

@@ -38,7 +38,8 @@ apt-get -y install jq
 echo "--- e2e, inline test"
 RUST_LOG="debug,risingwave_stream=info,risingwave_batch=info,risingwave_storage=info" \
 risedev ci-start ci-inline-source-test
-risedev slt './e2e_test/source_inline/**/*.slt'
+risedev slt './e2e_test/source_inline/**/*.slt' -j16
+risedev slt './e2e_test/source_inline/**/*.slt.serial'
 echo "--- Kill cluster"
 risedev ci-kill
 

ci/scripts/run-backfill-tests.sh

@@ -27,7 +27,7 @@ CLUSTER_PROFILE='ci-1cn-1fe-user-kafka-with-recovery'
 echo "--- Configuring cluster profiles"
 if [[ -n "${BUILDKITE:-}" ]]; then
   echo "Running in buildkite"
-  RUNTIME_CLUSTER_PROFILE='ci-3cn-1fe'
+  RUNTIME_CLUSTER_PROFILE='ci-backfill-3cn-1fe'
   MINIO_RATE_LIMIT_CLUSTER_PROFILE='ci-backfill-3cn-1fe-with-minio-rate-limit'
 else
   echo "Running locally"

e2e_test/batch/catalog/pg_settings.slt.part

@@ -33,7 +33,6 @@ user create_compaction_group_for_mv
 user datestyle
 user enable_join_ordering
 user enable_share_plan
-user enable_shared_source
 user enable_two_phase_agg
 user extra_float_digits
 user force_split_distinct_agg
@@ -61,6 +60,7 @@ user streaming_max_parallelism
 user streaming_over_window_cache_policy
 user streaming_parallelism
 user streaming_use_arrangement_backfill
+user streaming_use_shared_source
 user streaming_use_snapshot_backfill
 user synchronize_seqscans
 user timezone

e2e_test/source_inline/README.md

@@ -5,6 +5,19 @@ Compared with prior source tests ( `e2e_test/source` ), tests in this directory
 See the [connector development guide](http://risingwavelabs.github.io/risingwave/connector/intro.html#end-to-end-tests) for more information about how to set up the test environment,
 run tests, and write tests.
 
+## Serial Tests
+
+Tests ending with `.slt.serial` can only be run in serial, e.g., it contains `RECOVER` statement which will break other connections, or it has some special `system` commands.
+
+Other tests can be run in parallel.
+
+```bash
+# run all parallel tests
+risedev slt './e2e_test/source_inline/**/*.slt' -j16
+# run all serial tests
+risedev slt './e2e_test/source_inline/**/*.slt.serial'
+```
+
 ## Install Dependencies
 
 Some additional tools are needed to run the `system` commands in tests.

e2e_test/source_inline/kafka/alter/add_column.slt

@@ -1,5 +1,8 @@
 control substitution on
 
+statement ok
+SET streaming_use_shared_source TO false;
+
 system ok
 rpk topic delete kafka_alter || true
 
@@ -269,3 +272,6 @@ select * from t
 
 statement ok
 drop table t;
+
+statement ok
+SET streaming_use_shared_source TO true;

e2e_test/source_inline/kafka/alter/rate_limit_source_kafka.slt

@@ -1,5 +1,8 @@
 control substitution on
 
+statement ok
+SET streaming_use_shared_source TO false;
+
 ############## Create kafka seed data
 
 statement ok
@@ -127,3 +130,6 @@ drop sink kafka_sink;
 
 statement ok
 drop table kafka_seed_data;
+
+statement ok
+SET streaming_use_shared_source TO true;

e2e_test/source_inline/kafka/avro/alter_source.slt

@@ -1,5 +1,8 @@
 control substitution on
 
+statement ok
+SET streaming_use_shared_source TO false;
+
 # https://github.com/risingwavelabs/risingwave/issues/16486
 
 # cleanup
@@ -66,3 +69,6 @@ ABC	1
 
 statement ok
 drop source s cascade;
+
+statement ok
+SET streaming_use_shared_source TO true;

e2e_test/source_inline/kafka/avro/glue.slt

@@ -1,5 +1,8 @@
 control substitution on
 
+statement ok
+SET streaming_use_shared_source TO false;
+
 system ok
 rpk topic delete 'glue-sample-my-event'
 
@@ -14,6 +17,8 @@ EOF
 statement ok
 ALTER SYSTEM SET license_key TO '';
 
+sleep 1s
+
 statement error
 create source t with (
   connector = 'kafka',
@@ -146,3 +151,6 @@ drop source t;
 
 system ok
 rpk topic delete 'glue-sample-my-event'
+
+statement ok
+SET streaming_use_shared_source TO true;

e2e_test/source_inline/kafka/protobuf/alter_source.slt

@@ -1,18 +1,21 @@
 control substitution on
 
+statement ok
+SET streaming_use_shared_source TO false;
+
 system ok
-rpk topic delete sr_pb_test || true; \
-(rpk sr subject delete 'sr_pb_test-value' && rpk sr subject delete 'sr_pb_test-value' --permanent) || true;
+rpk topic delete pb_alter_source_test || true; \
+(rpk sr subject delete 'pb_alter_source_test-value' && rpk sr subject delete 'pb_alter_source_test-value' --permanent) || true;
 
 system ok
-python3 e2e_test/source_inline/kafka/protobuf/pb.py "${RISEDEV_KAFKA_BOOTSTRAP_SERVERS}" "${RISEDEV_SCHEMA_REGISTRY_URL}" "sr_pb_test" 20 user
+python3 e2e_test/source_inline/kafka/protobuf/pb.py "${RISEDEV_KAFKA_BOOTSTRAP_SERVERS}" "${RISEDEV_SCHEMA_REGISTRY_URL}" "pb_alter_source_test" 20 user
 
 statement ok
 CREATE SOURCE src_user
 INCLUDE timestamp -- include explicitly here to test a bug found in https://github.com/risingwavelabs/risingwave/pull/17293
 WITH (
     ${RISEDEV_KAFKA_WITH_OPTIONS_COMMON},
-    topic = 'sr_pb_test',
+    topic = 'pb_alter_source_test',
     scan.startup.mode = 'earliest'
 )
 FORMAT PLAIN ENCODE PROTOBUF(
@@ -26,7 +29,7 @@ CREATE MATERIALIZED VIEW mv_user AS SELECT * FROM src_user;
 statement ok
 CREATE TABLE t_user WITH (
     ${RISEDEV_KAFKA_WITH_OPTIONS_COMMON},
-    topic = 'sr_pb_test',
+    topic = 'pb_alter_source_test',
     scan.startup.mode = 'earliest'
 )
 FORMAT PLAIN ENCODE PROTOBUF(
@@ -42,7 +45,7 @@ SELECT age FROM t_user;
 
 # Push more events with extended fields
 system ok
-python3 e2e_test/source_inline/kafka/protobuf/pb.py "${RISEDEV_KAFKA_BOOTSTRAP_SERVERS}" "${RISEDEV_SCHEMA_REGISTRY_URL}" "sr_pb_test" 5 user_with_more_fields
+python3 e2e_test/source_inline/kafka/protobuf/pb.py "${RISEDEV_KAFKA_BOOTSTRAP_SERVERS}" "${RISEDEV_SCHEMA_REGISTRY_URL}" "pb_alter_source_test" 5 user_with_more_fields
 
 sleep 5s
 
@@ -69,7 +72,7 @@ SELECT COUNT(*), MAX(age), MIN(age), SUM(age) FROM t_user;
 
 # Push more events with extended fields
 system ok
-python3 e2e_test/source_inline/kafka/protobuf/pb.py "${RISEDEV_KAFKA_BOOTSTRAP_SERVERS}" "${RISEDEV_SCHEMA_REGISTRY_URL}" "sr_pb_test" 5 user_with_more_fields
+python3 e2e_test/source_inline/kafka/protobuf/pb.py "${RISEDEV_KAFKA_BOOTSTRAP_SERVERS}" "${RISEDEV_SCHEMA_REGISTRY_URL}" "pb_alter_source_test" 5 user_with_more_fields
 
 sleep 5s
 
@@ -89,3 +92,6 @@ DROP MATERIALIZED VIEW mv_user;
 
 statement ok
 DROP SOURCE src_user;
+
+statement ok
+SET streaming_use_shared_source TO true;