Merge branch 'main' into yingjunwu-patch-2

Makefile.toml

@@ -1332,7 +1332,7 @@ echo "All processes has exited."
 
 [tasks.slt]
 category = "RiseDev - SQLLogicTest"
-install_crate = { version = "0.18.0", crate_name = "sqllogictest-bin", binary = "sqllogictest", test_arg = [
+install_crate = { version = "0.19.1", crate_name = "sqllogictest-bin", binary = "sqllogictest", test_arg = [
   "--help",
 ], install_command = "binstall" }
 dependencies = ["check-risedev-env-file"]

backwards-compat-tests/slt/nexmark-backwards-compat/validate_restart.slt

@@ -1,9 +1,17 @@
 include ../nexmark/test_mv_result.slt.part
 
 include ./delete.slt.part
+
+statement ok
+flush;
+
 include ../nexmark/insert_person.slt.part
 include ../nexmark/insert_auction.slt.part
 include ../nexmark/insert_bid.slt.part
+
+statement ok
+flush;
+
 include ../nexmark/test_mv_result.slt.part
 
 include ../nexmark/drop_views.slt.part

backwards-compat-tests/slt/tpch-backwards-compat/validate_restart.slt

@@ -23,7 +23,15 @@ include ../tpch/q22.slt.part
 
 # Test deletes and updates should work as per normal.
 include ./delete.slt.part
+
+statement ok
+flush;
+
 include ./insert.slt.part
+
+statement ok
+flush;
+
 include ../tpch/q1.slt.part
 include ../tpch/q2.slt.part
 include ../tpch/q3.slt.part

ci/Dockerfile

@@ -48,8 +48,8 @@ ENV CARGO_REGISTRIES_CRATES_IO_PROTOCOL=sparse
 RUN curl -L --proto '=https' --tlsv1.2 -sSf https://raw.githubusercontent.com/cargo-bins/cargo-binstall/main/install-from-binstall-release.sh | bash
 RUN cargo binstall -y --no-symlinks cargo-llvm-cov cargo-nextest cargo-hakari cargo-sort cargo-cache cargo-audit \
     [email protected] \
-    sqllogictest-bin@0.18.0 \
-    && cargo install sccache \
+    sqllogictest-bin@0.19.1 \
+    sccache@0.7.4 \
     && cargo cache -a \
     && rm -rf "/root/.cargo/registry/index" \
     && rm -rf "/root/.cargo/registry/cache" \

ci/build-ci-image.sh

@@ -10,7 +10,7 @@ cat ../rust-toolchain
 # shellcheck disable=SC2155
 
 # REMEMBER TO ALSO UPDATE ci/docker-compose.yml
-export BUILD_ENV_VERSION=v20231226
+export BUILD_ENV_VERSION=v20240104_1
 
 export BUILD_TAG="public.ecr.aws/x5u3w5h6/rw-build-env:${BUILD_ENV_VERSION}"
 

ci/docker-compose.yml

@@ -71,7 +71,7 @@ services:
       retries: 5
 
   source-test-env:
-    image: public.ecr.aws/x5u3w5h6/rw-build-env:v20231226
+    image: public.ecr.aws/x5u3w5h6/rw-build-env:v20240104_1
     depends_on:
       - mysql
       - db
@@ -81,7 +81,7 @@ services:
       - ..:/risingwave
 
   sink-test-env:
-    image: public.ecr.aws/x5u3w5h6/rw-build-env:v20231226
+    image: public.ecr.aws/x5u3w5h6/rw-build-env:v20240104_1
     depends_on:
       - mysql
       - db
@@ -93,12 +93,12 @@ services:
       - ..:/risingwave
 
   rw-build-env:
-    image: public.ecr.aws/x5u3w5h6/rw-build-env:v20231226
+    image: public.ecr.aws/x5u3w5h6/rw-build-env:v20240104_1
     volumes:
       - ..:/risingwave
 
   ci-flamegraph-env:
-    image: public.ecr.aws/x5u3w5h6/rw-build-env:v20231226
+    image: public.ecr.aws/x5u3w5h6/rw-build-env:v20240104_1
     # NOTE(kwannoel): This is used in order to permit
     # syscalls for `nperf` (perf_event_open),
     # so it can do CPU profiling.
@@ -109,7 +109,7 @@ services:
       - ..:/risingwave
 
   regress-test-env:
-    image: public.ecr.aws/x5u3w5h6/rw-build-env:v20231226
+    image: public.ecr.aws/x5u3w5h6/rw-build-env:v20240104_1
     depends_on:
       db:
         condition: service_healthy

ci/scripts/cron-e2e-test.sh

@@ -4,7 +4,7 @@
 set -euo pipefail
 
 source ci/scripts/common.sh
-export RUN_COMPACTION=1;
+export RUN_COMPACTION=0;
 export RUN_META_BACKUP=1;
 export RUN_DELETE_RANGE=1;
 source ci/scripts/run-e2e-test.sh

ci/scripts/e2e-source-test.sh

@@ -56,6 +56,9 @@ echo "--- inline cdc test"
 export MYSQL_HOST=mysql MYSQL_TCP_PORT=3306 MYSQL_PWD=123456
 sqllogictest -p 4566 -d dev './e2e_test/source/cdc_inline/**/*.slt'
 
+echo "--- opendal source test"
+sqllogictest -p 4566 -d dev './e2e_test/source/opendal/**/*.slt'
+
 echo "--- mysql & postgres cdc validate test"
 sqllogictest -p 4566 -d dev './e2e_test/source/cdc/cdc.validate.mysql.slt'
 sqllogictest -p 4566 -d dev './e2e_test/source/cdc/cdc.validate.postgres.slt'

ci/scripts/notify.py

@@ -19,7 +19,8 @@
     "e2e-java-binding-tests": ["yiming"],
     "e2e-clickhouse-sink-tests": ["bohan"],
     "e2e-pulsar-sink-tests": ["renjie"],
-    "s3-source-test-for-opendal-fs-engine": ["congyi"],
+    "s3-source-test-for-opendal-fs-engine": ["congyi", "kexiang"],
+    "s3-source-tests": ["congyi", "kexiang"],
     "pulsar-source-tests": ["renjie"],
     "connector-node-integration-test": ["siyuan"],
 }

ci/workflows/main-cron.yml

@@ -489,6 +489,22 @@ steps:
     timeout_in_minutes: 25
     retry: *auto-retry
 
+  - label: "PosixFs source on OpenDAL fs engine (csv parser)"
+    command: "ci/scripts/s3-source-test.sh -p ci-release -s 'posix_fs_source.py csv_without_header'"
+    if: |
+      !(build.pull_request.labels includes "ci/main-cron/skip-ci") && build.env("CI_STEPS") == null
+      || build.pull_request.labels includes "ci/run-s3-source-tests"
+      || build.env("CI_STEPS") =~ /(^|,)s3-source-tests?(,|$$)/
+    depends_on: build
+    plugins:
+      - docker-compose#v4.9.0:
+          run: rw-build-env
+          config: ci/docker-compose.yml
+          mount-buildkite-agent: true
+      - ./ci/plugins/upload-failure-logs
+    timeout_in_minutes: 25
+    retry: *auto-retry
+
   - label: "S3 source on OpenDAL fs engine"
     key: "s3-source-test-for-opendal-fs-engine"
     command: "ci/scripts/s3-source-test-for-opendal-fs-engine.sh -p ci-release -s run"