Merge branch 'main' into yiming/non-async-inject-barrier

Cargo.toml

@@ -133,8 +133,8 @@ arrow-flight = "50"
 arrow-select = "50"
 arrow-ord = "50"
 arrow-row = "50"
-arrow-udf-js = { git = "https://github.com/risingwavelabs/arrow-udf.git", rev = "7ba1c22" }
-arrow-udf-wasm = "0.1"
+arrow-udf-js = "0.1"
+arrow-udf-wasm = { version = "0.1.2", features = ["build"] }
 arrow-array-deltalake = { package = "arrow-array", version = "48.0.1" }
 arrow-buffer-deltalake = { package = "arrow-buffer", version = "48.0.1" }
 arrow-cast-deltalake = { package = "arrow-cast", version = "48.0.1" }

ci/Dockerfile

@@ -41,6 +41,7 @@ RUN curl -sSL https://install.python-poetry.org | python3 -
 
 # add required rustup components
 RUN rustup component add rustfmt llvm-tools-preview clippy
+RUN rustup target add wasm32-wasi
 
 ENV CARGO_REGISTRIES_CRATES_IO_PROTOCOL=sparse
 

ci/build-ci-image.sh

@@ -10,7 +10,7 @@ cat ../rust-toolchain
 # shellcheck disable=SC2155
 
 # REMEMBER TO ALSO UPDATE ci/docker-compose.yml
-export BUILD_ENV_VERSION=v20240124_1
+export BUILD_ENV_VERSION=v20240204
 
 export BUILD_TAG="public.ecr.aws/x5u3w5h6/rw-build-env:${BUILD_ENV_VERSION}"
 

ci/docker-compose.yml

@@ -71,7 +71,7 @@ services:
       retries: 5
 
   source-test-env:
-    image: public.ecr.aws/x5u3w5h6/rw-build-env:v20240124_1
+    image: public.ecr.aws/x5u3w5h6/rw-build-env:v20240204
     depends_on:
       - mysql
       - db
@@ -81,7 +81,7 @@ services:
       - ..:/risingwave
 
   sink-test-env:
-    image: public.ecr.aws/x5u3w5h6/rw-build-env:v20240124_1
+    image: public.ecr.aws/x5u3w5h6/rw-build-env:v20240204
     depends_on:
       - mysql
       - db
@@ -93,12 +93,12 @@ services:
       - ..:/risingwave
 
   rw-build-env:
-    image: public.ecr.aws/x5u3w5h6/rw-build-env:v20240124_1
+    image: public.ecr.aws/x5u3w5h6/rw-build-env:v20240204
     volumes:
       - ..:/risingwave
 
   ci-flamegraph-env:
-    image: public.ecr.aws/x5u3w5h6/rw-build-env:v20240124_1
+    image: public.ecr.aws/x5u3w5h6/rw-build-env:v20240204
     # NOTE(kwannoel): This is used in order to permit
     # syscalls for `nperf` (perf_event_open),
     # so it can do CPU profiling.
@@ -109,7 +109,7 @@ services:
       - ..:/risingwave
 
   regress-test-env:
-    image: public.ecr.aws/x5u3w5h6/rw-build-env:v20240124_1
+    image: public.ecr.aws/x5u3w5h6/rw-build-env:v20240204
     depends_on:
       db:
         condition: service_healthy

ci/scripts/build-other.sh

@@ -8,7 +8,6 @@ source ci/scripts/common.sh
 
 echo "--- Build Rust UDF"
 cd e2e_test/udf/wasm
-rustup target add wasm32-wasi
 cargo build --release
 cd ../../..
 

ci/workflows/main-cron.yml

@@ -40,7 +40,7 @@ steps:
     timeout_in_minutes: 12
     retry: *auto-retry
 
-  - label: "build (deterministic simulation)"
+  - label: "build simulation test"
     command: "ci/scripts/build-simulation.sh"
     if: |
       !(build.pull_request.labels includes "ci/main-cron/skip-ci") && build.env("CI_STEPS") == null
@@ -214,7 +214,7 @@ steps:
     timeout_in_minutes: 22
     retry: *auto-retry
 
-  - label: "unit test (deterministic simulation)"
+  - label: "unit test (madsim)"
     key: "unit-test-deterministic"
     command: "MADSIM_TEST_NUM=100 timeout 15m ci/scripts/deterministic-unit-test.sh"
     if: |
@@ -229,7 +229,7 @@ steps:
     timeout_in_minutes: 15
     retry: *auto-retry
 
-  - label: "integration test (deterministic simulation) - scale"
+  - label: "integration test (madsim) - scale"
     key: "integration-test-deterministic-scale"
     command: "TEST_NUM=60 ci/scripts/deterministic-it-test.sh scale::"
     if: |
@@ -246,7 +246,7 @@ steps:
     timeout_in_minutes: 70
     retry: *auto-retry
 
-  - label: "integration test (deterministic simulation) - recovery"
+  - label: "integration test (madsim) - recovery"
     key: "integration-test-deterministic-recovery"
     command: "TEST_NUM=60 ci/scripts/deterministic-it-test.sh recovery::"
     if: |
@@ -263,9 +263,9 @@ steps:
     timeout_in_minutes: 70
     retry: *auto-retry
 
-  - label: "integration test (deterministic simulation) - others"
-    key: "integration-test-deterministic-others"
-    command: "TEST_NUM=30 ci/scripts/deterministic-it-test.sh backfill_tests:: storage:: sink::"
+  - label: "integration test (madsim) - backfill"
+    key: "integration-test-deterministic-backfill"
+    command: "TEST_NUM=30 ci/scripts/deterministic-it-test.sh backfill_tests::"
     if: |
       !(build.pull_request.labels includes "ci/main-cron/skip-ci") && build.env("CI_STEPS") == null
       || build.pull_request.labels includes "ci/run-integration-test-deterministic-simulation"
@@ -280,7 +280,41 @@ steps:
     timeout_in_minutes: 70
     retry: *auto-retry
 
-  - label: "end-to-end test (deterministic simulation)"
+  - label: "integration test (madsim) - storage"
+    key: "integration-test-deterministic-storage"
+    command: "TEST_NUM=30 ci/scripts/deterministic-it-test.sh storage::"
+    if: |
+      !(build.pull_request.labels includes "ci/main-cron/skip-ci") && build.env("CI_STEPS") == null
+      || build.pull_request.labels includes "ci/run-integration-test-deterministic-simulation"
+      || build.env("CI_STEPS") =~ /(^|,)integration-tests?-deterministic-simulation(,|$$)/
+    depends_on: "build-simulation"
+    plugins:
+      - docker-compose#v4.9.0:
+          run: rw-build-env
+          config: ci/docker-compose.yml
+          mount-buildkite-agent: true
+      - ./ci/plugins/upload-failure-logs
+    timeout_in_minutes: 70
+    retry: *auto-retry
+
+  - label: "integration test (madsim) - sink"
+    key: "integration-test-deterministic-sink"
+    command: "TEST_NUM=30 ci/scripts/deterministic-it-test.sh sink::"
+    if: |
+      !(build.pull_request.labels includes "ci/main-cron/skip-ci") && build.env("CI_STEPS") == null
+      || build.pull_request.labels includes "ci/run-integration-test-deterministic-simulation"
+      || build.env("CI_STEPS") =~ /(^|,)integration-tests?-deterministic-simulation(,|$$)/
+    depends_on: "build-simulation"
+    plugins:
+      - docker-compose#v4.9.0:
+          run: rw-build-env
+          config: ci/docker-compose.yml
+          mount-buildkite-agent: true
+      - ./ci/plugins/upload-failure-logs
+    timeout_in_minutes: 70
+    retry: *auto-retry
+
+  - label: "end-to-end test (madsim)"
     key: "e2e-test-deterministic"
     command: "TEST_NUM=64 timeout 55m ci/scripts/deterministic-e2e-test.sh"
     if: |
@@ -302,7 +336,7 @@ steps:
     timeout_in_minutes: 60
     retry: *auto-retry
 
-  - label: "recovery test (deterministic simulation)"
+  - label: "recovery test (madsim)"
     key: "recovery-test-deterministic"
     command: "TEST_NUM=12 KILL_RATE=1.0 BACKGROUND_DDL_RATE=0.0 timeout 55m ci/scripts/deterministic-recovery-test.sh"
     if: |
@@ -321,7 +355,7 @@ steps:
     retry: *auto-retry
 
   # Ddl statements will randomly run with background_ddl.
-  - label: "background_ddl, arrangement_backfill recovery test (deterministic simulation)"
+  - label: "background_ddl, arrangement_backfill recovery test (madsim)"
     key: "background-ddl-arrangement-backfill-recovery-test-deterministic"
     command: "TEST_NUM=12 KILL_RATE=1.0 BACKGROUND_DDL_RATE=0.8 USE_ARRANGEMENT_BACKFILL=--use-arrangement-backfill timeout 55m ci/scripts/deterministic-recovery-test.sh"
     if: |
@@ -340,7 +374,7 @@ steps:
     retry: *auto-retry
 
   # Ddl statements will randomly run with background_ddl.
-  - label: "background_ddl recovery test (deterministic simulation)"
+  - label: "background_ddl recovery test (madsim)"
     key: "background-ddl-recovery-test-deterministic"
     command: "TEST_NUM=12 KILL_RATE=1.0 BACKGROUND_DDL_RATE=0.8 timeout 55m ci/scripts/deterministic-recovery-test.sh"
     if: |

dashboard/next.config.js

@@ -19,7 +19,7 @@
  * @type {import('next').NextConfig}
  */
 const nextConfig = {
-  output: 'export',
+  output: "export",
   trailingSlash: true,
 }
 

docker/Dockerfile

@@ -33,7 +33,8 @@ COPY rust-toolchain rust-toolchain
 RUN rustup self update \
   && rustup set profile minimal \
   && rustup show \
-  && rustup component add rustfmt
+  && rustup component add rustfmt \
+  && rustup target add wasm32-wasi
 
 RUN cargo install flamegraph
 # TODO: cargo-chef doesn't work well now, because we update Cargo.lock very often.

docker/Dockerfile.hdfs

@@ -34,7 +34,8 @@ RUN unzip dashboard-artifact.zip && mv risingwave-dashboard-artifact /risingwave
 RUN rustup self update \
   && rustup set profile minimal \
   && rustup show \
-  && rustup component add rustfmt
+  && rustup component add rustfmt \
+  && rustup target add wasm32-wasi
 
 RUN cargo fetch
 

e2e_test/udf/wasm_udf.slt

@@ -91,3 +91,24 @@ drop function jsonb_access;
 
 statement ok
 drop function series;
+
+# inlined rust function
+statement ok
+create function gcd(int, int) returns int language rust as $$
+    fn gcd(mut a: i32, mut b: i32) -> i32 {
+        while b != 0 {
+            let t = b;
+            b = a % b;
+            a = t;
+        }
+        a
+    }
+$$;
+
+query I
+select gcd(25, 15);
+----
+5
+
+statement ok
+drop function gcd;

grafana/risingwave-dev-dashboard.dashboard.py

@@ -3678,6 +3678,26 @@ def section_memory_manager(outer_panels):
                         ),
                     ],
                 ),
+                panels.timeseries_memory(
+                    "The resident memory of jemalloc",
+                    "",
+                    [
+                        panels.target(
+                            f"{metric('jemalloc_resident_bytes')}",
+                            "",
+                        ),
+                    ],
+                ),
+                panels.timeseries_memory(
+                    "The metadata memory of jemalloc",
+                    "",
+                    [
+                        panels.target(
+                            f"{metric('jemalloc_metadata_bytes')}",
+                            "",
+                        ),
+                    ],
+                ),
                 panels.timeseries_memory(
                     "The allocated memory of jvm",
                     "",

src/batch/Cargo.toml

@@ -15,6 +15,8 @@ normal = ["workspace-hack"]
 
 [dependencies]
 anyhow = "1"
+arrow-array = { workspace = true }
+arrow-schema = { workspace = true }
 assert_matches = "1"
 async-recursion = "1"
 async-trait = "0.1"
@@ -24,6 +26,7 @@ futures-async-stream = { workspace = true }
 futures-util = "0.3"
 hashbrown = { workspace = true }
 hytra = "0.1.2"
+icelake = { workspace = true }
 itertools = "0.12"
 memcomparable = "0.2"
 parking_lot = { version = "0.12", features = ["arc_lock"] }

src/batch/src/error.rs

@@ -109,6 +109,13 @@ pub enum BatchError {
         DmlError,
     ),
 
+    #[error(transparent)]
+    Iceberg(
+        #[from]
+        #[backtrace]
+        icelake::Error,
+    ),
+
     // Make the ref-counted type to be a variant for easier code structuring.
     // TODO(error-handling): replace with `thiserror_ext::Arc`
     #[error(transparent)]