Merge branch 'main' into bump-deps-libpq-test

Makefile.toml

@@ -35,6 +35,7 @@ is_release = get_env ENABLE_RELEASE_PROFILE
 is_not_release = not ${is_release}
 is_dynamic_linking = get_env ENABLE_DYNAMIC_LINKING
 is_hummock_trace = get_env ENABLE_HUMMOCK_TRACE
+is_python_udf_enabled = get_env ENABLE_PYTHON_UDF
 
 if ${is_sanitizer_enabled}
   set_env RISEDEV_CARGO_BUILD_EXTRA_ARGS "-Zbuild-std --target ${CARGO_MAKE_RUST_TARGET_TRIPLE}"
@@ -65,6 +66,11 @@ else
   set_env RISINGWAVE_FEATURE_FLAGS "--features rw-static-link"
 end
 
+if ${is_python_udf_enabled}
+  flags = get_env RISINGWAVE_FEATURE_FLAGS
+  set_env RISINGWAVE_FEATURE_FLAGS "${flags} --features embedded-python-udf"
+end
+
 if ${is_hummock_trace}
   set_env BUILD_HUMMOCK_TRACE_CMD "-p risingwave_storage --features hm-trace"
 else

ci/docker-compose.yml

@@ -96,18 +96,6 @@ services:
     volumes:
       - ..:/risingwave
 
-  sink-doris-env:
-    image: public.ecr.aws/x5u3w5h6/rw-build-env:v20231109
-    depends_on:
-      - doris-fe-server
-      - doris-be-server
-    volumes:
-      - ..:/risingwave
-    command: >
-      sh -c "sudo sysctl -w vm.max_map_count=2000000"
-    networks:
-      mynetwork:
-        ipv4_address: 172.121.0.4
 
   rw-build-env:
     image: public.ecr.aws/x5u3w5h6/rw-build-env:v20240223
@@ -189,39 +177,6 @@ services:
       timeout: 30s
       retries: 50
 
-  doris-fe-server:
-    platform: linux/amd64
-    image: apache/doris:2.0.0_alpha-fe-x86_64
-    hostname: doris-fe-server
-    command: >
-      sh -c "sudo sysctl -w vm.max_map_count=2000000"
-    environment:
-     - FE_SERVERS=fe1:172.121.0.2:9010
-     - FE_ID=1
-    ports:
-      - "8030:8030"
-      - "9030:9030"
-    networks:
-      mynetwork:
-        ipv4_address: 172.121.0.2
-
-  doris-be-server:
-    platform: linux/amd64
-    image: apache/doris:2.0.0_alpha-be-x86_64
-    hostname: doris-be-server
-    command: >
-      sh -c "sudo sysctl -w vm.max_map_count=2000000"
-    environment:
-     - FE_SERVERS=fe1:172.121.0.2:9010
-     - BE_ADDR=172.121.0.3:9050
-    depends_on:
-      - doris-fe-server
-    ports:
-      - "9050:9050"
-    networks:
-      mynetwork:
-        ipv4_address: 172.121.0.3
-
   cassandra-server:
     container_name: cassandra-server
     image: cassandra:4.0
@@ -319,9 +274,3 @@ services:
       interval: 5s
       timeout: 5s
       retries: 5
-networks:
-  mynetwork:
-    ipam:
-      config:
-        - subnet: 172.121.80.0/16
-  default:

ci/scripts/build.sh

@@ -53,6 +53,7 @@ cargo build \
     -p risingwave_compaction_test \
     -p risingwave_e2e_extended_mode_test \
     $RISINGWAVE_FEATURE_FLAGS \
+    --features embedded-python-udf \
     --profile "$profile"
 
 

ci/workflows/main-cron.yml

@@ -834,36 +834,37 @@ steps:
     timeout_in_minutes: 10
     retry: *auto-retry
 
-  - label: "set vm_max_map_count_2000000"
-    key: "set-vm_max_map_count"
-    if: |
-      !(build.pull_request.labels includes "ci/main-cron/skip-ci") && build.env("CI_STEPS") == null
-      || build.pull_request.labels includes "ci/run-e2e-doris-sink-tests"
-      || build.env("CI_STEPS") =~ /(^|,)e2e-doris-sink-tests?(,|$$)/
-    command: "sudo sysctl -w vm.max_map_count=2000000"
-    depends_on:
-      - "build"
-      - "build-other"
+  # Causes ci error, close it first, fix it later
+  # - label: "set vm_max_map_count_2000000"
+  #   key: "set-vm_max_map_count"
+  #   if: |
+  #     !(build.pull_request.labels includes "ci/main-cron/skip-ci") && build.env("CI_STEPS") == null
+  #     || build.pull_request.labels includes "ci/run-e2e-doris-sink-tests"
+  #     || build.env("CI_STEPS") =~ /(^|,)e2e-doris-sink-tests?(,|$$)/
+  #   command: "sudo sysctl -w vm.max_map_count=2000000"
+  #   depends_on:
+  #     - "build"
+  #     - "build-other"
 
-  - label: "end-to-end doris sink test"
-    key: "e2e-doris-sink-tests"
-    command: "ci/scripts/e2e-doris-sink-test.sh -p ci-release"
-    if: |
-      !(build.pull_request.labels includes "ci/main-cron/skip-ci") && build.env("CI_STEPS") == null
-      || build.pull_request.labels includes "ci/run-e2e-doris-sink-tests"
-      || build.env("CI_STEPS") =~ /(^|,)e2e-doris-sink-tests?(,|$$)/
-    depends_on:
-      - "build"
-      - "build-other"
-      - "set-vm_max_map_count"
-    plugins:
-      - docker-compose#v5.1.0:
-          run: sink-doris-env
-          config: ci/docker-compose.yml
-          mount-buildkite-agent: true
-      - ./ci/plugins/upload-failure-logs
-    timeout_in_minutes: 10
-    retry: *auto-retry
+  # - label: "end-to-end doris sink test"
+  #   key: "e2e-doris-sink-tests"
+  #   command: "ci/scripts/e2e-doris-sink-test.sh -p ci-release"
+  #   if: |
+  #     !(build.pull_request.labels includes "ci/main-cron/skip-ci") && build.env("CI_STEPS") == null
+  #     || build.pull_request.labels includes "ci/run-e2e-doris-sink-tests"
+  #     || build.env("CI_STEPS") =~ /(^|,)e2e-doris-sink-tests?(,|$$)/
+  #   depends_on:
+  #     - "build"
+  #     - "build-other"
+  #     - "set-vm_max_map_count"
+  #   plugins:
+  #     - docker-compose#v5.1.0:
+  #         run: sink-doris-env
+  #         config: ci/docker-compose.yml
+  #         mount-buildkite-agent: true
+  #     - ./ci/plugins/upload-failure-logs
+  #   timeout_in_minutes: 10
+  #   retry: *auto-retry
 
   - label: "end-to-end starrocks sink test"
     key: "e2e-starrocks-sink-tests"

ci/workflows/pull-request.yml

@@ -307,29 +307,30 @@ steps:
     timeout_in_minutes: 10
     retry: *auto-retry
 
-  - label: "set vm_max_map_count_2000000"
-    key: "set-vm_max_map_count"
-    if: build.pull_request.labels includes "ci/run-e2e-doris-sink-tests" || build.env("CI_STEPS") =~ /(^|,) e2e-doris-sink-tests?(,|$$)/
-    command: "sudo sysctl -w vm.max_map_count=2000000"
-    depends_on:
-      - "build"
-      - "build-other"
-
-  - label: "end-to-end doris sink test"
-    if: build.pull_request.labels includes "ci/run-e2e-doris-sink-tests" || build.env("CI_STEPS") =~ /(^|,) e2e-doris-sink-tests?(,|$$)/
-    command: "ci/scripts/e2e-doris-sink-test.sh -p ci-dev"
-    depends_on:
-      - "build"
-      - "build-other"
-      - "set-vm_max_map_count"
-    plugins:
-      - docker-compose#v5.1.0:
-          run: sink-doris-env
-          config: ci/docker-compose.yml
-          mount-buildkite-agent: true
-      - ./ci/plugins/upload-failure-logs
-    timeout_in_minutes: 10
-    retry: *auto-retry
+  # Causes ci error, close it first, fix it later
+  # - label: "set vm_max_map_count_2000000"
+  #   key: "set-vm_max_map_count"
+  #   if: build.pull_request.labels includes "ci/run-e2e-doris-sink-tests" || build.env("CI_STEPS") =~ /(^|,) e2e-doris-sink-tests?(,|$$)/
+  #   command: "sudo sysctl -w vm.max_map_count=2000000"
+  #   depends_on:
+  #     - "build"
+  #     - "build-other"
+
+  # - label: "end-to-end doris sink test"
+  #   if: build.pull_request.labels includes "ci/run-e2e-doris-sink-tests" || build.env("CI_STEPS") =~ /(^|,) e2e-doris-sink-tests?(,|$$)/
+  #   command: "ci/scripts/e2e-doris-sink-test.sh -p ci-dev"
+  #   depends_on:
+  #     - "build"
+  #     - "build-other"
+  #     - "set-vm_max_map_count"
+  #   plugins:
+  #     - docker-compose#v5.1.0:
+  #         run: sink-doris-env
+  #         config: ci/docker-compose.yml
+  #         mount-buildkite-agent: true
+  #     - ./ci/plugins/upload-failure-logs
+  #   timeout_in_minutes: 10
+  #   retry: *auto-retry
 
   - label: "end-to-end starrocks sink test"
     if: build.pull_request.labels includes "ci/run-e2e-starrocks-sink-tests" || build.env("CI_STEPS") =~ /(^|,) e2e-starrocks-sink-tests?(,|$$)/

docker/Dockerfile

@@ -5,6 +5,13 @@ ENV LANG en_US.utf8
 RUN apt-get update \
   && apt-get -y install ca-certificates build-essential libsasl2-dev openjdk-11-jdk
 
+# Install Python 3.12
+RUN apt-get -y install software-properties-common
+RUN add-apt-repository ppa:deadsnakes/ppa -y && \
+    apt-get update -yy && \
+    DEBIAN_FRONTEND=noninteractive apt-get install python3.12 python3.12-dev -yy
+ENV PYO3_PYTHON=python3.12
+
 FROM base AS dashboard-builder
 
 RUN apt-get update && apt-get install -y curl gnupg protobuf-compiler && mkdir -p /etc/apt/keyrings \
@@ -70,7 +77,7 @@ COPY ./ /risingwave
 WORKDIR /risingwave
 
 RUN cargo fetch && \
-  cargo build -p risingwave_cmd_all --release --features "rw-static-link" && \
+  cargo build -p risingwave_cmd_all --release --features "rw-static-link" --features embedded-python-udf && \
   mkdir -p /risingwave/bin && \
     mv /risingwave/target/release/risingwave /risingwave/bin/ && \
     mv /risingwave/target/release/risingwave.dwp /risingwave/bin/ && \

docker/Dockerfile.hdfs

@@ -5,6 +5,13 @@ ENV LANG en_US.utf8
 RUN apt-get update \
   && apt-get -y install ca-certificates build-essential libsasl2-dev openjdk-11-jdk
 
+# Install Python 3.12
+RUN apt-get -y install software-properties-common
+RUN add-apt-repository ppa:deadsnakes/ppa -y && \
+    apt-get update -yy && \
+    DEBIAN_FRONTEND=noninteractive apt-get install python3.12 python3.12-dev -yy
+ENV PYO3_PYTHON=python3.12
+
 FROM base AS builder
 
 RUN apt-get update && apt-get -y install make cmake protobuf-compiler curl bash lld maven unzip
@@ -45,7 +52,7 @@ ENV JAVA_HOME ${JAVA_HOME_PATH}
 ENV LD_LIBRARY_PATH ${JAVA_HOME_PATH}/lib/server:${LD_LIBRARY_PATH}
 
 RUN cargo fetch && \
-  cargo build -p risingwave_cmd_all --release -p risingwave_object_store --features hdfs-backend --features "rw-static-link"  && \
+  cargo build -p risingwave_cmd_all --release -p risingwave_object_store --features hdfs-backend --features "rw-static-link" --features embedded-python-udf && \
   mkdir -p /risingwave/bin && \
     mv /risingwave/target/release/risingwave /risingwave/bin/ && \
     mv /risingwave/target/release/risingwave.dwp /risingwave/bin/ && \

docs/developer-guide.md

@@ -75,6 +75,7 @@ RiseDev is the development mode of RisingWave. To develop RisingWave, you need t
 * PostgreSQL (psql) (>= 14.1)
 * Tmux (>= v3.2a)
 * LLVM 16 (For macOS only, to workaround some bugs in macOS toolchain. See https://github.com/risingwavelabs/risingwave/issues/6205)
+* Python (>= 3.12) (Optional, only required by `embedded-python-udf` feature)
 
 To install the dependencies on macOS, run:
 
@@ -96,6 +97,31 @@ Then you'll be able to compile and start RiseDev!
 >
 > `.cargo/config.toml` contains `rustflags` configurations like `-Clink-arg` and `-Ctarget-feature`. Since it will be [merged](https://doc.rust-lang.org/cargo/reference/config.html#hierarchical-structure) with `$HOME/.cargo/config.toml`, check the config files and make sure they don't conflict if you have global `rustflags` configurations for e.g. linker there.
 
+> [!INFO]
+>
+> If you want to build RisingWave with `embedded-python-udf` feature, you need to install Python 3.12.
+>
+> To install Python 3.12 on macOS, run:
+>
+> ```shell
+> brew install [email protected]
+> ```
+>
+> To install Python 3.12 on Debian-based Linux systems, run:
+>
+> ```shell
+> sudo apt install software-properties-common
+> sudo add-apt-repository ppa:deadsnakes/ppa
+> sudo apt-get update
+> sudo apt-get install python3.12 python3.12-dev
+> ```
+>
+> If the default `python3` version is not 3.12, please set the `PYO3_PYTHON` environment variable:
+>
+> ```shell
+> export PYO3_PYTHON=python3.12
+> ```
+
 ## Start and monitor a dev cluster
 
 You can now build RiseDev and start a dev cluster. It is as simple as:

src/cmd_all/Cargo.toml

@@ -10,6 +10,7 @@ repository = { workspace = true }
 [features]
 rw-static-link = ["workspace-config/rw-static-link"]
 rw-dynamic-link = ["workspace-config/rw-dynamic-link"]
+embedded-python-udf = ["risingwave_expr/embedded-python-udf"]
 default = ["rw-static-link"]
 
 [package.metadata.cargo-machete]
@@ -30,6 +31,7 @@ risingwave_common = { workspace = true }
 risingwave_compactor = { workspace = true }
 risingwave_compute = { workspace = true }
 risingwave_ctl = { workspace = true }
+risingwave_expr = { workspace = true }
 risingwave_expr_impl = { workspace = true }
 risingwave_frontend = { workspace = true }
 risingwave_meta_node = { workspace = true }

src/expr/core/Cargo.toml

@@ -15,12 +15,15 @@ ignored = ["workspace-hack", "ctor"]
 [package.metadata.cargo-udeps.ignore]
 normal = ["workspace-hack", "ctor"]
 
+[features]
+embedded-python-udf = ["arrow-udf-python"]
+
 [dependencies]
 anyhow = "1"
 arrow-array = { workspace = true }
 arrow-schema = { workspace = true }
 arrow-udf-js = { workspace = true }
-arrow-udf-python = { workspace = true }
+arrow-udf-python = { workspace = true, optional = true }
 arrow-udf-wasm = { workspace = true }
 async-trait = "0.1"
 auto_impl = "1"