fix(udf): add embedded-python-udf feature and fix dockerfile

Makefile.toml

@@ -35,6 +35,7 @@ is_release = get_env ENABLE_RELEASE_PROFILE
 is_not_release = not ${is_release}
 is_dynamic_linking = get_env ENABLE_DYNAMIC_LINKING
 is_hummock_trace = get_env ENABLE_HUMMOCK_TRACE
+is_python_udf_enabled = get_env ENABLE_PYTHON_UDF
 
 if ${is_sanitizer_enabled}
   set_env RISEDEV_CARGO_BUILD_EXTRA_ARGS "-Zbuild-std --target ${CARGO_MAKE_RUST_TARGET_TRIPLE}"
@@ -65,6 +66,11 @@ else
   set_env RISINGWAVE_FEATURE_FLAGS "--features rw-static-link"
 end
 
+if ${is_python_udf_enabled}
+  flags = get_env RISINGWAVE_FEATURE_FLAGS
+  set_env RISINGWAVE_FEATURE_FLAGS "${flags} --features embedded-python-udf"
+end
+
 if ${is_hummock_trace}
   set_env BUILD_HUMMOCK_TRACE_CMD "-p risingwave_storage --features hm-trace"
 else

ci/scripts/build.sh

@@ -40,9 +40,9 @@ cargo fmt --all -- --check
 echo "--- Build Rust components"
 
 if [[ "$profile" == "ci-dev" ]]; then
-    RISINGWAVE_FEATURE_FLAGS="--features rw-dynamic-link --no-default-features"
+    RISINGWAVE_FEATURE_FLAGS="--features rw-dynamic-link embedded-python-udf --no-default-features"
 else
-    RISINGWAVE_FEATURE_FLAGS="--features rw-static-link"
+    RISINGWAVE_FEATURE_FLAGS="--features rw-static-link embedded-python-udf"
 fi
 
 cargo build \

docker/Dockerfile

@@ -21,6 +21,13 @@ FROM base AS rust-base
 
 RUN apt-get update && apt-get -y install make cmake protobuf-compiler curl bash lld unzip
 
+# Install Python 3.12
+RUN apt-get -y install software-properties-common
+RUN add-apt-repository ppa:deadsnakes/ppa -y && \
+    apt-get update -yy && \
+    DEBIAN_FRONTEND=noninteractive apt-get install python3.12 python3.12-dev -yy
+ENV PYO3_PYTHON=python3.12
+
 SHELL ["/bin/bash", "-c"]
 
 RUN curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- --no-modify-path --default-toolchain none -y
@@ -70,7 +77,7 @@ COPY ./ /risingwave
 WORKDIR /risingwave
 
 RUN cargo fetch && \
-  cargo build -p risingwave_cmd_all --release --features "rw-static-link" && \
+  cargo build -p risingwave_cmd_all --release --features "rw-static-link" --features embedded-python-udf && \
   mkdir -p /risingwave/bin && \
     mv /risingwave/target/release/risingwave /risingwave/bin/ && \
     mv /risingwave/target/release/risingwave.dwp /risingwave/bin/ && \

docker/Dockerfile.hdfs

@@ -9,6 +9,13 @@ FROM base AS builder
 
 RUN apt-get update && apt-get -y install make cmake protobuf-compiler curl bash lld maven unzip
 
+# Install Python 3.12
+RUN apt-get -y install software-properties-common
+RUN add-apt-repository ppa:deadsnakes/ppa -y && \
+    apt-get update -yy && \
+    DEBIAN_FRONTEND=noninteractive apt-get install python3.12 python3.12-dev -yy
+ENV PYO3_PYTHON=python3.12
+
 SHELL ["/bin/bash", "-c"]
 
 RUN curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- --no-modify-path --default-toolchain none -y
@@ -45,7 +52,7 @@ ENV JAVA_HOME ${JAVA_HOME_PATH}
 ENV LD_LIBRARY_PATH ${JAVA_HOME_PATH}/lib/server:${LD_LIBRARY_PATH}
 
 RUN cargo fetch && \
-  cargo build -p risingwave_cmd_all --release -p risingwave_object_store --features hdfs-backend --features "rw-static-link"  && \
+  cargo build -p risingwave_cmd_all --release -p risingwave_object_store --features hdfs-backend --features "rw-static-link" --features embedded-python-udf && \
   mkdir -p /risingwave/bin && \
     mv /risingwave/target/release/risingwave /risingwave/bin/ && \
     mv /risingwave/target/release/risingwave.dwp /risingwave/bin/ && \

docs/developer-guide.md

@@ -75,6 +75,7 @@ RiseDev is the development mode of RisingWave. To develop RisingWave, you need t
 * PostgreSQL (psql) (>= 14.1)
 * Tmux (>= v3.2a)
 * LLVM 16 (For macOS only, to workaround some bugs in macOS toolchain. See https://github.com/risingwavelabs/risingwave/issues/6205)
+* Python (>= 3.12) (Optional, only required by `embedded-python-udf` feature)
 
 To install the dependencies on macOS, run:
 
@@ -96,6 +97,31 @@ Then you'll be able to compile and start RiseDev!
 >
 > `.cargo/config.toml` contains `rustflags` configurations like `-Clink-arg` and `-Ctarget-feature`. Since it will be [merged](https://doc.rust-lang.org/cargo/reference/config.html#hierarchical-structure) with `$HOME/.cargo/config.toml`, check the config files and make sure they don't conflict if you have global `rustflags` configurations for e.g. linker there.
 
+> [!INFO]
+> 
+> If you want to build RisingWave with `embedded-python-udf` feature, you need to install Python 3.12.
+>
+> To install Python 3.12 on macOS, run:
+>
+> ```shell
+> brew install [email protected]
+> ```
+>
+> To install Python 3.12 on Debian-based Linux systems, run:
+>
+> ```shell
+> sudo apt install software-properties-common
+> sudo add-apt-repository ppa:deadsnakes/ppa
+> sudo apt-get update
+> sudo apt-get install python3.12 python3.12-dev
+> ```
+>
+> If the default `python3` version is not 3.12, please set the `PYO3_PYTHON` environment variable:
+>
+> ```shell
+> export PYO3_PYTHON=python3.12
+> ```
+
 ## Start and monitor a dev cluster
 
 You can now build RiseDev and start a dev cluster. It is as simple as:

src/cmd_all/Cargo.toml

@@ -10,6 +10,7 @@ repository = { workspace = true }
 [features]
 rw-static-link = ["workspace-config/rw-static-link"]
 rw-dynamic-link = ["workspace-config/rw-dynamic-link"]
+embedded-python-udf = ["risingwave_expr/embedded-python-udf"]
 default = ["rw-static-link"]
 
 [package.metadata.cargo-machete]
@@ -30,6 +31,7 @@ risingwave_common = { workspace = true }
 risingwave_compactor = { workspace = true }
 risingwave_compute = { workspace = true }
 risingwave_ctl = { workspace = true }
+risingwave_expr = { workspace = true }
 risingwave_expr_impl = { workspace = true }
 risingwave_frontend = { workspace = true }
 risingwave_meta_node = { workspace = true }

src/expr/core/Cargo.toml

@@ -15,12 +15,15 @@ ignored = ["workspace-hack", "ctor"]
 [package.metadata.cargo-udeps.ignore]
 normal = ["workspace-hack", "ctor"]
 
+[features]
+embedded-python-udf = ["arrow-udf-python"]
+
 [dependencies]
 anyhow = "1"
 arrow-array = { workspace = true }
 arrow-schema = { workspace = true }
 arrow-udf-js = { workspace = true }
-arrow-udf-python = { workspace = true }
+arrow-udf-python = { workspace = true, optional = true }
 arrow-udf-wasm = { workspace = true }
 async-trait = "0.1"
 auto_impl = "1"

src/expr/core/src/expr/expr_udf.rs

@@ -21,6 +21,7 @@ use std::time::Duration;
 use anyhow::Context;
 use arrow_schema::{Field, Fields, Schema};
 use arrow_udf_js::{CallMode as JsCallMode, Runtime as JsRuntime};
+#[cfg(feature = "embedded-python-udf")]
 use arrow_udf_python::{CallMode as PythonCallMode, Runtime as PythonRuntime};
 use arrow_udf_wasm::Runtime as WasmRuntime;
 use await_tree::InstrumentAwait;
@@ -60,10 +61,11 @@ pub struct UserDefinedFunction {
 const INITIAL_RETRY_COUNT: u8 = 16;
 
 #[derive(Debug)]
-enum UdfImpl {
+pub enum UdfImpl {
     External(Arc<ArrowFlightUdfClient>),
     Wasm(Arc<WasmRuntime>),
     JavaScript(JsRuntime),
+    #[cfg(feature = "embedded-python-udf")]
     Python(PythonRuntime),
 }
 
@@ -110,6 +112,7 @@ impl UserDefinedFunction {
         let arrow_output: arrow_array::RecordBatch = match &self.imp {
             UdfImpl::Wasm(runtime) => runtime.call(&self.identifier, &arrow_input)?,
             UdfImpl::JavaScript(runtime) => runtime.call(&self.identifier, &arrow_input)?,
+            #[cfg(feature = "embedded-python-udf")]
             UdfImpl::Python(runtime) => runtime.call(&self.identifier, &arrow_input)?,
             UdfImpl::External(client) => {
                 let disable_retry_count = self.disable_retry_count.load(Ordering::Relaxed);
@@ -210,6 +213,7 @@ impl Build for UserDefinedFunction {
                 )?;
                 UdfImpl::JavaScript(rt)
             }
+            #[cfg(feature = "embedded-python-udf")]
             "python" if udf.body.is_some() => {
                 let mut rt = PythonRuntime::builder().sandboxed(true).build()?;
                 let body = udf.get_body()?;

src/expr/core/src/table_function/user_defined.rs

@@ -18,16 +18,16 @@ use anyhow::Context;
 use arrow_array::RecordBatch;
 use arrow_schema::{Field, Fields, Schema, SchemaRef};
 use arrow_udf_js::{CallMode as JsCallMode, Runtime as JsRuntime};
+#[cfg(feature = "embedded-python-udf")]
 use arrow_udf_python::{CallMode as PythonCallMode, Runtime as PythonRuntime};
-use arrow_udf_wasm::Runtime as WasmRuntime;
 use cfg_or_panic::cfg_or_panic;
 use futures_util::stream;
 use risingwave_common::array::{ArrayError, DataChunk, I32Array};
 use risingwave_common::bail;
-use risingwave_udf::ArrowFlightUdfClient;
 use thiserror_ext::AsReport;
 
 use super::*;
+use crate::expr::expr_udf::UdfImpl;
 
 #[derive(Debug)]
 pub struct UserDefinedTableFunction {
@@ -41,14 +41,6 @@ pub struct UserDefinedTableFunction {
     chunk_size: usize,
 }
 
-#[derive(Debug)]
-enum UdfImpl {
-    External(Arc<ArrowFlightUdfClient>),
-    Wasm(Arc<WasmRuntime>),
-    JavaScript(JsRuntime),
-    Python(PythonRuntime),
-}
-
 #[async_trait::async_trait]
 impl TableFunction for UserDefinedTableFunction {
     fn return_type(&self) -> DataType {
@@ -80,6 +72,7 @@ impl UdfImpl {
                     yield res?;
                 }
             }
+            #[cfg(feature = "embedded-python-udf")]
             UdfImpl::Python(runtime) => {
                 for res in runtime.call_table_function(identifier, &input, 1024)? {
                     yield res?;
@@ -219,6 +212,7 @@ pub fn new_user_defined(prost: &PbTableFunction, chunk_size: usize) -> Result<Bo
             )?;
             UdfImpl::JavaScript(rt)
         }
+        #[cfg(feature = "embedded-python-udf")]
         "python" if udtf.body.is_some() => {
             let mut rt = PythonRuntime::builder().sandboxed(true).build()?;
             let body = udtf.get_body()?;