Merge branch 'main' into bz/better-locate-java-lib

Makefile.toml

@@ -35,6 +35,7 @@ is_release = get_env ENABLE_RELEASE_PROFILE
 is_not_release = not ${is_release}
 is_dynamic_linking = get_env ENABLE_DYNAMIC_LINKING
 is_hummock_trace = get_env ENABLE_HUMMOCK_TRACE
+is_python_udf_enabled = get_env ENABLE_PYTHON_UDF
 
 if ${is_sanitizer_enabled}
   set_env RISEDEV_CARGO_BUILD_EXTRA_ARGS "-Zbuild-std --target ${CARGO_MAKE_RUST_TARGET_TRIPLE}"
@@ -65,6 +66,11 @@ else
   set_env RISINGWAVE_FEATURE_FLAGS "--features rw-static-link"
 end
 
+if ${is_python_udf_enabled}
+  flags = get_env RISINGWAVE_FEATURE_FLAGS
+  set_env RISINGWAVE_FEATURE_FLAGS "${flags} --features embedded-python-udf"
+end
+
 if ${is_hummock_trace}
   set_env BUILD_HUMMOCK_TRACE_CMD "-p risingwave_storage --features hm-trace"
 else
@@ -621,6 +627,25 @@ if [ ! -f "${RC_ENV_FILE}" ]; then
 fi
 '''
 
+[tasks.psql-env]
+category = "RiseDev - Start/Stop"
+description = "Dump env configuration for psql"
+dependencies = ["check-risedev-env-file"]
+env_files = ["${PREFIX_CONFIG}/risedev-env"]
+script = '''
+#!/usr/bin/env bash
+cat <<EOF > "${PREFIX_CONFIG}/psql-env"
+export PGHOST=$RW_FRONTEND_LISTEN_ADDRESS
+export PGPORT=$RW_FRONTEND_PORT
+export PGUSER=root
+export PGDATABASE=dev
+EOF
+
+echo "psql environment file is created at ${PREFIX_CONFIG}/psql-env"
+echo "You can source this file to use psql with default connection parameters."
+echo "   $(tput setaf 4)source ${PREFIX_CONFIG}/psql-env$(tput sgr0)"
+'''
+
 [tasks.psql]
 category = "RiseDev - Start/Stop"
 description = "Run local psql client with default connection parameters. You can pass extra arguments to psql."

backwards-compat-tests/scripts/utils.sh

@@ -49,6 +49,23 @@ kill_zookeeper() {
   wait_zookeeper_exit
 }
 
+wait_for_process() {
+    process_name="$1"
+
+    while pgrep -x "$process_name" > /dev/null; do
+        echo "Process $process_name is still running... Wait for 1 sec"
+        sleep 1
+    done
+}
+
+wait_all_process_exit() {
+  wait_for_process meta-node
+  wait_for_process compute-node
+  wait_for_process frontend
+  wait_for_process compactor
+  echo "All processes has exited."
+}
+
 # Older versions of RW may not gracefully kill kafka.
 # So we duplicate the definition here.
 kill_cluster() {
@@ -75,6 +92,7 @@ kill_cluster() {
 
   tmux kill-session -t risedev
   test $? -eq 0 || { echo "Failed to stop all RiseDev components."; exit 1; }
+  wait_all_process_exit
 }
 
 run_sql () {

ci/docker-compose.yml

@@ -96,18 +96,6 @@ services:
     volumes:
       - ..:/risingwave
 
-  sink-doris-env:
-    image: public.ecr.aws/x5u3w5h6/rw-build-env:v20231109
-    depends_on:
-      - doris-fe-server
-      - doris-be-server
-    volumes:
-      - ..:/risingwave
-    command: >
-      sh -c "sudo sysctl -w vm.max_map_count=2000000"
-    networks:
-      mynetwork:
-        ipv4_address: 172.121.0.4
 
   rw-build-env:
     image: public.ecr.aws/x5u3w5h6/rw-build-env:v20240223
@@ -189,39 +177,6 @@ services:
       timeout: 30s
       retries: 50
 
-  doris-fe-server:
-    platform: linux/amd64
-    image: apache/doris:2.0.0_alpha-fe-x86_64
-    hostname: doris-fe-server
-    command: >
-      sh -c "sudo sysctl -w vm.max_map_count=2000000"
-    environment:
-     - FE_SERVERS=fe1:172.121.0.2:9010
-     - FE_ID=1
-    ports:
-      - "8030:8030"
-      - "9030:9030"
-    networks:
-      mynetwork:
-        ipv4_address: 172.121.0.2
-
-  doris-be-server:
-    platform: linux/amd64
-    image: apache/doris:2.0.0_alpha-be-x86_64
-    hostname: doris-be-server
-    command: >
-      sh -c "sudo sysctl -w vm.max_map_count=2000000"
-    environment:
-     - FE_SERVERS=fe1:172.121.0.2:9010
-     - BE_ADDR=172.121.0.3:9050
-    depends_on:
-      - doris-fe-server
-    ports:
-      - "9050:9050"
-    networks:
-      mynetwork:
-        ipv4_address: 172.121.0.3
-
   cassandra-server:
     container_name: cassandra-server
     image: cassandra:4.0
@@ -319,9 +274,3 @@ services:
       interval: 5s
       timeout: 5s
       retries: 5
-networks:
-  mynetwork:
-    ipam:
-      config:
-        - subnet: 172.121.80.0/16
-  default:

ci/scripts/build.sh

@@ -53,6 +53,7 @@ cargo build \
     -p risingwave_compaction_test \
     -p risingwave_e2e_extended_mode_test \
     $RISINGWAVE_FEATURE_FLAGS \
+    --features embedded-python-udf \
     --profile "$profile"
 
 

ci/workflows/main-cron.yml

@@ -834,36 +834,37 @@ steps:
     timeout_in_minutes: 10
     retry: *auto-retry
 
-  - label: "set vm_max_map_count_2000000"
-    key: "set-vm_max_map_count"
-    if: |
-      !(build.pull_request.labels includes "ci/main-cron/skip-ci") && build.env("CI_STEPS") == null
-      || build.pull_request.labels includes "ci/run-e2e-doris-sink-tests"
-      || build.env("CI_STEPS") =~ /(^|,)e2e-doris-sink-tests?(,|$$)/
-    command: "sudo sysctl -w vm.max_map_count=2000000"
-    depends_on:
-      - "build"
-      - "build-other"
+  # Causes ci error, close it first, fix it later
+  # - label: "set vm_max_map_count_2000000"
+  #   key: "set-vm_max_map_count"
+  #   if: |
+  #     !(build.pull_request.labels includes "ci/main-cron/skip-ci") && build.env("CI_STEPS") == null
+  #     || build.pull_request.labels includes "ci/run-e2e-doris-sink-tests"
+  #     || build.env("CI_STEPS") =~ /(^|,)e2e-doris-sink-tests?(,|$$)/
+  #   command: "sudo sysctl -w vm.max_map_count=2000000"
+  #   depends_on:
+  #     - "build"
+  #     - "build-other"
 
-  - label: "end-to-end doris sink test"
-    key: "e2e-doris-sink-tests"
-    command: "ci/scripts/e2e-doris-sink-test.sh -p ci-release"
-    if: |
-      !(build.pull_request.labels includes "ci/main-cron/skip-ci") && build.env("CI_STEPS") == null
-      || build.pull_request.labels includes "ci/run-e2e-doris-sink-tests"
-      || build.env("CI_STEPS") =~ /(^|,)e2e-doris-sink-tests?(,|$$)/
-    depends_on:
-      - "build"
-      - "build-other"
-      - "set-vm_max_map_count"
-    plugins:
-      - docker-compose#v5.1.0:
-          run: sink-doris-env
-          config: ci/docker-compose.yml
-          mount-buildkite-agent: true
-      - ./ci/plugins/upload-failure-logs
-    timeout_in_minutes: 10
-    retry: *auto-retry
+  # - label: "end-to-end doris sink test"
+  #   key: "e2e-doris-sink-tests"
+  #   command: "ci/scripts/e2e-doris-sink-test.sh -p ci-release"
+  #   if: |
+  #     !(build.pull_request.labels includes "ci/main-cron/skip-ci") && build.env("CI_STEPS") == null
+  #     || build.pull_request.labels includes "ci/run-e2e-doris-sink-tests"
+  #     || build.env("CI_STEPS") =~ /(^|,)e2e-doris-sink-tests?(,|$$)/
+  #   depends_on:
+  #     - "build"
+  #     - "build-other"
+  #     - "set-vm_max_map_count"
+  #   plugins:
+  #     - docker-compose#v5.1.0:
+  #         run: sink-doris-env
+  #         config: ci/docker-compose.yml
+  #         mount-buildkite-agent: true
+  #     - ./ci/plugins/upload-failure-logs
+  #   timeout_in_minutes: 10
+  #   retry: *auto-retry
 
   - label: "end-to-end starrocks sink test"
     key: "e2e-starrocks-sink-tests"

ci/workflows/pull-request.yml

@@ -96,7 +96,7 @@ steps:
           config: ci/docker-compose.yml
           mount-buildkite-agent: true
       - ./ci/plugins/upload-failure-logs
-    timeout_in_minutes: 17
+    timeout_in_minutes: 18
     retry: *auto-retry
 
   - label: "end-to-end test (parallel)"
@@ -307,29 +307,30 @@ steps:
     timeout_in_minutes: 10
     retry: *auto-retry
 
-  - label: "set vm_max_map_count_2000000"
-    key: "set-vm_max_map_count"
-    if: build.pull_request.labels includes "ci/run-e2e-doris-sink-tests" || build.env("CI_STEPS") =~ /(^|,) e2e-doris-sink-tests?(,|$$)/
-    command: "sudo sysctl -w vm.max_map_count=2000000"
-    depends_on:
-      - "build"
-      - "build-other"
-
-  - label: "end-to-end doris sink test"
-    if: build.pull_request.labels includes "ci/run-e2e-doris-sink-tests" || build.env("CI_STEPS") =~ /(^|,) e2e-doris-sink-tests?(,|$$)/
-    command: "ci/scripts/e2e-doris-sink-test.sh -p ci-dev"
-    depends_on:
-      - "build"
-      - "build-other"
-      - "set-vm_max_map_count"
-    plugins:
-      - docker-compose#v5.1.0:
-          run: sink-doris-env
-          config: ci/docker-compose.yml
-          mount-buildkite-agent: true
-      - ./ci/plugins/upload-failure-logs
-    timeout_in_minutes: 10
-    retry: *auto-retry
+  # Causes ci error, close it first, fix it later
+  # - label: "set vm_max_map_count_2000000"
+  #   key: "set-vm_max_map_count"
+  #   if: build.pull_request.labels includes "ci/run-e2e-doris-sink-tests" || build.env("CI_STEPS") =~ /(^|,) e2e-doris-sink-tests?(,|$$)/
+  #   command: "sudo sysctl -w vm.max_map_count=2000000"
+  #   depends_on:
+  #     - "build"
+  #     - "build-other"
+
+  # - label: "end-to-end doris sink test"
+  #   if: build.pull_request.labels includes "ci/run-e2e-doris-sink-tests" || build.env("CI_STEPS") =~ /(^|,) e2e-doris-sink-tests?(,|$$)/
+  #   command: "ci/scripts/e2e-doris-sink-test.sh -p ci-dev"
+  #   depends_on:
+  #     - "build"
+  #     - "build-other"
+  #     - "set-vm_max_map_count"
+  #   plugins:
+  #     - docker-compose#v5.1.0:
+  #         run: sink-doris-env
+  #         config: ci/docker-compose.yml
+  #         mount-buildkite-agent: true
+  #     - ./ci/plugins/upload-failure-logs
+  #   timeout_in_minutes: 10
+  #   retry: *auto-retry
 
   - label: "end-to-end starrocks sink test"
     if: build.pull_request.labels includes "ci/run-e2e-starrocks-sink-tests" || build.env("CI_STEPS") =~ /(^|,) e2e-starrocks-sink-tests?(,|$$)/

docker/Dockerfile

@@ -5,6 +5,13 @@ ENV LANG en_US.utf8
 RUN apt-get update \
   && apt-get -y install ca-certificates build-essential libsasl2-dev openjdk-11-jdk
 
+# Install Python 3.12
+RUN apt-get -y install software-properties-common
+RUN add-apt-repository ppa:deadsnakes/ppa -y && \
+    apt-get update -yy && \
+    DEBIAN_FRONTEND=noninteractive apt-get install python3.12 python3.12-dev -yy
+ENV PYO3_PYTHON=python3.12
+
 FROM base AS dashboard-builder
 
 RUN apt-get update && apt-get install -y curl gnupg protobuf-compiler && mkdir -p /etc/apt/keyrings \
@@ -70,7 +77,7 @@ COPY ./ /risingwave
 WORKDIR /risingwave
 
 RUN cargo fetch && \
-  cargo build -p risingwave_cmd_all --release --features "rw-static-link" && \
+  cargo build -p risingwave_cmd_all --release --features "rw-static-link" --features embedded-python-udf && \
   mkdir -p /risingwave/bin && \
     mv /risingwave/target/release/risingwave /risingwave/bin/ && \
     mv /risingwave/target/release/risingwave.dwp /risingwave/bin/ && \

docker/Dockerfile.hdfs

@@ -5,6 +5,13 @@ ENV LANG en_US.utf8
 RUN apt-get update \
   && apt-get -y install ca-certificates build-essential libsasl2-dev openjdk-11-jdk
 
+# Install Python 3.12
+RUN apt-get -y install software-properties-common
+RUN add-apt-repository ppa:deadsnakes/ppa -y && \
+    apt-get update -yy && \
+    DEBIAN_FRONTEND=noninteractive apt-get install python3.12 python3.12-dev -yy
+ENV PYO3_PYTHON=python3.12
+
 FROM base AS builder
 
 RUN apt-get update && apt-get -y install make cmake protobuf-compiler curl bash lld maven unzip
@@ -45,7 +52,7 @@ ENV JAVA_HOME ${JAVA_HOME_PATH}
 ENV LD_LIBRARY_PATH ${JAVA_HOME_PATH}/lib/server:${LD_LIBRARY_PATH}
 
 RUN cargo fetch && \
-  cargo build -p risingwave_cmd_all --release -p risingwave_object_store --features hdfs-backend --features "rw-static-link"  && \
+  cargo build -p risingwave_cmd_all --release -p risingwave_object_store --features hdfs-backend --features "rw-static-link" --features embedded-python-udf && \
   mkdir -p /risingwave/bin && \
     mv /risingwave/target/release/risingwave /risingwave/bin/ && \
     mv /risingwave/target/release/risingwave.dwp /risingwave/bin/ && \