Merge branch 'main' into yiming/disable-test-epoch-in-release

.github/workflows/cherry-pick-to-release-branch.yml

@@ -6,6 +6,22 @@ on:
     types: ["closed", "labeled"]
 
 jobs:
+  release_pull_request_1_7_standalone:
+    if: "contains(github.event.pull_request.labels.*.name, 'need-cherry-pick-release-1.7-standalone')  &&  github.event.pull_request.merged == true"
+    runs-on: ubuntu-latest
+    name: release_pull_request
+    steps:
+      - name: checkout
+        uses: actions/checkout@v1
+      - name: Create PR to branch
+        uses: risingwavelabs/github-action-cherry-pick@master
+        with:
+          pr_branch: 'release-1.7.0-standalone'
+          pr_labels: 'cherry-pick'
+          pr_body: ${{ format('Cherry picking \#{0} onto branch release-1.7.0-standalone', github.event.number) }}
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+
   release_pull_request_1_7:
     if: "contains(github.event.pull_request.labels.*.name, 'need-cherry-pick-release-1.7')  &&  github.event.pull_request.merged == true"
     runs-on: ubuntu-latest

Cargo.toml

@@ -122,7 +122,7 @@ tonic = { package = "madsim-tonic", version = "0.4.1" }
 tonic-build = { package = "madsim-tonic-build", version = "0.4.2" }
 otlp-embedded = { git = "https://github.com/risingwavelabs/otlp-embedded", rev = "58c1f003484449d7c6dd693b348bf19dd44889cb" }
 prost = { version = "0.12" }
-icelake = { git = "https://github.com/icelake-io/icelake", rev = "56943d203464d32701c20d74b278b6b3ef2eefc7", features = [
+icelake = { git = "https://github.com/icelake-io/icelake", rev = "54fd72fbd1dd8c592f05eeeb79223c8a6a33c297", features = [
   "prometheus",
 ] }
 arrow-array = "50"
@@ -136,6 +136,7 @@ arrow-ord = "50"
 arrow-row = "50"
 arrow-udf-js = "0.1"
 arrow-udf-wasm = { version = "0.1.2", features = ["build"] }
+arrow-udf-python = { git = "https://github.com/risingwavelabs/arrow-udf.git", rev = "6c32f71" }
 arrow-array-deltalake = { package = "arrow-array", version = "48.0.1" }
 arrow-buffer-deltalake = { package = "arrow-buffer", version = "48.0.1" }
 arrow-cast-deltalake = { package = "arrow-cast", version = "48.0.1" }

Makefile.toml

@@ -29,12 +29,12 @@ env_scripts = [
 set_env ENABLE_TELEMETRY "false"
 
 is_sanitizer_enabled = get_env ENABLE_SANITIZER
-is_all_in_one_enabled = get_env ENABLE_ALL_IN_ONE
 is_hdfs_backend = get_env ENABLE_HDFS
 is_release = get_env ENABLE_RELEASE_PROFILE
 is_not_release = not ${is_release}
 is_dynamic_linking = get_env ENABLE_DYNAMIC_LINKING
 is_hummock_trace = get_env ENABLE_HUMMOCK_TRACE
+is_python_udf_enabled = get_env ENABLE_PYTHON_UDF
 
 if ${is_sanitizer_enabled}
   set_env RISEDEV_CARGO_BUILD_EXTRA_ARGS "-Zbuild-std --target ${CARGO_MAKE_RUST_TARGET_TRIPLE}"
@@ -45,13 +45,8 @@ else
   set_env RISEDEV_BUILD_TARGET_DIR ""
 end
 
-if ${is_all_in_one_enabled}
-  set_env RISEDEV_CARGO_BUILD_CRATE "risingwave_cmd_all"
-  set_env RISEDEV_CTL_RUN_CMD "-- risectl"
-else
-  set_env RISEDEV_CARGO_BUILD_CRATE "risingwave_cmd"
-  set_env RISEDEV_CTL_RUN_CMD "--bin risectl --"
-end
+set_env RISEDEV_CARGO_BUILD_CRATE "risingwave_cmd_all"
+set_env RISEDEV_CTL_RUN_CMD "-- risectl"
 
 if ${is_hdfs_backend}
   set_env BUILD_HDFS_BACKEND_CMD "-p risingwave_object_store --features hdfs-backend"
@@ -65,6 +60,11 @@ else
   set_env RISINGWAVE_FEATURE_FLAGS "--features rw-static-link"
 end
 
+if ${is_python_udf_enabled}
+  flags = get_env RISINGWAVE_FEATURE_FLAGS
+  set_env RISINGWAVE_FEATURE_FLAGS "${flags} --features embedded-python-udf"
+end
+
 if ${is_hummock_trace}
   set_env BUILD_HUMMOCK_TRACE_CMD "-p risingwave_storage --features hm-trace"
 else
@@ -262,31 +262,10 @@ set -e
 python -mwebbrowser file://$(pwd)/target/doc/index.html
 '''
 
-[tasks.link-standalone-binaries]
-private = true
-category = "RiseDev - Build"
-description = "Link standalone cmds to RiseDev bin"
-condition = { env_not_set = ["ENABLE_ALL_IN_ONE"] }
-script = '''
-#!/usr/bin/env bash
-set -e
-rm -f "${PREFIX_BIN}/compute-node"
-rm -f "${PREFIX_BIN}/meta-node"
-rm -f "${PREFIX_BIN}/frontend"
-rm -f "${PREFIX_BIN}/compactor"
-rm -f "${PREFIX_BIN}/risectl"
-ln -s "$(pwd)/target/${RISEDEV_BUILD_TARGET_DIR}${BUILD_MODE_DIR}/compute-node" "${PREFIX_BIN}/compute-node"
-ln -s "$(pwd)/target/${RISEDEV_BUILD_TARGET_DIR}${BUILD_MODE_DIR}/meta-node" "${PREFIX_BIN}/meta-node"
-ln -s "$(pwd)/target/${RISEDEV_BUILD_TARGET_DIR}${BUILD_MODE_DIR}/frontend" "${PREFIX_BIN}/frontend"
-ln -s "$(pwd)/target/${RISEDEV_BUILD_TARGET_DIR}${BUILD_MODE_DIR}/compactor" "${PREFIX_BIN}/compactor"
-ln -s "$(pwd)/target/${RISEDEV_BUILD_TARGET_DIR}${BUILD_MODE_DIR}/risectl" "${PREFIX_BIN}/risectl"
-'''
-
 [tasks.link-all-in-one-binaries]
 private = true
 category = "RiseDev - Build"
 description = "Link all-in-one cmds to RiseDev bin"
-condition = { env_set = ["ENABLE_ALL_IN_ONE"] }
 script = '''
 #!/usr/bin/env bash
 set -e
@@ -333,13 +312,7 @@ script = '''
 #!/usr/bin/env bash
 set -e
 
-binaries=()
-
-if [[ "$ENABLE_ALL_IN_ONE" == "true" ]]; then
-  binaries=("risingwave")
-else
-  binaries=("meta-node" "compute-node" "frontend" "compactor")
-fi
+binaries=("risingwave")
 
 set -ex
 echo -n "${binaries[*]}" | parallel -d ' ' \
@@ -350,7 +323,6 @@ echo -n "${binaries[*]}" | parallel -d ' ' \
 private = true
 category = "RiseDev - Build"
 description = "Link all binaries to .bin"
-condition = { env_set = ["ENABLE_ALL_IN_ONE"] }
 script = '''
 #!/usr/bin/env bash
 set -e
@@ -369,7 +341,6 @@ category = "RiseDev - Build"
 description = "Copy RisngWave binaries to bin"
 condition = { env_set = ["ENABLE_BUILD_RUST"] }
 dependencies = [
-  "link-standalone-binaries",
   "link-all-in-one-binaries",
   "link-user-bin",
   "codesign-binaries",
@@ -621,6 +592,25 @@ if [ ! -f "${RC_ENV_FILE}" ]; then
 fi
 '''
 
+[tasks.psql-env]
+category = "RiseDev - Start/Stop"
+description = "Dump env configuration for psql"
+dependencies = ["check-risedev-env-file"]
+env_files = ["${PREFIX_CONFIG}/risedev-env"]
+script = '''
+#!/usr/bin/env bash
+cat <<EOF > "${PREFIX_CONFIG}/psql-env"
+export PGHOST=$RW_FRONTEND_LISTEN_ADDRESS
+export PGPORT=$RW_FRONTEND_PORT
+export PGUSER=root
+export PGDATABASE=dev
+EOF
+
+echo "psql environment file is created at ${PREFIX_CONFIG}/psql-env"
+echo "You can source this file to use psql with default connection parameters."
+echo "   $(tput setaf 4)source ${PREFIX_CONFIG}/psql-env$(tput sgr0)"
+'''
+
 [tasks.psql]
 category = "RiseDev - Start/Stop"
 description = "Run local psql client with default connection parameters. You can pass extra arguments to psql."

backwards-compat-tests/scripts/run_local.sh

@@ -37,9 +37,6 @@ ENABLE_KAFKA=true
 # Fetch risingwave binary from release.
 ENABLE_BUILD_RUST=true
 
-# Ensure it will link the all-in-one binary from our release.
-ENABLE_ALL_IN_ONE=true
-
 # ENABLE_RELEASE_PROFILE=true
 EOF
 }