From 9629fddebdaabd49a94e3ac9fb0935995fd3049a Mon Sep 17 00:00:00 2001
From: Noel Kwan <47273164+kwannoel@users.noreply.github.com>
Date: Wed, 25 Sep 2024 14:56:33 +0800
Subject: [PATCH] fix(ci): fix release static build (#18689)

---
 Cargo.lock                            | 11 ++++
 ci/scripts/common.sh                  |  5 +-
 ci/scripts/release.sh                 | 78 +++++++++++++++------------
 ci/workflows/main-cron.yml            | 42 +++++++++++++++
 src/cmd/Cargo.toml                    |  1 +
 src/cmd_all/Cargo.toml                |  1 +
 src/utils/workspace-config/Cargo.toml |  2 +
 7 files changed, 103 insertions(+), 37 deletions(-)

diff --git a/Cargo.lock b/Cargo.lock
index 17d40f4b5d462..83d02b79355ff 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -8177,6 +8177,15 @@ version = "0.1.5"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "ff011a302c396a5197692431fc1948019154afc178baf7d8e37367442a4601cf"
 
+[[package]]
+name = "openssl-src"
+version = "300.3.2+3.3.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "a211a18d945ef7e648cc6e0058f4c548ee46aab922ea203e0d30e966ea23647b"
+dependencies = [
+ "cc",
+]
+
 [[package]]
 name = "openssl-sys"
 version = "0.9.103"
@@ -8185,6 +8194,7 @@ checksum = "7f9e8deee91df40a943c71b917e5874b951d32a802526c85721ce3b776c929d6"
 dependencies = [
  "cc",
  "libc",
+ "openssl-src",
  "pkg-config",
  "vcpkg",
 ]
@@ -16240,6 +16250,7 @@ dependencies = [
  "libz-sys",
  "log",
  "lzma-sys",
+ "openssl-sys",
  "sasl2-sys",
  "tracing",
  "zstd-sys",
diff --git a/ci/scripts/common.sh b/ci/scripts/common.sh
index 656bb66cc934a..9593a54aeaf54 100755
--- a/ci/scripts/common.sh
+++ b/ci/scripts/common.sh
@@ -139,10 +139,11 @@ configure_static_openssl() {
 }
 
 check_link_info() {
-  ldd_output=$(ldd target/"$1"/risingwave)
+  profile=$1
+  ldd_output=$(ldd target/"$profile"/risingwave)
   echo "$ldd_output"
   # enforce that libssl is not present if we are building with static openssl
-  if [[ "$profile" == "ci-release" ]] && [[ "$ldd_output" == *"libssl"* ]]; then
+  if [[ "$profile" == "ci-release" || "$profile" == "production" ]] && [[ "$ldd_output" == *"libssl"* ]]; then
       echo "libssl should not be dynamically linked"
       exit 1
   fi
diff --git a/ci/scripts/release.sh b/ci/scripts/release.sh
index 6ffafbb4a30dd..c5013314fc543 100755
--- a/ci/scripts/release.sh
+++ b/ci/scripts/release.sh
@@ -3,6 +3,7 @@
 # Exits as soon as any line fails.
 set -euo pipefail
 
+SKIP_RELEASE=${SKIP_RELEASE:-0}
 REPO_ROOT=${PWD}
 ARCH="$(uname -m)"
 
@@ -72,22 +73,25 @@ if [ "${ARCH}" == "aarch64" ]; then
   export JEMALLOC_SYS_WITH_LG_PAGE=16
 fi
 
-configure_static_openssl
-
-cargo build -p risingwave_cmd_all --features "rw-static-link" --features external-udf --features wasm-udf --features js-udf --profile production
-cargo build -p risingwave_cmd --bin risectl --features "rw-static-link" --profile production
+cargo build -p risingwave_cmd_all --features "rw-static-link" --features external-udf --features wasm-udf --features js-udf --features openssl-vendored --profile production
+cargo build -p risingwave_cmd --bin risectl --features "rw-static-link" --features openssl-vendored --profile production
 
+echo "--- check link info"
 check_link_info production
 
 cd target/production && chmod +x risingwave risectl
 
-echo "--- Upload nightly binary to s3"
-if [ "${BUILDKITE_SOURCE}" == "schedule" ]; then
-  tar -czvf risingwave-"$(date '+%Y%m%d')"-"${ARCH}"-unknown-linux.tar.gz risingwave
-  aws s3 cp risingwave-"$(date '+%Y%m%d')"-"${ARCH}"-unknown-linux.tar.gz s3://rw-nightly-pre-built-binary
-elif [[ -n "${BINARY_NAME+x}" ]]; then
-  tar -czvf risingwave-"${BINARY_NAME}"-"${ARCH}"-unknown-linux.tar.gz risingwave
-  aws s3 cp risingwave-"${BINARY_NAME}"-"${ARCH}"-unknown-linux.tar.gz s3://rw-nightly-pre-built-binary
+if [ "${SKIP_RELEASE}" -ne 1 ]; then
+  echo "--- Upload nightly binary to s3"
+  if [ "${BUILDKITE_SOURCE}" == "schedule" ]; then
+    tar -czvf risingwave-"$(date '+%Y%m%d')"-"${ARCH}"-unknown-linux.tar.gz risingwave
+    aws s3 cp risingwave-"$(date '+%Y%m%d')"-"${ARCH}"-unknown-linux.tar.gz s3://rw-nightly-pre-built-binary
+  elif [[ -n "${BINARY_NAME+x}" ]]; then
+    tar -czvf risingwave-"${BINARY_NAME}"-"${ARCH}"-unknown-linux.tar.gz risingwave
+    aws s3 cp risingwave-"${BINARY_NAME}"-"${ARCH}"-unknown-linux.tar.gz s3://rw-nightly-pre-built-binary
+  fi
+else
+  echo "--- Skipped upload nightly binary"
 fi
 
 echo "--- Build connector node"
@@ -106,30 +110,34 @@ if [[ -n "${BUILDKITE_TAG}" ]]; then
   dnf config-manager --add-repo https://cli.github.com/packages/rpm/gh-cli.repo
   dnf install -y gh
 
-  echo "--- Release create"
-  set +e
-  response=$(gh release view -R risingwavelabs/risingwave "${BUILDKITE_TAG}" 2>&1)
-  set -euo pipefail
-  if [[ $response == *"not found"* ]]; then
-    echo "Tag ${BUILDKITE_TAG} does not exist. Creating release..."
-    gh release create "${BUILDKITE_TAG}" --notes "release ${BUILDKITE_TAG}" -d -p
+  if [ "${SKIP_RELEASE}" -ne 1 ]; then
+    echo "--- Release create"
+    set +e
+    response=$(gh release view -R risingwavelabs/risingwave "${BUILDKITE_TAG}" 2>&1)
+    set -euo pipefail
+    if [[ $response == *"not found"* ]]; then
+      echo "Tag ${BUILDKITE_TAG} does not exist. Creating release..."
+      gh release create "${BUILDKITE_TAG}" --notes "release ${BUILDKITE_TAG}" -d -p
+    else
+      echo "Tag ${BUILDKITE_TAG} already exists. Skipping release creation."
+    fi
+
+    echo "--- Release upload risingwave asset"
+    tar -czvf risingwave-"${BUILDKITE_TAG}"-"${ARCH}"-unknown-linux.tar.gz risingwave
+    gh release upload "${BUILDKITE_TAG}" risingwave-"${BUILDKITE_TAG}"-"${ARCH}"-unknown-linux.tar.gz
+
+    echo "--- Release upload risingwave debug info"
+    tar -czvf risingwave-"${BUILDKITE_TAG}"-"${ARCH}"-unknown-linux.dwp.tar.gz risingwave.dwp
+    gh release upload "${BUILDKITE_TAG}" risingwave-"${BUILDKITE_TAG}"-"${ARCH}"-unknown-linux.dwp.tar.gz
+
+    echo "--- Release upload risectl asset"
+    tar -czvf risectl-"${BUILDKITE_TAG}"-"${ARCH}"-unknown-linux.tar.gz risectl
+    gh release upload "${BUILDKITE_TAG}" risectl-"${BUILDKITE_TAG}"-"${ARCH}"-unknown-linux.tar.gz
+
+    echo "--- Release upload risingwave-all-in-one asset"
+    tar -czvf risingwave-"${BUILDKITE_TAG}"-"${ARCH}"-unknown-linux-all-in-one.tar.gz risingwave libs
+    gh release upload "${BUILDKITE_TAG}" risingwave-"${BUILDKITE_TAG}"-"${ARCH}"-unknown-linux-all-in-one.tar.gz
   else
-    echo "Tag ${BUILDKITE_TAG} already exists. Skipping release creation."
+    echo "--- Skipped upload RW assets"
   fi
-
-  echo "--- Release upload risingwave asset"
-  tar -czvf risingwave-"${BUILDKITE_TAG}"-"${ARCH}"-unknown-linux.tar.gz risingwave
-  gh release upload "${BUILDKITE_TAG}" risingwave-"${BUILDKITE_TAG}"-"${ARCH}"-unknown-linux.tar.gz
-
-  echo "--- Release upload risingwave debug info"
-  tar -czvf risingwave-"${BUILDKITE_TAG}"-"${ARCH}"-unknown-linux.dwp.tar.gz risingwave.dwp
-  gh release upload "${BUILDKITE_TAG}" risingwave-"${BUILDKITE_TAG}"-"${ARCH}"-unknown-linux.dwp.tar.gz
-
-  echo "--- Release upload risectl asset"
-  tar -czvf risectl-"${BUILDKITE_TAG}"-"${ARCH}"-unknown-linux.tar.gz risectl
-  gh release upload "${BUILDKITE_TAG}" risectl-"${BUILDKITE_TAG}"-"${ARCH}"-unknown-linux.tar.gz
-
-  echo "--- Release upload risingwave-all-in-one asset"
-  tar -czvf risingwave-"${BUILDKITE_TAG}"-"${ARCH}"-unknown-linux-all-in-one.tar.gz risingwave libs
-  gh release upload "${BUILDKITE_TAG}" risingwave-"${BUILDKITE_TAG}"-"${ARCH}"-unknown-linux-all-in-one.tar.gz
 fi
diff --git a/ci/workflows/main-cron.yml b/ci/workflows/main-cron.yml
index a627c12dd88cd..d1086092f7e2d 100644
--- a/ci/workflows/main-cron.yml
+++ b/ci/workflows/main-cron.yml
@@ -1061,6 +1061,26 @@ steps:
     timeout_in_minutes: 10
     retry: *auto-retry
 
+  - label: "release amd64 (dry-run)"
+    command: "SKIP_RELEASE=1 ci/scripts/release.sh"
+    if: |
+      build.pull_request.labels includes "ci/run-release-dry-run" || build.env("CI_STEPS") =~ /(^|,)release-dry-run(,|$$)/
+    plugins:
+      - seek-oss/aws-sm#v2.3.1:
+          env:
+            GITHUB_TOKEN: github-token
+      - docker-compose#v5.1.0:
+          run: release-env-x86
+          config: ci/docker-compose.yml
+          mount-buildkite-agent: true
+          environment:
+            - BINARY_NAME
+            - GITHUB_TOKEN
+            - BUILDKITE_TAG
+            - BUILDKITE_SOURCE
+    timeout_in_minutes: 60
+    retry: *auto-retry
+
   - label: "release amd64"
     command: "ci/scripts/release.sh"
     if: build.tag != null
@@ -1080,6 +1100,28 @@ steps:
     timeout_in_minutes: 60
     retry: *auto-retry
 
+  - label: "release aarch64 (dry-run)"
+    command: "SKIP_RELEASE=1 ci/scripts/release.sh"
+    if: |
+      build.pull_request.labels includes "ci/run-release-dry-run" || build.env("CI_STEPS") =~ /(^|,)release-dry-run(,|$$)/
+    plugins:
+      - seek-oss/aws-sm#v2.3.1:
+          env:
+            GITHUB_TOKEN: github-token
+      - docker-compose#v5.1.0:
+          run: release-env-arm
+          config: ci/docker-compose.yml
+          mount-buildkite-agent: true
+          environment:
+            - BINARY_NAME
+            - GITHUB_TOKEN
+            - BUILDKITE_TAG
+            - BUILDKITE_SOURCE
+    agents:
+      queue: "linux-arm64"
+    timeout_in_minutes: 60
+    retry: *auto-retry
+
   - label: "release aarch64"
     command: "ci/scripts/release.sh"
     if: build.tag != null
diff --git a/src/cmd/Cargo.toml b/src/cmd/Cargo.toml
index 33f1efd18a2ff..0e06897855a11 100644
--- a/src/cmd/Cargo.toml
+++ b/src/cmd/Cargo.toml
@@ -10,6 +10,7 @@ repository = { workspace = true }
 [features]
 rw-static-link = ["workspace-config/rw-static-link"]
 rw-dynamic-link = ["workspace-config/rw-dynamic-link"]
+openssl-vendored = ["workspace-config/openssl-vendored"]
 default = ["rw-static-link"]
 
 [package.metadata.cargo-machete]
diff --git a/src/cmd_all/Cargo.toml b/src/cmd_all/Cargo.toml
index aaa9f2886eaa8..7d8e771889bd3 100644
--- a/src/cmd_all/Cargo.toml
+++ b/src/cmd_all/Cargo.toml
@@ -11,6 +11,7 @@ repository = { workspace = true }
 default = ["rw-static-link"]
 rw-static-link = ["workspace-config/rw-static-link"]
 rw-dynamic-link = ["workspace-config/rw-dynamic-link"]
+openssl-vendored = ["workspace-config/openssl-vendored"]
 all-udf = ["external-udf", "wasm-udf", "js-udf", "python-udf"]
 external-udf = ["risingwave_expr_impl/external-udf"]
 wasm-udf = ["risingwave_expr_impl/wasm-udf"]
diff --git a/src/utils/workspace-config/Cargo.toml b/src/utils/workspace-config/Cargo.toml
index 8972cb7c08475..e82e7910fadbf 100644
--- a/src/utils/workspace-config/Cargo.toml
+++ b/src/utils/workspace-config/Cargo.toml
@@ -11,6 +11,7 @@ repository = { workspace = true }
 [features]
 # some crates opt-in static linking, while some opt-in dynamic linking,
 # so they are two features :)
+openssl-vendored = ["vendored-openssl-sys"]
 rw-static-link = ["static-libz-sys", "static-lzma-sys", "static-sasl2-sys"]
 rw-dynamic-link = ["dynamic-zstd-sys"]
 
@@ -23,6 +24,7 @@ tracing = { version = "0.1", features = ["release_max_level_debug"] }
 static-libz-sys = { package = "libz-sys", version = "1", optional = true, features = ["static"] }
 static-lzma-sys = { package = "lzma-sys", version = "0.1", optional = true, features = ["static"] }
 static-sasl2-sys = { package = "sasl2-sys", version = "0.1", optional = true, features = ["gssapi-vendored"] }
+vendored-openssl-sys = { package = "openssl-sys", version = "0.9.96", optional = true, features = ["vendored"] }
 
 # Dynamic linking
 dynamic-zstd-sys = { package = "zstd-sys", version = "2", optional = true, default-features = false, features = ["pkg-config"] }