fix(ci): fix release static build (#18689)

risingwavelabs · Sep 25, 2024 · 9629fdd · 9629fdd
1 parent a7e60cb
commit 9629fdd
Show file tree

Hide file tree

Showing 7 changed files with 103 additions and 37 deletions.
diff --git a/Cargo.lock b/Cargo.lock
diff --git a/ci/scripts/common.sh b/ci/scripts/common.sh
@@ -139,10 +139,11 @@ configure_static_openssl() {
 }
 
 check_link_info() {
-  ldd_output=$(ldd target/"$1"/risingwave)
+  profile=$1
+  ldd_output=$(ldd target/"$profile"/risingwave)
   echo "$ldd_output"
   # enforce that libssl is not present if we are building with static openssl
-  if [[ "$profile" == "ci-release" ]] && [[ "$ldd_output" == *"libssl"* ]]; then
+  if [[ "$profile" == "ci-release" || "$profile" == "production" ]] && [[ "$ldd_output" == *"libssl"* ]]; then
       echo "libssl should not be dynamically linked"
       exit 1
   fi

diff --git a/ci/scripts/release.sh b/ci/scripts/release.sh
@@ -3,6 +3,7 @@
 # Exits as soon as any line fails.
 set -euo pipefail
 
+SKIP_RELEASE=${SKIP_RELEASE:-0}
 REPO_ROOT=${PWD}
 ARCH="$(uname -m)"
 
@@ -72,22 +73,25 @@ if [ "${ARCH}" == "aarch64" ]; then
   export JEMALLOC_SYS_WITH_LG_PAGE=16
 fi
 
-configure_static_openssl
-
-cargo build -p risingwave_cmd_all --features "rw-static-link" --features external-udf --features wasm-udf --features js-udf --profile production
-cargo build -p risingwave_cmd --bin risectl --features "rw-static-link" --profile production
+cargo build -p risingwave_cmd_all --features "rw-static-link" --features external-udf --features wasm-udf --features js-udf --features openssl-vendored --profile production
+cargo build -p risingwave_cmd --bin risectl --features "rw-static-link" --features openssl-vendored --profile production
 
+echo "--- check link info"
 check_link_info production
 
 cd target/production && chmod +x risingwave risectl
 
-echo "--- Upload nightly binary to s3"
-if [ "${BUILDKITE_SOURCE}" == "schedule" ]; then
-  tar -czvf risingwave-"$(date '+%Y%m%d')"-"${ARCH}"-unknown-linux.tar.gz risingwave
-  aws s3 cp risingwave-"$(date '+%Y%m%d')"-"${ARCH}"-unknown-linux.tar.gz s3://rw-nightly-pre-built-binary
-elif [[ -n "${BINARY_NAME+x}" ]]; then
-  tar -czvf risingwave-"${BINARY_NAME}"-"${ARCH}"-unknown-linux.tar.gz risingwave
-  aws s3 cp risingwave-"${BINARY_NAME}"-"${ARCH}"-unknown-linux.tar.gz s3://rw-nightly-pre-built-binary
+if [ "${SKIP_RELEASE}" -ne 1 ]; then
+  echo "--- Upload nightly binary to s3"
+  if [ "${BUILDKITE_SOURCE}" == "schedule" ]; then
+    tar -czvf risingwave-"$(date '+%Y%m%d')"-"${ARCH}"-unknown-linux.tar.gz risingwave
+    aws s3 cp risingwave-"$(date '+%Y%m%d')"-"${ARCH}"-unknown-linux.tar.gz s3://rw-nightly-pre-built-binary
+  elif [[ -n "${BINARY_NAME+x}" ]]; then
+    tar -czvf risingwave-"${BINARY_NAME}"-"${ARCH}"-unknown-linux.tar.gz risingwave
+    aws s3 cp risingwave-"${BINARY_NAME}"-"${ARCH}"-unknown-linux.tar.gz s3://rw-nightly-pre-built-binary
+  fi
+else
+  echo "--- Skipped upload nightly binary"
 fi
 
 echo "--- Build connector node"
@@ -106,30 +110,34 @@ if [[ -n "${BUILDKITE_TAG}" ]]; then
   dnf config-manager --add-repo https://cli.github.com/packages/rpm/gh-cli.repo
   dnf install -y gh
 
-  echo "--- Release create"
-  set +e
-  response=$(gh release view -R risingwavelabs/risingwave "${BUILDKITE_TAG}" 2>&1)
-  set -euo pipefail
-  if [[ $response == *"not found"* ]]; then
-    echo "Tag ${BUILDKITE_TAG} does not exist. Creating release..."
-    gh release create "${BUILDKITE_TAG}" --notes "release ${BUILDKITE_TAG}" -d -p
+  if [ "${SKIP_RELEASE}" -ne 1 ]; then
+    echo "--- Release create"
+    set +e
+    response=$(gh release view -R risingwavelabs/risingwave "${BUILDKITE_TAG}" 2>&1)
+    set -euo pipefail
+    if [[ $response == *"not found"* ]]; then
+      echo "Tag ${BUILDKITE_TAG} does not exist. Creating release..."
+      gh release create "${BUILDKITE_TAG}" --notes "release ${BUILDKITE_TAG}" -d -p
+    else
+      echo "Tag ${BUILDKITE_TAG} already exists. Skipping release creation."
+    fi
+
+    echo "--- Release upload risingwave asset"
+    tar -czvf risingwave-"${BUILDKITE_TAG}"-"${ARCH}"-unknown-linux.tar.gz risingwave
+    gh release upload "${BUILDKITE_TAG}" risingwave-"${BUILDKITE_TAG}"-"${ARCH}"-unknown-linux.tar.gz
+
+    echo "--- Release upload risingwave debug info"
+    tar -czvf risingwave-"${BUILDKITE_TAG}"-"${ARCH}"-unknown-linux.dwp.tar.gz risingwave.dwp
+    gh release upload "${BUILDKITE_TAG}" risingwave-"${BUILDKITE_TAG}"-"${ARCH}"-unknown-linux.dwp.tar.gz
+
+    echo "--- Release upload risectl asset"
+    tar -czvf risectl-"${BUILDKITE_TAG}"-"${ARCH}"-unknown-linux.tar.gz risectl
+    gh release upload "${BUILDKITE_TAG}" risectl-"${BUILDKITE_TAG}"-"${ARCH}"-unknown-linux.tar.gz
+
+    echo "--- Release upload risingwave-all-in-one asset"
+    tar -czvf risingwave-"${BUILDKITE_TAG}"-"${ARCH}"-unknown-linux-all-in-one.tar.gz risingwave libs
+    gh release upload "${BUILDKITE_TAG}" risingwave-"${BUILDKITE_TAG}"-"${ARCH}"-unknown-linux-all-in-one.tar.gz
   else
-    echo "Tag ${BUILDKITE_TAG} already exists. Skipping release creation."
+    echo "--- Skipped upload RW assets"
   fi
-
-  echo "--- Release upload risingwave asset"
-  tar -czvf risingwave-"${BUILDKITE_TAG}"-"${ARCH}"-unknown-linux.tar.gz risingwave
-  gh release upload "${BUILDKITE_TAG}" risingwave-"${BUILDKITE_TAG}"-"${ARCH}"-unknown-linux.tar.gz
-
-  echo "--- Release upload risingwave debug info"
-  tar -czvf risingwave-"${BUILDKITE_TAG}"-"${ARCH}"-unknown-linux.dwp.tar.gz risingwave.dwp
-  gh release upload "${BUILDKITE_TAG}" risingwave-"${BUILDKITE_TAG}"-"${ARCH}"-unknown-linux.dwp.tar.gz
-
-  echo "--- Release upload risectl asset"
-  tar -czvf risectl-"${BUILDKITE_TAG}"-"${ARCH}"-unknown-linux.tar.gz risectl
-  gh release upload "${BUILDKITE_TAG}" risectl-"${BUILDKITE_TAG}"-"${ARCH}"-unknown-linux.tar.gz
-
-  echo "--- Release upload risingwave-all-in-one asset"
-  tar -czvf risingwave-"${BUILDKITE_TAG}"-"${ARCH}"-unknown-linux-all-in-one.tar.gz risingwave libs
-  gh release upload "${BUILDKITE_TAG}" risingwave-"${BUILDKITE_TAG}"-"${ARCH}"-unknown-linux-all-in-one.tar.gz
 fi
diff --git a/ci/workflows/main-cron.yml b/ci/workflows/main-cron.yml
@@ -1061,6 +1061,26 @@ steps:
     timeout_in_minutes: 10
     retry: *auto-retry
 
+  - label: "release amd64 (dry-run)"
+    command: "SKIP_RELEASE=1 ci/scripts/release.sh"
+    if: |
+      build.pull_request.labels includes "ci/run-release-dry-run" || build.env("CI_STEPS") =~ /(^|,)release-dry-run(,|$$)/
+    plugins:
+      - seek-oss/aws-sm#v2.3.1:
+          env:
+            GITHUB_TOKEN: github-token
+      - docker-compose#v5.1.0:
+          run: release-env-x86
+          config: ci/docker-compose.yml
+          mount-buildkite-agent: true
+          environment:
+            - BINARY_NAME
+            - GITHUB_TOKEN
+            - BUILDKITE_TAG
+            - BUILDKITE_SOURCE
+    timeout_in_minutes: 60
+    retry: *auto-retry
+
   - label: "release amd64"
     command: "ci/scripts/release.sh"
     if: build.tag != null
@@ -1080,6 +1100,28 @@ steps:
     timeout_in_minutes: 60
     retry: *auto-retry
 
+  - label: "release aarch64 (dry-run)"
+    command: "SKIP_RELEASE=1 ci/scripts/release.sh"
+    if: |
+      build.pull_request.labels includes "ci/run-release-dry-run" || build.env("CI_STEPS") =~ /(^|,)release-dry-run(,|$$)/
+    plugins:
+      - seek-oss/aws-sm#v2.3.1:
+          env:
+            GITHUB_TOKEN: github-token
+      - docker-compose#v5.1.0:
+          run: release-env-arm
+          config: ci/docker-compose.yml
+          mount-buildkite-agent: true
+          environment:
+            - BINARY_NAME
+            - GITHUB_TOKEN
+            - BUILDKITE_TAG
+            - BUILDKITE_SOURCE
+    agents:
+      queue: "linux-arm64"
+    timeout_in_minutes: 60
+    retry: *auto-retry
+
   - label: "release aarch64"
     command: "ci/scripts/release.sh"
     if: build.tag != null

diff --git a/src/cmd/Cargo.toml b/src/cmd/Cargo.toml
@@ -10,6 +10,7 @@ repository = { workspace = true }
 [features]
 rw-static-link = ["workspace-config/rw-static-link"]
 rw-dynamic-link = ["workspace-config/rw-dynamic-link"]
+openssl-vendored = ["workspace-config/openssl-vendored"]
 default = ["rw-static-link"]
 
 [package.metadata.cargo-machete]

diff --git a/src/cmd_all/Cargo.toml b/src/cmd_all/Cargo.toml
@@ -11,6 +11,7 @@ repository = { workspace = true }
 default = ["rw-static-link"]
 rw-static-link = ["workspace-config/rw-static-link"]
 rw-dynamic-link = ["workspace-config/rw-dynamic-link"]
+openssl-vendored = ["workspace-config/openssl-vendored"]
 all-udf = ["external-udf", "wasm-udf", "js-udf", "python-udf"]
 external-udf = ["risingwave_expr_impl/external-udf"]
 wasm-udf = ["risingwave_expr_impl/wasm-udf"]

diff --git a/src/utils/workspace-config/Cargo.toml b/src/utils/workspace-config/Cargo.toml
@@ -11,6 +11,7 @@ repository = { workspace = true }
 [features]
 # some crates opt-in static linking, while some opt-in dynamic linking,
 # so they are two features :)
+openssl-vendored = ["vendored-openssl-sys"]
 rw-static-link = ["static-libz-sys", "static-lzma-sys", "static-sasl2-sys"]
 rw-dynamic-link = ["dynamic-zstd-sys"]
 
@@ -23,6 +24,7 @@ tracing = { version = "0.1", features = ["release_max_level_debug"] }
 static-libz-sys = { package = "libz-sys", version = "1", optional = true, features = ["static"] }
 static-lzma-sys = { package = "lzma-sys", version = "0.1", optional = true, features = ["static"] }
 static-sasl2-sys = { package = "sasl2-sys", version = "0.1", optional = true, features = ["gssapi-vendored"] }
+vendored-openssl-sys = { package = "openssl-sys", version = "0.9.96", optional = true, features = ["vendored"] }
 
 # Dynamic linking
 dynamic-zstd-sys = { package = "zstd-sys", version = "2", optional = true, default-features = false, features = ["pkg-config"] }