From 1ecea639b746ee61f741456b2a3d6a98b637f897 Mon Sep 17 00:00:00 2001 From: Bugen Zhao Date: Thu, 14 Sep 2023 16:43:14 +0800 Subject: [PATCH] refactor(risedev): split the steps for building and running playground (#12279) Signed-off-by: Bugen Zhao Co-authored-by: xxchan --- .cargo/config.toml | 1 - Cargo.lock | 10 +++++ Makefile.toml | 69 ++++++++++++++++++++++------- scripts/coredump/sign-and-run | 10 ----- src/risedevtool/common.toml | 1 + src/utils/runtime/Cargo.toml | 1 + src/utils/runtime/src/panic_hook.rs | 4 ++ 7 files changed, 69 insertions(+), 27 deletions(-) delete mode 100755 scripts/coredump/sign-and-run diff --git a/.cargo/config.toml b/.cargo/config.toml index ce84940299190..2c41ca0110f64 100644 --- a/.cargo/config.toml +++ b/.cargo/config.toml @@ -24,7 +24,6 @@ rustflags = [ ] [target.aarch64-apple-darwin] -runner = "scripts/coredump/sign-and-run" rustflags = [ # neon is enabled by default "-Clink-arg=-fuse-ld=/opt/homebrew/opt/llvm/bin/ld64.lld", diff --git a/Cargo.lock b/Cargo.lock index edabb832341fb..faec2fdfb3aa8 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -7367,6 +7367,7 @@ dependencies = [ "prometheus", "risingwave_common", "risingwave_variables", + "rlimit", "time", "tracing", "tracing-opentelemetry", @@ -7708,6 +7709,15 @@ version = "1.0.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3582f63211428f83597b51b2ddb88e2a91a9d52d12831f9d08f5e624e8977422" +[[package]] +name = "rlimit" +version = "0.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3560f70f30a0f16d11d01ed078a07740fe6b489667abc7c7b029155d9f21c3d8" +dependencies = [ + "libc", +] + [[package]] name = "rsa" version = "0.9.2" diff --git a/Makefile.toml b/Makefile.toml index bbcc0df84243a..1583f7185e179 100644 --- a/Makefile.toml +++ b/Makefile.toml @@ -281,6 +281,23 @@ ln -s "$(pwd)/target/${RISEDEV_BUILD_TARGET_DIR}${BUILD_MODE_DIR}/risingwave" "$ ln -s "$(pwd)/target/${RISEDEV_BUILD_TARGET_DIR}${BUILD_MODE_DIR}/risingwave" "${PREFIX_BIN}/risingwave/standalone" ''' +[tasks.codesign-playground] +private = true +category = "RiseDev - Build" +description = "Codesign playground binary to support coredump" +# If core dump is enabled by RiseDev and we're on an Apple Silicon platform, +# codesign the binary before running. +# https://developer.apple.com/forums/thread/694233?answerId=695943022#695943022 +condition = { env_set = [ + "ENABLE_COREDUMP", +], env = { "SYSTEM" = "darwin-arm64" } } +script = ''' +#!/usr/bin/env bash + +set -ex +codesign -s - -f --entitlements scripts/coredump/coredump.entitlements "target/${RISEDEV_BUILD_TARGET_DIR}${BUILD_MODE_DIR}/risingwave" +''' + [tasks.codesign-binaries] private = true category = "RiseDev - Build" @@ -303,6 +320,7 @@ else binaries=("meta-node" "compute-node" "frontend" "compactor") fi +set -ex echo -n "${binaries[*]}" | parallel -d ' ' \ "codesign -s - -f --entitlements scripts/coredump/coredump.entitlements \"target/${RISEDEV_BUILD_TARGET_DIR}${BUILD_MODE_DIR}/{}\"" ''' @@ -393,6 +411,24 @@ ${BUILD_HUMMOCK_TRACE_CMD}\ ${RISEDEV_CARGO_BUILD_EXTRA_ARGS} ''' +[tasks.build-risingwave-playground] +category = "RiseDev - Build" +description = "Build RisingWave playground" +condition = { env_true = ["ENABLE_BUILD_RUST"] } +script = ''' +#!/usr/bin/env bash + +set -e +[[ -z "${RISEDEV_RUSTFLAGS}" ]] || export RUSTFLAGS="${RISEDEV_RUSTFLAGS}" +echo + RUSTFLAGS="${RUSTFLAGS:-}" +set -xe + +cargo build -p risingwave_cmd_all \ + --profile "${RISINGWAVE_BUILD_PROFILE}" \ + ${RISINGWAVE_FEATURE_FLAGS} \ + ${RISEDEV_CARGO_BUILD_EXTRA_ARGS} +''' + [tasks.clean] private = true category = "RiseDev - Build" @@ -508,7 +544,11 @@ dependencies = [ [tasks.pre-start-playground] category = "RiseDev - Prepare" description = "Preparation steps for playground" -dependencies = ["build-connector-node"] +dependencies = [ + "build-risingwave-playground", + "codesign-playground", + "build-connector-node", +] [tasks.check-risedev-env-file] private = true @@ -557,35 +597,31 @@ dependencies = ["pre-start-playground"] script = ''' #!/usr/bin/env bash -set -ex - +set -e if [[ $ENABLE_COREDUMP == "true" ]]; then + echo "+ ulimit -c unlimited" ulimit -c unlimited fi -cargo run -p risingwave_cmd_all \ - --profile "${RISINGWAVE_BUILD_PROFILE}" \ - ${RISINGWAVE_FEATURE_FLAGS} \ - -- playground +set -x +target/${RISEDEV_BUILD_TARGET_DIR}${BUILD_MODE_DIR}/risingwave playground $@ ''' [tasks.standalone] category = "RiseDev - Start/Stop" description = "🌟 Start a RisingWave standalone instance" -dependencies = ["build"] +dependencies = ["pre-start-playground"] script = ''' #!/usr/bin/env bash -set -euo pipefail - +set -e if [[ $ENABLE_COREDUMP == "true" ]]; then + echo "+ ulimit -c unlimited" ulimit -c unlimited fi -cargo run -p risingwave_cmd_all \ - --profile "${RISINGWAVE_BUILD_PROFILE}" \ - ${RISINGWAVE_FEATURE_FLAGS} \ - -- standalone $@ +set -x +target/${RISEDEV_BUILD_TARGET_DIR}${BUILD_MODE_DIR}/risingwave standalone $@ ''' # TODO(kwannoel): Support `tasks.standalone-dev` as well. @@ -612,12 +648,13 @@ description = "🌟 Start a full RisingWave dev cluster using risedev-dev" script = ''' #!/usr/bin/env bash -set -ex - +set -e if [[ $ENABLE_COREDUMP == "true" ]]; then + echo "+ ulimit -c unlimited" ulimit -c unlimited fi +set -x target/${BUILD_MODE_DIR}/risedev-dev ${@} ''' diff --git a/scripts/coredump/sign-and-run b/scripts/coredump/sign-and-run deleted file mode 100755 index 9f2bf69471d61..0000000000000 --- a/scripts/coredump/sign-and-run +++ /dev/null @@ -1,10 +0,0 @@ -#!/usr/bin/env zsh - -# If core dump is enabled by RiseDev and we're on an Apple Silicon platform, -# codesign the binary before running. -# https://developer.apple.com/forums/thread/694233?answerId=695943022#695943022 -if [[ "$ENABLE_COREDUMP" == "true" ]]; then - codesign -s - -f --entitlements scripts/coredump/coredump.entitlements "$1" -fi - -exec "$@" diff --git a/src/risedevtool/common.toml b/src/risedevtool/common.toml index a664f13635a08..9e0b30ae7e561 100644 --- a/src/risedevtool/common.toml +++ b/src/risedevtool/common.toml @@ -1,5 +1,6 @@ [env] RISEDEV = "1" +RUST_BACKTRACE = "1" OS = { source = "${CARGO_MAKE_RUST_TARGET_OS}", mapping = { linux = "linux", macos = "darwin" } } ARCH = { source = "${CARGO_MAKE_RUST_TARGET_ARCH}", mapping = { x86_64 = "amd64", aarch64 = "arm64" } } SYSTEM = "${OS}-${ARCH}" diff --git a/src/utils/runtime/Cargo.toml b/src/utils/runtime/Cargo.toml index 6a11fce54ffb6..f63f7d63d7e7b 100644 --- a/src/utils/runtime/Cargo.toml +++ b/src/utils/runtime/Cargo.toml @@ -32,6 +32,7 @@ pprof = { version = "0.12", features = ["flamegraph"] } prometheus = { version = "0.13" } risingwave_common = { workspace = true } risingwave_variables = { workspace = true } +rlimit = "0.10" time = { version = "0.3", features = ["formatting", "local-offset"] } tokio = { version = "0.2", package = "madsim-tokio", features = [ "rt", diff --git a/src/utils/runtime/src/panic_hook.rs b/src/utils/runtime/src/panic_hook.rs index 992126e196f58..848e7df8509c7 100644 --- a/src/utils/runtime/src/panic_hook.rs +++ b/src/utils/runtime/src/panic_hook.rs @@ -15,6 +15,10 @@ /// Set panic hook to abort the process if we're not catching unwind, without losing the information /// of stack trace and await-tree. pub fn set_panic_hook() { + if let Ok(limit) = rlimit::Resource::CORE.get_soft() && limit > 0 { + tracing::info!(limit, "coredump on panic is likely to be enabled"); + }; + std::panic::update_hook(|default_hook, info| { default_hook(info);