feat(risedev): core dump on panics for development (#12235)

Signed-off-by: Bugen Zhao <[email protected]>

.cargo/config.toml

@@ -24,6 +24,7 @@ rustflags = [
 ]
 
 [target.aarch64-apple-darwin]
+runner = "scripts/coredump/sign-and-run"
 rustflags = [
     # neon is enabled by default
     "-Clink-arg=-fuse-ld=/opt/homebrew/opt/llvm/bin/ld64.lld",
@@ -37,7 +38,6 @@ rustflags = [
     # uncomment the following two lines to enable `TaskLocalAlloc`
     # "--cfg",
     # "enable_task_local_alloc",
-
 ]
 
 [unstable]

.vscode/launch.json.example

@@ -13,6 +13,21 @@
             ],
             "cwd": "${workspaceRoot}",
             "preLaunchTask": "build rw bin"
+        },
+        {
+            "name": "Open playground coredump",
+            "type": "lldb",
+            "request": "custom",
+            "targetCreateCommands": [
+                "target create ${workspaceFolder}/target/debug/risingwave --core ${input:coreFileName}"
+            ],
+        }
+    ],
+    "inputs": [
+        {
+            "id": "coreFileName",
+            "type": "promptString",
+            "description": "Enter core file path"
         }
     ]
-}
+}

Makefile.toml

@@ -281,6 +281,32 @@ ln -s "$(pwd)/target/${RISEDEV_BUILD_TARGET_DIR}${BUILD_MODE_DIR}/risingwave" "$
 ln -s "$(pwd)/target/${RISEDEV_BUILD_TARGET_DIR}${BUILD_MODE_DIR}/risingwave" "${PREFIX_BIN}/risingwave/standalone"
 '''
 
+[tasks.codesign-binaries]
+private = true
+category = "RiseDev - Build"
+description = "Codesign all binaries to support coredump"
+# If core dump is enabled by RiseDev and we're on an Apple Silicon platform,
+# codesign the binary before running.
+# https://developer.apple.com/forums/thread/694233?answerId=695943022#695943022
+condition = { env_set = [
+  "ENABLE_COREDUMP",
+], env = { "SYSTEM" = "darwin-arm64" } }
+script = '''
+#!/usr/bin/env bash
+set -e
+
+binaries=()
+
+if [[ "$ENABLE_ALL_IN_ONE" == "true" ]]; then
+  binaries=("risingwave")
+else
+  binaries=("meta-node" "compute-node" "frontend" "compactor")
+fi
+
+echo -n "${binaries[*]}" | parallel -d ' ' \
+  "codesign -s - -f --entitlements scripts/coredump/coredump.entitlements \"target/${RISEDEV_BUILD_TARGET_DIR}${BUILD_MODE_DIR}/{}\""
+'''
+
 [tasks.link-user-bin]
 private = true
 category = "RiseDev - Build"
@@ -307,6 +333,7 @@ dependencies = [
   "link-standalone-binaries",
   "link-all-in-one-binaries",
   "link-user-bin",
+  "codesign-binaries",
 ]
 
 [tasks.b]
@@ -478,6 +505,11 @@ dependencies = [
   "download-redis",
 ]
 
+[tasks.pre-start-playground]
+category = "RiseDev - Prepare"
+description = "Preparation steps for playground"
+dependencies = ["build-connector-node"]
+
 [tasks.check-risedev-env-file]
 private = true
 category = "RiseDev - Prepare"
@@ -521,13 +553,16 @@ alias = "playground"
 [tasks.playground]
 category = "RiseDev - Start/Stop"
 description = "🌟 Start a lite RisingWave playground using risingwave all-in-one binary"
-dependencies = ["build-connector-node"]
+dependencies = ["pre-start-playground"]
 script = '''
 #!/usr/bin/env bash
 
 set -ex
 
-RUST_BACKTRACE=1 \
+if [[ $ENABLE_COREDUMP == "true" ]]; then
+  ulimit -c unlimited
+fi
+
 cargo run -p risingwave_cmd_all \
           --profile "${RISINGWAVE_BUILD_PROFILE}" \
           ${RISINGWAVE_FEATURE_FLAGS} \
@@ -543,7 +578,10 @@ script = '''
 
 set -euo pipefail
 
-RUST_BACKTRACE=1 \
+if [[ $ENABLE_COREDUMP == "true" ]]; then
+  ulimit -c unlimited
+fi
+
 cargo run -p risingwave_cmd_all \
           --profile "${RISINGWAVE_BUILD_PROFILE}" \
           ${RISINGWAVE_FEATURE_FLAGS} \
@@ -570,8 +608,18 @@ alias = "dev"
 [tasks.dev]
 category = "RiseDev - Start/Stop"
 dependencies = ["pre-start-dev"]
-script = "RUST_BACKTRACE=1 target/${BUILD_MODE_DIR}/risedev-dev ${@}"
 description = "🌟 Start a full RisingWave dev cluster using risedev-dev"
+script = '''
+#!/usr/bin/env bash
+
+set -ex
+
+if [[ $ENABLE_COREDUMP == "true" ]]; then
+  ulimit -c unlimited
+fi
+
+target/${BUILD_MODE_DIR}/risedev-dev ${@}
+'''
 
 [tasks.kill-risedev]
 category = "RiseDev - Start/Stop"
@@ -700,6 +748,10 @@ echo
 echo "check: $(tput setaf 4)protoc >= 3.12.0$(tput sgr0)"
 protoc --version || echo "$(tput setaf 3)protoc$(tput sgr0) not found."
 echo
+
+echo "check: $(tput setaf 4)parallel >= 2022XXXX$(tput sgr0)"
+parallel --version || echo "$(tput setaf 3)parallel$(tput sgr0) not found."
+echo
 """
 description = "Install (or upgrade) required tools to do pre-CI check and run e2e tests"
 
@@ -1288,4 +1340,4 @@ cargo run -p risingwave_common --bin example-config >> src/config/example.toml
 [tasks.backwards-compat-test]
 category = "RiseDev - Backwards Compatibility Test"
 description = "Run backwards compatibility test"
-script = "./backwards-compat-tests/scripts/run_local.sh"
+script = "./backwards-compat-tests/scripts/run_local.sh"

scripts/coredump/coredump.entitlements

@@ -0,0 +1,8 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!DOCTYPE plist PUBLIC "-//Apple//DTD PLIST 1.0//EN" "http://www.apple.com/DTDs/PropertyList-1.0.dtd">
+<plist version="1.0">
+<dict>
+	<key>com.apple.security.get-task-allow</key>
+	<true/>
+</dict>
+</plist>

scripts/coredump/sign-and-run

@@ -0,0 +1,10 @@
+#!/usr/bin/env zsh
+
+# If core dump is enabled by RiseDev and we're on an Apple Silicon platform,
+# codesign the binary before running.
+# https://developer.apple.com/forums/thread/694233?answerId=695943022#695943022
+if [[ "$ENABLE_COREDUMP" == "true" ]]; then
+    codesign -s - -f --entitlements scripts/coredump/coredump.entitlements "$1"
+fi
+
+exec "$@"

src/risedevtool/config/src/main.rs

@@ -73,6 +73,7 @@ pub enum Components {
     Sanitizer,
     DynamicLinking,
     HummockTrace,
+    Coredump,
 }
 
 impl Components {
@@ -94,6 +95,7 @@ impl Components {
             Self::Sanitizer => "[Build] Enable sanitizer",
             Self::DynamicLinking => "[Build] Enable dynamic linking",
             Self::HummockTrace => "[Build] Hummock Trace",
+            Self::Coredump => "[Runtime] Enable coredump",
         }
         .into()
     }
@@ -179,7 +181,18 @@ but you might need the expertise to install dependencies correctly.
                 "
             }
             Self::HummockTrace => {
-            "With this option enabled, RiseDev will enable tracing for Hummock. See storage/hummock_trace for details."
+                "
+With this option enabled, RiseDev will enable tracing for Hummock.
+See storage/hummock_trace for details.
+                "
+            }
+            Self::Coredump => {
+                "
+With this option enabled, RiseDev will unlimit the size of core
+files before launching RisingWave. On Apple Silicon platforms,
+the binaries will also be codesigned with `get-task-allow` enabled.
+As a result, RisingWave will dump the core on panics.
+                "
             }
         }
         .into()
@@ -225,6 +238,7 @@ but you might need the expertise to install dependencies correctly.
             Self::BuildConnectorNode => "ENABLE_BUILD_RW_CONNECTOR",
             Self::DynamicLinking => "ENABLE_DYNAMIC_LINKING",
             Self::HummockTrace => "ENABLE_HUMMOCK_TRACE",
+            Self::Coredump => "ENABLE_COREDUMP",
         }
         .into()
     }