From 3399f4860beb461a3611791d78506b8591a142b7 Mon Sep 17 00:00:00 2001 From: jspc Date: Sun, 26 Feb 2023 16:49:05 +0000 Subject: [PATCH 1/2] Drop to a shell on panic Note: there is no real great way to test this in a unit test without starting a shell and so this change needs to be taken on faith, or tested by hand --- main.go | 24 ++++++++++++++++++++++++ 1 file changed, 24 insertions(+) diff --git a/main.go b/main.go index fc4831e..39cc4d3 100644 --- a/main.go +++ b/main.go @@ -7,6 +7,7 @@ import ( "os" "os/exec" "path/filepath" + "runtime/debug" "github.com/grpc-ecosystem/go-grpc-middleware" "github.com/grpc-ecosystem/go-grpc-middleware/tags" @@ -24,6 +25,8 @@ var ( ) func main() { + defer panicHandler() + var err error sugar, err = NewLogger(kmesgF) if err != nil { @@ -141,6 +144,27 @@ func loadTLSCredentials() (credentials.TransportCredentials, error) { return credentials.NewTLS(config), nil } +func panicHandler() { + err := recover() + if err != nil { + stack := string(debug.Stack()) + errStr := err.(error).Error() + + if sugar.c != nil { + sugar.Errorw("vinit panic!", + "error", errStr, + "trace", stack, + ) + } else { + // This branch occurs on any panic before our + // kmesg handler is enabled + fmt.Printf("panic:\n%s\n%s", errStr, stack) + } + + recoveryShell() + } +} + func recoveryShell() { fmt.Println("Press Ctrl+D to reboot") From 323951ac39fa9f846a0cf416bf10cadbf01ec049 Mon Sep 17 00:00:00 2001 From: jspc Date: Sun, 26 Feb 2023 18:30:33 +0000 Subject: [PATCH 2/2] Stop all services on a panic --- main.go | 19 +++++++++++++++++-- 1 file changed, 17 insertions(+), 2 deletions(-) diff --git a/main.go b/main.go index 39cc4d3..44a0193 100644 --- a/main.go +++ b/main.go @@ -18,6 +18,19 @@ import ( ) var ( + // supervisor is the thing that handles all of the + // running of services, including tracking whether or not + // they're even running + // + // Why is this a global variable? + // Because we need to be able to stop all services and drop to + // a shell if our init process panics, so we can suss out what + // happened. + // + // Without a global, this needs all kinds of software architecture + // changes + supervisor *Supervisor + sockAddr = "/run/vinit.sock" svcDir = envOrDefault("SVC_DIR", "/etc/vinit/services") certDir = "certs" @@ -78,8 +91,6 @@ func main() { } func Setup() (grpcServer *grpc.Server, err error) { - var supervisor *Supervisor - supervisor, err = New(svcDir) if err != nil { if _, ok := err.(ConfigParseError); !ok { @@ -147,6 +158,10 @@ func loadTLSCredentials() (credentials.TransportCredentials, error) { func panicHandler() { err := recover() if err != nil { + if supervisor != nil { + _ = supervisor.StopAll() + } + stack := string(debug.Stack()) errStr := err.(error).Error()