From c02edeea8bd4d51bc760029aef70bd0de158f486 Mon Sep 17 00:00:00 2001 From: "tomas.panik" Date: Thu, 5 Oct 2023 15:42:40 +0200 Subject: [PATCH 1/3] NH-57813: Updating image to container wrapper around otel collector that try to detect checkpoint corruption --- build/docker/Dockerfile | 9 ++++- src/wrapper/go.mod | 3 ++ src/wrapper/main.go | 86 +++++++++++++++++++++++++++++++++++++++++ 3 files changed, 96 insertions(+), 2 deletions(-) create mode 100644 src/wrapper/go.mod create mode 100644 src/wrapper/main.go diff --git a/build/docker/Dockerfile b/build/docker/Dockerfile index ca9abd50..e3b1d62e 100644 --- a/build/docker/Dockerfile +++ b/build/docker/Dockerfile @@ -23,6 +23,10 @@ FROM debian:11.7@sha256:432f545c6ba13b79e2681f4cc4858788b0ab099fc1cca799cc0fae46 RUN apt update RUN apt install -y systemd +FROM base as wrapper +WORKDIR /src/src/wrapper +RUN CGO_ENABLED=0 GOEXPERIMENT=boringcrypto go build -a -tags netgo -ldflags '-w -extldflags "-static"' -o /bin/wrapper && chmod +x /bin/wrapper + FROM scratch ARG USER_UID=10001 @@ -30,6 +34,7 @@ USER ${USER_UID} COPY --from=prep /etc/ssl/certs/ca-certificates.crt /etc/ssl/certs/ca-certificates.crt COPY --from=builder /src/swi-k8s-opentelemetry-collector /swi-otelcol +COPY --from=wrapper /bin/wrapper /wrapper # dynamically linked libraries that are required for journalctl and the journalctl binary itself # use `ldd /bin/journalctl` to get dynamically linked libraries from the binary @@ -60,6 +65,6 @@ COPY --from=journal /usr/lib/x86_64-linux-gnu/libpcre2-8.so.0 /usr/lib/x86_64-li COPY --from=journal /lib/x86_64-linux-gnu/libcap-ng.so.0 /lib/x86_64-linux-gnu/libcap-ng.so.0 COPY --from=journal /bin/journalctl /bin/journalctl -ENTRYPOINT ["/swi-otelcol"] -CMD ["--config=/opt/default-config.yaml"] +ENTRYPOINT ["/wrapper"] +CMD ["/swi-otelcol", "--config=/opt/default-config.yaml"] diff --git a/src/wrapper/go.mod b/src/wrapper/go.mod new file mode 100644 index 00000000..5f017f7c --- /dev/null +++ b/src/wrapper/go.mod @@ -0,0 +1,3 @@ +module wrapper + +go 1.20 diff --git a/src/wrapper/main.go b/src/wrapper/main.go new file mode 100644 index 00000000..09c73d3f --- /dev/null +++ b/src/wrapper/main.go @@ -0,0 +1,86 @@ +/* +The goal of this wrapper is to detect panic that indicates corruption of checkpoints folder +(which can happend during kernel crash or system restart) and in case it is detected it removes checkpoint folder +*/ + +package main + +import ( + "bufio" + "fmt" + "os" + "os/exec" + "path/filepath" + "strings" + "sync" +) + +const ( + panicMessage = "panic: assertion failed: Page expected to be" +) + +func main() { + checkpointDir := os.Getenv("CHECKPOINT_DIR") + if checkpointDir == "" { + fmt.Println("Error: CHECKPOINT_DIR environment variable not set") + os.Exit(1) + } + + if len(os.Args) < 2 { + fmt.Println("Error: Missing command arguments") + os.Exit(1) + } + + cmd := exec.Command(os.Args[1], os.Args[2:]...) + + // Setup stderr pipe to capture output + stderr, err := cmd.StderrPipe() + if err != nil { + fmt.Println("Error:", err) + os.Exit(1) + } + + // Start the command + err = cmd.Start() + if err != nil { + fmt.Println("Error:", err) + os.Exit(1) + } + + var wg sync.WaitGroup + wg.Add(1) + + // Goroutine to monitor the output + go func() { + defer wg.Done() + scanner := bufio.NewScanner(stderr) + for scanner.Scan() { + line := scanner.Text() + fmt.Println(line) // Forward the output to stdout + if strings.Contains(line, panicMessage) { + fmt.Println("Specific panic detected, deleting all files in checkpoint folder...") + err := filepath.Walk(checkpointDir, func(path string, info os.FileInfo, err error) error { + if err != nil { + return err + } + if path != checkpointDir { // Skip the root directory + return os.RemoveAll(path) + } + return nil + }) + if err != nil { + fmt.Println("Error:", err) + } + } + } + }() + + // Wait for the command to exit + err = cmd.Wait() + if err != nil { + fmt.Println("Error:", err) + } + + // Wait for the output monitoring goroutine to finish + wg.Wait() +} \ No newline at end of file From a34f69bc4f46edb6ff3812e2551e6d2da054de23 Mon Sep 17 00:00:00 2001 From: "tomas.panik" Date: Thu, 5 Oct 2023 15:48:59 +0200 Subject: [PATCH 2/3] NH-57813: Updating Windows image --- build/docker/Dockerfile.Windows | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/build/docker/Dockerfile.Windows b/build/docker/Dockerfile.Windows index c1554bd2..d0501135 100644 --- a/build/docker/Dockerfile.Windows +++ b/build/docker/Dockerfile.Windows @@ -12,10 +12,20 @@ ARG GOEXPERIMENT=boringcrypto RUN /go/bin/builder --config ./swi-k8s-opentelemetry-collector.yaml --output-path ./ +FROM base as wrapper +WORKDIR /src/src/wrapper + +ARG CGO_ENABLED=0 +ARG GOEXPERIMENT=boringcrypto + +RUN go build -a -o /wrapper.exe + FROM mcr.microsoft.com/windows/nanoserver:ltsc2022 COPY --from=builder /src/swi-k8s-opentelemetry-collector /swi-otelcol.exe -ENTRYPOINT ["swi-otelcol.exe"] -CMD ["--config=c:/config/default-config.yaml"] +COPY --from=wrapper /wrapper.exe /wrapper.exe + +ENTRYPOINT ["wrapper.exe"] +CMD ["swi-otelcol.exe", "--config=/opt/default-config.yaml"] From 30b7a19fa173c11cee4f3e3777abe0ef97fc2496 Mon Sep 17 00:00:00 2001 From: "tomas.panik" Date: Fri, 6 Oct 2023 10:54:27 +0200 Subject: [PATCH 3/3] NH-57813: Updating code so that stderr is kept in stderr --- src/wrapper/main.go | 63 ++++++++++++++++++++++----------------------- 1 file changed, 31 insertions(+), 32 deletions(-) diff --git a/src/wrapper/main.go b/src/wrapper/main.go index 09c73d3f..559d8967 100644 --- a/src/wrapper/main.go +++ b/src/wrapper/main.go @@ -8,6 +8,7 @@ package main import ( "bufio" "fmt" + "io" "os" "os/exec" "path/filepath" @@ -21,66 +22,64 @@ const ( func main() { checkpointDir := os.Getenv("CHECKPOINT_DIR") - if checkpointDir == "" { - fmt.Println("Error: CHECKPOINT_DIR environment variable not set") - os.Exit(1) - } if len(os.Args) < 2 { - fmt.Println("Error: Missing command arguments") + fmt.Fprintln(os.Stderr, "Error: Missing command arguments") os.Exit(1) } cmd := exec.Command(os.Args[1], os.Args[2:]...) + cmd.Stdout = os.Stdout // Redirect stdout of cmd to stdout of wrapper - // Setup stderr pipe to capture output - stderr, err := cmd.StderrPipe() - if err != nil { - fmt.Println("Error:", err) - os.Exit(1) - } - - // Start the command - err = cmd.Start() - if err != nil { - fmt.Println("Error:", err) - os.Exit(1) - } + r, w := io.Pipe() + cmd.Stderr = w // Redirect stderr of cmd to the writer end of the pipe var wg sync.WaitGroup wg.Add(1) - // Goroutine to monitor the output + // Goroutine to monitor the stderr output for panic message go func() { defer wg.Done() - scanner := bufio.NewScanner(stderr) + scanner := bufio.NewScanner(r) // Read from the reader end of the pipe for scanner.Scan() { line := scanner.Text() - fmt.Println(line) // Forward the output to stdout + fmt.Fprintln(os.Stderr, line) // Forward the stderr output to stderr of wrapper if strings.Contains(line, panicMessage) { - fmt.Println("Specific panic detected, deleting all files in checkpoint folder...") - err := filepath.Walk(checkpointDir, func(path string, info os.FileInfo, err error) error { + fmt.Fprintln(os.Stderr, "Specific panic detected, deleting all files in checkpoint folder...") + if checkpointDir != "" { + err := filepath.Walk(checkpointDir, func(path string, info os.FileInfo, err error) error { + if err != nil { + return err + } + if path != checkpointDir { // Skip the root directory + return os.RemoveAll(path) + } + return nil + }) if err != nil { - return err - } - if path != checkpointDir { // Skip the root directory - return os.RemoveAll(path) + fmt.Fprintln(os.Stderr, "Error:", err) } - return nil - }) - if err != nil { - fmt.Println("Error:", err) } } } }() + // Start the command + err := cmd.Start() + if err != nil { + fmt.Fprintln(os.Stderr, "Error:", err) + os.Exit(1) + } + // Wait for the command to exit err = cmd.Wait() if err != nil { - fmt.Println("Error:", err) + fmt.Fprintln(os.Stderr, "Error:", err) } + // Close the writer end of the pipe to signal the end of data + w.Close() + // Wait for the output monitoring goroutine to finish wg.Wait() } \ No newline at end of file