From 68ba7b99e6b3aeae491f6dd5973bb6007f766276 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?St=C3=A9phane=20Duchesneau?=
 <stephane.duchesneau@streamingfast.io>
Date: Mon, 17 Jun 2024 12:04:32 -0400
Subject: [PATCH] truncate output error message a bit above 18k characters to
 prevent losing error messages to load balancer

---
 docs/release-notes/change-log.md |  6 ++++++
 pipeline/exec/errorexec.go       | 28 ++++++++++++++++++----------
 2 files changed, 24 insertions(+), 10 deletions(-)

diff --git a/docs/release-notes/change-log.md b/docs/release-notes/change-log.md
index 9da0285f4..37b962452 100644
--- a/docs/release-notes/change-log.md
+++ b/docs/release-notes/change-log.md
@@ -11,6 +11,12 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 
 ## Unreleased
 
+### Fixed
+
+* Truncate error messages log lines to 18k characters to prevent them from disappearing through some load balancers.
+
+### Added
+
 * Add a substreams `live back filler` once substreams tier1 is requested in `production mode`. 
   It enables, create `substreams cache` when block are processed live on tier1. 
 
diff --git a/pipeline/exec/errorexec.go b/pipeline/exec/errorexec.go
index 87e3179ac..4469bbf29 100644
--- a/pipeline/exec/errorexec.go
+++ b/pipeline/exec/errorexec.go
@@ -7,19 +7,27 @@ type ErrorExecutor struct {
 	stackTrace []string
 }
 
+const maxErrorSize = 18000 // Some load balancer will fail close to 20k
+
 func (e *ErrorExecutor) Error() string {
-	b := bytes.NewBuffer(nil)
+	if len(e.stackTrace) == 0 {
+		return e.message
+	}
 
-	b.WriteString(e.message)
+	b := bytes.NewBuffer(nil)
+	// stack trace section will also contain the logs of the execution
+	for _, stackTraceLine := range e.stackTrace {
+		b.WriteString(stackTraceLine)
+		b.WriteString("\n")
+	}
+	traces := b.String()
 
-	if len(e.stackTrace) > 0 {
-		// stack trace section will also contain the logs of the execution
-		b.WriteString("\n----- stack trace -----\n")
-		for _, stackTraceLine := range e.stackTrace {
-			b.WriteString(stackTraceLine)
-			b.WriteString("\n")
-		}
+	out := e.message + "\n\n----- stack trace / logs -----\n"
+	if length := len(traces); length > maxErrorSize {
+		out += "[TRUNCATED]\n" + traces[length-maxErrorSize:length]
+	} else {
+		out += traces
 	}
 
-	return b.String()
+	return out
 }