truncate output error message a bit above 18k characters to prevent l…

…osing error messages to load balancer

docs/release-notes/change-log.md

@@ -11,6 +11,12 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 
 ## Unreleased
 
+### Fixed
+
+* Truncate error messages log lines to 18k characters to prevent them from disappearing through some load balancers.
+
+### Added
+
 * Add a substreams `live back filler` once substreams tier1 is requested in `production mode`. 
   It enables, create `substreams cache` when block are processed live on tier1. 
 

pipeline/exec/errorexec.go

@@ -7,19 +7,27 @@ type ErrorExecutor struct {
 	stackTrace []string
 }
 
+const maxErrorSize = 18000 // Some load balancer will fail close to 20k
+
 func (e *ErrorExecutor) Error() string {
-	b := bytes.NewBuffer(nil)
+	if len(e.stackTrace) == 0 {
+		return e.message
+	}
 
-	b.WriteString(e.message)
+	b := bytes.NewBuffer(nil)
+	// stack trace section will also contain the logs of the execution
+	for _, stackTraceLine := range e.stackTrace {
+		b.WriteString(stackTraceLine)
+		b.WriteString("\n")
+	}
+	traces := b.String()
 
-	if len(e.stackTrace) > 0 {
-		// stack trace section will also contain the logs of the execution
-		b.WriteString("\n----- stack trace -----\n")
-		for _, stackTraceLine := range e.stackTrace {
-			b.WriteString(stackTraceLine)
-			b.WriteString("\n")
-		}
+	out := e.message + "\n\n----- stack trace / logs -----\n"
+	if length := len(traces); length > maxErrorSize {
+		out += "[TRUNCATED]\n" + traces[length-maxErrorSize:length]
+	} else {
+		out += traces
 	}
 
-	return b.String()
+	return out
 }