HPCC-30413 Add option to always capture post-mortem info

Enabled either globally (i.e. for all components), or for individual components (e.g. Thor). Usage: expert: alwaysPostMortem: true Signed-off-by: Jake Smith <[email protected]>
hpcc-systems · Oct 4, 2023 · df794a0 · df794a0
1 parent 9218e73
commit df794a0
Show file tree

Hide file tree

Showing 2 changed files with 18 additions and 2 deletions.
diff --git a/helm/hpcc/templates/_helpers.tpl b/helm/hpcc/templates/_helpers.tpl
@@ -713,6 +713,12 @@ Add extra args for a component
  {{- $debugPlane := .me.debugPlane | default (include "hpcc.getFirstPlaneForCategory"  (dict "root" .root "category" "debug")) -}}
  {{- include "hpcc.checkPlaneExists" (dict "root" .root "planeName" $debugPlane) -}}
  {{- $prefix := include "hpcc.getPlanePrefix" (dict "root" .root "planeName" $debugPlane) -}}
+ {{- $meExpert := .me.expert | default dict -}}
+ {{- $globalExpert := .root.Values.global.expert | default dict -}}
+ {{- $alwaysPostMortem := (hasKey $meExpert "alwaysPostMortem") | ternary $meExpert.alwaysPostMortem ($globalExpert.alwaysPostMortem | default false) -}}
+ {{- if $alwaysPostMortem -}}
+"-a",{{ "\n" }}
+ {{- end -}}
 "-d", {{ $prefix }},
 "--",
 {{ .process | quote }},
@@ -1670,7 +1676,14 @@ args:
  {{- $debugPlane := .me.debugPlane | default (include "hpcc.getFirstPlaneForCategory"  (dict "root" .root "category" "debug")) -}}
  {{- include "hpcc.checkPlaneExists" (dict "root" .root "planeName" $debugPlane) -}}
  {{- $prefix := include "hpcc.getPlanePrefix" (dict "root" .root "planeName" $debugPlane) -}}
- {{- $_ := set $check_cmd "command" (printf "check_executes -d %s -- %s" $prefix .command) -}}
+ {{- $pmd_always_opt := "" -}}
+ {{- $globalExpert := .root.Values.global.expert | default dict -}}
+ {{- $meExpert := .me.expert | default dict -}}
+ {{- $alwaysPostMortem := (hasKey $meExpert "alwaysPostMortem") | ternary $meExpert.alwaysPostMortem ($globalExpert.alwaysPostMortem | default false) -}}
+ {{- if $alwaysPostMortem -}}
+  {{- $pmd_always_opt = "-a " -}}
+ {{- end -}}
+ {{- $_ := set $check_cmd "command" (printf "check_executes %s-d %s -- %s" $pmd_always_opt $prefix .command) -}}
 {{- end }}
 - >-
     {{ $check_cmd.command }};

diff --git a/initfiles/bin/check_executes b/initfiles/bin/check_executes
@@ -11,6 +11,7 @@ PMD_PROGNAME=
 PMD_COPYFILES=()
 PMD_DALISERVER=
 PMD_WORKUNIT=
+PMD_ALWAYS=false
 
 while [ "$#" -gt 0 ]; do
   arg=$1
@@ -27,6 +28,8 @@ while [ "$#" -gt 0 ]; do
       f) shift;
          PMD_COPYFILES+=($1)
          ;;
+      a) PMD_ALWAYS=true
+         ;;
       *) usage
          exit
          ;;
@@ -66,7 +69,7 @@ ${PMD_PROGNAME} --logging.postMortem=1000 "$@"
 
 # If it did not exit cleanly, copy some post-mortem info
 retVal=$?
-if [ $retVal -ne 0 ]; then
+if [ $PMD_ALWAYS = true ] || [ $retVal -ne 0 ]; then
   POST_MORTEM_DIR=${PMD_DIRECTORYBASE}/$(hostname)/$(date -Iseconds)
   mkdir -p ${POST_MORTEM_DIR}
   echo "Post-mortem info gathered in $POST_MORTEM_DIR"