Merge pull request #18595 from ghalliday/issue31661

HPCC-31661 Add options to generate cpu resources as limits Reviewed-by: Jake Smith <[email protected]> Merged-by: Gavin Halliday <[email protected]>
hpcc-systems · May 3, 2024 · 13987dc · 13987dc
2 parents 15eaff2 + 45d6ba6
commit 13987dc
Show file tree

Hide file tree

Showing 13 changed files with 940 additions and 105 deletions.
diff --git a/helm/hpcc/templates/_helpers.tpl b/helm/hpcc/templates/_helpers.tpl
@@ -1316,18 +1316,38 @@ Pass in a dictionary with me defined
 */}}
 {{- define "hpcc.addResources" }}
 {{- $resources := .me | default .defaults }}
-{{- if $resources }}
- {{- $limits := omit $resources "cpu" }}
- {{- $requests := pick $resources "cpu" }}
+{{- $omitResources := .root.Values.global.omitResources | default false }}
+{{- $resourceCpusWithLimits := hasKey .root.Values.global "resourceCpusWithLimits" | ternary .root.Values.global.resourceCpusWithLimits false -}}
+{{- $resourceWholeCpusWithLimits := hasKey .root.Values.global "resourceWholeCpusWithLimits" | ternary .root.Values.global.resourceWholeCpusWithLimits false -}}
+{{- if not $omitResources }}
+{{-  if $resources }}
+{{-   $limits := omit $resources "cpu" }}
+{{-   $requests := dict }}
+{{-   if hasKey $resources "cpu"  -}}
+{{-    $cpu := $resources.cpu }}
+{{-    if $resourceCpusWithLimits -}}
+{{-     $_ := set $limits "cpu" $cpu -}}
+{{-    else if $resourceWholeCpusWithLimits -}}
+{{-     $milliCPUs := int (include "hpcc.k8sCPUStringToMilliCPU" $cpu) }}
+{{-     if eq (mod $milliCPUs 1000) 0 -}}
+{{-      $_ := set $limits "cpu" $cpu -}}
+{{-     else -}}
+{{-      $_ := set $requests "cpu" $cpu -}}
+{{-     end -}}
+{{-    else -}}
+{{-     $_ := set $requests "cpu" $cpu -}}
+{{-    end -}}
+{{-   end }}
 resources:
- {{- if $limits }}
+{{-   if $limits }}
   limits:
-  {{- toYaml $limits | nindent 4 }}
- {{- end -}}
- {{- if $requests }}
+{{-    toYaml $limits | nindent 4 }}
+{{-   end -}}
+{{-   if $requests }}
   requests:
-  {{- toYaml $requests | nindent 4 -}}
- {{- end -}}
+{{-    toYaml $requests | nindent 4 -}}
+{{-   end -}}
+{{-  end -}}
 {{- end -}}
 {{- end -}}
 
@@ -1336,16 +1356,11 @@ Add resources object for stub pods
 Pass in dict with root, me and instances defined
 */}}
 {{- define "hpcc.addStubResources" -}}
-{{- $stubInstanceResources := .stubResources | default .root.Values.global.stubInstanceResources | default dict }}
-{{- $milliCPUText := $stubInstanceResources.cpu | default "200m" }}
-{{- $milliCPUs := int (include "hpcc.k8sCPUStringToMilliCPU" $milliCPUText) }}
-{{- $memoryText := $stubInstanceResources.memory | default "400Mi" }}
-{{- $memory := int64 (include "hpcc.k8sMemoryStringToBytes" $memoryText) }}
-resources:
-  limits:
-    memory: {{ include "hpcc.bytesToK8sMemoryString" $memory | quote }}
-  requests:
-    cpu: {{ printf "%dm" $milliCPUs | quote }}
+{{- $stubInstanceResources := .me | default .root.Values.global.stubInstanceResources | default dict }}
+{{- $cpuResource := $stubInstanceResources.cpu | default "200m" }}
+{{- $memoryResource := $stubInstanceResources.memory | default "400Mi" }}
+{{- $resources := dict "memory" $memoryResource "cpu" $cpuResource -}}
+{{- include "hpcc.addResources" (dict "me" $resources "root" .root) }}
 {{- end -}}
 
 {{/*
@@ -1404,10 +1419,7 @@ Pass in dict with root, me and dali if container in dali pod
           "--service={{ .me.name }}",
 {{ include "hpcc.daliArg" (dict "root" .root "component" "Sasha" "optional" false "overrideDaliHost" $overrideDaliHost "overrideDaliPort" $overrideDaliPort) | indent 10 }}
         ]
-{{- $omitResources := .root.Values.global.omitResources | default false }}
-{{- if not $omitResources }}
-{{- include "hpcc.addResources" (dict "me" .me.resources) | indent 2 }}
-{{- end }}
+{{- include "hpcc.addResources" (dict "me" .me.resources "root" .root) | indent 2 }}
 {{- include "hpcc.addSecurityContext" . | indent 2 }}
   env:
 {{ include "hpcc.mergeEnvironments" $env | indent 2 -}}

diff --git a/helm/hpcc/templates/dafilesrv.yaml b/helm/hpcc/templates/dafilesrv.yaml
@@ -74,10 +74,7 @@ spec:
           value: "/tmp/{{ .name }}.sentinel"
 {{ include "hpcc.addSentinelProbes" . | indent 8 }}
 {{- include "hpcc.addSecurityContext" $commonCtx | indent 8 }}
-{{- $omitResources := $.Values.global.omitResources | default false }}
-{{- if not $omitResources }}
-{{- include "hpcc.addResources" (dict "me" .resources) | indent 8 }}
-{{- end }}
+{{- include "hpcc.addResources" (dict "me" .resources "root" $) | indent 8 }}
 {{ include "hpcc.addImageAttrs" $commonCtx | indent 8 }}
         volumeMounts:
 {{ include "hpcc.addConfigMapVolumeMount" . | indent 8 }}

diff --git a/helm/hpcc/templates/dali.yaml b/helm/hpcc/templates/dali.yaml
@@ -116,10 +116,7 @@ spec:
           value: "/tmp/{{ $dali.name }}.sentinel"
 {{ include "hpcc.addSentinelProbes" $dali | indent 8 }}
 {{- include "hpcc.addSecurityContext" $commonCtx | indent 8 }}
-{{- $omitResources := $.Values.global.omitResources | default false }}
-{{- if not $omitResources }}
-{{- include "hpcc.addResources" (dict "me" $dali.resources) | indent 8 }}
-{{- end }}
+{{- include "hpcc.addResources" (dict "me" $dali.resources "root" $) | indent 8 }}
 {{ include "hpcc.addImageAttrs" $commonCtx | indent 8 }}
         volumeMounts:
 {{ include "hpcc.addConfigMapVolumeMount" $dali | indent 8 }}

diff --git a/helm/hpcc/templates/dfuserver.yaml b/helm/hpcc/templates/dfuserver.yaml
@@ -84,10 +84,7 @@ spec:
           value: "/tmp/{{ .name }}.sentinel"
 {{ include "hpcc.addSentinelProbes" . | indent 8 }}
 {{- include "hpcc.addSecurityContext" $commonCtx | indent 8 }}
-{{- $omitResources := $.Values.global.omitResources | default false }}
-{{- if not $omitResources }}
-{{- include "hpcc.addResources" (dict "me" .resources) | indent 8 }}
-{{- end }}
+{{- include "hpcc.addResources" (dict "me" .resources "root" $) | indent 8 }}
 {{ include "hpcc.addImageAttrs" $commonCtx | indent 8 }}
         volumeMounts:
 {{- include "hpcc.addVolumeMounts" $commonCtx | indent 8 }}

diff --git a/helm/hpcc/templates/eclagent.yaml b/helm/hpcc/templates/eclagent.yaml
@@ -82,10 +82,7 @@ data:
           - name: {{ $appJobName }}
 {{- include "hpcc.addSecurityContext" . | indent 12 }}
 {{ include "hpcc.addImageAttrs" . | indent 12 }}
-{{- $omitResources := .root.Values.global.omitResources | default false }}
-{{- if not $omitResources }}
-{{- include "hpcc.addResources" (dict "me" .me.resources) | indent 12 }}
-{{- end }}
+{{- include "hpcc.addResources" (dict "me" .me.resources "root" .root) | indent 12 }}
 {{- $appCmd := printf "%s %s %s _HPCC_ARGS_" $apptype (include "hpcc.configArg" .me) (include "hpcc.daliArg" (dict "root" .root "component" "ECL Agent" "optional" false )) }}
 {{ include "hpcc.addCommandAndLifecycle" (. | merge (dict "command" $appCmd)) | indent 12 }}
             env:
@@ -167,13 +164,10 @@ spec:
           value: "/tmp/{{ .name }}.sentinel"
 {{ include "hpcc.addSentinelProbes" . | indent 8 }}
 {{- include "hpcc.addSecurityContext" $commonCtx | indent 8 }}
-{{- $omitResources := $.Values.global.omitResources | default false }}
-{{- if not $omitResources }}
-{{-  if .useChildProcesses }}
-{{- include "hpcc.addResources" (dict "me" .resources) | indent 8 }}
-{{-  else }}
-{{- include "hpcc.addStubResources" ($commonCtx | merge (dict "stubResources" .stubResources)) | indent 8 }}
-{{-  end }}
+{{- if .useChildProcesses }}
+{{- include "hpcc.addResources" (dict "me" .resources "root" $) | indent 8 }}
+{{- else }}
+{{- include "hpcc.addStubResources" (dict "me" .stubResources "root" $) | indent 8 }}
 {{- end }}
 {{ include "hpcc.addImageAttrs" $commonCtx | indent 8 }}
         volumeMounts:

diff --git a/helm/hpcc/templates/eclccserver.yaml b/helm/hpcc/templates/eclccserver.yaml
@@ -77,10 +77,7 @@ data:
 {{- end }}
           - name: {{ $compileJobName }}
 {{- include "hpcc.addSecurityContext" . | indent 12 }}
-{{- $omitResources := .root.Values.global.omitResources | default false }}
-{{- if not $omitResources }}
-{{- include "hpcc.addResources" (dict "me" .me.resources) | indent 12 }}
-{{- end }}
+{{- include "hpcc.addResources" (dict "me" .me.resources "root" .root) | indent 12 }}
 {{ include "hpcc.addImageAttrs" . | indent 12 }}
 {{- $misc := .root.Values.global.misc | default dict -}}
 {{- $postJobCommand := $misc.postJobCommand | default "" }}
@@ -175,14 +172,11 @@ spec:
           value: "/tmp/{{ .name }}.sentinel"
 {{ include "hpcc.addSentinelProbes" . | indent 8 }}
 {{- include "hpcc.addSecurityContext" $commonCtx | indent 8 }}
-{{- $omitResources := $.Values.global.omitResources | default false }}
-{{- if not $omitResources }}
-{{-  if .useChildProcesses }}
-{{-   include "hpcc.addResources" (dict "me" .resources) | indent 8 }}
-{{-  else }}
-{{-   $defaultResources := dict "cpu" "1" "memory" "1Gi" }}
-{{-   include "hpcc.addResources" (dict "me" .timedChildResources "defaults" $defaultResources) | indent 8 }}
-{{-  end }}
+{{- if .useChildProcesses }}
+{{-  include "hpcc.addResources" (dict "me" .resources "root" $) | indent 8 }}
+{{- else }}
+{{-  $defaultResources := dict "cpu" "1" "memory" "1Gi" }}
+{{-  include "hpcc.addResources" (dict "me" .timedChildResources "defaults" $defaultResources "root" $) | indent 8 }}
 {{- end }}
 {{ include "hpcc.addImageAttrs" $commonCtx | indent 8 }}
         volumeMounts:

diff --git a/helm/hpcc/templates/eclscheduler.yaml b/helm/hpcc/templates/eclscheduler.yaml
@@ -93,11 +93,8 @@ spec:
           value: "/tmp/{{ .name }}.sentinel"
 {{ include "hpcc.addSentinelProbes" . | indent 8 }}
 {{- include "hpcc.addSecurityContext" $commonCtx | indent 8 }}
-{{- $omitResources := $.Values.global.omitResources | default false }}
-{{- if not $omitResources }}
 {{- $defaultResources := dict "cpu" "500m" "memory" "200Mi" }}
-{{- include "hpcc.addResources" (dict "me" .resources "defaults" $defaultResources) | indent 8 }}
-{{- end }}
+{{- include "hpcc.addResources" (dict "me" .resources "defaults" $defaultResources "root" $) | indent 8 }}
 {{ include "hpcc.addImageAttrs" $commonCtx | indent 8 }}
         volumeMounts:
 {{ include "hpcc.addConfigMapVolumeMount" . | indent 8 }}

diff --git a/helm/hpcc/templates/esp.yaml b/helm/hpcc/templates/esp.yaml
@@ -155,10 +155,7 @@ spec:
           value: "/tmp/{{ .name }}.sentinel"
 {{ include "hpcc.addSentinelProbes" . | indent 8 }}
 {{- include "hpcc.addSecurityContext" $commonCtx | indent 8 }}
-{{- $omitResources := $.Values.global.omitResources | default false }}
-{{- if not $omitResources }}
-{{- include "hpcc.addResources" (dict "me" .resources) | indent 8 }}
-{{- end }}
+{{- include "hpcc.addResources" (dict "me" .resources "root" $) | indent 8 }}
 {{ include "hpcc.addImageAttrs" $commonCtx | indent 8 }}
         volumeMounts:
 {{ include "hpcc.addConfigMapVolumeMount" . | indent 8 }}

diff --git a/helm/hpcc/templates/localroxie.yaml b/helm/hpcc/templates/localroxie.yaml
@@ -114,10 +114,7 @@ spec:
 {{- end }}
 {{- end }}
 {{- include "hpcc.addSecurityContext" $commonCtx | indent 8 }}
-{{- $omitResources := $.Values.global.omitResources | default false }}
-{{- if not $omitResources }}
-{{- include "hpcc.addResources" (dict "me" $roxie.resources) | indent 8 }}
-{{- end }}
+{{- include "hpcc.addResources" (dict "me" $roxie.resources "root" $) | indent 8 }}
 {{ include "hpcc.addImageAttrs" $commonCtx | indent 8 }}
         volumeMounts:
 {{ include "hpcc.addConfigMapVolumeMount" . | indent 8 }}

diff --git a/helm/hpcc/templates/roxie.yaml b/helm/hpcc/templates/roxie.yaml
@@ -146,11 +146,8 @@ spec:
       - name: {{ $commonCtx.toponame | quote }}
 {{ include "hpcc.addSentinelProbes" $toposerver | indent 8 }}
 {{- include "hpcc.addSecurityContext" $commonCtx | indent 8 }}
-{{- $omitResources := $.Values.global.omitResources | default false }}
-{{- if not $omitResources }}
 {{- $defaultResources := dict "cpu" "500m" "memory" "200Mi" }}
-{{- include "hpcc.addResources" (dict "me" .topoResources "defaults" $defaultResources) | indent 8 }}
-{{- end }}
+{{- include "hpcc.addResources" (dict "me" .topoResources "defaults" $defaultResources "root" $) | indent 8 }}
 {{ include "hpcc.addImageAttrs" $commonCtx | indent 8 }}
         workingDir: /var/lib/HPCCSystems
         command: [ {{ include "hpcc.componentCommand" (dict "me" $toposerver "root" $ "process" "toposerver") }} ] 
@@ -300,10 +297,7 @@ spec:
               command: ["testsocket", ".", "control:closedown"]        
 {{ include "hpcc.addSentinelProbes" ( $roxie | merge (dict "readyProbeName" ".ready" )) | indent 8 }}
 {{ include "hpcc.addSecurityContext" (dict "root" $ "me" .) | indent 8 }}
-{{- $omitResources := $.Values.global.omitResources | default false }}
-{{- if not $omitResources }}
-{{- include "hpcc.addResources" (dict "me"  ($roxie.serverResources | default $roxie.resources)) | indent 8 }}
-{{- end }}
+{{- include "hpcc.addResources" (dict "me"  ($roxie.serverResources | default $roxie.resources) "root" $) | indent 8 }}
 {{ include "hpcc.addImageAttrs" $commonCtx | indent 8 }}
         volumeMounts:
 {{ include "hpcc.addConfigMapVolumeMount" $roxie | indent 8 }}
@@ -411,10 +405,7 @@ spec:
 {{ include "hpcc.addSentinelProbes" ( $roxie | merge (dict "readyProbeName" ".ready" )) | indent 8 }}
 {{- end }}
 {{ include "hpcc.addSecurityContext" (dict "root" $ "me" .) | indent 8 }}
-{{- $omitResources := $.Values.global.omitResources | default false }}
-{{- if not $omitResources }}
-{{- include "hpcc.addResources" (dict "me" ($roxie.channelResources | default $roxie.resources)) | indent 8 }}
-{{- end }}
+{{- include "hpcc.addResources" (dict "me" ($roxie.channelResources | default $roxie.resources) "root" $) | indent 8 }}
 {{ include "hpcc.addImageAttrs" $commonCtx | indent 8 }}
         volumeMounts:
 {{ include "hpcc.addConfigMapVolumeMount" $roxie | indent 8 }}

diff --git a/helm/hpcc/templates/thor.yaml b/helm/hpcc/templates/thor.yaml
@@ -109,10 +109,7 @@ data:
           - name: {{ $eclAgentJobName }}
 {{- include "hpcc.addSecurityContext" . | indent 12 }}
 {{ include "hpcc.addImageAttrs" . | indent 12 }}
-{{- $omitResources := .root.Values.global.omitResources | default false }}
-{{- if not $omitResources }}
-{{- include "hpcc.addResources" (dict "me" .eclAgentResources) | indent 12 }}
-{{- end }}
+{{- include "hpcc.addResources" (dict "me" .eclAgentResources "root" .root) | indent 12 }}
 {{- $agentCmd := printf "%s %s %s _HPCC_ARGS_" $eclAgentType (include "hpcc.configArg" .me) (include "hpcc.daliArg" (dict "root" .root "component" "Thor" "optional" false)) }}
 {{ include "hpcc.addCommandAndLifecycle" (. | merge (dict "command" $agentCmd)) | indent 12 }}
             env:
@@ -179,10 +176,7 @@ data:
           - name: {{ $thorManagerJobName }}
 {{- include "hpcc.addSecurityContext" . | indent 12 }}
 {{ include "hpcc.addImageAttrs" . | indent 12 }}
-{{- $omitResources := .root.Values.global.omitResources | default false }}
-{{- if not $omitResources }}
-{{- include "hpcc.addResources" (dict "me" $thorScope.managerResources) | indent 12 }}
-{{- end }}
+{{- include "hpcc.addResources" (dict "me" $thorScope.managerResources "root" .root) | indent 12 }}
 {{- $thorManagerCmd := printf "thormaster_lcr %s %s _HPCC_ARGS_" (include "hpcc.configArg" .me) (include "hpcc.daliArg" (dict "root" .root "component" "Thor" "optional" false)) }}
 {{ include "hpcc.addCommandAndLifecycle" (. | merge (dict "command" $thorManagerCmd)) | indent 12 }}
             env:
@@ -251,10 +245,7 @@ data:
           - name: {{ $thorWorkerJobName }}-{{ $containerNum }}
 {{- include "hpcc.addSecurityContext" $configCtx | indent 12 }}
 {{ include "hpcc.addImageAttrs" $configCtx | indent 12 }}
-{{- $omitResources := $configCtx.root.Values.global.omitResources | default false }}
-{{- if not $omitResources }}
-{{- include "hpcc.addResources" (dict "me" $thorScope.workerResources) | indent 12 }}
-{{- end }}
+{{- include "hpcc.addResources" (dict "me" $thorScope.workerResources "root" $configCtx.root) | indent 12 }}
 {{- $thorWorkerCmd := printf "thorslave_lcr %s %s _HPCC_ARGS_ --slaveport=%d" (include "hpcc.configArg" $configCtx.me) (include "hpcc.daliArg" (dict "root" $configCtx.root "component" "Thor" "optional" false)) $slavePort }}
 {{ include "hpcc.addCommandAndLifecycle" ($configCtx | merge (dict "command" $thorWorkerCmd)) | indent 12 }}
             env:
@@ -390,13 +381,10 @@ spec:
         - name: "SENTINEL"
           value: "/tmp/{{ $commonCtx.eclAgentName }}.sentinel"
 {{- include "hpcc.addSecurityContext" $commonCtx | indent 8 }}
-{{- $omitResources := $.Values.global.omitResources | default false }}
-{{- if not $omitResources }}
-{{-  if $commonCtx.eclAgentUseChildProcesses }}
-{{-   include "hpcc.addResources" (dict "me" .eclAgentResources) | indent 8 }}
-{{-  else }}
-{{-   include "hpcc.addStubResources" ($commonCtx | merge (dict "stubResources" .stubResources)) | indent 8 }}
-{{-  end }}
+{{- if $commonCtx.eclAgentUseChildProcesses }}
+{{-  include "hpcc.addResources" (dict "me" .eclAgentResources "root" $) | indent 8 }}
+{{- else }}
+{{-  include "hpcc.addStubResources" (dict "me" .stubResources "root" $) | indent 8 }}
 {{- end }}
 {{ include "hpcc.addImageAttrs" $commonCtx | indent 8 }}
         volumeMounts:
@@ -456,10 +444,7 @@ spec:
         - name: "SENTINEL"
           value: "/tmp/{{ $commonCtx.thorAgentName }}.sentinel"
 {{- include "hpcc.addSecurityContext" $commonCtx | indent 8 }}
-{{- $omitResources := $.Values.global.omitResources | default false }}
-{{- if not $omitResources }}
-{{- include "hpcc.addStubResources" ($commonCtx | merge (dict "stubResources" .stubResources)) | indent 8 }}
-{{- end }}
+{{- include "hpcc.addStubResources" (dict "me" .stubResources "root" $ ) | indent 8 }}
 {{ include "hpcc.addImageAttrs" $commonCtx | indent 8 }}
         volumeMounts:
 {{ include "hpcc.addConfigMapVolumeMount" . | indent 8 }}

diff --git a/helm/hpcc/values.schema.json b/helm/hpcc/values.schema.json
@@ -273,6 +273,14 @@
           "description": "if set, no resource definitions are generated from the helm charts",
           "type": "boolean"
         },
+        "resourceCpusWithLimits": {
+          "description": "if set, cpu resources are provided as limits rather than requests.  Not recommended because this can lead to painful throttling on roxie and thor",
+          "type": "boolean"
+        },
+        "resourceWholeCpusWithLimits": {
+          "description": "if set, whole cpu resource are provided as limits rather than requests.  On K8s systems where cpuManagerPolicy=static this allows binding using affinities.",
+          "type": "boolean"
+        },
         "privileged": {
           "type": "boolean"
         },