diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 1a7fe6b0..9fa3913d 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -3,7 +3,6 @@ repos: - repo: https://github.com/pre-commit/pre-commit-hooks rev: v3.1.0 hooks: - - id: end-of-file-fixer - id: no-commit-to-branch - id: trailing-whitespace - repo: https://github.com/norwoodj/helm-docs diff --git a/charts/prefect-agent/README.md b/charts/prefect-agent/README.md index 7f4b455e..2388edf4 100644 --- a/charts/prefect-agent/README.md +++ b/charts/prefect-agent/README.md @@ -80,5 +80,3 @@ Prefect Agent application bundle | serviceAccount.create | bool | `true` | specifies whether a ServiceAccount should be created | | serviceAccount.name | string | `""` | the name of the ServiceAccount to use. if not set and create is true, a name is generated using the common.names.fullname template | ----------------------------------------------- -Autogenerated from chart metadata using [helm-docs v1.13.1](https://github.com/norwoodj/helm-docs/releases/v1.13.1) diff --git a/charts/prefect-server/README.md b/charts/prefect-server/README.md index 0c602934..d1c07cce 100644 --- a/charts/prefect-server/README.md +++ b/charts/prefect-server/README.md @@ -198,5 +198,3 @@ No secrets are created when providing an existing secret. | serviceAccount.create | bool | `true` | specifies whether a ServiceAccount should be created | | serviceAccount.name | string | `""` | the name of the ServiceAccount to use. if not set and create is true, a name is generated using the common.names.fullname template | ----------------------------------------------- -Autogenerated from chart metadata using [helm-docs v1.13.1](https://github.com/norwoodj/helm-docs/releases/v1.13.1) diff --git a/charts/prefect-worker/README.md b/charts/prefect-worker/README.md index eb4a0fb6..77b1307c 100644 --- a/charts/prefect-worker/README.md +++ b/charts/prefect-worker/README.md @@ -255,7 +255,8 @@ helm install prefect-worker prefect/prefect-worker -f values.yaml --set-file wor | worker.cloudApiConfig.cloudUrl | string | `"https://api.prefect.cloud/api"` | prefect cloud API url; the full URL is constructed as https://cloudUrl/accounts/accountId/workspaces/workspaceId | | worker.cloudApiConfig.workspaceId | string | `""` | prefect workspace ID | | worker.clusterUid | string | `""` | unique cluster identifier, if none is provided this value will be infered at time of helm install | -| worker.config.baseJobTemplate | string | `nil` | JSON formatted base job template. If unspecified, Prefect will use the default base job template for the given worker type. If the work pool already exists, this will be ignored. | +| worker.config.baseJobTemplate.configuration | string | `nil` | JSON formatted base job template. If data is provided here, the chart will generate a configmap and mount it to the worker pod | +| worker.config.baseJobTemplate.existingConfigMapName | string | `""` | the name of an existing ConfigMap containing a base job template. NOTE - the key must be 'baseJobTemplate.json' | | worker.config.http2 | bool | `true` | connect using HTTP/2 if the server supports it (experimental) | | worker.config.installPolicy | string | `"prompt"` | install policy to use workers from Prefect integration packages. | | worker.config.limit | string | `nil` | maximum number of flow runs to start simultaneously (default: unlimited) | @@ -307,5 +308,3 @@ helm install prefect-worker prefect/prefect-worker -f values.yaml --set-file wor | worker.serverApiConfig.uiUrl | string | `"http://localhost:4200"` | prefect UI url | | worker.tolerations | list | `[]` | tolerations for worker pods assignment | ----------------------------------------------- -Autogenerated from chart metadata using [helm-docs v1.13.1](https://github.com/norwoodj/helm-docs/releases/v1.13.1) diff --git a/charts/prefect-worker/templates/_helpers.tpl b/charts/prefect-worker/templates/_helpers.tpl index 0937d0bb..841264b7 100644 --- a/charts/prefect-worker/templates/_helpers.tpl +++ b/charts/prefect-worker/templates/_helpers.tpl @@ -103,3 +103,15 @@ Require Prefect Server API URL {{- (lookup "v1" "Namespace" "" "kube-system" | default $defaultDict).metadata.uid | quote }} {{- end }} {{- end }} + +{{/* +Determine the name of the ConfigMap for the baseJobTemplate +*/}} +{{- define "worker.baseJobTemplateName" -}} +{{- if .Values.worker.config.baseJobTemplate.configuration -}} + {{ default "prefect-worker-base-job-template" .Values.worker.config.baseJobTemplate.name . }} +{{- else if .Values.worker.config.baseJobTemplate.existingConfigMapName -}} + {{ .Values.worker.config.baseJobTemplate.existingConfigMapName }} +{{- else -}} +{{- end -}} +{{- end -}} diff --git a/charts/prefect-worker/templates/configmap.yaml b/charts/prefect-worker/templates/configmap.yaml index d78f5ca5..16aec893 100644 --- a/charts/prefect-worker/templates/configmap.yaml +++ b/charts/prefect-worker/templates/configmap.yaml @@ -1,8 +1,8 @@ -{{- if .Values.worker.config.baseJobTemplate }} +{{- if .Values.worker.config.baseJobTemplate.configuration }} apiVersion: v1 kind: ConfigMap metadata: - name: {{ default "prefect-worker-base-job-template" .Values.worker.config.baseJobTemplateName . }} + name: {{ include "worker.baseJobTemplateName" . }} namespace: {{ include "common.names.namespace" . | quote }} labels: {{- include "common.labels.standard" ( dict "customLabels" .Values.commonLabels "context" $ ) | nindent 4 }} app.kubernetes.io/component: worker @@ -10,5 +10,5 @@ metadata: annotations: {{- include "common.tplvalues.render" ( dict "value" .Values.commonAnnotations "context" $ ) | nindent 4 }} {{- end }} data: - baseJobTemplate.json: {{ .Values.worker.config.baseJobTemplate | toJson }} + baseJobTemplate.json: {{ .Values.worker.config.baseJobTemplate.configuration | toJson }} {{- end }} diff --git a/charts/prefect-worker/templates/deployment.yaml b/charts/prefect-worker/templates/deployment.yaml index 1969b2ce..1999bd5f 100644 --- a/charts/prefect-worker/templates/deployment.yaml +++ b/charts/prefect-worker/templates/deployment.yaml @@ -85,7 +85,7 @@ spec: - --name - {{ .Values.worker.config.name | quote }} {{- end }} - {{- if .Values.worker.config.baseJobTemplate }} + {{- if include "worker.baseJobTemplateName" . }} - --base-job-template - baseJobTemplate.json {{- end }} @@ -153,7 +153,7 @@ spec: - mountPath: /tmp name: scratch subPathExpr: tmp - {{- if .Values.worker.config.baseJobTemplate }} + {{- if include "worker.baseJobTemplateName" . }} - mountPath: /home/prefect/baseJobTemplate.json name: base-job-template-file subPath: baseJobTemplate.json @@ -170,8 +170,8 @@ spec: {{- if .Values.worker.extraVolumes }} {{- include "common.tplvalues.render" (dict "value" .Values.worker.extraVolumes "context" $) | nindent 8 }} {{- end }} - {{- if .Values.worker.config.baseJobTemplate }} + {{ if include "worker.baseJobTemplateName" . }} - name: base-job-template-file configMap: - name: {{ default "prefect-worker-base-job-template" .Values.worker.config.baseJobTemplateName . }} + name: {{ include "worker.baseJobTemplateName" . }} {{- end }} diff --git a/charts/prefect-worker/values.schema.json b/charts/prefect-worker/values.schema.json index 7a7bd1ad..5f734944 100644 --- a/charts/prefect-worker/values.schema.json +++ b/charts/prefect-worker/values.schema.json @@ -38,38 +38,32 @@ "type": "object", "title": "Autoscaling", "description": "autoscaling configuration for the worker", - "form": true, "additionalProperties": false, "properties": { "enabled": { "type": "boolean", "title": "Enabled", - "description": "enable autoscaling for the worker", - "form": true + "description": "enable autoscaling for the worker" }, "minReplicas": { "type": "integer", "title": "Min Replicas", - "description": "minimum number of replicas to scale down to", - "form": true + "description": "minimum number of replicas to scale down to" }, "maxReplicas": { "type": "integer", "title": "Max Replicas", - "description": "maximum number of replicas to scale up to", - "form": true + "description": "maximum number of replicas to scale up to" }, "targetCPUUtilizationPercentage": { "type": "integer", "title": "Target CPU Utilization Percentage", - "description": "target CPU utilization percentage for scaling the worker", - "form": true + "description": "target CPU utilization percentage for scaling the worker" }, "targetMemoryUtilizationPercentage": { "type": "integer", "title": "Target Memory Utilization Percentage", - "description": "target memory utilization percentage for scaling the worker", - "form": true + "description": "target memory utilization percentage for scaling the worker" } } }, @@ -168,15 +162,28 @@ "title": "Type", "description": "the type of worker to start i.e kubernetes" }, - "baseJobTemplateName": { - "type": ["string", "null"], - "title": "Base Job Template Name", - "description": "Template name of the base job template. If unspecified, Prefect will use the default base job template name for the given worker type." - }, "baseJobTemplate": { - "type": ["string", "null"], + "type": "object", "title": "Base Job Template", - "description": "JSON formatted base job template. If unspecified, Prefect will use the default base job template for the given worker type. If the work pool already exists, this will be ignored." + "description": "If unspecified, Prefect will use the default base job template for the given worker type. If the work pool already exists, this will be ignored.", + "additionalProperties": false, + "properties": { + "existingConfigMapName": { + "type": "string", + "title": "Existing ConfigMap Name", + "description": "the name of an existing ConfigMap containing a base job template. NOTE - the key must be 'baseJobTemplate.json'" + }, + "configuration": { + "type": ["string", "null"], + "title": "Base Job Template Configuration", + "description": "JSON formatted base job template. If data is provided here, the chart will generate a configmap and mount it to the worker pod" + }, + "name": { + "type": ["string", "null"], + "title": "Name", + "description": "optionally override the default name of the generated configmap." + } + } } } }, diff --git a/charts/prefect-worker/values.yaml b/charts/prefect-worker/values.yaml index 41df3cd2..447ede1a 100644 --- a/charts/prefect-worker/values.yaml +++ b/charts/prefect-worker/values.yaml @@ -72,8 +72,29 @@ worker: name: null # -- maximum number of flow runs to start simultaneously (default: unlimited) limit: null - # -- JSON formatted base job template. If unspecified, Prefect will use the default base job template for the given worker type. If the work pool already exists, this will be ignored. - baseJobTemplate: null + + ## If unspecified, Prefect will use the default base job template for the given worker type. If the work pool already exists, this will be ignored. + ## e.g.: + ## baseJobTemplate: + ## configuration: | + ## { + ## "variables": { + ## ... + ## }, + ## "job_configuration": { + ## ... + ## } + ## } + ## OR + ## baseJobTemplate: + ## existingConfigMapName: "my-existing-config-map" + baseJobTemplate: + # -- the name of an existing ConfigMap containing a base job template. NOTE - the key must be 'baseJobTemplate.json' + existingConfigMapName: "" + # -- JSON formatted base job template. If data is provided here, the chart will generate a configmap and mount it to the worker pod + configuration: null + # -- optionally override the default name of the generated configmap + # name: "" ## connection settings # -- one of 'cloud', 'selfHosted', or 'server' diff --git a/charts/prometheus-prefect-exporter/README.md b/charts/prometheus-prefect-exporter/README.md index 28d5db5f..b2610530 100644 --- a/charts/prometheus-prefect-exporter/README.md +++ b/charts/prometheus-prefect-exporter/README.md @@ -89,5 +89,3 @@ Shoutout to @ialejandro for the original work on this chart! | serviceMonitor.enabled | bool | `false` | Enable or disable | | tolerations | list | `[]` | Tolerations for pod assignment | ----------------------------------------------- -Autogenerated from chart metadata using [helm-docs v1.13.1](https://github.com/norwoodj/helm-docs/releases/v1.13.1)