diff --git a/control-plane/roles/gardener/defaults/main/gardener.yaml b/control-plane/roles/gardener/defaults/main/gardener.yaml index d5f2b63e..3bd07929 100644 --- a/control-plane/roles/gardener/defaults/main/gardener.yaml +++ b/control-plane/roles/gardener/defaults/main/gardener.yaml @@ -46,6 +46,10 @@ gardener_gardenlet_shoot_concurrent_syncs: 20 gardener_gardenlet_shoot_reconcile_in_maintenance_only: false gardener_gardenlet_shoot_respect_sync_period_overwrite: true +gardener_gardenlet_feature_gates: + HVPA: false + HVPAForShootedSeed: false + gardener_shooted_seeds: [] # - name: shoot-1 # project_id: 00000000-0000-0000-0000-000000000001 diff --git a/control-plane/roles/gardener/files/10-crd-autoscaling.k8s.io_verticalpodautoscalers.yaml b/control-plane/roles/gardener/files/10-crd-autoscaling.k8s.io_verticalpodautoscalers.yaml new file mode 100644 index 00000000..7be2e9ce --- /dev/null +++ b/control-plane/roles/gardener/files/10-crd-autoscaling.k8s.io_verticalpodautoscalers.yaml @@ -0,0 +1,543 @@ +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + api-approved.kubernetes.io: unapproved, temporarily squatting + controller-gen.kubebuilder.io/version: v0.13.0 + name: verticalpodautoscalers.autoscaling.k8s.io +spec: + group: autoscaling.k8s.io + names: + kind: VerticalPodAutoscaler + listKind: VerticalPodAutoscalerList + plural: verticalpodautoscalers + shortNames: + - vpa + singular: verticalpodautoscaler + scope: Namespaced + versions: + - additionalPrinterColumns: + - jsonPath: .spec.updatePolicy.updateMode + name: Mode + type: string + - jsonPath: .status.recommendation.containerRecommendations[0].target.cpu + name: CPU + type: string + - jsonPath: .status.recommendation.containerRecommendations[0].target.memory + name: Mem + type: string + - jsonPath: .status.conditions[?(@.type=='RecommendationProvided')].status + name: Provided + type: string + - jsonPath: .metadata.creationTimestamp + name: Age + type: date + name: v1 + schema: + openAPIV3Schema: + description: VerticalPodAutoscaler is the configuration for a vertical pod + autoscaler, which automatically manages pod resources based on historical + and real time resource utilization. + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation + of an object. Servers should convert recognized schemas to the latest + internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this + object represents. Servers may infer this from the endpoint the client + submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + spec: + description: 'Specification of the behavior of the autoscaler. More info: + https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#spec-and-status.' + properties: + recommenders: + description: Recommender responsible for generating recommendation + for this object. List should be empty (then the default recommender + will generate the recommendation) or contain exactly one recommender. + items: + description: VerticalPodAutoscalerRecommenderSelector points to + a specific Vertical Pod Autoscaler recommender. In the future + it might pass parameters to the recommender. + properties: + name: + description: Name of the recommender responsible for generating + recommendation for this object. + type: string + required: + - name + type: object + type: array + resourcePolicy: + description: Controls how the autoscaler computes recommended resources. + The resource policy may be used to set constraints on the recommendations + for individual containers. If not specified, the autoscaler computes + recommended resources for all containers in the pod, without additional + constraints. + properties: + containerPolicies: + description: Per-container resource policies. + items: + description: ContainerResourcePolicy controls how autoscaler + computes the recommended resources for a specific container. + properties: + containerName: + description: Name of the container or DefaultContainerResourcePolicy, + in which case the policy is used by the containers that + don't have their own policy specified. + type: string + controlledResources: + description: Specifies the type of recommendations that + will be computed (and possibly applied) by VPA. If not + specified, the default of [ResourceCPU, ResourceMemory] + will be used. + items: + description: ResourceName is the name identifying various + resources in a ResourceList. + type: string + type: array + controlledValues: + description: Specifies which resource values should be controlled. + The default is "RequestsAndLimits". + enum: + - RequestsAndLimits + - RequestsOnly + type: string + maxAllowed: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: Specifies the maximum amount of resources that + will be recommended for the container. The default is + no maximum. + type: object + minAllowed: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: Specifies the minimal amount of resources that + will be recommended for the container. The default is + no minimum. + type: object + mode: + description: Whether autoscaler is enabled for the container. + The default is "Auto". + enum: + - Auto + - "Off" + type: string + type: object + type: array + type: object + targetRef: + description: TargetRef points to the controller managing the set of + pods for the autoscaler to control - e.g. Deployment, StatefulSet. + VerticalPodAutoscaler can be targeted at controller implementing + scale subresource (the pod set is retrieved from the controller's + ScaleStatus) or some well known controllers (e.g. for DaemonSet + the pod set is read from the controller's spec). If VerticalPodAutoscaler + cannot use specified target it will report ConfigUnsupported condition. + Note that VerticalPodAutoscaler does not require full implementation + of scale subresource - it will not use it to modify the replica + count. The only thing retrieved is a label selector matching pods + grouped by the target resource. + properties: + apiVersion: + description: apiVersion is the API version of the referent + type: string + kind: + description: 'kind is the kind of the referent; More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + name: + description: 'name is the name of the referent; More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names' + type: string + required: + - kind + - name + type: object + x-kubernetes-map-type: atomic + updatePolicy: + description: Describes the rules on how changes are applied to the + pods. If not specified, all fields in the `PodUpdatePolicy` are + set to their default values. + properties: + minReplicas: + description: Minimal number of replicas which need to be alive + for Updater to attempt pod eviction (pending other checks like + PDB). Only positive values are allowed. Overrides global '--min-replicas' + flag. + format: int32 + type: integer + updateMode: + description: Controls when autoscaler applies changes to the pod + resources. The default is 'Auto'. + enum: + - "Off" + - Initial + - Recreate + - Auto + type: string + type: object + required: + - targetRef + type: object + status: + description: Current information about the autoscaler. + properties: + conditions: + description: Conditions is the set of conditions required for this + autoscaler to scale its target, and indicates whether or not those + conditions are met. + items: + description: VerticalPodAutoscalerCondition describes the state + of a VerticalPodAutoscaler at a certain point. + properties: + lastTransitionTime: + description: lastTransitionTime is the last time the condition + transitioned from one status to another + format: date-time + type: string + message: + description: message is a human-readable explanation containing + details about the transition + type: string + reason: + description: reason is the reason for the condition's last transition. + type: string + status: + description: status is the status of the condition (True, False, + Unknown) + type: string + type: + description: type describes the current condition + type: string + required: + - status + - type + type: object + type: array + recommendation: + description: The most recently computed amount of resources recommended + by the autoscaler for the controlled pods. + properties: + containerRecommendations: + description: Resources recommended by the autoscaler for each + container. + items: + description: RecommendedContainerResources is the recommendation + of resources computed by autoscaler for a specific container. + Respects the container resource policy if present in the spec. + In particular the recommendation is not produced for containers + with `ContainerScalingMode` set to 'Off'. + properties: + containerName: + description: Name of the container. + type: string + lowerBound: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: Minimum recommended amount of resources. Observes + ContainerResourcePolicy. This amount is not guaranteed + to be sufficient for the application to operate in a stable + way, however running with less resources is likely to + have significant impact on performance/availability. + type: object + target: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: Recommended amount of resources. Observes ContainerResourcePolicy. + type: object + uncappedTarget: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: The most recent recommended resources target + computed by the autoscaler for the controlled pods, based + only on actual resource usage, not taking into account + the ContainerResourcePolicy. May differ from the Recommendation + if the actual resource usage causes the target to violate + the ContainerResourcePolicy (lower than MinAllowed or + higher that MaxAllowed). Used only as status indication, + will not affect actual resource assignment. + type: object + upperBound: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: Maximum recommended amount of resources. Observes + ContainerResourcePolicy. Any resources allocated beyond + this value are likely wasted. This value may be larger + than the maximum amount of application is actually capable + of consuming. + type: object + required: + - target + type: object + type: array + type: object + type: object + required: + - spec + type: object + served: true + storage: true + subresources: {} + - deprecated: true + deprecationWarning: autoscaling.k8s.io/v1beta2 API is deprecated + name: v1beta2 + schema: + openAPIV3Schema: + description: VerticalPodAutoscaler is the configuration for a vertical pod + autoscaler, which automatically manages pod resources based on historical + and real time resource utilization. + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation + of an object. Servers should convert recognized schemas to the latest + internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this + object represents. Servers may infer this from the endpoint the client + submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + spec: + description: 'Specification of the behavior of the autoscaler. More info: + https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#spec-and-status.' + properties: + resourcePolicy: + description: Controls how the autoscaler computes recommended resources. + The resource policy may be used to set constraints on the recommendations + for individual containers. If not specified, the autoscaler computes + recommended resources for all containers in the pod, without additional + constraints. + properties: + containerPolicies: + description: Per-container resource policies. + items: + description: ContainerResourcePolicy controls how autoscaler + computes the recommended resources for a specific container. + properties: + containerName: + description: Name of the container or DefaultContainerResourcePolicy, + in which case the policy is used by the containers that + don't have their own policy specified. + type: string + maxAllowed: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: Specifies the maximum amount of resources that + will be recommended for the container. The default is + no maximum. + type: object + minAllowed: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: Specifies the minimal amount of resources that + will be recommended for the container. The default is + no minimum. + type: object + mode: + description: Whether autoscaler is enabled for the container. + The default is "Auto". + enum: + - Auto + - "Off" + type: string + type: object + type: array + type: object + targetRef: + description: TargetRef points to the controller managing the set of + pods for the autoscaler to control - e.g. Deployment, StatefulSet. + VerticalPodAutoscaler can be targeted at controller implementing + scale subresource (the pod set is retrieved from the controller's + ScaleStatus) or some well known controllers (e.g. for DaemonSet + the pod set is read from the controller's spec). If VerticalPodAutoscaler + cannot use specified target it will report ConfigUnsupported condition. + Note that VerticalPodAutoscaler does not require full implementation + of scale subresource - it will not use it to modify the replica + count. The only thing retrieved is a label selector matching pods + grouped by the target resource. + properties: + apiVersion: + description: apiVersion is the API version of the referent + type: string + kind: + description: 'kind is the kind of the referent; More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + name: + description: 'name is the name of the referent; More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names' + type: string + required: + - kind + - name + type: object + x-kubernetes-map-type: atomic + updatePolicy: + description: Describes the rules on how changes are applied to the + pods. If not specified, all fields in the `PodUpdatePolicy` are + set to their default values. + properties: + updateMode: + description: Controls when autoscaler applies changes to the pod + resources. The default is 'Auto'. + enum: + - "Off" + - Initial + - Recreate + - Auto + type: string + type: object + required: + - targetRef + type: object + status: + description: Current information about the autoscaler. + properties: + conditions: + description: Conditions is the set of conditions required for this + autoscaler to scale its target, and indicates whether or not those + conditions are met. + items: + description: VerticalPodAutoscalerCondition describes the state + of a VerticalPodAutoscaler at a certain point. + properties: + lastTransitionTime: + description: lastTransitionTime is the last time the condition + transitioned from one status to another + format: date-time + type: string + message: + description: message is a human-readable explanation containing + details about the transition + type: string + reason: + description: reason is the reason for the condition's last transition. + type: string + status: + description: status is the status of the condition (True, False, + Unknown) + type: string + type: + description: type describes the current condition + type: string + required: + - status + - type + type: object + type: array + recommendation: + description: The most recently computed amount of resources recommended + by the autoscaler for the controlled pods. + properties: + containerRecommendations: + description: Resources recommended by the autoscaler for each + container. + items: + description: RecommendedContainerResources is the recommendation + of resources computed by autoscaler for a specific container. + Respects the container resource policy if present in the spec. + In particular the recommendation is not produced for containers + with `ContainerScalingMode` set to 'Off'. + properties: + containerName: + description: Name of the container. + type: string + lowerBound: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: Minimum recommended amount of resources. Observes + ContainerResourcePolicy. This amount is not guaranteed + to be sufficient for the application to operate in a stable + way, however running with less resources is likely to + have significant impact on performance/availability. + type: object + target: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: Recommended amount of resources. Observes ContainerResourcePolicy. + type: object + uncappedTarget: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: The most recent recommended resources target + computed by the autoscaler for the controlled pods, based + only on actual resource usage, not taking into account + the ContainerResourcePolicy. May differ from the Recommendation + if the actual resource usage causes the target to violate + the ContainerResourcePolicy (lower than MinAllowed or + higher that MaxAllowed). Used only as status indication, + will not affect actual resource assignment. + type: object + upperBound: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: Maximum recommended amount of resources. Observes + ContainerResourcePolicy. Any resources allocated beyond + this value are likely wasted. This value may be larger + than the maximum amount of application is actually capable + of consuming. + type: object + required: + - target + type: object + type: array + type: object + type: object + required: + - spec + type: object + served: true + storage: false diff --git a/control-plane/roles/gardener/tasks/main.yaml b/control-plane/roles/gardener/tasks/main.yaml index 27b40451..2d0b0be3 100644 --- a/control-plane/roles/gardener/tasks/main.yaml +++ b/control-plane/roles/gardener/tasks/main.yaml @@ -56,13 +56,19 @@ - gardener_dns_provider is not none - gardener_cloud_profile_metal_api_url is not none - gardener_cloud_profile_metal_api_hmac is not none -# Gardener expects a VPA to be present, otherwise VPA enabled and soil bootstrapping do not work -# maybe this can be removed in a future version of Gardener? -- name: Deploy VerticalPodAutoscaler CRD if required + +- name: Deploy required Seed CRDs k8s: - definition: "{{ lookup('template', 'vpa.yaml') }}" + definition: "{{ item.definition }}" apply: yes - when: not lookup('k8s', api_version='apiextensions.k8s.io/v1', kind='CustomResourceDefinition', resource_name='verticalpodautoscalers.autoscaling.k8s.io') + loop: + - name: hvpas + definition: "{{ (lookup('url', 'https://raw.githubusercontent.com/gardener/gardener/' + gardener_repo_ref + '/example/seed-crds/10-crd-autoscaling.k8s.io_hvpas.yaml', split_lines=False) | from_yaml_all | list)[0] }}" + # TODO: starting from g/g 1.82, this resource will be located in seed-crds, too + - name: vpas + definition: "{{ lookup('file', '10-crd-autoscaling.k8s.io_verticalpodautoscalers.yaml') }}" + loop_control: + label: "{{ item.name }}" - name: Create garden namespace k8s: diff --git a/control-plane/roles/gardener/templates/gardener-control-plane-values.j2 b/control-plane/roles/gardener/templates/gardener-control-plane-values.j2 index 10b263df..b9dec2c0 100644 --- a/control-plane/roles/gardener/templates/gardener-control-plane-values.j2 +++ b/control-plane/roles/gardener/templates/gardener-control-plane-values.j2 @@ -51,38 +51,8 @@ global: shootViewerKubeconfigMaxExpiration: {{ gardener_apiserver_shoot_kubeconfig_max_expiration }} vpa: {{ gardener_apiserver_vpa }} - # this requires the Hvpa resource in autoscaling.k8s.io/v1alpha1, which is not in the GKE cluster... how to actually use this? - # hvpa: - # enabled: true - # maxReplicas: 4 - # minReplicas: 1 - # targetAverageUtilizationCpu: 400 - # targetAverageUtilizationMemory: 400 - # vpaScaleUpStabilization: - # stabilizationDuration: "3m" - # minChange: - # cpu: - # value: 300m - # percentage: 80 - # memory: - # value: 600M - # percentage: 80 - # vpaScaleDownStabilization: - # stabilizationDuration: "15m" - # minChange: - # cpu: - # value: 200m - # percentage: 80 - # memory: - # value: 300M - # percentage: 80 - # limitsRequestsGapScaleParams: - # cpu: - # value: "1" - # percentage: 40 - # memory: - # value: "2G" - # percentage: 40 + hvpa: + enabled: true scheduler: resources: {{ gardener_scheduler_resources | to_json }} diff --git a/control-plane/roles/gardener/templates/gardenlet-values.j2 b/control-plane/roles/gardener/templates/gardenlet-values.j2 index d2e6937c..d041d83c 100644 --- a/control-plane/roles/gardener/templates/gardenlet-values.j2 +++ b/control-plane/roles/gardener/templates/gardenlet-values.j2 @@ -66,9 +66,7 @@ config: - key: seed.gardener.cloud/invisible - key: seed.gardener.cloud/disable-capacity-reservation - featureGates: - HVPA: false - HVPAForShootedSeed: false + featureGates: {{ gardener_gardenlet_feature_gates | to_json }} vpa: {{ gardener_soil_vertical_pod_autoscaler_enabled }} diff --git a/control-plane/roles/gardener/templates/vpa.yaml b/control-plane/roles/gardener/templates/vpa.yaml deleted file mode 100644 index 138e9dee..00000000 --- a/control-plane/roles/gardener/templates/vpa.yaml +++ /dev/null @@ -1,124 +0,0 @@ -apiVersion: apiextensions.k8s.io/v1 -kind: CustomResourceDefinition -metadata: - annotations: - api-approved.kubernetes.io: https://github.com/kubernetes/kubernetes/pull/63797 - resources.gardener.cloud/keep-object: "true" - labels: - gardener.cloud/role: vpa - name: verticalpodautoscalers.autoscaling.k8s.io -spec: - conversion: - strategy: None - group: autoscaling.k8s.io - names: - kind: VerticalPodAutoscaler - listKind: VerticalPodAutoscalerList - plural: verticalpodautoscalers - shortNames: - - vpa - singular: verticalpodautoscaler - scope: Namespaced - versions: - - name: v1 - schema: - openAPIV3Schema: - properties: - spec: - properties: - resourcePolicy: - properties: - containerPolicies: - items: - properties: - containerName: - type: string - controlledResources: - items: - enum: - - cpu - - memory - type: string - type: array - maxAllowed: - type: object - x-kubernetes-preserve-unknown-fields: true - minAllowed: - type: object - x-kubernetes-preserve-unknown-fields: true - mode: - enum: - - Auto - - "Off" - type: string - type: object - x-kubernetes-preserve-unknown-fields: true - type: array - type: object - x-kubernetes-preserve-unknown-fields: true - targetRef: - type: object - x-kubernetes-preserve-unknown-fields: true - updatePolicy: - properties: - updateMode: - type: string - type: object - x-kubernetes-preserve-unknown-fields: true - type: object - x-kubernetes-preserve-unknown-fields: true - type: object - x-kubernetes-preserve-unknown-fields: true - served: true - storage: true - - name: v1beta2 - schema: - openAPIV3Schema: - properties: - spec: - properties: - resourcePolicy: - properties: - containerPolicies: - items: - properties: - containerName: - type: string - controlledResources: - items: - enum: - - cpu - - memory - type: string - type: array - maxAllowed: - type: object - x-kubernetes-preserve-unknown-fields: true - minAllowed: - type: object - x-kubernetes-preserve-unknown-fields: true - mode: - enum: - - Auto - - "Off" - type: string - type: object - x-kubernetes-preserve-unknown-fields: true - type: array - type: object - x-kubernetes-preserve-unknown-fields: true - targetRef: - type: object - x-kubernetes-preserve-unknown-fields: true - updatePolicy: - properties: - updateMode: - type: string - type: object - x-kubernetes-preserve-unknown-fields: true - type: object - x-kubernetes-preserve-unknown-fields: true - type: object - x-kubernetes-preserve-unknown-fields: true - served: true - storage: false