diff --git a/charts/m4d-crd/Chart.yaml b/charts/m4d-crd/Chart.yaml index cf54d8b..985e61b 100644 --- a/charts/m4d-crd/Chart.yaml +++ b/charts/m4d-crd/Chart.yaml @@ -5,12 +5,10 @@ apiVersion: v2 name: m4d-crd description: M4D CustomResourceDefinitions type: application - # This is the chart version. This version number should be incremented each time you make changes # to the chart and its templates, including the app version. # Versions are expected to follow Semantic Versioning (https://semver.org/) -version: 0.1.0 - +version: 0.2.0 # This is the version number of the application being deployed. This version number should be # incremented each time you make changes to the application. -appVersion: "0.1.0" +appVersion: 0.2.0 diff --git a/charts/m4d-crd/templates/app.m4d.ibm.com_blueprints.yaml b/charts/m4d-crd/templates/app.m4d.ibm.com_blueprints.yaml index f5a2564..2fc787a 100644 --- a/charts/m4d-crd/templates/app.m4d.ibm.com_blueprints.yaml +++ b/charts/m4d-crd/templates/app.m4d.ibm.com_blueprints.yaml @@ -1,20 +1,13 @@ --- -apiVersion: apiextensions.k8s.io/v1beta1 +apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: - controller-gen.kubebuilder.io/version: v0.3.0 + controller-gen.kubebuilder.io/version: v0.5.0 creationTimestamp: null name: blueprints.app.m4d.ibm.com spec: - additionalPrinterColumns: - - JSONPath: .metadata.creationTimestamp - name: Age - type: date - - JSONPath: .status.observedState.ready - name: Ready - type: string group: app.m4d.ibm.com names: kind: Blueprint @@ -22,133 +15,54 @@ spec: plural: blueprints singular: blueprint scope: Namespaced - subresources: - status: {} - validation: - openAPIV3Schema: - description: Blueprint is the Schema for the blueprints API - properties: - apiVersion: - description: 'APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' - type: string - kind: - description: 'Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' - type: string - metadata: - type: object - spec: - description: 'BlueprintSpec defines the desired state of Blueprint, which is the runtime environment which provides the Data Scientist''s application with secure and governed access to the data requested in the M4DApplication. The blueprint uses an "argo like" syntax which indicates the components and the flow of data between them as steps TODO: Add an indication of the communication relationships between the components' - properties: - entrypoint: - type: string - flow: - description: DataFlow indicates the flow of the data between the components Currently we assume this is linear and thus use steps, but other more complex graphs could be defined as per how it is done in argo workflow - properties: - name: - type: string - steps: - items: - description: FlowStep is one step indicates an instance of a module in the blueprint, It includes the name of the module template (spec) and the parameters received by the component instance that is initiated by the orchestrator. - properties: - arguments: - description: Arguments are the input parameters for a specific instance of a module. - properties: - copy: - description: CopyArgs are parameters specific to modules that copy data from one data store to another. - properties: - destination: - description: Destination is the data store to which the data will be copied - properties: - connection: - description: Connection has the relevant details for accesing the data (url, table, ssl, etc.) - type: object - format: - description: Format represents data format (e.g. parquet) as received from catalog connectors - type: string - vault: - description: Holds details for retrieving credentials by the modules from Vault store. - properties: - address: - description: Address is Vault address - type: string - authPath: - description: AuthPath is the path to auth method i.e. kubernetes - type: string - role: - description: Role is the Vault role used for retrieving the credentials - type: string - secretPath: - description: SecretPath is the path of the secret holding the Credentials in Vault - type: string - required: - - address - - authPath - - role - - secretPath - type: object - required: - - connection - - format - - vault - type: object - source: - description: Source is the where the data currently resides - properties: - connection: - description: Connection has the relevant details for accesing the data (url, table, ssl, etc.) - type: object - format: - description: Format represents data format (e.g. parquet) as received from catalog connectors - type: string - vault: - description: Holds details for retrieving credentials by the modules from Vault store. - properties: - address: - description: Address is Vault address - type: string - authPath: - description: AuthPath is the path to auth method i.e. kubernetes - type: string - role: - description: Role is the Vault role used for retrieving the credentials - type: string - secretPath: - description: SecretPath is the path of the secret holding the Credentials in Vault - type: string - required: - - address - - authPath - - role - - secretPath - type: object - required: - - connection - - format - - vault - type: object - transformations: - description: Transformations are different types of processing that may be done to the data as it is copied. - items: - type: object - type: array - required: - - destination - - source - type: object - read: - description: ReadArgs are parameters that are specific to modules that enable an application to read data - items: - description: ReadModuleArgs define the input parameters for modules that read data from location A + versions: + - additionalPrinterColumns: + - jsonPath: .metadata.creationTimestamp + name: Age + type: date + - jsonPath: .status.observedState.ready + name: Ready + type: string + name: v1alpha1 + schema: + openAPIV3Schema: + description: Blueprint is the Schema for the blueprints API + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + spec: + description: 'BlueprintSpec defines the desired state of Blueprint, which is the runtime environment which provides the Data Scientist''s application with secure and governed access to the data requested in the M4DApplication. The blueprint uses an "argo like" syntax which indicates the components and the flow of data between them as steps TODO: Add an indication of the communication relationships between the components' + properties: + entrypoint: + type: string + flow: + description: DataFlow indicates the flow of the data between the components Currently we assume this is linear and thus use steps, but other more complex graphs could be defined as per how it is done in argo workflow + properties: + name: + type: string + steps: + items: + description: FlowStep is one step indicates an instance of a module in the blueprint, It includes the name of the module template (spec) and the parameters received by the component instance that is initiated by the orchestrator. + properties: + arguments: + description: Arguments are the input parameters for a specific instance of a module. + properties: + copy: + description: CopyArgs are parameters specific to modules that copy data from one data store to another. properties: - assetID: - description: AssetID identifies the asset to be used for accessing the data when it is ready It is copied from the M4DApplication resource - type: string - source: - description: Source of the read path module + destination: + description: Destination is the data store to which the data will be copied properties: connection: description: Connection has the relevant details for accesing the data (url, table, ssl, etc.) type: object + x-kubernetes-preserve-unknown-fields: true format: description: Format represents data format (e.g. parquet) as received from catalog connectors type: string @@ -178,27 +92,13 @@ spec: - format - vault type: object - transformations: - description: Transformations are different types of processing that may be done to the data - items: - type: object - type: array - required: - - assetID - - source - type: object - type: array - write: - description: WriteArgs are parameters that are specific to modules that enable an application to write data - items: - description: WriteModuleArgs define the input parameters for modules that write data to location B - properties: - destination: - description: Destination is the data store to which the data will be written + source: + description: Source is the where the data currently resides properties: connection: description: Connection has the relevant details for accesing the data (url, table, ssl, etc.) type: object + x-kubernetes-preserve-unknown-fields: true format: description: Format represents data format (e.g. parquet) as received from catalog connectors type: string @@ -229,98 +129,204 @@ spec: - vault type: object transformations: - description: Transformations are different types of processing that may be done to the data as it is written. + description: Transformations are different types of processing that may be done to the data as it is copied. items: type: object + x-kubernetes-preserve-unknown-fields: true type: array required: - destination + - source type: object - type: array - type: object - name: - description: Name is the name of the instance of the module. For example, if the application is named "notebook" and an implicitcopy module is deemed necessary. The FlowStep name would be notebook-implicitcopy. - type: string - template: - description: 'Template is the name of the specification in the Blueprint describing how to instantiate a component indicated by the module. It is the name of a M4DModule CRD. For example: implicit-copy-db2wh-to-s3-latest' - type: string - required: - - name - - template - type: object - type: array - required: - - name - - steps - type: object - templates: - items: - description: ComponentTemplate is a copy of a M4DModule Custom Resource. It contains the information necessary to instantiate a component in a FlowStep, which provides the functionality described by the module. There are 3 different module types. - properties: - chart: - description: Chart contains the location of the helm chart with info detailing how to deploy - properties: - name: - description: Name of helm chart - type: string - values: - additionalProperties: + read: + description: ReadArgs are parameters that are specific to modules that enable an application to read data + items: + description: ReadModuleArgs define the input parameters for modules that read data from location A + properties: + assetID: + description: AssetID identifies the asset to be used for accessing the data when it is ready It is copied from the M4DApplication resource + type: string + source: + description: Source of the read path module + properties: + connection: + description: Connection has the relevant details for accesing the data (url, table, ssl, etc.) + type: object + x-kubernetes-preserve-unknown-fields: true + format: + description: Format represents data format (e.g. parquet) as received from catalog connectors + type: string + vault: + description: Holds details for retrieving credentials by the modules from Vault store. + properties: + address: + description: Address is Vault address + type: string + authPath: + description: AuthPath is the path to auth method i.e. kubernetes + type: string + role: + description: Role is the Vault role used for retrieving the credentials + type: string + secretPath: + description: SecretPath is the path of the secret holding the Credentials in Vault + type: string + required: + - address + - authPath + - role + - secretPath + type: object + required: + - connection + - format + - vault + type: object + transformations: + description: Transformations are different types of processing that may be done to the data + items: + type: object + x-kubernetes-preserve-unknown-fields: true + type: array + required: + - assetID + - source + type: object + type: array + write: + description: WriteArgs are parameters that are specific to modules that enable an application to write data + items: + description: WriteModuleArgs define the input parameters for modules that write data to location B + properties: + destination: + description: Destination is the data store to which the data will be written + properties: + connection: + description: Connection has the relevant details for accesing the data (url, table, ssl, etc.) + type: object + x-kubernetes-preserve-unknown-fields: true + format: + description: Format represents data format (e.g. parquet) as received from catalog connectors + type: string + vault: + description: Holds details for retrieving credentials by the modules from Vault store. + properties: + address: + description: Address is Vault address + type: string + authPath: + description: AuthPath is the path to auth method i.e. kubernetes + type: string + role: + description: Role is the Vault role used for retrieving the credentials + type: string + secretPath: + description: SecretPath is the path of the secret holding the Credentials in Vault + type: string + required: + - address + - authPath + - role + - secretPath + type: object + required: + - connection + - format + - vault + type: object + transformations: + description: Transformations are different types of processing that may be done to the data as it is written. + items: + type: object + x-kubernetes-preserve-unknown-fields: true + type: array + required: + - destination + type: object + type: array + type: object + name: + description: Name is the name of the instance of the module. For example, if the application is named "notebook" and an implicitcopy module is deemed necessary. The FlowStep name would be notebook-implicitcopy. type: string - description: Values to pass to helm chart installation - type: object - required: - - name - type: object - kind: - description: Kind of k8s resource - type: string - name: - description: Name of the template - type: string + template: + description: 'Template is the name of the specification in the Blueprint describing how to instantiate a component indicated by the module. It is the name of a M4DModule CRD. For example: implicit-copy-db2wh-to-s3-latest' + type: string + required: + - name + - template + type: object + type: array required: - - chart - - kind - name + - steps type: object - type: array - required: - - entrypoint - - flow - - templates - type: object - status: - description: BlueprintStatus defines the observed state of Blueprint This includes readiness, error message, and indicators forthe Kubernetes resources owned by the Blueprint for cleanup and status monitoring - properties: - observedGeneration: - description: ObservedGeneration is taken from the Blueprint metadata. This is used to determine during reconcile whether reconcile was called because the desired state changed, or whether status of the allocated resources should be checked. - format: int64 - type: integer - observedState: - description: ObservedState includes information to be reported back to the M4DApplication resource It includes readiness and error indications, as well as user instructions - properties: - dataAccessInstructions: - description: DataAccessInstructions indicate how the data user or his application may access the data. Instructions are available upon successful orchestration. - type: string - error: - description: Error indicates that there has been an error to orchestrate the modules and provides the error message - type: string - ready: - description: Ready represents that the modules have been orchestrated successfully and the data is ready for usage - type: boolean - type: object - releases: - additionalProperties: + templates: + items: + description: ComponentTemplate is a copy of a M4DModule Custom Resource. It contains the information necessary to instantiate a component in a FlowStep, which provides the functionality described by the module. There are 3 different module types. + properties: + chart: + description: Chart contains the location of the helm chart with info detailing how to deploy + properties: + name: + description: Name of helm chart + type: string + values: + additionalProperties: + type: string + description: Values to pass to helm chart installation + type: object + required: + - name + type: object + kind: + description: Kind of k8s resource + type: string + name: + description: Name of the template + type: string + required: + - chart + - kind + - name + type: object + type: array + required: + - entrypoint + - flow + - templates + type: object + status: + description: BlueprintStatus defines the observed state of Blueprint This includes readiness, error message, and indicators forthe Kubernetes resources owned by the Blueprint for cleanup and status monitoring + properties: + observedGeneration: + description: ObservedGeneration is taken from the Blueprint metadata. This is used to determine during reconcile whether reconcile was called because the desired state changed, or whether status of the allocated resources should be checked. format: int64 type: integer - description: Releases map each release to the observed generation of the blueprint containing this release. At the end of reconcile, each release should be mapped to the latest blueprint version or be uninstalled. - type: object - type: object - type: object - version: v1alpha1 - versions: - - name: v1alpha1 + observedState: + description: ObservedState includes information to be reported back to the M4DApplication resource It includes readiness and error indications, as well as user instructions + properties: + dataAccessInstructions: + description: DataAccessInstructions indicate how the data user or his application may access the data. Instructions are available upon successful orchestration. + type: string + error: + description: Error indicates that there has been an error to orchestrate the modules and provides the error message + type: string + ready: + description: Ready represents that the modules have been orchestrated successfully and the data is ready for usage + type: boolean + type: object + releases: + additionalProperties: + format: int64 + type: integer + description: Releases map each release to the observed generation of the blueprint containing this release. At the end of reconcile, each release should be mapped to the latest blueprint version or be uninstalled. + type: object + type: object + type: object served: true storage: true + subresources: + status: {} status: acceptedNames: kind: "" diff --git a/charts/m4d-crd/templates/app.m4d.ibm.com_m4dapplications.yaml b/charts/m4d-crd/templates/app.m4d.ibm.com_m4dapplications.yaml index 9cc00c8..cee2406 100644 --- a/charts/m4d-crd/templates/app.m4d.ibm.com_m4dapplications.yaml +++ b/charts/m4d-crd/templates/app.m4d.ibm.com_m4dapplications.yaml @@ -1,10 +1,10 @@ --- -apiVersion: apiextensions.k8s.io/v1beta1 +apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: - controller-gen.kubebuilder.io/version: v0.3.0 + controller-gen.kubebuilder.io/version: v0.5.0 creationTimestamp: null name: m4dapplications.app.m4d.ibm.com spec: @@ -15,217 +15,242 @@ spec: plural: m4dapplications singular: m4dapplication scope: Namespaced - subresources: - status: {} - validation: - openAPIV3Schema: - description: M4DApplication provides information about the application being used by a Data Scientist, the nature of the processing, and the data sets that the Data Scientist has chosen for processing by the application. The M4DApplication controller (aka pilot) obtains instructions regarding any governance related changes that must be performed on the data, identifies the modules capable of performing such changes, and finally generates the Blueprint which defines the secure runtime environment and all the components in it. This runtime environment provides the Data Scientist's application with access to the data requested in a secure manner and without having to provide any credentials for the data sets. The credentials are obtained automatically by the manager from an external credential management system, which may or may not be part of a data catalog. - properties: - apiVersion: - description: 'APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' - type: string - kind: - description: 'Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' - type: string - metadata: - type: object - spec: - description: M4DApplicationSpec defines the desired state of M4DApplication. - properties: - appInfo: - additionalProperties: + versions: + - name: v1alpha1 + schema: + openAPIV3Schema: + description: M4DApplication provides information about the application being used by a Data Scientist, the nature of the processing, and the data sets that the Data Scientist has chosen for processing by the application. The M4DApplication controller (aka pilot) obtains instructions regarding any governance related changes that must be performed on the data, identifies the modules capable of performing such changes, and finally generates the Blueprint which defines the secure runtime environment and all the components in it. This runtime environment provides the Data Scientist's application with access to the data requested in a secure manner and without having to provide any credentials for the data sets. The credentials are obtained automatically by the manager from an external credential management system, which may or may not be part of a data catalog. + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + spec: + description: M4DApplicationSpec defines the desired state of M4DApplication. + properties: + appInfo: + additionalProperties: + type: string + description: AppInfo contains information describing the reasons for the processing that will be done by the Data Scientist's application. + type: object + data: + description: Data contains the identifiers of the data to be used by the Data Scientist's application, and the protocol used to access it and the format expected. + items: + description: DataContext indicates data set chosen by the Data Scientist to be used by his application, and includes information about the data format and technologies used by the application to access the data. + properties: + catalogService: + description: CatalogService represents the catalog service for accessing the requested dataset. If not specified, the enterprise catalog service will be used. + type: string + dataSetID: + description: DataSetID is a unique identifier of the dataset chosen from the data catalog for processing by the data user application. + minLength: 1 + type: string + requirements: + description: Requirements from the system + properties: + copy: + description: CopyRequrements include the requirements for copying the data + properties: + catalog: + description: Catalog indicates that the data asset must be cataloged. + properties: + catalogID: + description: CatalogID specifies the catalog where the data will be cataloged. + type: string + service: + description: CatalogService specifies the datacatalog service that will be used for catalogging the data into. + type: string + type: object + required: + description: Required indicates that the data must be copied. + type: boolean + type: object + interface: + description: Interface indicates the protocol and format expected by the data user + properties: + dataformat: + description: DataFormatType defines data format type + enum: + - parquet + - table + - csv + - json + - avro + - orc + - binary + - arrow + type: string + protocol: + description: IFProtocol defines interface protocol for data transactions + enum: + - s3 + - kafka + - jdbc-db2 + - m4d-arrow-flight + type: string + required: + - protocol + type: object + required: + - interface + type: object + required: + - dataSetID + - requirements + type: object + minItems: 1 + type: array + secretRef: + description: SecretRef points to the secret that holds credentials for each system the user has been authenticated with. The secret is deployed in M4dApplication namespace. type: string - description: AppInfo contains information describing the reasons for the processing that will be done by the Data Scientist's application. - type: object - data: - description: Data contains the identifiers of the data to be used by the Data Scientist's application, and the protocol used to access it and the format expected. - items: - description: DataContext indicates data set chosen by the Data Scientist to be used by his application, and includes information about the data format and technologies used by the application to access the data. + selector: + description: Selector enables to connect the resource to the application Application labels should match the labels in the selector. For some flows the selector may not be used. properties: - catalogService: - description: CatalogService represents the catalog service for accessing the requested dataset. If not specified, the enterprise catalog service will be used. - type: string - dataSetID: - description: DataSetID is a unique identifier of the dataset chosen from the data catalog for processing by the data user application. - minLength: 1 + clusterName: + description: Cluster name type: string - requirements: - description: Requirements from the system + workloadSelector: + description: WorkloadSelector enables to connect the resource to the application Application labels should match the labels in the selector. properties: - copy: - description: CopyRequrements include the requirements for copying the data - properties: - catalog: - description: Catalog indicates that the data asset must be cataloged. - properties: - catalogID: - description: CatalogID specifies the catalog where the data will be cataloged. - type: string - service: - description: CatalogService specifies the datacatalog service that will be used for catalogging the data into. + matchExpressions: + description: matchExpressions is a list of label selector requirements. The requirements are ANDed. + items: + description: A label selector requirement is a selector that contains values, a key, and an operator that relates the key and values. + properties: + key: + description: key is the label key that the selector applies to. + type: string + operator: + description: operator represents a key's relationship to a set of values. Valid operators are In, NotIn, Exists and DoesNotExist. + type: string + values: + description: values is an array of string values. If the operator is In or NotIn, the values array must be non-empty. If the operator is Exists or DoesNotExist, the values array must be empty. This array is replaced during a strategic merge patch. + items: type: string - type: object + type: array required: - description: Required indicates that the data must be copied. - type: boolean - type: object - interface: - description: Interface indicates the protocol and format expected by the data user - properties: - dataformat: - description: DataFormatType defines data format type - enum: - - parquet - - table - - csv - - json - - avro - - binary - - arrow - type: string - protocol: - description: IFProtocol defines interface protocol for data transactions - enum: - - s3 - - kafka - - jdbc-db2 - - m4d-arrow-flight - type: string - required: - - protocol + - key + - operator + type: object + type: array + matchLabels: + additionalProperties: + type: string + description: matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels map is equivalent to an element of matchExpressions, whose key field is "key", the operator is "In", and the values array contains only "value". The requirements are ANDed. type: object - required: - - interface type: object required: - - dataSetID - - requirements + - workloadSelector type: object - minItems: 1 - type: array - secretRef: - description: SecretRef points to the secret that holds credentials for each system the user has been authenticated with. The secret is deployed in M4dApplication namespace. - type: string - selector: - description: Selector enables to connect the resource to the application Application labels should match the labels in the selector. For some flows the selector may not be used. - properties: - clusterName: - description: Cluster name + required: + - appInfo + - data + type: object + status: + description: M4DApplicationStatus defines the observed state of M4DApplication. + properties: + catalogedAssets: + additionalProperties: type: string - workloadSelector: - description: WorkloadSelector enables to connect the resource to the application Application labels should match the labels in the selector. + description: CatalogedAssets provide the new asset identifiers after being registered in the enterprise catalog It maps the original asset id to the cataloged asset id. + type: object + conditions: + description: Conditions represent the possible error and failure conditions + items: + description: Condition describes the state of a M4DApplication at a certain point. properties: - matchExpressions: - description: matchExpressions is a list of label selector requirements. The requirements are ANDed. - items: - description: A label selector requirement is a selector that contains values, a key, and an operator that relates the key and values. - properties: - key: - description: key is the label key that the selector applies to. - type: string - operator: - description: operator represents a key's relationship to a set of values. Valid operators are In, NotIn, Exists and DoesNotExist. - type: string - values: - description: values is an array of string values. If the operator is In or NotIn, the values array must be non-empty. If the operator is Exists or DoesNotExist, the values array must be empty. This array is replaced during a strategic merge patch. - items: - type: string - type: array - required: - - key - - operator - type: object - type: array - matchLabels: - additionalProperties: - type: string - description: matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels map is equivalent to an element of matchExpressions, whose key field is "key", the operator is "In", and the values array contains only "value". The requirements are ANDed. - type: object + message: + description: Message contains the details of the current condition + type: string + status: + description: 'Status of the condition: true or false' + type: string + type: + description: Type of the condition + type: string + required: + - status + - type type: object - required: - - workloadSelector - type: object - required: - - appInfo - - data - type: object - status: - description: M4DApplicationStatus defines the observed state of M4DApplication. - properties: - catalogedAssets: - additionalProperties: + type: array + dataAccessInstructions: + description: DataAccessInstructions indicate how the data user or his application may access the data. Instructions are available upon successful orchestration. type: string - description: CatalogedAssets provide the new asset identifiers after being registered in the enterprise catalog It maps the original asset id to the cataloged asset id. - type: object - conditions: - description: Conditions represent the possible error and failure conditions - items: - description: Condition describes the state of a M4DApplication at a certain point. + generated: + description: Generated resource identifier properties: - message: - description: Message contains the details of the current condition + appVersion: + description: Version of M4DApplication that has generated this resource + format: int64 + type: integer + kind: + description: Kind of the resource (Blueprint, Plotter) type: string - status: - description: 'Status of the condition: true or false' + name: + description: Name of the resource type: string - type: - description: Type of the condition + namespace: + description: Namespace of the resource type: string required: - - status - - type + - appVersion + - kind + - name + - namespace type: object - type: array - dataAccessInstructions: - description: DataAccessInstructions indicate how the data user or his application may access the data. Instructions are available upon successful orchestration. - type: string - generated: - description: Generated resource identifier - properties: - kind: - description: Kind of the resource (Blueprint, Plotter) - type: string - name: - description: Name of the resource - type: string - namespace: - description: Namespace of the resource - type: string - required: - - kind - - name - - namespace - type: object - observedGeneration: - description: ObservedGeneration is taken from the M4DApplication metadata. This is used to determine during reconcile whether reconcile was called because the desired state changed, or whether the Blueprint status changed. - format: int64 - type: integer - provisionedStorage: - additionalProperties: - description: DatasetDetails contain dataset connection and metadata required to register this dataset in the enterprise catalog - properties: - datasetRef: - description: Reference to a Dataset resource containing the request to provision storage - type: string - details: - description: Dataset information - type: object - secretRef: - description: Reference to a secret where the credentials are stored - type: string + observedGeneration: + description: ObservedGeneration is taken from the M4DApplication metadata. This is used to determine during reconcile whether reconcile was called because the desired state changed, or whether the Blueprint status changed. + format: int64 + type: integer + provisionedStorage: + additionalProperties: + description: DatasetDetails contain dataset connection and metadata required to register this dataset in the enterprise catalog + properties: + datasetRef: + description: Reference to a Dataset resource containing the request to provision storage + type: string + details: + description: Dataset information + type: object + x-kubernetes-preserve-unknown-fields: true + secretRef: + description: Reference to a secret where the credentials are stored + type: string + type: object + description: ProvisionedStorage maps a dataset (identified by AssetID) to the new provisioned bucket. It allows M4DApplication controller to manage buckets in case the spec has been modified, an error has occurred, or a delete event has been received. ProvisionedStorage has the information required to register the dataset once the owned plotter resource is ready type: object - description: ProvisionedStorage maps a dataset (identified by DataSetID) to the new provisioned bucket. It allows M4DApplication controller to manage buckets in case the spec has been modified, an error has occurred, or a delete event has been received. ProvisionedStorage has the information required to register the dataset once the owned plotter resource is ready - type: object - ready: - description: Ready is true if a blueprint has been successfully orchestrated - type: boolean - type: object - type: object - version: v1alpha1 - versions: - - name: v1alpha1 + readEndpointsMap: + additionalProperties: + description: EndpointSpec is used both by the module creator and by the status of the m4dapplication + properties: + hostname: + description: Always equals the release name. Can be omitted. + type: string + port: + format: int32 + type: integer + scheme: + description: 'For example: http, https, grpc, grpc+tls, jdbc:oracle:thin:@ etc' + type: string + required: + - port + - scheme + type: object + description: ReadEndpointsMap maps an datasetID (after parsing from json to a string with dashes) to the endpoint spec from which the asset will be served to the application + type: object + ready: + description: Ready is true if a blueprint has been successfully orchestrated + type: boolean + type: object + type: object served: true storage: true + subresources: + status: {} status: acceptedNames: kind: "" diff --git a/charts/m4d-crd/templates/app.m4d.ibm.com_m4dmodules.yaml b/charts/m4d-crd/templates/app.m4d.ibm.com_m4dmodules.yaml index f561b6a..2f5ab3a 100644 --- a/charts/m4d-crd/templates/app.m4d.ibm.com_m4dmodules.yaml +++ b/charts/m4d-crd/templates/app.m4d.ibm.com_m4dmodules.yaml @@ -1,10 +1,10 @@ --- -apiVersion: apiextensions.k8s.io/v1beta1 +apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: - controller-gen.kubebuilder.io/version: v0.3.0 + controller-gen.kubebuilder.io/version: v0.5.0 creationTimestamp: null name: m4dmodules.app.m4d.ibm.com spec: @@ -15,207 +15,226 @@ spec: plural: m4dmodules singular: m4dmodule scope: Namespaced - validation: - openAPIV3Schema: - description: M4DModule is a description of an injectable component. the parameters it requires, as well as the specification of how to instantiate such a component. It is used as metadata only. There is no status nor reconciliation. - properties: - apiVersion: - description: 'APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' - type: string - kind: - description: 'Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' - type: string - metadata: - type: object - spec: - description: M4DModuleSpec contains the info common to all modules, which are one of the components that process, load, write, audit, monitor the data used by the data scientist's application. - properties: - capabilities: - description: Capabilities declares what this module knows how to do and the types of data it knows how to handle - properties: - actions: - description: Actions are the data transformations that the module supports - items: - description: SupportedAction declares an action that the module supports (action identifier and its scope) - properties: - id: - type: string - level: - format: int32 - type: integer - type: object - type: array - api: - description: API indicates to the application how to access/write the data - properties: - dataformat: - description: DataFormatType defines data format type - enum: - - parquet - - table - - csv - - json - - avro - - binary - - arrow - type: string - protocol: - description: IFProtocol defines interface protocol for data transactions - enum: - - s3 - - kafka - - jdbc-db2 - - m4d-arrow-flight - type: string - required: - - protocol - type: object - supportedInterfaces: - description: Copy should have one or more instances in the list, and its content should have source and sink Read should have one or more instances in the list, each with source populated Write should have one or more instances in the list, each with sink populated TODO - In the future if we have a module type that doesn't interface directly with data then this list could be empty - items: - description: ModuleInOut specifies the protocol and format of the data input and output by the module - if any + versions: + - name: v1alpha1 + schema: + openAPIV3Schema: + description: M4DModule is a description of an injectable component. the parameters it requires, as well as the specification of how to instantiate such a component. It is used as metadata only. There is no status nor reconciliation. + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + spec: + description: M4DModuleSpec contains the info common to all modules, which are one of the components that process, load, write, audit, monitor the data used by the data scientist's application. + properties: + capabilities: + description: Capabilities declares what this module knows how to do and the types of data it knows how to handle + properties: + actions: + description: Actions are the data transformations that the module supports + items: + description: SupportedAction declares an action that the module supports (action identifier and its scope) + properties: + id: + type: string + level: + format: int32 + type: integer + type: object + type: array + api: + description: API indicates to the application how to access/write the data properties: - flow: - description: Flow for which this interface is supported + dataformat: + description: DataFormatType defines data format type enum: - - copy - - read - - write + - parquet + - table + - csv + - json + - avro + - orc + - binary + - arrow type: string - sink: - description: Sink specifies the output data protocol and format + endpoint: + description: EndpointSpec is used both by the module creator and by the status of the m4dapplication properties: - dataformat: - description: DataFormatType defines data format type - enum: - - parquet - - table - - csv - - json - - avro - - binary - - arrow + hostname: + description: Always equals the release name. Can be omitted. type: string - protocol: - description: IFProtocol defines interface protocol for data transactions - enum: - - s3 - - kafka - - jdbc-db2 - - m4d-arrow-flight + port: + format: int32 + type: integer + scheme: + description: 'For example: http, https, grpc, grpc+tls, jdbc:oracle:thin:@ etc' type: string required: - - protocol - type: object - source: - description: Source specifies the input data protocol and format - properties: - dataformat: - description: DataFormatType defines data format type - enum: - - parquet - - table - - csv - - json - - avro - - binary - - arrow - type: string - protocol: - description: IFProtocol defines interface protocol for data transactions - enum: - - s3 - - kafka - - jdbc-db2 - - m4d-arrow-flight - type: string - required: - - protocol + - port + - scheme type: object + protocol: + description: IFProtocol defines interface protocol for data transactions + enum: + - s3 + - kafka + - jdbc-db2 + - m4d-arrow-flight + type: string required: - - flow + - endpoint + - protocol type: object - type: array - required: - - supportedInterfaces - type: object - chart: - description: Reference to a Helm chart that allows deployment of the resources required for this module - properties: - name: - description: Name of helm chart - type: string - values: - additionalProperties: - type: string - description: Values to pass to helm chart installation - type: object - required: - - name - type: object - dependencies: - description: Other components that must be installed in order for this module to work - items: - description: Dependency details another component on which this module relies - i.e. a pre-requisit - properties: - name: - description: Name is the name of the dependent component - type: string - type: - description: Type provides information used in determining how to instantiate the component - enum: - - module - - connector - - feature - type: string + supportedInterfaces: + description: Copy should have one or more instances in the list, and its content should have source and sink Read should have one or more instances in the list, each with source populated Write should have one or more instances in the list, each with sink populated TODO - In the future if we have a module type that doesn't interface directly with data then this list could be empty + items: + description: ModuleInOut specifies the protocol and format of the data input and output by the module - if any + properties: + flow: + description: Flow for which this interface is supported + enum: + - copy + - read + - write + type: string + sink: + description: Sink specifies the output data protocol and format + properties: + dataformat: + description: DataFormatType defines data format type + enum: + - parquet + - table + - csv + - json + - avro + - orc + - binary + - arrow + type: string + protocol: + description: IFProtocol defines interface protocol for data transactions + enum: + - s3 + - kafka + - jdbc-db2 + - m4d-arrow-flight + type: string + required: + - protocol + type: object + source: + description: Source specifies the input data protocol and format + properties: + dataformat: + description: DataFormatType defines data format type + enum: + - parquet + - table + - csv + - json + - avro + - orc + - binary + - arrow + type: string + protocol: + description: IFProtocol defines interface protocol for data transactions + enum: + - s3 + - kafka + - jdbc-db2 + - m4d-arrow-flight + type: string + required: + - protocol + type: object + required: + - flow + type: object + type: array required: - - name - - type + - supportedInterfaces type: object - type: array - flows: - description: Flows is a list of the types of capabilities supported by the module - copy, read, write - items: - description: ModuleFlow indicates what data flow is performed by the module - enum: - - copy - - read - - write - type: string - type: array - statusIndicators: - description: StatusIndicators allow to check status of a non-standard resource that can not be computed by helm/kstatus - items: - description: ResourceStatusIndicator is used to determine the status of an orchestrated resource + chart: + description: Reference to a Helm chart that allows deployment of the resources required for this module properties: - errorMessage: - description: ErrorMessage specifies the resource field to check for an error, e.g. status.errorMsg - type: string - failureCondition: - description: FailureCondition specifies a condition that indicates the resource failure It uses kubernetes label selection syntax (https://kubernetes.io/docs/concepts/overview/working-with-objects/labels/) - type: string - kind: - description: Kind provides information about the resource kind - type: string - successCondition: - description: SuccessCondition specifies a condition that indicates that the resource is ready It uses kubernetes label selection syntax (https://kubernetes.io/docs/concepts/overview/working-with-objects/labels/) + name: + description: Name of helm chart type: string + values: + additionalProperties: + type: string + description: Values to pass to helm chart installation + type: object required: - - kind - - successCondition + - name type: object - type: array - required: - - capabilities - - chart - - flows - type: object - required: - - spec - type: object - version: v1alpha1 - versions: - - name: v1alpha1 + dependencies: + description: Other components that must be installed in order for this module to work + items: + description: Dependency details another component on which this module relies - i.e. a pre-requisit + properties: + name: + description: Name is the name of the dependent component + type: string + type: + description: Type provides information used in determining how to instantiate the component + enum: + - module + - connector + - feature + type: string + required: + - name + - type + type: object + type: array + flows: + description: Flows is a list of the types of capabilities supported by the module - copy, read, write + items: + description: ModuleFlow indicates what data flow is performed by the module + enum: + - copy + - read + - write + type: string + type: array + statusIndicators: + description: StatusIndicators allow to check status of a non-standard resource that can not be computed by helm/kstatus + items: + description: ResourceStatusIndicator is used to determine the status of an orchestrated resource + properties: + errorMessage: + description: ErrorMessage specifies the resource field to check for an error, e.g. status.errorMsg + type: string + failureCondition: + description: FailureCondition specifies a condition that indicates the resource failure It uses kubernetes label selection syntax (https://kubernetes.io/docs/concepts/overview/working-with-objects/labels/) + type: string + kind: + description: Kind provides information about the resource kind + type: string + successCondition: + description: SuccessCondition specifies a condition that indicates that the resource is ready It uses kubernetes label selection syntax (https://kubernetes.io/docs/concepts/overview/working-with-objects/labels/) + type: string + required: + - kind + - successCondition + type: object + type: array + required: + - capabilities + - chart + - flows + type: object + required: + - spec + type: object served: true storage: true status: diff --git a/charts/m4d-crd/templates/app.m4d.ibm.com_m4dstorageaccounts.yaml b/charts/m4d-crd/templates/app.m4d.ibm.com_m4dstorageaccounts.yaml index d8b3803..ec4a04a 100644 --- a/charts/m4d-crd/templates/app.m4d.ibm.com_m4dstorageaccounts.yaml +++ b/charts/m4d-crd/templates/app.m4d.ibm.com_m4dstorageaccounts.yaml @@ -1,10 +1,10 @@ --- -apiVersion: apiextensions.k8s.io/v1beta1 +apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: - controller-gen.kubebuilder.io/version: v0.3.0 + controller-gen.kubebuilder.io/version: v0.5.0 creationTimestamp: null name: m4dstorageaccounts.app.m4d.ibm.com spec: @@ -15,45 +15,44 @@ spec: plural: m4dstorageaccounts singular: m4dstorageaccount scope: Namespaced - validation: - openAPIV3Schema: - description: M4DStorageAccount defines a storage account used for copying data. Only S3 based storage is supported. It contains endpoint, region and a reference to the credentials a Owner of the asset is responsible to store the credentials - properties: - apiVersion: - description: 'APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' - type: string - kind: - description: 'Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' - type: string - metadata: - type: object - spec: - description: M4DStorageAccountSpec defines the desired state of M4DStorageAccount - properties: - endpoint: - description: Endpoint - type: string - regions: - description: Regions - items: - type: string - minItems: 1 - type: array - secretRef: - description: A name of k8s secret deployed in the control plane. This secret includes secretKey and accessKey credentials for S3 bucket - type: string - required: - - endpoint - - regions - - secretRef - type: object - status: - description: M4DStorageAccountStatus defines the observed state of M4DStorageAccount - type: object - type: object - version: v1alpha1 versions: - name: v1alpha1 + schema: + openAPIV3Schema: + description: M4DStorageAccount defines a storage account used for copying data. Only S3 based storage is supported. It contains endpoint, region and a reference to the credentials a Owner of the asset is responsible to store the credentials + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + spec: + description: M4DStorageAccountSpec defines the desired state of M4DStorageAccount + properties: + endpoint: + description: Endpoint + type: string + regions: + description: Regions + items: + type: string + minItems: 1 + type: array + secretRef: + description: A name of k8s secret deployed in the control plane. This secret includes secretKey and accessKey credentials for S3 bucket + type: string + required: + - endpoint + - regions + - secretRef + type: object + status: + description: M4DStorageAccountStatus defines the observed state of M4DStorageAccount + type: object + type: object served: true storage: true status: diff --git a/charts/m4d-crd/templates/app.m4d.ibm.com_plotters.yaml b/charts/m4d-crd/templates/app.m4d.ibm.com_plotters.yaml index 1407628..434bd5b 100644 --- a/charts/m4d-crd/templates/app.m4d.ibm.com_plotters.yaml +++ b/charts/m4d-crd/templates/app.m4d.ibm.com_plotters.yaml @@ -1,23 +1,13 @@ --- -apiVersion: apiextensions.k8s.io/v1beta1 +apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: - controller-gen.kubebuilder.io/version: v0.3.0 + controller-gen.kubebuilder.io/version: v0.5.0 creationTimestamp: null name: plotters.app.m4d.ibm.com spec: - additionalPrinterColumns: - - JSONPath: .metadata.creationTimestamp - name: Age - type: date - - JSONPath: .status.observedState.ready - name: Ready - type: string - - JSONPath: .status.readyTimestamp - name: ReadySince - type: string group: app.m4d.ibm.com names: kind: Plotter @@ -25,137 +15,61 @@ spec: plural: plotters singular: plotter scope: Namespaced - subresources: - status: {} - validation: - openAPIV3Schema: - description: Plotter is the Schema for the plotters API - properties: - apiVersion: - description: 'APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' - type: string - kind: - description: 'Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' - type: string - metadata: - type: object - spec: - description: PlotterSpec defines the desired state of Plotter, which is applied in a multi-clustered environment. Plotter installs the runtime environment (as blueprints running on remote clusters) which provides the Data Scientist's application with secure and governed access to the data requested in the M4DApplication. - properties: - blueprints: - additionalProperties: - description: 'BlueprintSpec defines the desired state of Blueprint, which is the runtime environment which provides the Data Scientist''s application with secure and governed access to the data requested in the M4DApplication. The blueprint uses an "argo like" syntax which indicates the components and the flow of data between them as steps TODO: Add an indication of the communication relationships between the components' - properties: - entrypoint: - type: string - flow: - description: DataFlow indicates the flow of the data between the components Currently we assume this is linear and thus use steps, but other more complex graphs could be defined as per how it is done in argo workflow - properties: - name: - type: string - steps: - items: - description: FlowStep is one step indicates an instance of a module in the blueprint, It includes the name of the module template (spec) and the parameters received by the component instance that is initiated by the orchestrator. - properties: - arguments: - description: Arguments are the input parameters for a specific instance of a module. - properties: - copy: - description: CopyArgs are parameters specific to modules that copy data from one data store to another. - properties: - destination: - description: Destination is the data store to which the data will be copied - properties: - connection: - description: Connection has the relevant details for accesing the data (url, table, ssl, etc.) - type: object - format: - description: Format represents data format (e.g. parquet) as received from catalog connectors - type: string - vault: - description: Holds details for retrieving credentials by the modules from Vault store. - properties: - address: - description: Address is Vault address - type: string - authPath: - description: AuthPath is the path to auth method i.e. kubernetes - type: string - role: - description: Role is the Vault role used for retrieving the credentials - type: string - secretPath: - description: SecretPath is the path of the secret holding the Credentials in Vault - type: string - required: - - address - - authPath - - role - - secretPath - type: object - required: - - connection - - format - - vault - type: object - source: - description: Source is the where the data currently resides - properties: - connection: - description: Connection has the relevant details for accesing the data (url, table, ssl, etc.) - type: object - format: - description: Format represents data format (e.g. parquet) as received from catalog connectors - type: string - vault: - description: Holds details for retrieving credentials by the modules from Vault store. - properties: - address: - description: Address is Vault address - type: string - authPath: - description: AuthPath is the path to auth method i.e. kubernetes - type: string - role: - description: Role is the Vault role used for retrieving the credentials - type: string - secretPath: - description: SecretPath is the path of the secret holding the Credentials in Vault - type: string - required: - - address - - authPath - - role - - secretPath - type: object - required: - - connection - - format - - vault - type: object - transformations: - description: Transformations are different types of processing that may be done to the data as it is copied. - items: - type: object - type: array - required: - - destination - - source - type: object - read: - description: ReadArgs are parameters that are specific to modules that enable an application to read data - items: - description: ReadModuleArgs define the input parameters for modules that read data from location A + versions: + - additionalPrinterColumns: + - jsonPath: .metadata.creationTimestamp + name: Age + type: date + - jsonPath: .status.observedState.ready + name: Ready + type: string + - jsonPath: .status.readyTimestamp + name: ReadySince + type: string + name: v1alpha1 + schema: + openAPIV3Schema: + description: Plotter is the Schema for the plotters API + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + spec: + description: PlotterSpec defines the desired state of Plotter, which is applied in a multi-clustered environment. Plotter installs the runtime environment (as blueprints running on remote clusters) which provides the Data Scientist's application with secure and governed access to the data requested in the M4DApplication. + properties: + blueprints: + additionalProperties: + description: 'BlueprintSpec defines the desired state of Blueprint, which is the runtime environment which provides the Data Scientist''s application with secure and governed access to the data requested in the M4DApplication. The blueprint uses an "argo like" syntax which indicates the components and the flow of data between them as steps TODO: Add an indication of the communication relationships between the components' + properties: + entrypoint: + type: string + flow: + description: DataFlow indicates the flow of the data between the components Currently we assume this is linear and thus use steps, but other more complex graphs could be defined as per how it is done in argo workflow + properties: + name: + type: string + steps: + items: + description: FlowStep is one step indicates an instance of a module in the blueprint, It includes the name of the module template (spec) and the parameters received by the component instance that is initiated by the orchestrator. + properties: + arguments: + description: Arguments are the input parameters for a specific instance of a module. + properties: + copy: + description: CopyArgs are parameters specific to modules that copy data from one data store to another. properties: - assetID: - description: AssetID identifies the asset to be used for accessing the data when it is ready It is copied from the M4DApplication resource - type: string - source: - description: Source of the read path module + destination: + description: Destination is the data store to which the data will be copied properties: connection: description: Connection has the relevant details for accesing the data (url, table, ssl, etc.) type: object + x-kubernetes-preserve-unknown-fields: true format: description: Format represents data format (e.g. parquet) as received from catalog connectors type: string @@ -185,27 +99,13 @@ spec: - format - vault type: object - transformations: - description: Transformations are different types of processing that may be done to the data - items: - type: object - type: array - required: - - assetID - - source - type: object - type: array - write: - description: WriteArgs are parameters that are specific to modules that enable an application to write data - items: - description: WriteModuleArgs define the input parameters for modules that write data to location B - properties: - destination: - description: Destination is the data store to which the data will be written + source: + description: Source is the where the data currently resides properties: connection: description: Connection has the relevant details for accesing the data (url, table, ssl, etc.) type: object + x-kubernetes-preserve-unknown-fields: true format: description: Format represents data format (e.g. parquet) as received from catalog connectors type: string @@ -236,166 +136,278 @@ spec: - vault type: object transformations: - description: Transformations are different types of processing that may be done to the data as it is written. + description: Transformations are different types of processing that may be done to the data as it is copied. items: type: object + x-kubernetes-preserve-unknown-fields: true type: array required: - destination + - source type: object - type: array - type: object - name: - description: Name is the name of the instance of the module. For example, if the application is named "notebook" and an implicitcopy module is deemed necessary. The FlowStep name would be notebook-implicitcopy. - type: string - template: - description: 'Template is the name of the specification in the Blueprint describing how to instantiate a component indicated by the module. It is the name of a M4DModule CRD. For example: implicit-copy-db2wh-to-s3-latest' - type: string - required: - - name - - template - type: object - type: array - required: - - name - - steps - type: object - templates: - items: - description: ComponentTemplate is a copy of a M4DModule Custom Resource. It contains the information necessary to instantiate a component in a FlowStep, which provides the functionality described by the module. There are 3 different module types. - properties: - chart: - description: Chart contains the location of the helm chart with info detailing how to deploy - properties: - name: - description: Name of helm chart - type: string - values: - additionalProperties: + read: + description: ReadArgs are parameters that are specific to modules that enable an application to read data + items: + description: ReadModuleArgs define the input parameters for modules that read data from location A + properties: + assetID: + description: AssetID identifies the asset to be used for accessing the data when it is ready It is copied from the M4DApplication resource + type: string + source: + description: Source of the read path module + properties: + connection: + description: Connection has the relevant details for accesing the data (url, table, ssl, etc.) + type: object + x-kubernetes-preserve-unknown-fields: true + format: + description: Format represents data format (e.g. parquet) as received from catalog connectors + type: string + vault: + description: Holds details for retrieving credentials by the modules from Vault store. + properties: + address: + description: Address is Vault address + type: string + authPath: + description: AuthPath is the path to auth method i.e. kubernetes + type: string + role: + description: Role is the Vault role used for retrieving the credentials + type: string + secretPath: + description: SecretPath is the path of the secret holding the Credentials in Vault + type: string + required: + - address + - authPath + - role + - secretPath + type: object + required: + - connection + - format + - vault + type: object + transformations: + description: Transformations are different types of processing that may be done to the data + items: + type: object + x-kubernetes-preserve-unknown-fields: true + type: array + required: + - assetID + - source + type: object + type: array + write: + description: WriteArgs are parameters that are specific to modules that enable an application to write data + items: + description: WriteModuleArgs define the input parameters for modules that write data to location B + properties: + destination: + description: Destination is the data store to which the data will be written + properties: + connection: + description: Connection has the relevant details for accesing the data (url, table, ssl, etc.) + type: object + x-kubernetes-preserve-unknown-fields: true + format: + description: Format represents data format (e.g. parquet) as received from catalog connectors + type: string + vault: + description: Holds details for retrieving credentials by the modules from Vault store. + properties: + address: + description: Address is Vault address + type: string + authPath: + description: AuthPath is the path to auth method i.e. kubernetes + type: string + role: + description: Role is the Vault role used for retrieving the credentials + type: string + secretPath: + description: SecretPath is the path of the secret holding the Credentials in Vault + type: string + required: + - address + - authPath + - role + - secretPath + type: object + required: + - connection + - format + - vault + type: object + transformations: + description: Transformations are different types of processing that may be done to the data as it is written. + items: + type: object + x-kubernetes-preserve-unknown-fields: true + type: array + required: + - destination + type: object + type: array + type: object + name: + description: Name is the name of the instance of the module. For example, if the application is named "notebook" and an implicitcopy module is deemed necessary. The FlowStep name would be notebook-implicitcopy. type: string - description: Values to pass to helm chart installation - type: object - required: - - name - type: object - kind: - description: Kind of k8s resource - type: string - name: - description: Name of the template - type: string + template: + description: 'Template is the name of the specification in the Blueprint describing how to instantiate a component indicated by the module. It is the name of a M4DModule CRD. For example: implicit-copy-db2wh-to-s3-latest' + type: string + required: + - name + - template + type: object + type: array required: - - chart - - kind - name + - steps type: object - type: array - required: - - entrypoint - - flow - - templates - type: object - description: Blueprints structure represents remote blueprints mapped by the identifier of a cluster in which they will be running - type: object - selector: - description: Selector enables to connect the resource to the application Should match the selector of the owner - M4DApplication CRD. - properties: - matchExpressions: - description: matchExpressions is a list of label selector requirements. The requirements are ANDed. - items: - description: A label selector requirement is a selector that contains values, a key, and an operator that relates the key and values. - properties: - key: - description: key is the label key that the selector applies to. - type: string - operator: - description: operator represents a key's relationship to a set of values. Valid operators are In, NotIn, Exists and DoesNotExist. - type: string - values: - description: values is an array of string values. If the operator is In or NotIn, the values array must be non-empty. If the operator is Exists or DoesNotExist, the values array must be empty. This array is replaced during a strategic merge patch. - items: - type: string - type: array - required: - - key - - operator - type: object - type: array - matchLabels: - additionalProperties: - type: string - description: matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels map is equivalent to an element of matchExpressions, whose key field is "key", the operator is "In", and the values array contains only "value". The requirements are ANDed. - type: object - type: object - required: - - blueprints - - selector - type: object - status: - description: PlotterStatus defines the observed state of Plotter This includes readiness, error message, and indicators received from blueprint resources owned by the Plotter for cleanup and status monitoring - properties: - blueprints: - additionalProperties: - description: MetaBlueprint defines blueprint metadata (name, namespace) and status - properties: - metadata: - type: object - status: - description: BlueprintStatus defines the observed state of Blueprint This includes readiness, error message, and indicators forthe Kubernetes resources owned by the Blueprint for cleanup and status monitoring - properties: - observedGeneration: - description: ObservedGeneration is taken from the Blueprint metadata. This is used to determine during reconcile whether reconcile was called because the desired state changed, or whether status of the allocated resources should be checked. - format: int64 - type: integer - observedState: - description: ObservedState includes information to be reported back to the M4DApplication resource It includes readiness and error indications, as well as user instructions + templates: + items: + description: ComponentTemplate is a copy of a M4DModule Custom Resource. It contains the information necessary to instantiate a component in a FlowStep, which provides the functionality described by the module. There are 3 different module types. properties: - dataAccessInstructions: - description: DataAccessInstructions indicate how the data user or his application may access the data. Instructions are available upon successful orchestration. + chart: + description: Chart contains the location of the helm chart with info detailing how to deploy + properties: + name: + description: Name of helm chart + type: string + values: + additionalProperties: + type: string + description: Values to pass to helm chart installation + type: object + required: + - name + type: object + kind: + description: Kind of k8s resource type: string - error: - description: Error indicates that there has been an error to orchestrate the modules and provides the error message + name: + description: Name of the template type: string - ready: - description: Ready represents that the modules have been orchestrated successfully and the data is ready for usage - type: boolean + required: + - chart + - kind + - name type: object - releases: - additionalProperties: + type: array + required: + - entrypoint + - flow + - templates + type: object + description: Blueprints structure represents remote blueprints mapped by the identifier of a cluster in which they will be running + type: object + selector: + description: Selector enables to connect the resource to the application Should match the selector of the owner - M4DApplication CRD. + properties: + matchExpressions: + description: matchExpressions is a list of label selector requirements. The requirements are ANDed. + items: + description: A label selector requirement is a selector that contains values, a key, and an operator that relates the key and values. + properties: + key: + description: key is the label key that the selector applies to. + type: string + operator: + description: operator represents a key's relationship to a set of values. Valid operators are In, NotIn, Exists and DoesNotExist. + type: string + values: + description: values is an array of string values. If the operator is In or NotIn, the values array must be non-empty. If the operator is Exists or DoesNotExist, the values array must be empty. This array is replaced during a strategic merge patch. + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchLabels: + additionalProperties: + type: string + description: matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels map is equivalent to an element of matchExpressions, whose key field is "key", the operator is "In", and the values array contains only "value". The requirements are ANDed. + type: object + type: object + required: + - blueprints + - selector + type: object + status: + description: PlotterStatus defines the observed state of Plotter This includes readiness, error message, and indicators received from blueprint resources owned by the Plotter for cleanup and status monitoring + properties: + blueprints: + additionalProperties: + description: MetaBlueprint defines blueprint metadata (name, namespace) and status + properties: + name: + type: string + namespace: + type: string + status: + description: BlueprintStatus defines the observed state of Blueprint This includes readiness, error message, and indicators forthe Kubernetes resources owned by the Blueprint for cleanup and status monitoring + properties: + observedGeneration: + description: ObservedGeneration is taken from the Blueprint metadata. This is used to determine during reconcile whether reconcile was called because the desired state changed, or whether status of the allocated resources should be checked. format: int64 type: integer - description: Releases map each release to the observed generation of the blueprint containing this release. At the end of reconcile, each release should be mapped to the latest blueprint version or be uninstalled. - type: object - type: object + observedState: + description: ObservedState includes information to be reported back to the M4DApplication resource It includes readiness and error indications, as well as user instructions + properties: + dataAccessInstructions: + description: DataAccessInstructions indicate how the data user or his application may access the data. Instructions are available upon successful orchestration. + type: string + error: + description: Error indicates that there has been an error to orchestrate the modules and provides the error message + type: string + ready: + description: Ready represents that the modules have been orchestrated successfully and the data is ready for usage + type: boolean + type: object + releases: + additionalProperties: + format: int64 + type: integer + description: Releases map each release to the observed generation of the blueprint containing this release. At the end of reconcile, each release should be mapped to the latest blueprint version or be uninstalled. + type: object + type: object + required: + - name + - namespace + - status + type: object type: object - type: object - observedGeneration: - description: ObservedGeneration is taken from the Plotter metadata. This is used to determine during reconcile whether reconcile was called because the desired state changed, or whether status of the allocated blueprints should be checked. - format: int64 - type: integer - observedState: - description: ObservedState includes information to be reported back to the M4DApplication resource It includes readiness and error indications, as well as user instructions - properties: - dataAccessInstructions: - description: DataAccessInstructions indicate how the data user or his application may access the data. Instructions are available upon successful orchestration. - type: string - error: - description: Error indicates that there has been an error to orchestrate the modules and provides the error message - type: string - ready: - description: Ready represents that the modules have been orchestrated successfully and the data is ready for usage - type: boolean - type: object - readyTimestamp: - format: date-time - type: string - type: object - type: object - version: v1alpha1 - versions: - - name: v1alpha1 + observedGeneration: + description: ObservedGeneration is taken from the Plotter metadata. This is used to determine during reconcile whether reconcile was called because the desired state changed, or whether status of the allocated blueprints should be checked. + format: int64 + type: integer + observedState: + description: ObservedState includes information to be reported back to the M4DApplication resource It includes readiness and error indications, as well as user instructions + properties: + dataAccessInstructions: + description: DataAccessInstructions indicate how the data user or his application may access the data. Instructions are available upon successful orchestration. + type: string + error: + description: Error indicates that there has been an error to orchestrate the modules and provides the error message + type: string + ready: + description: Ready represents that the modules have been orchestrated successfully and the data is ready for usage + type: boolean + type: object + readyTimestamp: + format: date-time + type: string + type: object + type: object served: true storage: true + subresources: + status: {} status: acceptedNames: kind: "" diff --git a/charts/m4d-crd/templates/motion.m4d.ibm.com_batchtransfers.yaml b/charts/m4d-crd/templates/motion.m4d.ibm.com_batchtransfers.yaml index 32ae81d..ef291ec 100644 --- a/charts/m4d-crd/templates/motion.m4d.ibm.com_batchtransfers.yaml +++ b/charts/m4d-crd/templates/motion.m4d.ibm.com_batchtransfers.yaml @@ -1,26 +1,13 @@ --- -apiVersion: apiextensions.k8s.io/v1beta1 +apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: - controller-gen.kubebuilder.io/version: v0.3.0 + controller-gen.kubebuilder.io/version: v0.5.0 creationTimestamp: null name: batchtransfers.motion.m4d.ibm.com spec: - additionalPrinterColumns: - - JSONPath: .spec.source.description - name: Source - type: string - - JSONPath: .spec.destination.description - name: Destination - type: string - - JSONPath: .spec.schedule - name: Schedule - type: string - - JSONPath: .status.status - name: Status - type: string group: motion.m4d.ibm.com names: kind: BatchTransfer @@ -28,742 +15,732 @@ spec: plural: batchtransfers singular: batchtransfer scope: Namespaced - subresources: - status: {} - validation: - openAPIV3Schema: - description: BatchTransfer is the Schema for the batchtransfers API - properties: - apiVersion: - description: 'APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' - type: string - kind: - description: 'Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' - type: string - metadata: - type: object - spec: - description: BatchTransferSpec defines the state of a BatchTransfer. The state includes source/destination specification, a schedule and the means by which data movement is to be conducted. The means is given as a kubernetes job description. In addition, the state also contains a sketch of a transformation instruction. In future releases, the transformation description should be specified in a separate CRD. - properties: - destination: - description: Destination data store for this batch job - properties: - cloudant: - description: IBM Cloudant. Needs cloudant legacy credentials. - properties: - database: - description: Database to be read from/written to - type: string - host: - description: Host of cloudant instance - type: string - password: - description: Cloudant password. Can be retrieved from vault if specified in vaultPath parameter and is thus optional. - type: string - secretImport: - description: Define a secret import definition. - type: string - username: - description: Cloudant user. Can be retrieved from vault if specified in vaultPath parameter and is thus optional. - type: string - vault: - description: Define secrets that are fetched from a Vault instance - properties: - address: - description: Address is Vault address - type: string - authPath: - description: AuthPath is the path to auth method i.e. kubernetes - type: string - role: - description: Role is the Vault role used for retrieving the credentials - type: string - secretPath: - description: SecretPath is the path of the secret holding the Credentials in Vault - type: string - required: - - address - - authPath - - role - - secretPath - type: object - vaultPath: - description: Vault path where the user name/password are stored. If not specified user and password have to be specified! - type: string - required: - - database - - host - type: object - database: - description: Database data store. For the moment only Db2 is supported. - properties: - db2URL: - description: URL to Db2 instance in JDBC format Supported SSL certificates are currently certificates signed with IBM Intermediate CA or cloud signed certificates. - type: string - password: - description: Database password. Can be retrieved from vault if specified in vaultPath parameter and is thus optional. - type: string - secretImport: - description: Define a secret import definition. - type: string - table: - description: Table to be read - type: string - user: - description: Database user. Can be retrieved from vault if specified in vaultPath parameter and is thus optional. - type: string - vault: - description: Define secrets that are fetched from a Vault instance - properties: - address: - description: Address is Vault address - type: string - authPath: - description: AuthPath is the path to auth method i.e. kubernetes - type: string - role: - description: Role is the Vault role used for retrieving the credentials - type: string - secretPath: - description: SecretPath is the path of the secret holding the Credentials in Vault - type: string - required: - - address - - authPath - - role - - secretPath - type: object - vaultPath: - description: Vault path where the user name/password are stored. If not specified user and password have to be specified! - type: string - required: - - db2URL - - table - type: object - description: - description: Description of the transfer in human readable form that is displayed in the kubectl get If not provided this will be filled in depending on the datastore that is specified. - type: string - kafka: - description: Kafka data store. The supposed format within the given Kafka topic is a Confluent compatible format stored as Avro. A schema registry needs to be specified as well. - properties: - createSnapshot: - description: 'If a snapshot should be created of the topic. Records in Kafka are stored as key-value pairs. Updates/Deletes for the same key are appended to the Kafka topic and the last value for a given key is the valid key in a Snapshot. When this property is true only the last value will be written. If the property is false all values will be written out. As a CDC example: If the property is true a valid snapshot of the log stream will be created. If the property is false the CDC stream will be dumped as is like a change log.' - type: boolean - dataFormat: - description: Data format of the objects in S3. e.g. parquet or csv. Please refer to struct for allowed values. - type: string - kafkaBrokers: - description: Kafka broker URLs as a comma separated list. - type: string - kafkaTopic: - description: Kafka topic - type: string - keyDeserializer: - description: Deserializer to be used for the keys of the topic - type: string - password: - description: Kafka user password Can be retrieved from vault if specified in vaultPath parameter and is thus optional. - type: string - saslMechanism: - description: SASL Mechanism to be used (e.g. PLAIN or SCRAM-SHA-512) Default SCRAM-SHA-512 will be assumed if not specified - type: string - schemaRegistryURL: - description: URL to the schema registry. The registry has to be Confluent schema registry compatible. - type: string - secretImport: - description: Define a secret import definition. - type: string - securityProtocol: - description: Kafka security protocol one of (PLAINTEXT, SASL_PLAINTEXT, SASL_SSL, SSL) Default SASL_SSL will be assumed if not specified - type: string - sslTruststore: - description: A truststore or certificate encoded as base64. The format can be JKS or PKCS12. A truststore can be specified like this or in a predefined Kubernetes secret - type: string - sslTruststoreLocation: - description: SSL truststore location. - type: string - sslTruststorePassword: - description: SSL truststore password. - type: string - sslTruststoreSecret: - description: Kubernetes secret that contains the SSL truststore. The format can be JKS or PKCS12. A truststore can be specified like this or as - type: string - user: - description: Kafka user name. Can be retrieved from vault if specified in vaultPath parameter and is thus optional. - type: string - valueDeserializer: - description: Deserializer to be used for the values of the topic - type: string - vault: - description: Define secrets that are fetched from a Vault instance - properties: - address: - description: Address is Vault address - type: string - authPath: - description: AuthPath is the path to auth method i.e. kubernetes - type: string - role: - description: Role is the Vault role used for retrieving the credentials - type: string - secretPath: - description: SecretPath is the path of the secret holding the Credentials in Vault - type: string - required: - - address - - authPath - - role - - secretPath - type: object - vaultPath: - description: Vault path where the user name/password are stored. If not specified user and password have to be specified! - type: string - required: - - kafkaBrokers - - kafkaTopic - - schemaRegistryURL - type: object - s3: - description: An object store data store that is compatible with S3. This can be a COS bucket. - properties: - accessKey: - description: Access key of the HMAC credentials that can access the given bucket. Can be retrieved from vault if specified in vaultPath parameter and is thus optional. - type: string - bucket: - description: Bucket of S3 service - type: string - dataFormat: - description: Data format of the objects in S3. e.g. parquet or csv. Please refer to struct for allowed values. - type: string - endpoint: - description: Endpoint of S3 service - type: string - objectKey: - description: Object key of the object in S3. This is used as a prefix! Thus all objects that have the given objectKey as prefix will be used as input! - type: string - partitionBy: - description: Partition by partition (for target data stores) Defines the columns to partition the output by for a target data store. - items: + versions: + - additionalPrinterColumns: + - jsonPath: .spec.source.description + name: Source + type: string + - jsonPath: .spec.destination.description + name: Destination + type: string + - jsonPath: .spec.schedule + name: Schedule + type: string + - jsonPath: .status.status + name: Status + type: string + name: v1alpha1 + schema: + openAPIV3Schema: + description: BatchTransfer is the Schema for the batchtransfers API + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + spec: + description: BatchTransferSpec defines the state of a BatchTransfer. The state includes source/destination specification, a schedule and the means by which data movement is to be conducted. The means is given as a kubernetes job description. In addition, the state also contains a sketch of a transformation instruction. In future releases, the transformation description should be specified in a separate CRD. + properties: + destination: + description: Destination data store for this batch job + properties: + cloudant: + description: IBM Cloudant. Needs cloudant legacy credentials. + properties: + database: + description: Database to be read from/written to type: string - type: array - region: - description: Region of S3 service - type: string - secretImport: - description: Define a secret import definition. - type: string - secretKey: - description: Secret key of the HMAC credentials that can access the given bucket. Can be retrieved from vault if specified in vaultPath parameter and is thus optional. - type: string - vault: - description: Define secrets that are fetched from a Vault instance - properties: - address: - description: Address is Vault address - type: string - authPath: - description: AuthPath is the path to auth method i.e. kubernetes - type: string - role: - description: Role is the Vault role used for retrieving the credentials - type: string - secretPath: - description: SecretPath is the path of the secret holding the Credentials in Vault - type: string - required: - - address - - authPath - - role - - secretPath - type: object - vaultPath: - description: Vault path where the accessKey/secretKey are stored. If not specified accessKey and secretKey have to be specified! - type: string - required: - - bucket - - endpoint - - objectKey - type: object - type: object - failedJobHistoryLimit: - description: Maximal number of failed Kubernetes job objects that should be kept. This property will be defaulted by the webhook if not set. - maximum: 20 - minimum: 0 - type: integer - flowType: - description: Data flow type that specifies if this is a stream or a batch workflow - enum: - - Batch - - Stream - type: string - image: - description: Image that should be used for the actual batch job. This is usually a datamover image. This property will be defaulted by the webhook if not set. - type: string - imagePullPolicy: - description: Image pull policy that should be used for the actual job. This property will be defaulted by the webhook if not set. - type: string - maxFailedRetries: - description: Maximal number of failed retries until the batch job should stop trying. This property will be defaulted by the webhook if not set. - maximum: 10 - minimum: 0 - type: integer - noFinalizer: - description: If this batch job instance should have a finalizer or not. This property will be defaulted by the webhook if not set. - type: boolean - readDataType: - description: Data type of the data that is read from source (log data or change data) - enum: - - LogData - - ChangeData - type: string - schedule: - description: Cron schedule if this BatchTransfer job should run on a regular schedule. Values are specified like cron job schedules. A good translation to human language can be found here https://crontab.guru/ - type: string - secretProviderRole: - description: Secret provider role that should be used for the actual job. This property will be defaulted by the webhook if not set. - type: string - secretProviderURL: - description: Secret provider url that should be used for the actual job. This property will be defaulted by the webhook if not set. - type: string - source: - description: Source data store for this batch job - properties: - cloudant: - description: IBM Cloudant. Needs cloudant legacy credentials. - properties: - database: - description: Database to be read from/written to - type: string - host: - description: Host of cloudant instance - type: string - password: - description: Cloudant password. Can be retrieved from vault if specified in vaultPath parameter and is thus optional. - type: string - secretImport: - description: Define a secret import definition. - type: string - username: - description: Cloudant user. Can be retrieved from vault if specified in vaultPath parameter and is thus optional. - type: string - vault: - description: Define secrets that are fetched from a Vault instance - properties: - address: - description: Address is Vault address - type: string - authPath: - description: AuthPath is the path to auth method i.e. kubernetes - type: string - role: - description: Role is the Vault role used for retrieving the credentials - type: string - secretPath: - description: SecretPath is the path of the secret holding the Credentials in Vault - type: string - required: - - address - - authPath - - role - - secretPath - type: object - vaultPath: - description: Vault path where the user name/password are stored. If not specified user and password have to be specified! - type: string - required: - - database - - host - type: object - database: - description: Database data store. For the moment only Db2 is supported. - properties: - db2URL: - description: URL to Db2 instance in JDBC format Supported SSL certificates are currently certificates signed with IBM Intermediate CA or cloud signed certificates. - type: string - password: - description: Database password. Can be retrieved from vault if specified in vaultPath parameter and is thus optional. - type: string - secretImport: - description: Define a secret import definition. - type: string - table: - description: Table to be read - type: string - user: - description: Database user. Can be retrieved from vault if specified in vaultPath parameter and is thus optional. - type: string - vault: - description: Define secrets that are fetched from a Vault instance - properties: - address: - description: Address is Vault address - type: string - authPath: - description: AuthPath is the path to auth method i.e. kubernetes - type: string - role: - description: Role is the Vault role used for retrieving the credentials - type: string - secretPath: - description: SecretPath is the path of the secret holding the Credentials in Vault - type: string - required: - - address - - authPath - - role - - secretPath - type: object - vaultPath: - description: Vault path where the user name/password are stored. If not specified user and password have to be specified! - type: string - required: - - db2URL - - table - type: object - description: - description: Description of the transfer in human readable form that is displayed in the kubectl get If not provided this will be filled in depending on the datastore that is specified. - type: string - kafka: - description: Kafka data store. The supposed format within the given Kafka topic is a Confluent compatible format stored as Avro. A schema registry needs to be specified as well. - properties: - createSnapshot: - description: 'If a snapshot should be created of the topic. Records in Kafka are stored as key-value pairs. Updates/Deletes for the same key are appended to the Kafka topic and the last value for a given key is the valid key in a Snapshot. When this property is true only the last value will be written. If the property is false all values will be written out. As a CDC example: If the property is true a valid snapshot of the log stream will be created. If the property is false the CDC stream will be dumped as is like a change log.' - type: boolean - dataFormat: - description: Data format of the objects in S3. e.g. parquet or csv. Please refer to struct for allowed values. - type: string - kafkaBrokers: - description: Kafka broker URLs as a comma separated list. - type: string - kafkaTopic: - description: Kafka topic - type: string - keyDeserializer: - description: Deserializer to be used for the keys of the topic - type: string - password: - description: Kafka user password Can be retrieved from vault if specified in vaultPath parameter and is thus optional. - type: string - saslMechanism: - description: SASL Mechanism to be used (e.g. PLAIN or SCRAM-SHA-512) Default SCRAM-SHA-512 will be assumed if not specified - type: string - schemaRegistryURL: - description: URL to the schema registry. The registry has to be Confluent schema registry compatible. - type: string - secretImport: - description: Define a secret import definition. - type: string - securityProtocol: - description: Kafka security protocol one of (PLAINTEXT, SASL_PLAINTEXT, SASL_SSL, SSL) Default SASL_SSL will be assumed if not specified - type: string - sslTruststore: - description: A truststore or certificate encoded as base64. The format can be JKS or PKCS12. A truststore can be specified like this or in a predefined Kubernetes secret - type: string - sslTruststoreLocation: - description: SSL truststore location. - type: string - sslTruststorePassword: - description: SSL truststore password. - type: string - sslTruststoreSecret: - description: Kubernetes secret that contains the SSL truststore. The format can be JKS or PKCS12. A truststore can be specified like this or as - type: string - user: - description: Kafka user name. Can be retrieved from vault if specified in vaultPath parameter and is thus optional. - type: string - valueDeserializer: - description: Deserializer to be used for the values of the topic - type: string - vault: - description: Define secrets that are fetched from a Vault instance - properties: - address: - description: Address is Vault address - type: string - authPath: - description: AuthPath is the path to auth method i.e. kubernetes - type: string - role: - description: Role is the Vault role used for retrieving the credentials + host: + description: Host of cloudant instance + type: string + password: + description: Cloudant password. Can be retrieved from vault if specified in vault parameter and is thus optional. + type: string + secretImport: + description: Define a secret import definition. + type: string + username: + description: Cloudant user. Can be retrieved from vault if specified in vault parameter and is thus optional. + type: string + vault: + description: Define secrets that are fetched from a Vault instance + properties: + address: + description: Address is Vault address + type: string + authPath: + description: AuthPath is the path to auth method i.e. kubernetes + type: string + role: + description: Role is the Vault role used for retrieving the credentials + type: string + secretPath: + description: SecretPath is the path of the secret holding the Credentials in Vault + type: string + required: + - address + - authPath + - role + - secretPath + type: object + required: + - database + - host + type: object + database: + description: Database data store. For the moment only Db2 is supported. + properties: + db2URL: + description: URL to Db2 instance in JDBC format Supported SSL certificates are currently certificates signed with IBM Intermediate CA or cloud signed certificates. + type: string + password: + description: Database password. Can be retrieved from vault if specified in vault parameter and is thus optional. + type: string + secretImport: + description: Define a secret import definition. + type: string + table: + description: Table to be read + type: string + user: + description: Database user. Can be retrieved from vault if specified in vault parameter and is thus optional. + type: string + vault: + description: Define secrets that are fetched from a Vault instance + properties: + address: + description: Address is Vault address + type: string + authPath: + description: AuthPath is the path to auth method i.e. kubernetes + type: string + role: + description: Role is the Vault role used for retrieving the credentials + type: string + secretPath: + description: SecretPath is the path of the secret holding the Credentials in Vault + type: string + required: + - address + - authPath + - role + - secretPath + type: object + required: + - db2URL + - table + type: object + description: + description: Description of the transfer in human readable form that is displayed in the kubectl get If not provided this will be filled in depending on the datastore that is specified. + type: string + kafka: + description: Kafka data store. The supposed format within the given Kafka topic is a Confluent compatible format stored as Avro. A schema registry needs to be specified as well. + properties: + createSnapshot: + description: 'If a snapshot should be created of the topic. Records in Kafka are stored as key-value pairs. Updates/Deletes for the same key are appended to the Kafka topic and the last value for a given key is the valid key in a Snapshot. When this property is true only the last value will be written. If the property is false all values will be written out. As a CDC example: If the property is true a valid snapshot of the log stream will be created. If the property is false the CDC stream will be dumped as is like a change log.' + type: boolean + dataFormat: + description: Data format of the objects in S3. e.g. parquet or csv. Please refer to struct for allowed values. + type: string + kafkaBrokers: + description: Kafka broker URLs as a comma separated list. + type: string + kafkaTopic: + description: Kafka topic + type: string + keyDeserializer: + description: Deserializer to be used for the keys of the topic + type: string + password: + description: Kafka user password Can be retrieved from vault if specified in vault parameter and is thus optional. + type: string + saslMechanism: + description: SASL Mechanism to be used (e.g. PLAIN or SCRAM-SHA-512) Default SCRAM-SHA-512 will be assumed if not specified + type: string + schemaRegistryURL: + description: URL to the schema registry. The registry has to be Confluent schema registry compatible. + type: string + secretImport: + description: Define a secret import definition. + type: string + securityProtocol: + description: Kafka security protocol one of (PLAINTEXT, SASL_PLAINTEXT, SASL_SSL, SSL) Default SASL_SSL will be assumed if not specified + type: string + sslTruststore: + description: A truststore or certificate encoded as base64. The format can be JKS or PKCS12. A truststore can be specified like this or in a predefined Kubernetes secret + type: string + sslTruststoreLocation: + description: SSL truststore location. + type: string + sslTruststorePassword: + description: SSL truststore password. + type: string + sslTruststoreSecret: + description: Kubernetes secret that contains the SSL truststore. The format can be JKS or PKCS12. A truststore can be specified like this or as + type: string + user: + description: Kafka user name. Can be retrieved from vault if specified in vault parameter and is thus optional. + type: string + valueDeserializer: + description: Deserializer to be used for the values of the topic + type: string + vault: + description: Define secrets that are fetched from a Vault instance + properties: + address: + description: Address is Vault address + type: string + authPath: + description: AuthPath is the path to auth method i.e. kubernetes + type: string + role: + description: Role is the Vault role used for retrieving the credentials + type: string + secretPath: + description: SecretPath is the path of the secret holding the Credentials in Vault + type: string + required: + - address + - authPath + - role + - secretPath + type: object + required: + - kafkaBrokers + - kafkaTopic + - schemaRegistryURL + type: object + s3: + description: An object store data store that is compatible with S3. This can be a COS bucket. + properties: + accessKey: + description: Access key of the HMAC credentials that can access the given bucket. Can be retrieved from vault if specified in vault parameter and is thus optional. + type: string + bucket: + description: Bucket of S3 service + type: string + dataFormat: + description: Data format of the objects in S3. e.g. parquet or csv. Please refer to struct for allowed values. + type: string + endpoint: + description: Endpoint of S3 service + type: string + objectKey: + description: Object key of the object in S3. This is used as a prefix! Thus all objects that have the given objectKey as prefix will be used as input! + type: string + partitionBy: + description: Partition by partition (for target data stores) Defines the columns to partition the output by for a target data store. + items: type: string - secretPath: - description: SecretPath is the path of the secret holding the Credentials in Vault + type: array + region: + description: Region of S3 service + type: string + secretImport: + description: Define a secret import definition. + type: string + secretKey: + description: Secret key of the HMAC credentials that can access the given bucket. Can be retrieved from vault if specified in vault parameter and is thus optional. + type: string + vault: + description: Define secrets that are fetched from a Vault instance + properties: + address: + description: Address is Vault address + type: string + authPath: + description: AuthPath is the path to auth method i.e. kubernetes + type: string + role: + description: Role is the Vault role used for retrieving the credentials + type: string + secretPath: + description: SecretPath is the path of the secret holding the Credentials in Vault + type: string + required: + - address + - authPath + - role + - secretPath + type: object + required: + - bucket + - endpoint + - objectKey + type: object + type: object + failedJobHistoryLimit: + description: Maximal number of failed Kubernetes job objects that should be kept. This property will be defaulted by the webhook if not set. + maximum: 20 + minimum: 0 + type: integer + flowType: + description: Data flow type that specifies if this is a stream or a batch workflow + enum: + - Batch + - Stream + type: string + image: + description: Image that should be used for the actual batch job. This is usually a datamover image. This property will be defaulted by the webhook if not set. + type: string + imagePullPolicy: + description: Image pull policy that should be used for the actual job. This property will be defaulted by the webhook if not set. + type: string + maxFailedRetries: + description: Maximal number of failed retries until the batch job should stop trying. This property will be defaulted by the webhook if not set. + maximum: 10 + minimum: 0 + type: integer + noFinalizer: + description: If this batch job instance should have a finalizer or not. This property will be defaulted by the webhook if not set. + type: boolean + readDataType: + description: Data type of the data that is read from source (log data or change data) + enum: + - LogData + - ChangeData + type: string + schedule: + description: Cron schedule if this BatchTransfer job should run on a regular schedule. Values are specified like cron job schedules. A good translation to human language can be found here https://crontab.guru/ + type: string + secretProviderRole: + description: Secret provider role that should be used for the actual job. This property will be defaulted by the webhook if not set. + type: string + secretProviderURL: + description: Secret provider url that should be used for the actual job. This property will be defaulted by the webhook if not set. + type: string + source: + description: Source data store for this batch job + properties: + cloudant: + description: IBM Cloudant. Needs cloudant legacy credentials. + properties: + database: + description: Database to be read from/written to + type: string + host: + description: Host of cloudant instance + type: string + password: + description: Cloudant password. Can be retrieved from vault if specified in vault parameter and is thus optional. + type: string + secretImport: + description: Define a secret import definition. + type: string + username: + description: Cloudant user. Can be retrieved from vault if specified in vault parameter and is thus optional. + type: string + vault: + description: Define secrets that are fetched from a Vault instance + properties: + address: + description: Address is Vault address + type: string + authPath: + description: AuthPath is the path to auth method i.e. kubernetes + type: string + role: + description: Role is the Vault role used for retrieving the credentials + type: string + secretPath: + description: SecretPath is the path of the secret holding the Credentials in Vault + type: string + required: + - address + - authPath + - role + - secretPath + type: object + required: + - database + - host + type: object + database: + description: Database data store. For the moment only Db2 is supported. + properties: + db2URL: + description: URL to Db2 instance in JDBC format Supported SSL certificates are currently certificates signed with IBM Intermediate CA or cloud signed certificates. + type: string + password: + description: Database password. Can be retrieved from vault if specified in vault parameter and is thus optional. + type: string + secretImport: + description: Define a secret import definition. + type: string + table: + description: Table to be read + type: string + user: + description: Database user. Can be retrieved from vault if specified in vault parameter and is thus optional. + type: string + vault: + description: Define secrets that are fetched from a Vault instance + properties: + address: + description: Address is Vault address + type: string + authPath: + description: AuthPath is the path to auth method i.e. kubernetes + type: string + role: + description: Role is the Vault role used for retrieving the credentials + type: string + secretPath: + description: SecretPath is the path of the secret holding the Credentials in Vault + type: string + required: + - address + - authPath + - role + - secretPath + type: object + required: + - db2URL + - table + type: object + description: + description: Description of the transfer in human readable form that is displayed in the kubectl get If not provided this will be filled in depending on the datastore that is specified. + type: string + kafka: + description: Kafka data store. The supposed format within the given Kafka topic is a Confluent compatible format stored as Avro. A schema registry needs to be specified as well. + properties: + createSnapshot: + description: 'If a snapshot should be created of the topic. Records in Kafka are stored as key-value pairs. Updates/Deletes for the same key are appended to the Kafka topic and the last value for a given key is the valid key in a Snapshot. When this property is true only the last value will be written. If the property is false all values will be written out. As a CDC example: If the property is true a valid snapshot of the log stream will be created. If the property is false the CDC stream will be dumped as is like a change log.' + type: boolean + dataFormat: + description: Data format of the objects in S3. e.g. parquet or csv. Please refer to struct for allowed values. + type: string + kafkaBrokers: + description: Kafka broker URLs as a comma separated list. + type: string + kafkaTopic: + description: Kafka topic + type: string + keyDeserializer: + description: Deserializer to be used for the keys of the topic + type: string + password: + description: Kafka user password Can be retrieved from vault if specified in vault parameter and is thus optional. + type: string + saslMechanism: + description: SASL Mechanism to be used (e.g. PLAIN or SCRAM-SHA-512) Default SCRAM-SHA-512 will be assumed if not specified + type: string + schemaRegistryURL: + description: URL to the schema registry. The registry has to be Confluent schema registry compatible. + type: string + secretImport: + description: Define a secret import definition. + type: string + securityProtocol: + description: Kafka security protocol one of (PLAINTEXT, SASL_PLAINTEXT, SASL_SSL, SSL) Default SASL_SSL will be assumed if not specified + type: string + sslTruststore: + description: A truststore or certificate encoded as base64. The format can be JKS or PKCS12. A truststore can be specified like this or in a predefined Kubernetes secret + type: string + sslTruststoreLocation: + description: SSL truststore location. + type: string + sslTruststorePassword: + description: SSL truststore password. + type: string + sslTruststoreSecret: + description: Kubernetes secret that contains the SSL truststore. The format can be JKS or PKCS12. A truststore can be specified like this or as + type: string + user: + description: Kafka user name. Can be retrieved from vault if specified in vault parameter and is thus optional. + type: string + valueDeserializer: + description: Deserializer to be used for the values of the topic + type: string + vault: + description: Define secrets that are fetched from a Vault instance + properties: + address: + description: Address is Vault address + type: string + authPath: + description: AuthPath is the path to auth method i.e. kubernetes + type: string + role: + description: Role is the Vault role used for retrieving the credentials + type: string + secretPath: + description: SecretPath is the path of the secret holding the Credentials in Vault + type: string + required: + - address + - authPath + - role + - secretPath + type: object + required: + - kafkaBrokers + - kafkaTopic + - schemaRegistryURL + type: object + s3: + description: An object store data store that is compatible with S3. This can be a COS bucket. + properties: + accessKey: + description: Access key of the HMAC credentials that can access the given bucket. Can be retrieved from vault if specified in vault parameter and is thus optional. + type: string + bucket: + description: Bucket of S3 service + type: string + dataFormat: + description: Data format of the objects in S3. e.g. parquet or csv. Please refer to struct for allowed values. + type: string + endpoint: + description: Endpoint of S3 service + type: string + objectKey: + description: Object key of the object in S3. This is used as a prefix! Thus all objects that have the given objectKey as prefix will be used as input! + type: string + partitionBy: + description: Partition by partition (for target data stores) Defines the columns to partition the output by for a target data store. + items: type: string - required: - - address - - authPath - - role - - secretPath - type: object - vaultPath: - description: Vault path where the user name/password are stored. If not specified user and password have to be specified! + type: array + region: + description: Region of S3 service + type: string + secretImport: + description: Define a secret import definition. + type: string + secretKey: + description: Secret key of the HMAC credentials that can access the given bucket. Can be retrieved from vault if specified in vault parameter and is thus optional. + type: string + vault: + description: Define secrets that are fetched from a Vault instance + properties: + address: + description: Address is Vault address + type: string + authPath: + description: AuthPath is the path to auth method i.e. kubernetes + type: string + role: + description: Role is the Vault role used for retrieving the credentials + type: string + secretPath: + description: SecretPath is the path of the secret holding the Credentials in Vault + type: string + required: + - address + - authPath + - role + - secretPath + type: object + required: + - bucket + - endpoint + - objectKey + type: object + type: object + spark: + description: Optional Spark configuration for tuning + properties: + appName: + description: Name of the transaction. Mainly used for debugging and lineage tracking. + type: string + driverCores: + description: Number of cores that the driver should use + type: integer + driverMemory: + description: Memory that the driver should have + type: integer + executorCores: + description: Number of cores that each executor should have + type: integer + executorMemory: + description: Memory that each executor should have + type: string + image: + description: Image to be used for executors + type: string + imagePullPolicy: + description: Image pull policy to be used for executor + type: string + numExecutors: + description: Number of executors to be started + type: integer + options: + additionalProperties: type: string - required: - - kafkaBrokers - - kafkaTopic - - schemaRegistryURL - type: object - s3: - description: An object store data store that is compatible with S3. This can be a COS bucket. + description: Additional options for Spark configuration. + type: object + x-kubernetes-preserve-unknown-fields: true + shufflePartitions: + description: Number of shuffle partitions for Spark + type: integer + type: object + successfulJobHistoryLimit: + description: Maximal number of successful Kubernetes job objects that should be kept. This property will be defaulted by the webhook if not set. + maximum: 20 + minimum: 0 + type: integer + suspend: + description: If this batch job instance is run on a schedule the regular schedule can be suspended with this property. This property will be defaulted by the webhook if not set. + type: boolean + transformation: + description: Transformations to be applied to the source data before writing to destination + items: + description: to be refined... properties: - accessKey: - description: Access key of the HMAC credentials that can access the given bucket. Can be retrieved from vault if specified in vaultPath parameter and is thus optional. - type: string - bucket: - description: Bucket of S3 service - type: string - dataFormat: - description: Data format of the objects in S3. e.g. parquet or csv. Please refer to struct for allowed values. - type: string - endpoint: - description: Endpoint of S3 service - type: string - objectKey: - description: Object key of the object in S3. This is used as a prefix! Thus all objects that have the given objectKey as prefix will be used as input! - type: string - partitionBy: - description: Partition by partition (for target data stores) Defines the columns to partition the output by for a target data store. + action: + description: Transformation action that should be performed. + enum: + - RemoveColumns + - EncryptColumns + - DigestColumns + - RedactColumns + - SampleRows + - FilterRows + type: string + columns: + description: Columns that are involved in this action. This property is optional as for some actions no columns have to be specified. E.g. filter is a row based transformation. items: type: string type: array - region: - description: Region of S3 service - type: string - secretImport: - description: Define a secret import definition. - type: string - secretKey: - description: Secret key of the HMAC credentials that can access the given bucket. Can be retrieved from vault if specified in vaultPath parameter and is thus optional. + name: + description: Name of the transaction. Mainly used for debugging and lineage tracking. type: string - vault: - description: Define secrets that are fetched from a Vault instance - properties: - address: - description: Address is Vault address - type: string - authPath: - description: AuthPath is the path to auth method i.e. kubernetes - type: string - role: - description: Role is the Vault role used for retrieving the credentials - type: string - secretPath: - description: SecretPath is the path of the secret holding the Credentials in Vault - type: string - required: - - address - - authPath - - role - - secretPath + options: + additionalProperties: + type: string + description: Additional options for this transformation. type: object - vaultPath: - description: Vault path where the accessKey/secretKey are stored. If not specified accessKey and secretKey have to be specified! - type: string - required: - - bucket - - endpoint - - objectKey + x-kubernetes-preserve-unknown-fields: true type: object - type: object - spark: - description: Optional Spark configuration for tuning - properties: - appName: - description: Name of the transaction. Mainly used for debugging and lineage tracking. - type: string - driverCores: - description: Number of cores that the driver should use - type: integer - driverMemory: - description: Memory that the driver should have - type: integer - executorCores: - description: Number of cores that each executor should have - type: integer - executorMemory: - description: Memory that each executor should have - type: string - image: - description: Image to be used for executors - type: string - imagePullPolicy: - description: Image pull policy to be used for executor - type: string - numExecutors: - description: Number of executors to be started - type: integer - options: - additionalProperties: + type: array + writeDataType: + description: Data type of how the data should be written to the target (log data or change data) + enum: + - LogData + - ChangeData + type: string + writeOperation: + description: 'Write operation that should be performed when writing (overwrite,append,update) Caution: Some write operations are only available for batch and some only for stream.' + enum: + - Overwrite + - Append + - Update + type: string + required: + - destination + - source + type: object + status: + description: 'BatchTransferStatus defines the observed state of BatchTransfer This includes a reference to the job that implements the movement as well as the last schedule time. What is missing: Extended status information such as: - number of records moved - technical meta-data' + properties: + active: + description: A pointer to the currently running job (or nil) + properties: + apiVersion: + description: API version of the referent. type: string - description: Additional options for Spark configuration. - type: object - shufflePartitions: - description: Number of shuffle partitions for Spark - type: integer - type: object - successfulJobHistoryLimit: - description: Maximal number of successful Kubernetes job objects that should be kept. This property will be defaulted by the webhook if not set. - maximum: 20 - minimum: 0 - type: integer - suspend: - description: If this batch job instance is run on a schedule the regular schedule can be suspended with this property. This property will be defaulted by the webhook if not set. - type: boolean - transformation: - description: Transformations to be applied to the source data before writing to destination - items: - description: to be refined... + fieldPath: + description: 'If referring to a piece of an object instead of an entire object, this string should contain a valid JSON/Go field access statement, such as desiredState.manifest.containers[2]. For example, if the object reference is to a container within a pod, this would take on a value like: "spec.containers{name}" (where "name" refers to the name of the container that triggered the event) or if no container name is specified "spec.containers[2]" (container with index 2 in this pod). This syntax is chosen only to have some well-defined way of referencing a part of an object. TODO: this design is not final and this field is subject to change in the future.' + type: string + kind: + description: 'Kind of the referent. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + name: + description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names' + type: string + namespace: + description: 'Namespace of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/namespaces/' + type: string + resourceVersion: + description: 'Specific resourceVersion to which this reference is made, if any. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#concurrency-control-and-consistency' + type: string + uid: + description: 'UID of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#uids' + type: string + type: object + error: + type: string + lastCompleted: + description: 'ObjectReference contains enough information to let you inspect or modify the referred object. --- New uses of this type are discouraged because of difficulty describing its usage when embedded in APIs. 1. Ignored fields. It includes many fields which are not generally honored. For instance, ResourceVersion and FieldPath are both very rarely valid in actual usage. 2. Invalid usage help. It is impossible to add specific help for individual usage. In most embedded usages, there are particular restrictions like, "must refer only to types A and B" or "UID not honored" or "name must be restricted". Those cannot be well described when embedded. 3. Inconsistent validation. Because the usages are different, the validation rules are different by usage, which makes it hard for users to predict what will happen. 4. The fields are both imprecise and overly precise. Kind is not a precise mapping to a URL. This can produce ambiguity during interpretation and require a REST mapping. In most cases, the dependency is on the group,resource tuple and the version of the actual struct is irrelevant. 5. We cannot easily change it. Because this type is embedded in many locations, updates to this type will affect numerous schemas. Don''t make new APIs embed an underspecified API type they do not control. Instead of using this type, create a locally provided and used type that is well-focused on your reference. For example, ServiceReferences for admission registration: https://github.com/kubernetes/api/blob/release-1.17/admissionregistration/v1/types.go#L533 .' properties: - action: - description: Transformation action that should be performed. - enum: - - RemoveColumns - - EncryptColumns - - DigestColumns - - RedactColumns - - SampleRows - - FilterRows + apiVersion: + description: API version of the referent. + type: string + fieldPath: + description: 'If referring to a piece of an object instead of an entire object, this string should contain a valid JSON/Go field access statement, such as desiredState.manifest.containers[2]. For example, if the object reference is to a container within a pod, this would take on a value like: "spec.containers{name}" (where "name" refers to the name of the container that triggered the event) or if no container name is specified "spec.containers[2]" (container with index 2 in this pod). This syntax is chosen only to have some well-defined way of referencing a part of an object. TODO: this design is not final and this field is subject to change in the future.' + type: string + kind: + description: 'Kind of the referent. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' type: string - columns: - description: Columns that are involved in this action. This property is optional as for some actions no columns have to be specified. E.g. filter is a row based transformation. - items: - type: string - type: array name: - description: Name of the transaction. Mainly used for debugging and lineage tracking. + description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names' + type: string + namespace: + description: 'Namespace of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/namespaces/' + type: string + resourceVersion: + description: 'Specific resourceVersion to which this reference is made, if any. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#concurrency-control-and-consistency' + type: string + uid: + description: 'UID of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#uids' type: string - options: - additionalProperties: - type: string - description: Additional options for this transformation. - type: object type: object - type: array - writeDataType: - description: Data type of how the data should be written to the target (log data or change data) - enum: - - LogData - - ChangeData - type: string - writeOperation: - description: 'Write operation that should be performed when writing (overwrite,append,update) Caution: Some write operations are only available for batch and some only for stream.' - enum: - - Overwrite - - Append - - Update - type: string - required: - - destination - - source - type: object - status: - description: 'BatchTransferStatus defines the observed state of BatchTransfer This includes a reference to the job that implements the movement as well as the last schedule time. What is missing: Extended status information such as: - number of records moved - technical meta-data' - properties: - active: - description: A pointer to the currently running job (or nil) - properties: - apiVersion: - description: API version of the referent. - type: string - fieldPath: - description: 'If referring to a piece of an object instead of an entire object, this string should contain a valid JSON/Go field access statement, such as desiredState.manifest.containers[2]. For example, if the object reference is to a container within a pod, this would take on a value like: "spec.containers{name}" (where "name" refers to the name of the container that triggered the event) or if no container name is specified "spec.containers[2]" (container with index 2 in this pod). This syntax is chosen only to have some well-defined way of referencing a part of an object. TODO: this design is not final and this field is subject to change in the future.' - type: string - kind: - description: 'Kind of the referent. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' - type: string - name: - description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names' - type: string - namespace: - description: 'Namespace of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/namespaces/' - type: string - resourceVersion: - description: 'Specific resourceVersion to which this reference is made, if any. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#concurrency-control-and-consistency' - type: string - uid: - description: 'UID of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#uids' - type: string - type: object - error: - type: string - lastCompleted: - description: 'ObjectReference contains enough information to let you inspect or modify the referred object. --- New uses of this type are discouraged because of difficulty describing its usage when embedded in APIs. 1. Ignored fields. It includes many fields which are not generally honored. For instance, ResourceVersion and FieldPath are both very rarely valid in actual usage. 2. Invalid usage help. It is impossible to add specific help for individual usage. In most embedded usages, there are particular restrictions like, "must refer only to types A and B" or "UID not honored" or "name must be restricted". Those cannot be well described when embedded. 3. Inconsistent validation. Because the usages are different, the validation rules are different by usage, which makes it hard for users to predict what will happen. 4. The fields are both imprecise and overly precise. Kind is not a precise mapping to a URL. This can produce ambiguity during interpretation and require a REST mapping. In most cases, the dependency is on the group,resource tuple and the version of the actual struct is irrelevant. 5. We cannot easily change it. Because this type is embedded in many locations, updates to this type will affect numerous schemas. Don''t make new APIs embed an underspecified API type they do not control. Instead of using this type, create a locally provided and used type that is well-focused on your reference. For example, ServiceReferences for admission registration: https://github.com/kubernetes/api/blob/release-1.17/admissionregistration/v1/types.go#L533 .' - properties: - apiVersion: - description: API version of the referent. - type: string - fieldPath: - description: 'If referring to a piece of an object instead of an entire object, this string should contain a valid JSON/Go field access statement, such as desiredState.manifest.containers[2]. For example, if the object reference is to a container within a pod, this would take on a value like: "spec.containers{name}" (where "name" refers to the name of the container that triggered the event) or if no container name is specified "spec.containers[2]" (container with index 2 in this pod). This syntax is chosen only to have some well-defined way of referencing a part of an object. TODO: this design is not final and this field is subject to change in the future.' - type: string - kind: - description: 'Kind of the referent. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' - type: string - name: - description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names' - type: string - namespace: - description: 'Namespace of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/namespaces/' - type: string - resourceVersion: - description: 'Specific resourceVersion to which this reference is made, if any. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#concurrency-control-and-consistency' - type: string - uid: - description: 'UID of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#uids' - type: string - type: object - lastFailed: - description: 'ObjectReference contains enough information to let you inspect or modify the referred object. --- New uses of this type are discouraged because of difficulty describing its usage when embedded in APIs. 1. Ignored fields. It includes many fields which are not generally honored. For instance, ResourceVersion and FieldPath are both very rarely valid in actual usage. 2. Invalid usage help. It is impossible to add specific help for individual usage. In most embedded usages, there are particular restrictions like, "must refer only to types A and B" or "UID not honored" or "name must be restricted". Those cannot be well described when embedded. 3. Inconsistent validation. Because the usages are different, the validation rules are different by usage, which makes it hard for users to predict what will happen. 4. The fields are both imprecise and overly precise. Kind is not a precise mapping to a URL. This can produce ambiguity during interpretation and require a REST mapping. In most cases, the dependency is on the group,resource tuple and the version of the actual struct is irrelevant. 5. We cannot easily change it. Because this type is embedded in many locations, updates to this type will affect numerous schemas. Don''t make new APIs embed an underspecified API type they do not control. Instead of using this type, create a locally provided and used type that is well-focused on your reference. For example, ServiceReferences for admission registration: https://github.com/kubernetes/api/blob/release-1.17/admissionregistration/v1/types.go#L533 .' - properties: - apiVersion: - description: API version of the referent. - type: string - fieldPath: - description: 'If referring to a piece of an object instead of an entire object, this string should contain a valid JSON/Go field access statement, such as desiredState.manifest.containers[2]. For example, if the object reference is to a container within a pod, this would take on a value like: "spec.containers{name}" (where "name" refers to the name of the container that triggered the event) or if no container name is specified "spec.containers[2]" (container with index 2 in this pod). This syntax is chosen only to have some well-defined way of referencing a part of an object. TODO: this design is not final and this field is subject to change in the future.' - type: string - kind: - description: 'Kind of the referent. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' - type: string - name: - description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names' - type: string - namespace: - description: 'Namespace of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/namespaces/' - type: string - resourceVersion: - description: 'Specific resourceVersion to which this reference is made, if any. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#concurrency-control-and-consistency' - type: string - uid: - description: 'UID of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#uids' - type: string - type: object - lastRecordTime: - format: date-time - type: string - lastScheduleTime: - description: Information when was the last time the job was successfully scheduled. - format: date-time - type: string - lastSuccessTime: - format: date-time - type: string - numRecords: - format: int64 - minimum: 0 - type: integer - status: - enum: - - STARTING - - RUNNING - - SUCCEEDED - - FAILED - type: string - type: object - type: object - version: v1alpha1 - versions: - - name: v1alpha1 + lastFailed: + description: 'ObjectReference contains enough information to let you inspect or modify the referred object. --- New uses of this type are discouraged because of difficulty describing its usage when embedded in APIs. 1. Ignored fields. It includes many fields which are not generally honored. For instance, ResourceVersion and FieldPath are both very rarely valid in actual usage. 2. Invalid usage help. It is impossible to add specific help for individual usage. In most embedded usages, there are particular restrictions like, "must refer only to types A and B" or "UID not honored" or "name must be restricted". Those cannot be well described when embedded. 3. Inconsistent validation. Because the usages are different, the validation rules are different by usage, which makes it hard for users to predict what will happen. 4. The fields are both imprecise and overly precise. Kind is not a precise mapping to a URL. This can produce ambiguity during interpretation and require a REST mapping. In most cases, the dependency is on the group,resource tuple and the version of the actual struct is irrelevant. 5. We cannot easily change it. Because this type is embedded in many locations, updates to this type will affect numerous schemas. Don''t make new APIs embed an underspecified API type they do not control. Instead of using this type, create a locally provided and used type that is well-focused on your reference. For example, ServiceReferences for admission registration: https://github.com/kubernetes/api/blob/release-1.17/admissionregistration/v1/types.go#L533 .' + properties: + apiVersion: + description: API version of the referent. + type: string + fieldPath: + description: 'If referring to a piece of an object instead of an entire object, this string should contain a valid JSON/Go field access statement, such as desiredState.manifest.containers[2]. For example, if the object reference is to a container within a pod, this would take on a value like: "spec.containers{name}" (where "name" refers to the name of the container that triggered the event) or if no container name is specified "spec.containers[2]" (container with index 2 in this pod). This syntax is chosen only to have some well-defined way of referencing a part of an object. TODO: this design is not final and this field is subject to change in the future.' + type: string + kind: + description: 'Kind of the referent. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + name: + description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names' + type: string + namespace: + description: 'Namespace of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/namespaces/' + type: string + resourceVersion: + description: 'Specific resourceVersion to which this reference is made, if any. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#concurrency-control-and-consistency' + type: string + uid: + description: 'UID of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#uids' + type: string + type: object + lastRecordTime: + format: date-time + type: string + lastScheduleTime: + description: Information when was the last time the job was successfully scheduled. + format: date-time + type: string + lastSuccessTime: + format: date-time + type: string + numRecords: + format: int64 + minimum: 0 + type: integer + status: + enum: + - STARTING + - RUNNING + - SUCCEEDED + - FAILED + type: string + type: object + type: object served: true storage: true + subresources: + status: {} status: acceptedNames: kind: "" diff --git a/charts/m4d-crd/templates/motion.m4d.ibm.com_streamtransfers.yaml b/charts/m4d-crd/templates/motion.m4d.ibm.com_streamtransfers.yaml index 03404ed..e70999a 100644 --- a/charts/m4d-crd/templates/motion.m4d.ibm.com_streamtransfers.yaml +++ b/charts/m4d-crd/templates/motion.m4d.ibm.com_streamtransfers.yaml @@ -1,23 +1,13 @@ --- -apiVersion: apiextensions.k8s.io/v1beta1 +apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: - controller-gen.kubebuilder.io/version: v0.3.0 + controller-gen.kubebuilder.io/version: v0.5.0 creationTimestamp: null name: streamtransfers.motion.m4d.ibm.com spec: - additionalPrinterColumns: - - JSONPath: .spec.source.description - name: Source - type: string - - JSONPath: .spec.destination.description - name: Destination - type: string - - JSONPath: .status.status - name: Status - type: string group: motion.m4d.ibm.com names: kind: StreamTransfer @@ -25,627 +15,613 @@ spec: plural: streamtransfers singular: streamtransfer scope: Namespaced - subresources: - status: {} - validation: - openAPIV3Schema: - description: StreamTransfer is the Schema for the streamtransfers API - properties: - apiVersion: - description: 'APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' - type: string - kind: - description: 'Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' - type: string - metadata: - type: object - spec: - description: StreamTransferSpec defines the desired state of StreamTransfer - properties: - destination: - description: Destination data store for this batch job - properties: - cloudant: - description: IBM Cloudant. Needs cloudant legacy credentials. - properties: - database: - description: Database to be read from/written to - type: string - host: - description: Host of cloudant instance - type: string - password: - description: Cloudant password. Can be retrieved from vault if specified in vaultPath parameter and is thus optional. - type: string - secretImport: - description: Define a secret import definition. - type: string - username: - description: Cloudant user. Can be retrieved from vault if specified in vaultPath parameter and is thus optional. - type: string - vault: - description: Define secrets that are fetched from a Vault instance - properties: - address: - description: Address is Vault address - type: string - authPath: - description: AuthPath is the path to auth method i.e. kubernetes - type: string - role: - description: Role is the Vault role used for retrieving the credentials - type: string - secretPath: - description: SecretPath is the path of the secret holding the Credentials in Vault - type: string - required: - - address - - authPath - - role - - secretPath - type: object - vaultPath: - description: Vault path where the user name/password are stored. If not specified user and password have to be specified! - type: string - required: - - database - - host - type: object - database: - description: Database data store. For the moment only Db2 is supported. - properties: - db2URL: - description: URL to Db2 instance in JDBC format Supported SSL certificates are currently certificates signed with IBM Intermediate CA or cloud signed certificates. - type: string - password: - description: Database password. Can be retrieved from vault if specified in vaultPath parameter and is thus optional. - type: string - secretImport: - description: Define a secret import definition. - type: string - table: - description: Table to be read - type: string - user: - description: Database user. Can be retrieved from vault if specified in vaultPath parameter and is thus optional. - type: string - vault: - description: Define secrets that are fetched from a Vault instance - properties: - address: - description: Address is Vault address - type: string - authPath: - description: AuthPath is the path to auth method i.e. kubernetes - type: string - role: - description: Role is the Vault role used for retrieving the credentials - type: string - secretPath: - description: SecretPath is the path of the secret holding the Credentials in Vault - type: string - required: - - address - - authPath - - role - - secretPath - type: object - vaultPath: - description: Vault path where the user name/password are stored. If not specified user and password have to be specified! - type: string - required: - - db2URL - - table - type: object - description: - description: Description of the transfer in human readable form that is displayed in the kubectl get If not provided this will be filled in depending on the datastore that is specified. - type: string - kafka: - description: Kafka data store. The supposed format within the given Kafka topic is a Confluent compatible format stored as Avro. A schema registry needs to be specified as well. - properties: - createSnapshot: - description: 'If a snapshot should be created of the topic. Records in Kafka are stored as key-value pairs. Updates/Deletes for the same key are appended to the Kafka topic and the last value for a given key is the valid key in a Snapshot. When this property is true only the last value will be written. If the property is false all values will be written out. As a CDC example: If the property is true a valid snapshot of the log stream will be created. If the property is false the CDC stream will be dumped as is like a change log.' - type: boolean - dataFormat: - description: Data format of the objects in S3. e.g. parquet or csv. Please refer to struct for allowed values. - type: string - kafkaBrokers: - description: Kafka broker URLs as a comma separated list. - type: string - kafkaTopic: - description: Kafka topic - type: string - keyDeserializer: - description: Deserializer to be used for the keys of the topic - type: string - password: - description: Kafka user password Can be retrieved from vault if specified in vaultPath parameter and is thus optional. - type: string - saslMechanism: - description: SASL Mechanism to be used (e.g. PLAIN or SCRAM-SHA-512) Default SCRAM-SHA-512 will be assumed if not specified - type: string - schemaRegistryURL: - description: URL to the schema registry. The registry has to be Confluent schema registry compatible. - type: string - secretImport: - description: Define a secret import definition. - type: string - securityProtocol: - description: Kafka security protocol one of (PLAINTEXT, SASL_PLAINTEXT, SASL_SSL, SSL) Default SASL_SSL will be assumed if not specified - type: string - sslTruststore: - description: A truststore or certificate encoded as base64. The format can be JKS or PKCS12. A truststore can be specified like this or in a predefined Kubernetes secret - type: string - sslTruststoreLocation: - description: SSL truststore location. - type: string - sslTruststorePassword: - description: SSL truststore password. - type: string - sslTruststoreSecret: - description: Kubernetes secret that contains the SSL truststore. The format can be JKS or PKCS12. A truststore can be specified like this or as - type: string - user: - description: Kafka user name. Can be retrieved from vault if specified in vaultPath parameter and is thus optional. - type: string - valueDeserializer: - description: Deserializer to be used for the values of the topic - type: string - vault: - description: Define secrets that are fetched from a Vault instance - properties: - address: - description: Address is Vault address - type: string - authPath: - description: AuthPath is the path to auth method i.e. kubernetes - type: string - role: - description: Role is the Vault role used for retrieving the credentials - type: string - secretPath: - description: SecretPath is the path of the secret holding the Credentials in Vault - type: string - required: - - address - - authPath - - role - - secretPath - type: object - vaultPath: - description: Vault path where the user name/password are stored. If not specified user and password have to be specified! - type: string - required: - - kafkaBrokers - - kafkaTopic - - schemaRegistryURL - type: object - s3: - description: An object store data store that is compatible with S3. This can be a COS bucket. - properties: - accessKey: - description: Access key of the HMAC credentials that can access the given bucket. Can be retrieved from vault if specified in vaultPath parameter and is thus optional. - type: string - bucket: - description: Bucket of S3 service - type: string - dataFormat: - description: Data format of the objects in S3. e.g. parquet or csv. Please refer to struct for allowed values. - type: string - endpoint: - description: Endpoint of S3 service - type: string - objectKey: - description: Object key of the object in S3. This is used as a prefix! Thus all objects that have the given objectKey as prefix will be used as input! - type: string - partitionBy: - description: Partition by partition (for target data stores) Defines the columns to partition the output by for a target data store. - items: + versions: + - additionalPrinterColumns: + - jsonPath: .spec.source.description + name: Source + type: string + - jsonPath: .spec.destination.description + name: Destination + type: string + - jsonPath: .status.status + name: Status + type: string + name: v1alpha1 + schema: + openAPIV3Schema: + description: StreamTransfer is the Schema for the streamtransfers API + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + spec: + description: StreamTransferSpec defines the desired state of StreamTransfer + properties: + destination: + description: Destination data store for this batch job + properties: + cloudant: + description: IBM Cloudant. Needs cloudant legacy credentials. + properties: + database: + description: Database to be read from/written to type: string - type: array - region: - description: Region of S3 service - type: string - secretImport: - description: Define a secret import definition. - type: string - secretKey: - description: Secret key of the HMAC credentials that can access the given bucket. Can be retrieved from vault if specified in vaultPath parameter and is thus optional. - type: string - vault: - description: Define secrets that are fetched from a Vault instance - properties: - address: - description: Address is Vault address - type: string - authPath: - description: AuthPath is the path to auth method i.e. kubernetes - type: string - role: - description: Role is the Vault role used for retrieving the credentials - type: string - secretPath: - description: SecretPath is the path of the secret holding the Credentials in Vault - type: string - required: - - address - - authPath - - role - - secretPath - type: object - vaultPath: - description: Vault path where the accessKey/secretKey are stored. If not specified accessKey and secretKey have to be specified! - type: string - required: - - bucket - - endpoint - - objectKey - type: object - type: object - flowType: - description: Data flow type that specifies if this is a stream or a batch workflow - enum: - - Batch - - Stream - type: string - image: - description: Image that should be used for the actual batch job. This is usually a datamover image. This property will be defaulted by the webhook if not set. - type: string - imagePullPolicy: - description: Image pull policy that should be used for the actual job. This property will be defaulted by the webhook if not set. - type: string - noFinalizer: - description: If this batch job instance should have a finalizer or not. This property will be defaulted by the webhook if not set. - type: boolean - readDataType: - description: Data type of the data that is read from source (log data or change data) - enum: - - LogData - - ChangeData - type: string - secretProviderRole: - description: Secret provider role that should be used for the actual job. This property will be defaulted by the webhook if not set. - type: string - secretProviderURL: - description: Secret provider url that should be used for the actual job. This property will be defaulted by the webhook if not set. - type: string - source: - description: Source data store for this batch job - properties: - cloudant: - description: IBM Cloudant. Needs cloudant legacy credentials. - properties: - database: - description: Database to be read from/written to - type: string - host: - description: Host of cloudant instance - type: string - password: - description: Cloudant password. Can be retrieved from vault if specified in vaultPath parameter and is thus optional. - type: string - secretImport: - description: Define a secret import definition. - type: string - username: - description: Cloudant user. Can be retrieved from vault if specified in vaultPath parameter and is thus optional. - type: string - vault: - description: Define secrets that are fetched from a Vault instance - properties: - address: - description: Address is Vault address - type: string - authPath: - description: AuthPath is the path to auth method i.e. kubernetes - type: string - role: - description: Role is the Vault role used for retrieving the credentials - type: string - secretPath: - description: SecretPath is the path of the secret holding the Credentials in Vault - type: string - required: - - address - - authPath - - role - - secretPath - type: object - vaultPath: - description: Vault path where the user name/password are stored. If not specified user and password have to be specified! - type: string - required: - - database - - host - type: object - database: - description: Database data store. For the moment only Db2 is supported. - properties: - db2URL: - description: URL to Db2 instance in JDBC format Supported SSL certificates are currently certificates signed with IBM Intermediate CA or cloud signed certificates. - type: string - password: - description: Database password. Can be retrieved from vault if specified in vaultPath parameter and is thus optional. - type: string - secretImport: - description: Define a secret import definition. - type: string - table: - description: Table to be read - type: string - user: - description: Database user. Can be retrieved from vault if specified in vaultPath parameter and is thus optional. - type: string - vault: - description: Define secrets that are fetched from a Vault instance - properties: - address: - description: Address is Vault address - type: string - authPath: - description: AuthPath is the path to auth method i.e. kubernetes - type: string - role: - description: Role is the Vault role used for retrieving the credentials - type: string - secretPath: - description: SecretPath is the path of the secret holding the Credentials in Vault - type: string - required: - - address - - authPath - - role - - secretPath - type: object - vaultPath: - description: Vault path where the user name/password are stored. If not specified user and password have to be specified! - type: string - required: - - db2URL - - table - type: object - description: - description: Description of the transfer in human readable form that is displayed in the kubectl get If not provided this will be filled in depending on the datastore that is specified. - type: string - kafka: - description: Kafka data store. The supposed format within the given Kafka topic is a Confluent compatible format stored as Avro. A schema registry needs to be specified as well. - properties: - createSnapshot: - description: 'If a snapshot should be created of the topic. Records in Kafka are stored as key-value pairs. Updates/Deletes for the same key are appended to the Kafka topic and the last value for a given key is the valid key in a Snapshot. When this property is true only the last value will be written. If the property is false all values will be written out. As a CDC example: If the property is true a valid snapshot of the log stream will be created. If the property is false the CDC stream will be dumped as is like a change log.' - type: boolean - dataFormat: - description: Data format of the objects in S3. e.g. parquet or csv. Please refer to struct for allowed values. - type: string - kafkaBrokers: - description: Kafka broker URLs as a comma separated list. - type: string - kafkaTopic: - description: Kafka topic - type: string - keyDeserializer: - description: Deserializer to be used for the keys of the topic - type: string - password: - description: Kafka user password Can be retrieved from vault if specified in vaultPath parameter and is thus optional. - type: string - saslMechanism: - description: SASL Mechanism to be used (e.g. PLAIN or SCRAM-SHA-512) Default SCRAM-SHA-512 will be assumed if not specified - type: string - schemaRegistryURL: - description: URL to the schema registry. The registry has to be Confluent schema registry compatible. - type: string - secretImport: - description: Define a secret import definition. - type: string - securityProtocol: - description: Kafka security protocol one of (PLAINTEXT, SASL_PLAINTEXT, SASL_SSL, SSL) Default SASL_SSL will be assumed if not specified - type: string - sslTruststore: - description: A truststore or certificate encoded as base64. The format can be JKS or PKCS12. A truststore can be specified like this or in a predefined Kubernetes secret - type: string - sslTruststoreLocation: - description: SSL truststore location. - type: string - sslTruststorePassword: - description: SSL truststore password. - type: string - sslTruststoreSecret: - description: Kubernetes secret that contains the SSL truststore. The format can be JKS or PKCS12. A truststore can be specified like this or as - type: string - user: - description: Kafka user name. Can be retrieved from vault if specified in vaultPath parameter and is thus optional. - type: string - valueDeserializer: - description: Deserializer to be used for the values of the topic - type: string - vault: - description: Define secrets that are fetched from a Vault instance - properties: - address: - description: Address is Vault address - type: string - authPath: - description: AuthPath is the path to auth method i.e. kubernetes - type: string - role: - description: Role is the Vault role used for retrieving the credentials + host: + description: Host of cloudant instance + type: string + password: + description: Cloudant password. Can be retrieved from vault if specified in vault parameter and is thus optional. + type: string + secretImport: + description: Define a secret import definition. + type: string + username: + description: Cloudant user. Can be retrieved from vault if specified in vault parameter and is thus optional. + type: string + vault: + description: Define secrets that are fetched from a Vault instance + properties: + address: + description: Address is Vault address + type: string + authPath: + description: AuthPath is the path to auth method i.e. kubernetes + type: string + role: + description: Role is the Vault role used for retrieving the credentials + type: string + secretPath: + description: SecretPath is the path of the secret holding the Credentials in Vault + type: string + required: + - address + - authPath + - role + - secretPath + type: object + required: + - database + - host + type: object + database: + description: Database data store. For the moment only Db2 is supported. + properties: + db2URL: + description: URL to Db2 instance in JDBC format Supported SSL certificates are currently certificates signed with IBM Intermediate CA or cloud signed certificates. + type: string + password: + description: Database password. Can be retrieved from vault if specified in vault parameter and is thus optional. + type: string + secretImport: + description: Define a secret import definition. + type: string + table: + description: Table to be read + type: string + user: + description: Database user. Can be retrieved from vault if specified in vault parameter and is thus optional. + type: string + vault: + description: Define secrets that are fetched from a Vault instance + properties: + address: + description: Address is Vault address + type: string + authPath: + description: AuthPath is the path to auth method i.e. kubernetes + type: string + role: + description: Role is the Vault role used for retrieving the credentials + type: string + secretPath: + description: SecretPath is the path of the secret holding the Credentials in Vault + type: string + required: + - address + - authPath + - role + - secretPath + type: object + required: + - db2URL + - table + type: object + description: + description: Description of the transfer in human readable form that is displayed in the kubectl get If not provided this will be filled in depending on the datastore that is specified. + type: string + kafka: + description: Kafka data store. The supposed format within the given Kafka topic is a Confluent compatible format stored as Avro. A schema registry needs to be specified as well. + properties: + createSnapshot: + description: 'If a snapshot should be created of the topic. Records in Kafka are stored as key-value pairs. Updates/Deletes for the same key are appended to the Kafka topic and the last value for a given key is the valid key in a Snapshot. When this property is true only the last value will be written. If the property is false all values will be written out. As a CDC example: If the property is true a valid snapshot of the log stream will be created. If the property is false the CDC stream will be dumped as is like a change log.' + type: boolean + dataFormat: + description: Data format of the objects in S3. e.g. parquet or csv. Please refer to struct for allowed values. + type: string + kafkaBrokers: + description: Kafka broker URLs as a comma separated list. + type: string + kafkaTopic: + description: Kafka topic + type: string + keyDeserializer: + description: Deserializer to be used for the keys of the topic + type: string + password: + description: Kafka user password Can be retrieved from vault if specified in vault parameter and is thus optional. + type: string + saslMechanism: + description: SASL Mechanism to be used (e.g. PLAIN or SCRAM-SHA-512) Default SCRAM-SHA-512 will be assumed if not specified + type: string + schemaRegistryURL: + description: URL to the schema registry. The registry has to be Confluent schema registry compatible. + type: string + secretImport: + description: Define a secret import definition. + type: string + securityProtocol: + description: Kafka security protocol one of (PLAINTEXT, SASL_PLAINTEXT, SASL_SSL, SSL) Default SASL_SSL will be assumed if not specified + type: string + sslTruststore: + description: A truststore or certificate encoded as base64. The format can be JKS or PKCS12. A truststore can be specified like this or in a predefined Kubernetes secret + type: string + sslTruststoreLocation: + description: SSL truststore location. + type: string + sslTruststorePassword: + description: SSL truststore password. + type: string + sslTruststoreSecret: + description: Kubernetes secret that contains the SSL truststore. The format can be JKS or PKCS12. A truststore can be specified like this or as + type: string + user: + description: Kafka user name. Can be retrieved from vault if specified in vault parameter and is thus optional. + type: string + valueDeserializer: + description: Deserializer to be used for the values of the topic + type: string + vault: + description: Define secrets that are fetched from a Vault instance + properties: + address: + description: Address is Vault address + type: string + authPath: + description: AuthPath is the path to auth method i.e. kubernetes + type: string + role: + description: Role is the Vault role used for retrieving the credentials + type: string + secretPath: + description: SecretPath is the path of the secret holding the Credentials in Vault + type: string + required: + - address + - authPath + - role + - secretPath + type: object + required: + - kafkaBrokers + - kafkaTopic + - schemaRegistryURL + type: object + s3: + description: An object store data store that is compatible with S3. This can be a COS bucket. + properties: + accessKey: + description: Access key of the HMAC credentials that can access the given bucket. Can be retrieved from vault if specified in vault parameter and is thus optional. + type: string + bucket: + description: Bucket of S3 service + type: string + dataFormat: + description: Data format of the objects in S3. e.g. parquet or csv. Please refer to struct for allowed values. + type: string + endpoint: + description: Endpoint of S3 service + type: string + objectKey: + description: Object key of the object in S3. This is used as a prefix! Thus all objects that have the given objectKey as prefix will be used as input! + type: string + partitionBy: + description: Partition by partition (for target data stores) Defines the columns to partition the output by for a target data store. + items: type: string - secretPath: - description: SecretPath is the path of the secret holding the Credentials in Vault + type: array + region: + description: Region of S3 service + type: string + secretImport: + description: Define a secret import definition. + type: string + secretKey: + description: Secret key of the HMAC credentials that can access the given bucket. Can be retrieved from vault if specified in vault parameter and is thus optional. + type: string + vault: + description: Define secrets that are fetched from a Vault instance + properties: + address: + description: Address is Vault address + type: string + authPath: + description: AuthPath is the path to auth method i.e. kubernetes + type: string + role: + description: Role is the Vault role used for retrieving the credentials + type: string + secretPath: + description: SecretPath is the path of the secret holding the Credentials in Vault + type: string + required: + - address + - authPath + - role + - secretPath + type: object + required: + - bucket + - endpoint + - objectKey + type: object + type: object + flowType: + description: Data flow type that specifies if this is a stream or a batch workflow + enum: + - Batch + - Stream + type: string + image: + description: Image that should be used for the actual batch job. This is usually a datamover image. This property will be defaulted by the webhook if not set. + type: string + imagePullPolicy: + description: Image pull policy that should be used for the actual job. This property will be defaulted by the webhook if not set. + type: string + noFinalizer: + description: If this batch job instance should have a finalizer or not. This property will be defaulted by the webhook if not set. + type: boolean + readDataType: + description: Data type of the data that is read from source (log data or change data) + enum: + - LogData + - ChangeData + type: string + secretProviderRole: + description: Secret provider role that should be used for the actual job. This property will be defaulted by the webhook if not set. + type: string + secretProviderURL: + description: Secret provider url that should be used for the actual job. This property will be defaulted by the webhook if not set. + type: string + source: + description: Source data store for this batch job + properties: + cloudant: + description: IBM Cloudant. Needs cloudant legacy credentials. + properties: + database: + description: Database to be read from/written to + type: string + host: + description: Host of cloudant instance + type: string + password: + description: Cloudant password. Can be retrieved from vault if specified in vault parameter and is thus optional. + type: string + secretImport: + description: Define a secret import definition. + type: string + username: + description: Cloudant user. Can be retrieved from vault if specified in vault parameter and is thus optional. + type: string + vault: + description: Define secrets that are fetched from a Vault instance + properties: + address: + description: Address is Vault address + type: string + authPath: + description: AuthPath is the path to auth method i.e. kubernetes + type: string + role: + description: Role is the Vault role used for retrieving the credentials + type: string + secretPath: + description: SecretPath is the path of the secret holding the Credentials in Vault + type: string + required: + - address + - authPath + - role + - secretPath + type: object + required: + - database + - host + type: object + database: + description: Database data store. For the moment only Db2 is supported. + properties: + db2URL: + description: URL to Db2 instance in JDBC format Supported SSL certificates are currently certificates signed with IBM Intermediate CA or cloud signed certificates. + type: string + password: + description: Database password. Can be retrieved from vault if specified in vault parameter and is thus optional. + type: string + secretImport: + description: Define a secret import definition. + type: string + table: + description: Table to be read + type: string + user: + description: Database user. Can be retrieved from vault if specified in vault parameter and is thus optional. + type: string + vault: + description: Define secrets that are fetched from a Vault instance + properties: + address: + description: Address is Vault address + type: string + authPath: + description: AuthPath is the path to auth method i.e. kubernetes + type: string + role: + description: Role is the Vault role used for retrieving the credentials + type: string + secretPath: + description: SecretPath is the path of the secret holding the Credentials in Vault + type: string + required: + - address + - authPath + - role + - secretPath + type: object + required: + - db2URL + - table + type: object + description: + description: Description of the transfer in human readable form that is displayed in the kubectl get If not provided this will be filled in depending on the datastore that is specified. + type: string + kafka: + description: Kafka data store. The supposed format within the given Kafka topic is a Confluent compatible format stored as Avro. A schema registry needs to be specified as well. + properties: + createSnapshot: + description: 'If a snapshot should be created of the topic. Records in Kafka are stored as key-value pairs. Updates/Deletes for the same key are appended to the Kafka topic and the last value for a given key is the valid key in a Snapshot. When this property is true only the last value will be written. If the property is false all values will be written out. As a CDC example: If the property is true a valid snapshot of the log stream will be created. If the property is false the CDC stream will be dumped as is like a change log.' + type: boolean + dataFormat: + description: Data format of the objects in S3. e.g. parquet or csv. Please refer to struct for allowed values. + type: string + kafkaBrokers: + description: Kafka broker URLs as a comma separated list. + type: string + kafkaTopic: + description: Kafka topic + type: string + keyDeserializer: + description: Deserializer to be used for the keys of the topic + type: string + password: + description: Kafka user password Can be retrieved from vault if specified in vault parameter and is thus optional. + type: string + saslMechanism: + description: SASL Mechanism to be used (e.g. PLAIN or SCRAM-SHA-512) Default SCRAM-SHA-512 will be assumed if not specified + type: string + schemaRegistryURL: + description: URL to the schema registry. The registry has to be Confluent schema registry compatible. + type: string + secretImport: + description: Define a secret import definition. + type: string + securityProtocol: + description: Kafka security protocol one of (PLAINTEXT, SASL_PLAINTEXT, SASL_SSL, SSL) Default SASL_SSL will be assumed if not specified + type: string + sslTruststore: + description: A truststore or certificate encoded as base64. The format can be JKS or PKCS12. A truststore can be specified like this or in a predefined Kubernetes secret + type: string + sslTruststoreLocation: + description: SSL truststore location. + type: string + sslTruststorePassword: + description: SSL truststore password. + type: string + sslTruststoreSecret: + description: Kubernetes secret that contains the SSL truststore. The format can be JKS or PKCS12. A truststore can be specified like this or as + type: string + user: + description: Kafka user name. Can be retrieved from vault if specified in vault parameter and is thus optional. + type: string + valueDeserializer: + description: Deserializer to be used for the values of the topic + type: string + vault: + description: Define secrets that are fetched from a Vault instance + properties: + address: + description: Address is Vault address + type: string + authPath: + description: AuthPath is the path to auth method i.e. kubernetes + type: string + role: + description: Role is the Vault role used for retrieving the credentials + type: string + secretPath: + description: SecretPath is the path of the secret holding the Credentials in Vault + type: string + required: + - address + - authPath + - role + - secretPath + type: object + required: + - kafkaBrokers + - kafkaTopic + - schemaRegistryURL + type: object + s3: + description: An object store data store that is compatible with S3. This can be a COS bucket. + properties: + accessKey: + description: Access key of the HMAC credentials that can access the given bucket. Can be retrieved from vault if specified in vault parameter and is thus optional. + type: string + bucket: + description: Bucket of S3 service + type: string + dataFormat: + description: Data format of the objects in S3. e.g. parquet or csv. Please refer to struct for allowed values. + type: string + endpoint: + description: Endpoint of S3 service + type: string + objectKey: + description: Object key of the object in S3. This is used as a prefix! Thus all objects that have the given objectKey as prefix will be used as input! + type: string + partitionBy: + description: Partition by partition (for target data stores) Defines the columns to partition the output by for a target data store. + items: type: string - required: - - address - - authPath - - role - - secretPath - type: object - vaultPath: - description: Vault path where the user name/password are stored. If not specified user and password have to be specified! - type: string - required: - - kafkaBrokers - - kafkaTopic - - schemaRegistryURL - type: object - s3: - description: An object store data store that is compatible with S3. This can be a COS bucket. + type: array + region: + description: Region of S3 service + type: string + secretImport: + description: Define a secret import definition. + type: string + secretKey: + description: Secret key of the HMAC credentials that can access the given bucket. Can be retrieved from vault if specified in vault parameter and is thus optional. + type: string + vault: + description: Define secrets that are fetched from a Vault instance + properties: + address: + description: Address is Vault address + type: string + authPath: + description: AuthPath is the path to auth method i.e. kubernetes + type: string + role: + description: Role is the Vault role used for retrieving the credentials + type: string + secretPath: + description: SecretPath is the path of the secret holding the Credentials in Vault + type: string + required: + - address + - authPath + - role + - secretPath + type: object + required: + - bucket + - endpoint + - objectKey + type: object + type: object + suspend: + description: If this batch job instance is run on a schedule the regular schedule can be suspended with this property. This property will be defaulted by the webhook if not set. + type: boolean + transformation: + description: Transformations to be applied to the source data before writing to destination + items: + description: to be refined... properties: - accessKey: - description: Access key of the HMAC credentials that can access the given bucket. Can be retrieved from vault if specified in vaultPath parameter and is thus optional. - type: string - bucket: - description: Bucket of S3 service - type: string - dataFormat: - description: Data format of the objects in S3. e.g. parquet or csv. Please refer to struct for allowed values. - type: string - endpoint: - description: Endpoint of S3 service - type: string - objectKey: - description: Object key of the object in S3. This is used as a prefix! Thus all objects that have the given objectKey as prefix will be used as input! - type: string - partitionBy: - description: Partition by partition (for target data stores) Defines the columns to partition the output by for a target data store. + action: + description: Transformation action that should be performed. + enum: + - RemoveColumns + - EncryptColumns + - DigestColumns + - RedactColumns + - SampleRows + - FilterRows + type: string + columns: + description: Columns that are involved in this action. This property is optional as for some actions no columns have to be specified. E.g. filter is a row based transformation. items: type: string type: array - region: - description: Region of S3 service - type: string - secretImport: - description: Define a secret import definition. + name: + description: Name of the transaction. Mainly used for debugging and lineage tracking. type: string - secretKey: - description: Secret key of the HMAC credentials that can access the given bucket. Can be retrieved from vault if specified in vaultPath parameter and is thus optional. - type: string - vault: - description: Define secrets that are fetched from a Vault instance - properties: - address: - description: Address is Vault address - type: string - authPath: - description: AuthPath is the path to auth method i.e. kubernetes - type: string - role: - description: Role is the Vault role used for retrieving the credentials - type: string - secretPath: - description: SecretPath is the path of the secret holding the Credentials in Vault - type: string - required: - - address - - authPath - - role - - secretPath + options: + additionalProperties: + type: string + description: Additional options for this transformation. type: object - vaultPath: - description: Vault path where the accessKey/secretKey are stored. If not specified accessKey and secretKey have to be specified! - type: string - required: - - bucket - - endpoint - - objectKey + x-kubernetes-preserve-unknown-fields: true type: object - type: object - suspend: - description: If this batch job instance is run on a schedule the regular schedule can be suspended with this property. This property will be defaulted by the webhook if not set. - type: boolean - transformation: - description: Transformations to be applied to the source data before writing to destination - items: - description: to be refined... + type: array + triggerInterval: + description: Interval in which the Micro batches of this stream should be triggered The default is '5 seconds'. + type: string + writeDataType: + description: Data type of how the data should be written to the target (log data or change data) + enum: + - LogData + - ChangeData + type: string + writeOperation: + description: 'Write operation that should be performed when writing (overwrite,append,update) Caution: Some write operations are only available for batch and some only for stream.' + enum: + - Overwrite + - Append + - Update + type: string + required: + - destination + - source + type: object + status: + description: StreamTransferStatus defines the observed state of StreamTransfer + properties: + active: + description: A pointer to the currently running job (or nil) properties: - action: - description: Transformation action that should be performed. - enum: - - RemoveColumns - - EncryptColumns - - DigestColumns - - RedactColumns - - SampleRows - - FilterRows + apiVersion: + description: API version of the referent. + type: string + fieldPath: + description: 'If referring to a piece of an object instead of an entire object, this string should contain a valid JSON/Go field access statement, such as desiredState.manifest.containers[2]. For example, if the object reference is to a container within a pod, this would take on a value like: "spec.containers{name}" (where "name" refers to the name of the container that triggered the event) or if no container name is specified "spec.containers[2]" (container with index 2 in this pod). This syntax is chosen only to have some well-defined way of referencing a part of an object. TODO: this design is not final and this field is subject to change in the future.' + type: string + kind: + description: 'Kind of the referent. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' type: string - columns: - description: Columns that are involved in this action. This property is optional as for some actions no columns have to be specified. E.g. filter is a row based transformation. - items: - type: string - type: array name: - description: Name of the transaction. Mainly used for debugging and lineage tracking. + description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names' + type: string + namespace: + description: 'Namespace of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/namespaces/' + type: string + resourceVersion: + description: 'Specific resourceVersion to which this reference is made, if any. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#concurrency-control-and-consistency' + type: string + uid: + description: 'UID of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#uids' type: string - options: - additionalProperties: - type: string - description: Additional options for this transformation. - type: object type: object - type: array - triggerInterval: - description: Interval in which the Micro batches of this stream should be triggered The default is '5 seconds'. - type: string - writeDataType: - description: Data type of how the data should be written to the target (log data or change data) - enum: - - LogData - - ChangeData - type: string - writeOperation: - description: 'Write operation that should be performed when writing (overwrite,append,update) Caution: Some write operations are only available for batch and some only for stream.' - enum: - - Overwrite - - Append - - Update - type: string - required: - - destination - - source - type: object - status: - description: StreamTransferStatus defines the observed state of StreamTransfer - properties: - active: - description: A pointer to the currently running job (or nil) - properties: - apiVersion: - description: API version of the referent. - type: string - fieldPath: - description: 'If referring to a piece of an object instead of an entire object, this string should contain a valid JSON/Go field access statement, such as desiredState.manifest.containers[2]. For example, if the object reference is to a container within a pod, this would take on a value like: "spec.containers{name}" (where "name" refers to the name of the container that triggered the event) or if no container name is specified "spec.containers[2]" (container with index 2 in this pod). This syntax is chosen only to have some well-defined way of referencing a part of an object. TODO: this design is not final and this field is subject to change in the future.' - type: string - kind: - description: 'Kind of the referent. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' - type: string - name: - description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names' - type: string - namespace: - description: 'Namespace of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/namespaces/' - type: string - resourceVersion: - description: 'Specific resourceVersion to which this reference is made, if any. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#concurrency-control-and-consistency' - type: string - uid: - description: 'UID of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#uids' - type: string - type: object - error: - type: string - status: - enum: - - STARTING - - RUNNING - - STOPPED - - FAILING - type: string - type: object - type: object - version: v1alpha1 - versions: - - name: v1alpha1 + error: + type: string + status: + enum: + - STARTING + - RUNNING + - STOPPED + - FAILING + type: string + type: object + type: object served: true storage: true + subresources: + status: {} status: acceptedNames: kind: "" diff --git a/charts/m4d/Chart.yaml b/charts/m4d/Chart.yaml index 1c59276..efd7971 100644 --- a/charts/m4d/Chart.yaml +++ b/charts/m4d/Chart.yaml @@ -5,12 +5,10 @@ apiVersion: v2 name: m4d description: Mesh for Data Helm Chart type: application - # This is the chart version. This version number should be incremented each time you make changes # to the chart and its templates, including the app version. # Versions are expected to follow Semantic Versioning (https://semver.org/) -version: 0.1.0 - +version: 0.2.0 # This is the version number of the application being deployed. This version number should be # incremented each time you make changes to the application. -appVersion: 0.1.0 +appVersion: 0.2.0 diff --git a/charts/m4d/files/opa-server/policy-lib/helper_functions.rego b/charts/m4d/files/opa-server/policy-lib/helper_functions.rego index af88ce6..07dc8a2 100644 --- a/charts/m4d/files/opa-server/policy-lib/helper_functions.rego +++ b/charts/m4d/files/opa-server/policy-lib/helper_functions.rego @@ -40,8 +40,8 @@ column_has_tag(tag) { compare_str(tag, input.details.metadata.components_metadata[_].tags[_]) } -check_purpose(purpose) { - compare_str(purpose, Purpose()) +check_intent(intent) { + compare_str(intent, Intent()) } check_role(role) { diff --git a/charts/m4d/files/opa-server/policy-lib/input_reader.rego b/charts/m4d/files/opa-server/policy-lib/input_reader.rego index 5b5e18b..1c1cd34 100644 --- a/charts/m4d/files/opa-server/policy-lib/input_reader.rego +++ b/charts/m4d/files/opa-server/policy-lib/input_reader.rego @@ -1,6 +1,6 @@ package data_policies -#this file assumes input to be provided in specific format, in this case how data mesh provides it +#this file assumes input to be provided in specific format, in this case how data mesh provides it #similar file can be built for Egeria, at least for the metadata part, or any other catalog when we show how the input should be parsed correctly #Example structure: @@ -46,11 +46,11 @@ package data_policies # } Properties() = input.properties -Purpose() = Properties().intent +Intent() = Properties().intent Role() = Properties().role -AccessType() = input.type +AccessType() = input.type DatasetTags() = input.details.metadata.dataset_tags diff --git a/charts/m4d/files/opa-server/policy-lib/verify_correct_input.rego b/charts/m4d/files/opa-server/policy-lib/verify_correct_input.rego index 4a62ab5..dfaa972 100644 --- a/charts/m4d/files/opa-server/policy-lib/verify_correct_input.rego +++ b/charts/m4d/files/opa-server/policy-lib/verify_correct_input.rego @@ -8,8 +8,8 @@ incorrect_input[used_policy] { not verify_access_type used_policy := build_action_from_policies(build_policy_from_description("unknown access type")) } { - not verify_purpose - used_policy := build_action_from_policies(build_policy_from_description("unknown purpose")) + not verify_intent + used_policy := build_action_from_policies(build_policy_from_description("unknown intent")) } { not verify_role used_policy := build_action_from_policies(build_policy_from_description("unknown role")) diff --git a/charts/m4d/files/webhook-configs.yaml b/charts/m4d/files/webhook-configs.yaml index dcc4484..48a38e6 100644 --- a/charts/m4d/files/webhook-configs.yaml +++ b/charts/m4d/files/webhook-configs.yaml @@ -1,13 +1,16 @@ -apiVersion: admissionregistration.k8s.io/v1beta1 +apiVersion: admissionregistration.k8s.io/v1 kind: MutatingWebhookConfiguration metadata: creationTimestamp: null name: '{{ .Release.Namespace }}-mutating-webhook' annotations: cert-manager.io/inject-ca-from: '{{ .Release.Namespace }}/serving-cert' + certmanager.k8s.io/inject-ca-from: '{{ .Release.Namespace }}/serving-cert' webhooks: - - clientConfig: - caBundle: Cg== + - admissionReviewVersions: + - v1 + - v1beta1 + clientConfig: service: name: webhook-service namespace: '{{ .Release.Namespace }}' @@ -24,8 +27,11 @@ webhooks: - UPDATE resources: - batchtransfers - - clientConfig: - caBundle: Cg== + sideEffects: None + - admissionReviewVersions: + - v1 + - v1beta1 + clientConfig: service: name: webhook-service namespace: '{{ .Release.Namespace }}' @@ -42,17 +48,21 @@ webhooks: - UPDATE resources: - streamtransfers + sideEffects: None --- -apiVersion: admissionregistration.k8s.io/v1beta1 +apiVersion: admissionregistration.k8s.io/v1 kind: ValidatingWebhookConfiguration metadata: creationTimestamp: null name: '{{ .Release.Namespace }}-validating-webhook' annotations: cert-manager.io/inject-ca-from: '{{ .Release.Namespace }}/serving-cert' + certmanager.k8s.io/inject-ca-from: '{{ .Release.Namespace }}/serving-cert' webhooks: - - clientConfig: - caBundle: Cg== + - admissionReviewVersions: + - v1 + - v1beta1 + clientConfig: service: name: webhook-service namespace: '{{ .Release.Namespace }}' @@ -69,8 +79,11 @@ webhooks: - UPDATE resources: - batchtransfers - - clientConfig: - caBundle: Cg== + sideEffects: None + - admissionReviewVersions: + - v1 + - v1beta1 + clientConfig: service: name: webhook-service namespace: '{{ .Release.Namespace }}' @@ -87,3 +100,4 @@ webhooks: - UPDATE resources: - streamtransfers + sideEffects: None diff --git a/charts/m4d/templates/_helpers.tpl b/charts/m4d/templates/_helpers.tpl index a842e25..cb6c2e0 100644 --- a/charts/m4d/templates/_helpers.tpl +++ b/charts/m4d/templates/_helpers.tpl @@ -87,3 +87,15 @@ isRazeeEnabled checks if razee configuration is enabled true {{- end -}} {{- end }} + +{{/* +Detect the version of cert manager crd that is installed +Defaults to cert-manager.io/v1alpha2 +*/}} +{{- define "m4d.certManagerApiVersion" -}} +{{- if (.Capabilities.APIVersions.Has "certmanager.k8s.io/v1alpha1") -}} +certmanager.k8s.io/v1alpha1 +{{- else -}} +cert-manager.io/v1alpha2 +{{- end -}} +{{- end -}} diff --git a/charts/m4d/templates/cluster-metadata-config.yaml b/charts/m4d/templates/cluster-metadata-config.yaml index 6edb658..b297df6 100644 --- a/charts/m4d/templates/cluster-metadata-config.yaml +++ b/charts/m4d/templates/cluster-metadata-config.yaml @@ -1,4 +1,4 @@ -{{- if include "m4d.isEnabled" (tuple .Values.manager.enabled (or .Values.coordinator.enabled .Values.worker.enabled)) }} +{{- if or .Values.coordinator.enabled .Values.worker.enabled }} apiVersion: v1 kind: ConfigMap metadata: diff --git a/charts/m4d/templates/egeria-connector/egeria-connector-cm.yaml b/charts/m4d/templates/egeria-connector/egeria-connector-cm.yaml index 5f6f0d0..229bc28 100644 --- a/charts/m4d/templates/egeria-connector/egeria-connector-cm.yaml +++ b/charts/m4d/templates/egeria-connector/egeria-connector-cm.yaml @@ -1,4 +1,4 @@ -{{- $autoFlag := and .Values.manager.enabled .Values.coordinator.enabled (eq .Values.coordinator.catalog "egeria") }} +{{- $autoFlag := and .Values.coordinator.enabled (eq .Values.coordinator.catalog "egeria") }} {{- if include "m4d.isEnabled" (tuple .Values.egeriaConnector.enabled $autoFlag) }} apiVersion: v1 kind: ConfigMap diff --git a/charts/m4d/templates/egeria-connector/egeria-connector-deployment.yaml b/charts/m4d/templates/egeria-connector/egeria-connector-deployment.yaml index b4acd58..5b30aae 100644 --- a/charts/m4d/templates/egeria-connector/egeria-connector-deployment.yaml +++ b/charts/m4d/templates/egeria-connector/egeria-connector-deployment.yaml @@ -1,4 +1,4 @@ -{{- $autoFlag := and .Values.manager.enabled .Values.coordinator.enabled (eq .Values.coordinator.catalog "egeria") }} +{{- $autoFlag := and .Values.coordinator.enabled (eq .Values.coordinator.catalog "egeria") }} {{- if include "m4d.isEnabled" (tuple .Values.egeriaConnector.enabled $autoFlag) }} apiVersion: apps/v1 kind: Deployment diff --git a/charts/m4d/templates/egeria-connector/egeria-connector-hpa.yaml b/charts/m4d/templates/egeria-connector/egeria-connector-hpa.yaml index c45e62b..be4fd96 100644 --- a/charts/m4d/templates/egeria-connector/egeria-connector-hpa.yaml +++ b/charts/m4d/templates/egeria-connector/egeria-connector-hpa.yaml @@ -1,4 +1,4 @@ -{{- $autoFlag := and .Values.manager.enabled .Values.coordinator.enabled (eq .Values.coordinator.catalog "egeria") }} +{{- $autoFlag := and .Values.coordinator.enabled (eq .Values.coordinator.catalog "egeria") }} {{- if include "m4d.isEnabled" (tuple .Values.egeriaConnector.enabled $autoFlag) }} {{- if .Values.egeriaConnector.autoscaling.enabled }} apiVersion: autoscaling/v2beta1 diff --git a/charts/m4d/templates/egeria-connector/egeria-connector-service.yaml b/charts/m4d/templates/egeria-connector/egeria-connector-service.yaml index e4a07ce..e5c65f9 100644 --- a/charts/m4d/templates/egeria-connector/egeria-connector-service.yaml +++ b/charts/m4d/templates/egeria-connector/egeria-connector-service.yaml @@ -1,4 +1,4 @@ -{{- $autoFlag := and .Values.manager.enabled .Values.coordinator.enabled (eq .Values.coordinator.catalog "egeria") }} +{{- $autoFlag := and .Values.coordinator.enabled (eq .Values.coordinator.catalog "egeria") }} {{- if include "m4d.isEnabled" (tuple .Values.egeriaConnector.enabled $autoFlag) }} apiVersion: v1 kind: Service diff --git a/charts/m4d/templates/egeria-connector/egeria-connector-serviceaccount.yaml b/charts/m4d/templates/egeria-connector/egeria-connector-serviceaccount.yaml index 87d22db..d876a06 100644 --- a/charts/m4d/templates/egeria-connector/egeria-connector-serviceaccount.yaml +++ b/charts/m4d/templates/egeria-connector/egeria-connector-serviceaccount.yaml @@ -1,4 +1,4 @@ -{{- $autoFlag := and .Values.manager.enabled .Values.coordinator.enabled (eq .Values.coordinator.catalog "egeria") }} +{{- $autoFlag := and .Values.coordinator.enabled (eq .Values.coordinator.catalog "egeria") }} {{- if include "m4d.isEnabled" (tuple .Values.egeriaConnector.enabled $autoFlag) }} {{- if .Values.egeriaConnector.serviceAccount.create }} apiVersion: v1 diff --git a/charts/m4d/templates/katalog-connector/katalog-connector-deployment.yaml b/charts/m4d/templates/katalog-connector/katalog-connector-deployment.yaml index 6c4a6f9..5df4aed 100644 --- a/charts/m4d/templates/katalog-connector/katalog-connector-deployment.yaml +++ b/charts/m4d/templates/katalog-connector/katalog-connector-deployment.yaml @@ -1,4 +1,4 @@ -{{- $autoFlag := and .Values.manager.enabled .Values.coordinator.enabled (eq .Values.coordinator.catalog "katalog") }} +{{- $autoFlag := and .Values.coordinator.enabled (eq .Values.coordinator.catalog "katalog") }} {{- if include "m4d.isEnabled" (tuple .Values.katalogConnector.enabled $autoFlag) }} apiVersion: apps/v1 kind: Deployment diff --git a/charts/m4d/templates/katalog-connector/katalog-connector-hpa.yaml b/charts/m4d/templates/katalog-connector/katalog-connector-hpa.yaml index 3d086cf..6fc8c5b 100644 --- a/charts/m4d/templates/katalog-connector/katalog-connector-hpa.yaml +++ b/charts/m4d/templates/katalog-connector/katalog-connector-hpa.yaml @@ -1,4 +1,4 @@ -{{- $autoFlag := and .Values.manager.enabled .Values.coordinator.enabled (eq .Values.coordinator.catalog "katalog") }} +{{- $autoFlag := and .Values.coordinator.enabled (eq .Values.coordinator.catalog "katalog") }} {{- if include "m4d.isEnabled" (tuple .Values.katalogConnector.enabled $autoFlag) }} {{- if .Values.katalogConnector.autoscaling.enabled }} apiVersion: autoscaling/v2beta1 diff --git a/charts/m4d/templates/katalog-connector/katalog-connector-rbac.yaml b/charts/m4d/templates/katalog-connector/katalog-connector-rbac.yaml index 0b3228d..9ec092a 100644 --- a/charts/m4d/templates/katalog-connector/katalog-connector-rbac.yaml +++ b/charts/m4d/templates/katalog-connector/katalog-connector-rbac.yaml @@ -1,4 +1,4 @@ -{{- $autoFlag := and .Values.manager.enabled .Values.coordinator.enabled (eq .Values.coordinator.catalog "katalog") }} +{{- $autoFlag := and .Values.coordinator.enabled (eq .Values.coordinator.catalog "katalog") }} {{- if include "m4d.isEnabled" (tuple .Values.katalogConnector.enabled $autoFlag) }} {{- if .Values.clusterScoped }} # Grant katalog-connector the katalog-editor Role. diff --git a/charts/m4d/templates/katalog-connector/katalog-connector-service.yaml b/charts/m4d/templates/katalog-connector/katalog-connector-service.yaml index 0f1dd88..ad53219 100644 --- a/charts/m4d/templates/katalog-connector/katalog-connector-service.yaml +++ b/charts/m4d/templates/katalog-connector/katalog-connector-service.yaml @@ -1,4 +1,4 @@ -{{- $autoFlag := and .Values.manager.enabled .Values.coordinator.enabled (eq .Values.coordinator.catalog "katalog") }} +{{- $autoFlag := and .Values.coordinator.enabled (eq .Values.coordinator.catalog "katalog") }} {{- if include "m4d.isEnabled" (tuple .Values.katalogConnector.enabled $autoFlag) }} apiVersion: v1 kind: Service diff --git a/charts/m4d/templates/katalog-connector/katalog-connector-serviceaccount.yaml b/charts/m4d/templates/katalog-connector/katalog-connector-serviceaccount.yaml index 8dbeee5..3fc1ab8 100644 --- a/charts/m4d/templates/katalog-connector/katalog-connector-serviceaccount.yaml +++ b/charts/m4d/templates/katalog-connector/katalog-connector-serviceaccount.yaml @@ -1,4 +1,4 @@ -{{- $autoFlag := and .Values.manager.enabled .Values.coordinator.enabled (eq .Values.coordinator.catalog "katalog") }} +{{- $autoFlag := and .Values.coordinator.enabled (eq .Values.coordinator.catalog "katalog") }} {{- if include "m4d.isEnabled" (tuple .Values.katalogConnector.enabled $autoFlag) }} {{- if .Values.katalogConnector.serviceAccount.create }} apiVersion: v1 diff --git a/charts/m4d/templates/katalog-connector/katalog-rbac.yaml b/charts/m4d/templates/katalog-connector/katalog-rbac.yaml index 7aefe0f..b774fd4 100644 --- a/charts/m4d/templates/katalog-connector/katalog-rbac.yaml +++ b/charts/m4d/templates/katalog-connector/katalog-rbac.yaml @@ -1,4 +1,4 @@ -{{- $autoFlag := and .Values.manager.enabled .Values.coordinator.enabled (eq .Values.coordinator.catalog "katalog") }} +{{- $autoFlag := and .Values.coordinator.enabled (eq .Values.coordinator.catalog "katalog") }} {{- if include "m4d.isEnabled" (tuple .Values.katalogConnector.enabled $autoFlag) }} {{- if .Values.clusterScoped }} # ClusterRole katalog-editor allows managing assets. diff --git a/charts/m4d/templates/m4d-blueprints-ns.yaml b/charts/m4d/templates/m4d-blueprints-ns.yaml index 1372193..7079a61 100644 --- a/charts/m4d/templates/m4d-blueprints-ns.yaml +++ b/charts/m4d/templates/m4d-blueprints-ns.yaml @@ -1,4 +1,4 @@ -{{- if include "m4d.isEnabled" (tuple .Values.manager.enabled (or .Values.coordinator.enabled .Values.worker.enabled)) }} +{{- if or .Values.coordinator.enabled .Values.worker.enabled }} {{- if .Values.clusterScoped }} apiVersion: v1 kind: Namespace diff --git a/charts/m4d/templates/m4d-config.yaml b/charts/m4d/templates/m4d-config.yaml index a5a3c2e..89b631b 100644 --- a/charts/m4d/templates/m4d-config.yaml +++ b/charts/m4d/templates/m4d-config.yaml @@ -1,4 +1,3 @@ -{{- if include "m4d.isEnabled" (tuple .Values.manager.enabled (or .Values.coordinator.enabled .Values.worker.enabled)) }} apiVersion: v1 kind: ConfigMap metadata: @@ -8,17 +7,9 @@ data: CONNECTION_TIMEOUT: {{ .Values.manager.connectionTimeout | default .Values.global.connectionTimeout | quote }} CATALOG_PROVIDER_NAME: {{ .Values.coordinator.catalog | quote }} CATALOG_CONNECTOR_URL: {{ .Values.coordinator.catalogConnectorURL | default (printf "%s-connector:80" .Values.coordinator.catalog) | quote }} - CREDENTIALS_PROVIDER_NAME: {{ .Values.coordinator.credentialsManager | quote }} - CREDENTIALS_CONNECTOR_URL: {{ .Values.coordinator.credentialsManagerConnectorURL | default (printf "%s-connector:80" .Values.coordinator.catalog) | quote }} MAIN_POLICY_MANAGER_NAME: {{ .Values.coordinator.policyManager | quote }} MAIN_POLICY_MANAGER_CONNECTOR_URL: {{ .Values.coordinator.policyManagerConnectorURL | default (printf "%s-connector:80" .Values.coordinator.policyManager) | quote }} USE_EXTENSIONPOLICY_MANAGER: "false" # deprecated - VAULT_ADDRESS: {{ .Values.coordinator.vault.address | quote }} - VAULT_DATASET_HOME: {{ printf "m4d/dataset-creds/" | quote }} # temporary - VAULT_DATASET_MOUNT: {{ printf "/v1/sys/mounts/m4d/dataset-creds" | quote }} # temporary - USER_VAULT_ADDRESS: {{ .Values.coordinator.vault.address | quote }} # deprecated - USER_VAULT_PATH: "external" # deprecated - VAULT_TTL: "24h" # temporary + VAULT_ADDRESS: {{ tpl .Values.coordinator.vault.address . | quote }} VAULT_MODULES_ROLE: "module" # temporary {{- end }} -{{- end }} diff --git a/charts/m4d/templates/manager-deployment.yaml b/charts/m4d/templates/manager-deployment.yaml index bfbcf31..190d590 100644 --- a/charts/m4d/templates/manager-deployment.yaml +++ b/charts/m4d/templates/manager-deployment.yaml @@ -60,8 +60,8 @@ spec: {{- if .Values.manager.overrideArgs }} {{- toYaml .Values.manager.overrideArgs | nindent 12 }} {{- else }} - - "--metrics-addr=127.0.0.1:8080" - - "--enable-leader-election" + - "--metrics-bind-addr=127.0.0.1:8080" + - "--leader-elect" {{- if .Values.coordinator.enabled}} - "--enable-application-controller" - "--enable-plotter-controller" @@ -78,10 +78,6 @@ spec: - secretRef: name: razee-credentials {{- end }} - {{- if .Values.coordinator.enabled}} - - secretRef: - name: vault-credentials - {{- end }} env: - name: ENABLE_WEBHOOKS value: "true" diff --git a/charts/m4d/templates/opa-connector/opa-connector-cm.yaml b/charts/m4d/templates/opa-connector/opa-connector-cm.yaml index ac1839d..17970c7 100644 --- a/charts/m4d/templates/opa-connector/opa-connector-cm.yaml +++ b/charts/m4d/templates/opa-connector/opa-connector-cm.yaml @@ -1,4 +1,4 @@ -{{- $autoFlag := and .Values.manager.enabled .Values.coordinator.enabled (eq .Values.coordinator.policyManager "opa") }} +{{- $autoFlag := and .Values.coordinator.enabled (eq .Values.coordinator.policyManager "opa") }} {{- if include "m4d.isEnabled" (tuple .Values.opaConnector.enabled $autoFlag) }} apiVersion: v1 kind: ConfigMap diff --git a/charts/m4d/templates/opa-connector/opa-connector-deployment.yaml b/charts/m4d/templates/opa-connector/opa-connector-deployment.yaml index 752d441..a0cf4de 100644 --- a/charts/m4d/templates/opa-connector/opa-connector-deployment.yaml +++ b/charts/m4d/templates/opa-connector/opa-connector-deployment.yaml @@ -1,4 +1,4 @@ -{{- $autoFlag := and .Values.manager.enabled .Values.coordinator.enabled (eq .Values.coordinator.policyManager "opa") }} +{{- $autoFlag := and .Values.coordinator.enabled (eq .Values.coordinator.policyManager "opa") }} {{- if include "m4d.isEnabled" (tuple .Values.opaConnector.enabled $autoFlag) }} apiVersion: apps/v1 kind: Deployment diff --git a/charts/m4d/templates/opa-connector/opa-connector-hpa.yaml b/charts/m4d/templates/opa-connector/opa-connector-hpa.yaml index 76044e8..6dea867 100644 --- a/charts/m4d/templates/opa-connector/opa-connector-hpa.yaml +++ b/charts/m4d/templates/opa-connector/opa-connector-hpa.yaml @@ -1,4 +1,4 @@ -{{- $autoFlag := and .Values.manager.enabled .Values.coordinator.enabled (eq .Values.coordinator.policyManager "opa") }} +{{- $autoFlag := and .Values.coordinator.enabled (eq .Values.coordinator.policyManager "opa") }} {{- if include "m4d.isEnabled" (tuple .Values.opaConnector.enabled $autoFlag) }} {{- if .Values.opaConnector.autoscaling.enabled }} apiVersion: autoscaling/v2beta1 diff --git a/charts/m4d/templates/opa-connector/opa-connector-service.yaml b/charts/m4d/templates/opa-connector/opa-connector-service.yaml index 64b9c93..75a90cb 100644 --- a/charts/m4d/templates/opa-connector/opa-connector-service.yaml +++ b/charts/m4d/templates/opa-connector/opa-connector-service.yaml @@ -1,4 +1,4 @@ -{{- $autoFlag := and .Values.manager.enabled .Values.coordinator.enabled (eq .Values.coordinator.policyManager "opa") }} +{{- $autoFlag := and .Values.coordinator.enabled (eq .Values.coordinator.policyManager "opa") }} {{- if include "m4d.isEnabled" (tuple .Values.opaConnector.enabled $autoFlag) }} apiVersion: v1 kind: Service diff --git a/charts/m4d/templates/opa-connector/opa-connector-serviceaccount.yaml b/charts/m4d/templates/opa-connector/opa-connector-serviceaccount.yaml index e9961d1..7c25db1 100644 --- a/charts/m4d/templates/opa-connector/opa-connector-serviceaccount.yaml +++ b/charts/m4d/templates/opa-connector/opa-connector-serviceaccount.yaml @@ -1,4 +1,4 @@ -{{- $autoFlag := and .Values.manager.enabled .Values.coordinator.enabled (eq .Values.coordinator.policyManager "opa") }} +{{- $autoFlag := and .Values.coordinator.enabled (eq .Values.coordinator.policyManager "opa") }} {{- if include "m4d.isEnabled" (tuple .Values.opaConnector.enabled $autoFlag) }} {{- if .Values.opaConnector.serviceAccount.create }} apiVersion: v1 diff --git a/charts/m4d/templates/opa-server/opa-deployment.yaml b/charts/m4d/templates/opa-server/opa-deployment.yaml index 559264d..eac9299 100644 --- a/charts/m4d/templates/opa-server/opa-deployment.yaml +++ b/charts/m4d/templates/opa-server/opa-deployment.yaml @@ -1,4 +1,4 @@ -{{- $autoFlag := and .Values.manager.enabled .Values.coordinator.enabled (eq .Values.coordinator.policyManager "opa") }} +{{- $autoFlag := and .Values.coordinator.enabled (eq .Values.coordinator.policyManager "opa") }} {{- $opaConnectorEnabled := include "m4d.isEnabled" (tuple .Values.opaConnector.enabled $autoFlag) }} {{- if include "m4d.isEnabled" (tuple .Values.opaServer.enabled $opaConnectorEnabled) }} apiVersion: apps/v1 diff --git a/charts/m4d/templates/opa-server/opa-files-policy-lib-cm.yaml b/charts/m4d/templates/opa-server/opa-files-policy-lib-cm.yaml index 0217b23..523dc6b 100644 --- a/charts/m4d/templates/opa-server/opa-files-policy-lib-cm.yaml +++ b/charts/m4d/templates/opa-server/opa-files-policy-lib-cm.yaml @@ -1,4 +1,4 @@ -{{- $autoFlag := and .Values.manager.enabled .Values.coordinator.enabled (eq .Values.coordinator.policyManager "opa") }} +{{- $autoFlag := and .Values.coordinator.enabled (eq .Values.coordinator.policyManager "opa") }} {{- $opaConnectorEnabled := include "m4d.isEnabled" (tuple .Values.opaConnector.enabled $autoFlag) }} {{- if include "m4d.isEnabled" (tuple .Values.opaServer.enabled $opaConnectorEnabled) }} apiVersion: v1 diff --git a/charts/m4d/templates/opa-server/opa-hpa.yaml b/charts/m4d/templates/opa-server/opa-hpa.yaml index c722784..876194e 100644 --- a/charts/m4d/templates/opa-server/opa-hpa.yaml +++ b/charts/m4d/templates/opa-server/opa-hpa.yaml @@ -1,4 +1,4 @@ -{{- $autoFlag := and .Values.manager.enabled .Values.coordinator.enabled (eq .Values.coordinator.policyManager "opa") }} +{{- $autoFlag := and .Values.coordinator.enabled (eq .Values.coordinator.policyManager "opa") }} {{- $opaConnectorEnabled := include "m4d.isEnabled" (tuple .Values.opaConnector.enabled $autoFlag) }} {{- if include "m4d.isEnabled" (tuple .Values.opaServer.enabled $opaConnectorEnabled) }} {{- if .Values.opaServer.autoscaling.enabled }} diff --git a/charts/m4d/templates/opa-server/opa-server-rbac.yaml b/charts/m4d/templates/opa-server/opa-server-rbac.yaml index b3977d7..4aa4203 100644 --- a/charts/m4d/templates/opa-server/opa-server-rbac.yaml +++ b/charts/m4d/templates/opa-server/opa-server-rbac.yaml @@ -1,4 +1,4 @@ -{{- $autoFlag := and .Values.manager.enabled .Values.coordinator.enabled (eq .Values.coordinator.policyManager "opa") }} +{{- $autoFlag := and .Values.coordinator.enabled (eq .Values.coordinator.policyManager "opa") }} {{- $opaConnectorEnabled := include "m4d.isEnabled" (tuple .Values.opaConnector.enabled $autoFlag) }} {{- if include "m4d.isEnabled" (tuple .Values.opaServer.enabled $opaConnectorEnabled) }} kind: Role diff --git a/charts/m4d/templates/opa-server/opa-service.yaml b/charts/m4d/templates/opa-server/opa-service.yaml index 5399560..7a57e55 100644 --- a/charts/m4d/templates/opa-server/opa-service.yaml +++ b/charts/m4d/templates/opa-server/opa-service.yaml @@ -1,4 +1,4 @@ -{{- $autoFlag := and .Values.manager.enabled .Values.coordinator.enabled (eq .Values.coordinator.policyManager "opa") }} +{{- $autoFlag := and .Values.coordinator.enabled (eq .Values.coordinator.policyManager "opa") }} {{- $opaConnectorEnabled := include "m4d.isEnabled" (tuple .Values.opaConnector.enabled $autoFlag) }} {{- if include "m4d.isEnabled" (tuple .Values.opaServer.enabled $opaConnectorEnabled) }} apiVersion: v1 diff --git a/charts/m4d/templates/opa-server/opa-serviceaccount.yaml b/charts/m4d/templates/opa-server/opa-serviceaccount.yaml index 123ea0d..7faa76e 100644 --- a/charts/m4d/templates/opa-server/opa-serviceaccount.yaml +++ b/charts/m4d/templates/opa-server/opa-serviceaccount.yaml @@ -1,4 +1,4 @@ -{{- $autoFlag := and .Values.manager.enabled .Values.coordinator.enabled (eq .Values.coordinator.policyManager "opa") }} +{{- $autoFlag := and .Values.coordinator.enabled (eq .Values.coordinator.policyManager "opa") }} {{- $opaConnectorEnabled := include "m4d.isEnabled" (tuple .Values.opaConnector.enabled $autoFlag) }} {{- if include "m4d.isEnabled" (tuple .Values.opaServer.enabled $opaConnectorEnabled) }} {{- if .Values.opaServer.serviceAccount.create }} diff --git a/charts/m4d/templates/webhook-certificates.yaml b/charts/m4d/templates/webhook-certificates.yaml index 0f9830a..4ecc2b1 100644 --- a/charts/m4d/templates/webhook-certificates.yaml +++ b/charts/m4d/templates/webhook-certificates.yaml @@ -1,5 +1,5 @@ {{- if include "m4d.isEnabled" (tuple .Values.manager.enabled (or .Values.coordinator.enabled .Values.worker.enabled)) }} -apiVersion: cert-manager.io/v1alpha2 +apiVersion: {{ include "m4d.certManagerApiVersion" . }} kind: Issuer metadata: name: selfsigned-issuer @@ -8,7 +8,7 @@ spec: selfSigned: {} --- -apiVersion: cert-manager.io/v1alpha2 +apiVersion: {{ include "m4d.certManagerApiVersion" . }} kind: Certificate metadata: name: serving-cert @@ -21,4 +21,4 @@ spec: kind: Issuer name: selfsigned-issuer secretName: webhook-server-cert -{{- end }} \ No newline at end of file +{{- end }} diff --git a/charts/m4d/values.yaml b/charts/m4d/values.yaml index d6c7883..a570ffe 100644 --- a/charts/m4d/values.yaml +++ b/charts/m4d/values.yaml @@ -16,7 +16,7 @@ clusterScoped: true # Global configuration applies to multiple components installed by this chart global: # Default hub for Mesh for Data images. - hub: ghcr.io/the-mesh-for-data + hub: ghcr.io/mesh-for-data # Default tag for Mesh for Data images. # If no value is set, the chart's appVersion will be used. tag: "" @@ -62,21 +62,10 @@ coordinator: # Defaults to `-connector:80`. policyManagerConnectorURL: "" - # Configures the credentials manager system name to be used by the coordinator manager. - # Accepted values are "katalog" or any meaningful name if a third party connector is used. - # such as "vault". - # This field is subject to be removed in the upcoming release. - credentialsManager: "katalog" - - # Overrides the credentials manager connector URL. - # Defaults to `-connector:80`. - # This field is subject to be removed in a future release. - credentialsManagerConnectorURL: "" - # Configure the vault instance to be used by the coordinator manager vault: # Set to the Vault address. - address: "http://vault.m4d-system:8200" + address: "http://vault.{{ .Release.Namespace }}:8200" # Login method to Vault login: # Token authentication