From c4cdf98e23510d5937720ad74c5de7ad72bdc4c7 Mon Sep 17 00:00:00 2001 From: "Chang, Hui-Tang" Date: Wed, 6 Mar 2024 09:12:56 -0600 Subject: [PATCH] feat(vdp): remove openapi_specification and add data_specification (#282) Because - Currently, we have an `openapi_specification` field in the pipeline and connector/operator definition. The original purpose was that we could generate an OpenAPI schema and have some UI to let the user debug on an individual component. However, it turns out that we don't really need that OpenAPI schema, and it makes our response very large and hard to read. This commit - Simplifies `openapi_specification` into `data_specification` by preserving only the input and output JSON schema and removing the nested structure of the OpenAPI spec. - Adds `data_specifications` in iterator component. --------- Co-authored-by: droplet-bot --- openapiv2/vdp/service.swagger.yaml | 82 ++++++++++++------- .../v1beta/component_definition.proto | 22 ++++- vdp/pipeline/v1beta/pipeline.proto | 15 +++- 3 files changed, 81 insertions(+), 38 deletions(-) diff --git a/openapiv2/vdp/service.swagger.yaml b/openapiv2/vdp/service.swagger.yaml index 9f9f169e..02a9c3cd 100644 --- a/openapiv2/vdp/service.swagger.yaml +++ b/openapiv2/vdp/service.swagger.yaml @@ -407,10 +407,6 @@ paths: format: date-time description: Pipeline update time. readOnly: true - openapi_schema: - type: object - description: OpenAPI schema. - readOnly: true delete_time: type: string format: date-time @@ -448,6 +444,10 @@ paths: $ref: '#/definitions/v1betaOwner' description: Pipeline owner. readOnly: true + data_specification: + $ref: '#/definitions/v1betaDataSpecification' + description: Data specifications. + readOnly: true title: The pipeline fields that will replace the existing ones. tags: - PipelinePublicService @@ -908,10 +908,6 @@ paths: format: date-time description: Pipeline update time. readOnly: true - openapi_schema: - type: object - description: OpenAPI schema. - readOnly: true delete_time: type: string format: date-time @@ -929,6 +925,10 @@ paths: readme: type: string description: README. + data_specification: + $ref: '#/definitions/v1betaDataSpecification' + description: Data specifications. + readOnly: true title: |- The pipeline release fields that will replace the existing ones. A pipeline release resource to update @@ -1384,10 +1384,6 @@ paths: format: date-time description: Pipeline update time. readOnly: true - openapi_schema: - type: object - description: OpenAPI schema. - readOnly: true delete_time: type: string format: date-time @@ -1425,6 +1421,10 @@ paths: $ref: '#/definitions/v1betaOwner' description: Pipeline owner. readOnly: true + data_specification: + $ref: '#/definitions/v1betaDataSpecification' + description: Data specifications. + readOnly: true title: The pipeline fields that will replace the existing ones. tags: - PipelinePublicService @@ -1876,10 +1876,6 @@ paths: format: date-time description: Pipeline update time. readOnly: true - openapi_schema: - type: object - description: OpenAPI schema. - readOnly: true delete_time: type: string format: date-time @@ -1897,6 +1893,10 @@ paths: readme: type: string description: README. + data_specification: + $ref: '#/definitions/v1betaDataSpecification' + description: Data specifications. + readOnly: true title: |- The pipeline release fields that will replace the existing ones. A pipeline release resource to update @@ -4414,14 +4414,18 @@ definitions: component_specification: type: object description: Component specification. - openapi_specifications: + data_specifications: type: object - description: OpenAPI specification. + additionalProperties: + $ref: '#/definitions/v1betaDataSpecification' + description: |- + Data specifications. + The key represents the task, and the value is the corresponding data_specification. description: ConnectorSpec represents a specification data model. required: - resource_specification - component_specification - - openapi_specifications + - data_specifications v1betaConnectorState: type: string enum: @@ -4512,6 +4516,16 @@ definitions: $ref: '#/definitions/v1betaPipeline' description: The created pipeline resource. description: CreateUserPipelineResponse contains the created pipeline. + v1betaDataSpecification: + type: object + properties: + input: + type: object + description: JSON schema describing the component input data. + output: + type: object + description: JSON schema describing the component output data. + description: DataSpecification describes the JSON schema of component input and output. v1betaDeleteOrganizationConnectorResponse: type: object description: DeleteOrganizationConnectorResponse is an empty response. @@ -4689,6 +4703,10 @@ definitions: condition: type: string description: Condition statement determining whether the component is executed or not. + data_specification: + $ref: '#/definitions/v1betaDataSpecification' + description: DataSpecification returns the JSON schema for the iterator input and output. + readOnly: true description: |- IteratorComponent Configures an iterator component. An iterator takes an array and executes an @@ -5113,13 +5131,17 @@ definitions: component_specification: type: object description: Component specification. - openapi_specifications: + data_specifications: type: object - description: OpenAPI specification. + additionalProperties: + $ref: '#/definitions/v1betaDataSpecification' + description: |- + Data specifications. + The key represents the task, and the value is the corresponding data_specification. description: OperatorSpec represents a specification data model. required: - component_specification - - openapi_specifications + - data_specifications v1betaOrganization: type: object properties: @@ -5245,10 +5267,6 @@ definitions: format: date-time description: Pipeline update time. readOnly: true - openapi_schema: - type: object - description: OpenAPI schema. - readOnly: true delete_time: type: string format: date-time @@ -5286,6 +5304,10 @@ definitions: $ref: '#/definitions/v1betaOwner' description: Pipeline owner. readOnly: true + data_specification: + $ref: '#/definitions/v1betaDataSpecification' + description: Data specifications. + readOnly: true description: |- A Pipeline is an end-to-end workflow that automates a sequence of components to process data. @@ -5328,10 +5350,6 @@ definitions: format: date-time description: Pipeline update time. readOnly: true - openapi_schema: - type: object - description: OpenAPI schema. - readOnly: true delete_time: type: string format: date-time @@ -5349,6 +5367,10 @@ definitions: readme: type: string description: README. + data_specification: + $ref: '#/definitions/v1betaDataSpecification' + description: Data specifications. + readOnly: true description: |- Pipeline releases contain the version control information of a pipeline. This allows users to track changes in the pipeline over time. diff --git a/vdp/pipeline/v1beta/component_definition.proto b/vdp/pipeline/v1beta/component_definition.proto index 08e8b683..d2f4a06c 100644 --- a/vdp/pipeline/v1beta/component_definition.proto +++ b/vdp/pipeline/v1beta/component_definition.proto @@ -39,14 +39,25 @@ message ComponentDefinition { } } +// DataSpecification describes the JSON schema of component input and output. +message DataSpecification { + // JSON schema describing the component input data. + google.protobuf.Struct input = 1; + // JSON schema describing the component output data. + google.protobuf.Struct output = 2; +} + // ConnectorSpec represents a specification data model. message ConnectorSpec { // Resource specification. google.protobuf.Struct resource_specification = 2 [(google.api.field_behavior) = REQUIRED]; // Component specification. google.protobuf.Struct component_specification = 3 [(google.api.field_behavior) = REQUIRED]; - // OpenAPI specification. - google.protobuf.Struct openapi_specifications = 4 [(google.api.field_behavior) = REQUIRED]; + // Deleted field. + reserved 4; + // Data specifications. + // The key represents the task, and the value is the corresponding data_specification. + map data_specifications = 5 [(google.api.field_behavior) = REQUIRED]; } // ConnectorType defines the connector type based on its task features. @@ -138,8 +149,11 @@ message ConnectorDefinition { message OperatorSpec { // Component specification. google.protobuf.Struct component_specification = 1 [(google.api.field_behavior) = REQUIRED]; - // OpenAPI specification. - google.protobuf.Struct openapi_specifications = 2 [(google.api.field_behavior) = REQUIRED]; + // Deleted field. + reserved 2; + // Data specifications. + // The key represents the task, and the value is the corresponding data_specification. + map data_specifications = 3 [(google.api.field_behavior) = REQUIRED]; } // An Operator is a type of pipeline component that performs data injection and diff --git a/vdp/pipeline/v1beta/pipeline.proto b/vdp/pipeline/v1beta/pipeline.proto index 4c212563..4a7eee28 100644 --- a/vdp/pipeline/v1beta/pipeline.proto +++ b/vdp/pipeline/v1beta/pipeline.proto @@ -148,6 +148,9 @@ message IteratorComponent { // Condition statement determining whether the component is executed or not. string condition = 4 [(google.api.field_behavior) = OPTIONAL]; + + // DataSpecification returns the JSON schema for the iterator input and output. + DataSpecification data_specification = 5 [(google.api.field_behavior) = OUTPUT_ONLY]; } // Component @@ -272,8 +275,8 @@ message Pipeline { google.protobuf.Timestamp create_time = 10 [(google.api.field_behavior) = OUTPUT_ONLY]; // Pipeline update time. google.protobuf.Timestamp update_time = 11 [(google.api.field_behavior) = OUTPUT_ONLY]; - // OpenAPI schema. - google.protobuf.Struct openapi_schema = 13 [(google.api.field_behavior) = OUTPUT_ONLY]; + // Deleted field. + reserved 13; // Pipeline delete time. google.protobuf.Timestamp delete_time = 14 [(google.api.field_behavior) = OUTPUT_ONLY]; // Pipeline sharing information. @@ -297,6 +300,8 @@ message Pipeline { (google.api.field_behavior) = OPTIONAL, (google.api.field_behavior) = OUTPUT_ONLY ]; + // Data specifications. + DataSpecification data_specification = 24 [(google.api.field_behavior) = OUTPUT_ONLY]; } // TriggerMetadata contains the traces of the pipeline inference. @@ -360,8 +365,8 @@ message PipelineRelease { google.protobuf.Timestamp create_time = 6 [(google.api.field_behavior) = OUTPUT_ONLY]; // Pipeline update time. google.protobuf.Timestamp update_time = 7 [(google.api.field_behavior) = OUTPUT_ONLY]; - // OpenAPI schema. - google.protobuf.Struct openapi_schema = 9 [(google.api.field_behavior) = OUTPUT_ONLY]; + // Deleted field. + reserved 9; // Pipeline deletion time. google.protobuf.Timestamp delete_time = 10 [(google.api.field_behavior) = OUTPUT_ONLY]; // Alias. @@ -371,6 +376,8 @@ message PipelineRelease { google.protobuf.Struct metadata = 12; // README. string readme = 13 [(google.api.field_behavior) = OPTIONAL]; + // Data specifications. + DataSpecification data_specification = 14 [(google.api.field_behavior) = OUTPUT_ONLY]; } // ListPipelinesRequest represents a request to list pipelines.