feat(vdp): remove openapi_specification and add data_specification (#282

) Because - Currently, we have an `openapi_specification` field in the pipeline and connector/operator definition. The original purpose was that we could generate an OpenAPI schema and have some UI to let the user debug on an individual component. However, it turns out that we don't really need that OpenAPI schema, and it makes our response very large and hard to read. This commit - Simplifies `openapi_specification` into `data_specification` by preserving only the input and output JSON schema and removing the nested structure of the OpenAPI spec. - Adds `data_specifications` in iterator component. --------- Co-authored-by: droplet-bot <[email protected]>
instill-ai · Mar 6, 2024 · c4cdf98 · c4cdf98
1 parent e128375
commit c4cdf98
Show file tree

Hide file tree

Showing 3 changed files with 81 additions and 38 deletions.
diff --git a/openapiv2/vdp/service.swagger.yaml b/openapiv2/vdp/service.swagger.yaml
@@ -407,10 +407,6 @@ paths:
                 format: date-time
                 description: Pipeline update time.
                 readOnly: true
-              openapi_schema:
-                type: object
-                description: OpenAPI schema.
-                readOnly: true
               delete_time:
                 type: string
                 format: date-time
@@ -448,6 +444,10 @@ paths:
                 $ref: '#/definitions/v1betaOwner'
                 description: Pipeline owner.
                 readOnly: true
+              data_specification:
+                $ref: '#/definitions/v1betaDataSpecification'
+                description: Data specifications.
+                readOnly: true
             title: The pipeline fields that will replace the existing ones.
       tags:
         - PipelinePublicService
@@ -908,10 +908,6 @@ paths:
                 format: date-time
                 description: Pipeline update time.
                 readOnly: true
-              openapi_schema:
-                type: object
-                description: OpenAPI schema.
-                readOnly: true
               delete_time:
                 type: string
                 format: date-time
@@ -929,6 +925,10 @@ paths:
               readme:
                 type: string
                 description: README.
+              data_specification:
+                $ref: '#/definitions/v1betaDataSpecification'
+                description: Data specifications.
+                readOnly: true
             title: |-
               The pipeline release fields that will replace the existing ones.
               A pipeline release resource to update
@@ -1384,10 +1384,6 @@ paths:
                 format: date-time
                 description: Pipeline update time.
                 readOnly: true
-              openapi_schema:
-                type: object
-                description: OpenAPI schema.
-                readOnly: true
               delete_time:
                 type: string
                 format: date-time
@@ -1425,6 +1421,10 @@ paths:
                 $ref: '#/definitions/v1betaOwner'
                 description: Pipeline owner.
                 readOnly: true
+              data_specification:
+                $ref: '#/definitions/v1betaDataSpecification'
+                description: Data specifications.
+                readOnly: true
             title: The pipeline fields that will replace the existing ones.
       tags:
         - PipelinePublicService
@@ -1876,10 +1876,6 @@ paths:
                 format: date-time
                 description: Pipeline update time.
                 readOnly: true
-              openapi_schema:
-                type: object
-                description: OpenAPI schema.
-                readOnly: true
               delete_time:
                 type: string
                 format: date-time
@@ -1897,6 +1893,10 @@ paths:
               readme:
                 type: string
                 description: README.
+              data_specification:
+                $ref: '#/definitions/v1betaDataSpecification'
+                description: Data specifications.
+                readOnly: true
             title: |-
               The pipeline release fields that will replace the existing ones.
               A pipeline release resource to update
@@ -4414,14 +4414,18 @@ definitions:
       component_specification:
         type: object
         description: Component specification.
-      openapi_specifications:
+      data_specifications:
         type: object
-        description: OpenAPI specification.
+        additionalProperties:
+          $ref: '#/definitions/v1betaDataSpecification'
+        description: |-
+          Data specifications.
+          The key represents the task, and the value is the corresponding data_specification.
     description: ConnectorSpec represents a specification data model.
     required:
       - resource_specification
       - component_specification
-      - openapi_specifications
+      - data_specifications
   v1betaConnectorState:
     type: string
     enum:
@@ -4512,6 +4516,16 @@ definitions:
         $ref: '#/definitions/v1betaPipeline'
         description: The created pipeline resource.
     description: CreateUserPipelineResponse contains the created pipeline.
+  v1betaDataSpecification:
+    type: object
+    properties:
+      input:
+        type: object
+        description: JSON schema describing the component input data.
+      output:
+        type: object
+        description: JSON schema describing the component output data.
+    description: DataSpecification describes the JSON schema of component input and output.
   v1betaDeleteOrganizationConnectorResponse:
     type: object
     description: DeleteOrganizationConnectorResponse is an empty response.
@@ -4689,6 +4703,10 @@ definitions:
       condition:
         type: string
         description: Condition statement determining whether the component is executed or not.
+      data_specification:
+        $ref: '#/definitions/v1betaDataSpecification'
+        description: DataSpecification returns the JSON schema for the iterator input and output.
+        readOnly: true
     description: |-
       IteratorComponent
       Configures an iterator component. An iterator takes an array and executes an
@@ -5113,13 +5131,17 @@ definitions:
       component_specification:
         type: object
         description: Component specification.
-      openapi_specifications:
+      data_specifications:
         type: object
-        description: OpenAPI specification.
+        additionalProperties:
+          $ref: '#/definitions/v1betaDataSpecification'
+        description: |-
+          Data specifications.
+          The key represents the task, and the value is the corresponding data_specification.
     description: OperatorSpec represents a specification data model.
     required:
       - component_specification
-      - openapi_specifications
+      - data_specifications
   v1betaOrganization:
     type: object
     properties:
@@ -5245,10 +5267,6 @@ definitions:
         format: date-time
         description: Pipeline update time.
         readOnly: true
-      openapi_schema:
-        type: object
-        description: OpenAPI schema.
-        readOnly: true
       delete_time:
         type: string
         format: date-time
@@ -5286,6 +5304,10 @@ definitions:
         $ref: '#/definitions/v1betaOwner'
         description: Pipeline owner.
         readOnly: true
+      data_specification:
+        $ref: '#/definitions/v1betaDataSpecification'
+        description: Data specifications.
+        readOnly: true
     description: |-
       A Pipeline is an end-to-end workflow that automates a sequence of components
       to process data.
@@ -5328,10 +5350,6 @@ definitions:
         format: date-time
         description: Pipeline update time.
         readOnly: true
-      openapi_schema:
-        type: object
-        description: OpenAPI schema.
-        readOnly: true
       delete_time:
         type: string
         format: date-time
@@ -5349,6 +5367,10 @@ definitions:
       readme:
         type: string
         description: README.
+      data_specification:
+        $ref: '#/definitions/v1betaDataSpecification'
+        description: Data specifications.
+        readOnly: true
     description: |-
       Pipeline releases contain the version control information of a pipeline.
       This allows users to track changes in the pipeline over time.

diff --git a/vdp/pipeline/v1beta/component_definition.proto b/vdp/pipeline/v1beta/component_definition.proto
@@ -39,14 +39,25 @@ message ComponentDefinition {
   }
 }
 
+// DataSpecification describes the JSON schema of component input and output.
+message DataSpecification {
+  // JSON schema describing the component input data.
+  google.protobuf.Struct input = 1;
+  // JSON schema describing the component output data.
+  google.protobuf.Struct output = 2;
+}
+
 // ConnectorSpec represents a specification data model.
 message ConnectorSpec {
   // Resource specification.
   google.protobuf.Struct resource_specification = 2 [(google.api.field_behavior) = REQUIRED];
   // Component specification.
   google.protobuf.Struct component_specification = 3 [(google.api.field_behavior) = REQUIRED];
-  // OpenAPI specification.
-  google.protobuf.Struct openapi_specifications = 4 [(google.api.field_behavior) = REQUIRED];
+  // Deleted field.
+  reserved 4;
+  // Data specifications.
+  // The key represents the task, and the value is the corresponding data_specification.
+  map<string, DataSpecification> data_specifications = 5 [(google.api.field_behavior) = REQUIRED];
 }
 
 // ConnectorType defines the connector type based on its task features.
@@ -138,8 +149,11 @@ message ConnectorDefinition {
 message OperatorSpec {
   // Component specification.
   google.protobuf.Struct component_specification = 1 [(google.api.field_behavior) = REQUIRED];
-  // OpenAPI specification.
-  google.protobuf.Struct openapi_specifications = 2 [(google.api.field_behavior) = REQUIRED];
+  // Deleted field.
+  reserved 2;
+  // Data specifications.
+  // The key represents the task, and the value is the corresponding data_specification.
+  map<string, DataSpecification> data_specifications = 3 [(google.api.field_behavior) = REQUIRED];
 }
 
 // An Operator is a type of pipeline component that performs data injection and

diff --git a/vdp/pipeline/v1beta/pipeline.proto b/vdp/pipeline/v1beta/pipeline.proto
@@ -148,6 +148,9 @@ message IteratorComponent {
 
   // Condition statement determining whether the component is executed or not.
   string condition = 4 [(google.api.field_behavior) = OPTIONAL];
+
+  // DataSpecification returns the JSON schema for the iterator input and output.
+  DataSpecification data_specification = 5 [(google.api.field_behavior) = OUTPUT_ONLY];
 }
 
 // Component
@@ -272,8 +275,8 @@ message Pipeline {
   google.protobuf.Timestamp create_time = 10 [(google.api.field_behavior) = OUTPUT_ONLY];
   // Pipeline update time.
   google.protobuf.Timestamp update_time = 11 [(google.api.field_behavior) = OUTPUT_ONLY];
-  // OpenAPI schema.
-  google.protobuf.Struct openapi_schema = 13 [(google.api.field_behavior) = OUTPUT_ONLY];
+  // Deleted field.
+  reserved 13;
   // Pipeline delete time.
   google.protobuf.Timestamp delete_time = 14 [(google.api.field_behavior) = OUTPUT_ONLY];
   // Pipeline sharing information.
@@ -297,6 +300,8 @@ message Pipeline {
     (google.api.field_behavior) = OPTIONAL,
     (google.api.field_behavior) = OUTPUT_ONLY
   ];
+  // Data specifications.
+  DataSpecification data_specification = 24 [(google.api.field_behavior) = OUTPUT_ONLY];
 }
 
 // TriggerMetadata contains the traces of the pipeline inference.
@@ -360,8 +365,8 @@ message PipelineRelease {
   google.protobuf.Timestamp create_time = 6 [(google.api.field_behavior) = OUTPUT_ONLY];
   // Pipeline update time.
   google.protobuf.Timestamp update_time = 7 [(google.api.field_behavior) = OUTPUT_ONLY];
-  // OpenAPI schema.
-  google.protobuf.Struct openapi_schema = 9 [(google.api.field_behavior) = OUTPUT_ONLY];
+  // Deleted field.
+  reserved 9;
   // Pipeline deletion time.
   google.protobuf.Timestamp delete_time = 10 [(google.api.field_behavior) = OUTPUT_ONLY];
   // Alias.
@@ -371,6 +376,8 @@ message PipelineRelease {
   google.protobuf.Struct metadata = 12;
   // README.
   string readme = 13 [(google.api.field_behavior) = OPTIONAL];
+  // Data specifications.
+  DataSpecification data_specification = 14 [(google.api.field_behavior) = OUTPUT_ONLY];
 }
 
 // ListPipelinesRequest represents a request to list pipelines.