From 7116a98797b901e3819dca9dc94f8c17c15026dc Mon Sep 17 00:00:00 2001 From: Heiru Wu Date: Thu, 9 May 2024 04:58:56 +0800 Subject: [PATCH] chore: refactor resources --- instill/clients/pipeline.py | 2 +- instill/resources/__init__.py | 17 ++++++++-- instill/resources/component.py | 40 +++++++++++++++-------- instill/resources/connector_ai.py | 33 ++++++++++++++++--- instill/resources/connector_blockchain.py | 5 ++- instill/resources/connector_data.py | 14 ++++---- instill/resources/operator.py | 21 ++++++------ instill/resources/trigger.py | 33 +++++++++++-------- 8 files changed, 110 insertions(+), 55 deletions(-) diff --git a/instill/clients/pipeline.py b/instill/clients/pipeline.py index 680b757..1e81851 100644 --- a/instill/clients/pipeline.py +++ b/instill/clients/pipeline.py @@ -8,8 +8,8 @@ # pipeline import instill.protogen.vdp.pipeline.v1beta.pipeline_pb2 as pipeline_interface -import instill.protogen.vdp.pipeline.v1beta.secret_pb2 as secret_interface import instill.protogen.vdp.pipeline.v1beta.pipeline_public_service_pb2_grpc as pipeline_service +import instill.protogen.vdp.pipeline.v1beta.secret_pb2 as secret_interface from instill.clients.base import Client, RequestFactory from instill.clients.constant import DEFAULT_INSTANCE from instill.clients.instance import InstillInstance diff --git a/instill/resources/__init__.py b/instill/resources/__init__.py index 4dc4798..4f38e32 100644 --- a/instill/resources/__init__.py +++ b/instill/resources/__init__.py @@ -12,11 +12,24 @@ InstillModelConnector, OpenAIConnector, StabilityAIConnector, + HuggingfaceConnector, ) from instill.resources.connector_blockchain import NumbersConnector -from instill.resources.connector_data import PineconeConnector +from instill.resources.connector_data import ( + PineconeConnector, + RedisConnector, + WebsiteConnector, + BigQueryConnector, + GoogleSearchConnector, + GoogleCloudStorageConnector, +) +from instill.resources.operator import ( + Base64Operator, + ImageOperator, + TextOperator, + JSONOperator, +) from instill.resources.model import GithubModel, HugginfaceModel, Model -from instill.resources.operator import create_end_operator, create_start_operator from instill.resources.pipeline import Pipeline from instill.resources.recipe import create_recipe from instill.resources.schema.helper import populate_default_value diff --git a/instill/resources/component.py b/instill/resources/component.py index 661a13b..bb80d35 100644 --- a/instill/resources/component.py +++ b/instill/resources/component.py @@ -1,6 +1,12 @@ # pylint: disable=no-member,wrong-import-position,no-name-in-module from typing import Union -import instill.protogen.vdp.pipeline.v1beta.pipeline_pb2 as pipeline_interface + +from instill.protogen.vdp.pipeline.v1beta.pipeline_pb2 import Component as Comp +from instill.protogen.vdp.pipeline.v1beta.pipeline_pb2 import ( + ConnectorComponent, + IteratorComponent, + OperatorComponent, +) from instill.resources.errors import ComponentTypeExection @@ -8,26 +14,32 @@ class Component: def __init__( self, name: str, - component_type: str, component: Union[ - pipeline_interface.ConnectorComponent, - pipeline_interface.OperatorComponent, - pipeline_interface.IteratorComponent, + ConnectorComponent, + OperatorComponent, + IteratorComponent, ], ): - c = pipeline_interface.Component() - c.id = name - if component_type == "connector": - c.connector_component = component - elif component_type == "operator": - c.operator_component = component - elif component_type == "iterator": - c.iterator_component = component + if isinstance(component, ConnectorComponent): + c = Comp( + id=name, + connector_component=component, + ) + elif isinstance(component, OperatorComponent): + c = Comp( + id=name, + operator_component=component, + ) + elif isinstance(component, IteratorComponent): + c = Comp( + id=name, + iterator_component=component, + ) else: raise ComponentTypeExection self.c = c - def get_component(self) -> pipeline_interface.Component: + def get_component(self) -> Comp: return self.c diff --git a/instill/resources/connector_ai.py b/instill/resources/connector_ai.py index bfe73a8..e2e6046 100644 --- a/instill/resources/connector_ai.py +++ b/instill/resources/connector_ai.py @@ -39,6 +39,9 @@ openai_task_text_to_speech_input, stabilityai_task_image_to_image_input, stabilityai_task_text_to_image_input, + archetypeai_task_describe_input, + archetypeai_task_summarize_input, + archetypeai_task_upload_file_input, ) @@ -76,7 +79,7 @@ def __init__( component_type, definition_name, inp ) - super().__init__(name, component_type, component) + super().__init__(name, component) class InstillModelConnector(Component): @@ -105,7 +108,7 @@ def __init__( component_type, definition_name, inp ) - super().__init__(name, component_type, component) + super().__init__(name, component) class StabilityAIConnector(Component): @@ -126,7 +129,7 @@ def __init__( component_type, definition_name, inp ) - super().__init__(name, component_type, component) + super().__init__(name, component) class OpenAIConnector(Component): @@ -150,4 +153,26 @@ def __init__( component_type, definition_name, inp ) - super().__init__(name, component_type, component) + super().__init__(name, component) + + +class ArchetypeAIConnector(Component): + """ArchetypeAI Connector""" + + def __init__( + self, + name: str, + inp: Union[ + archetypeai_task_upload_file_input.Input, + archetypeai_task_describe_input.Input, + archetypeai_task_summarize_input.Input, + ], + ) -> None: + definition_name = "connector-definitions/archetype-ai" + component_type = "connector" + + component = helper.construct_component_config( + component_type, definition_name, inp + ) + + super().__init__(name, component) diff --git a/instill/resources/connector_blockchain.py b/instill/resources/connector_blockchain.py index a0e2e92..cd2c2bb 100644 --- a/instill/resources/connector_blockchain.py +++ b/instill/resources/connector_blockchain.py @@ -1,8 +1,7 @@ # pylint: disable=no-member,wrong-import-position,no-name-in-module import json -from instill.resources import Component -from instill.resources import const +from instill.resources import Component, const from instill.resources.schema import helper, numbers_task_commit_input @@ -21,4 +20,4 @@ def __init__( component_type, definition_name, inp ) - super().__init__(name, component_type, component) + super().__init__(name, component) diff --git a/instill/resources/connector_data.py b/instill/resources/connector_data.py index f2d5c25..4cc8c82 100644 --- a/instill/resources/connector_data.py +++ b/instill/resources/connector_data.py @@ -38,7 +38,7 @@ def __init__( component_type, definition_name, inp ) - super().__init__(name, component_type, component) + super().__init__(name, component) class PineconeConnector(Component): @@ -59,7 +59,7 @@ def __init__( component_type, definition_name, inp ) - super().__init__(name, component_type, component) + super().__init__(name, component) class GoogleCloudStorageConnector(Component): @@ -77,7 +77,7 @@ def __init__( component_type, definition_name, inp ) - super().__init__(name, component_type, component) + super().__init__(name, component) class GoogleSearchConnector(Component): @@ -95,7 +95,7 @@ def __init__( component_type, definition_name, inp ) - super().__init__(name, component_type, component) + super().__init__(name, component) class RedisConnector(Component): @@ -117,7 +117,7 @@ def __init__( component_type, definition_name, inp ) - super().__init__(name, component_type, component) + super().__init__(name, component) class RestAPIConnector(Component): @@ -144,7 +144,7 @@ def __init__( component_type, definition_name, inp ) - super().__init__(name, component_type, component) + super().__init__(name, component) class WebsiteConnector(Component): @@ -162,4 +162,4 @@ def __init__( component_type, definition_name, inp ) - super().__init__(name, component_type, component) + super().__init__(name, component) diff --git a/instill/resources/operator.py b/instill/resources/operator.py index 2dbeb97..1d0a906 100644 --- a/instill/resources/operator.py +++ b/instill/resources/operator.py @@ -1,20 +1,21 @@ # pylint: disable=no-member,wrong-import-position from typing import Union + from instill.resources import Component from instill.resources.schema import ( base64_task_decode_input, base64_task_encode_input, - json_task_marshal_input, - json_task_unmarshal_input, + helper, image_task_draw_classification_input, image_task_draw_detection_input, image_task_draw_instance_segmentation_input, - image_task_draw_semantic_segmentation_input, - image_task_draw_ocr_input, image_task_draw_keypoint_input, - text_task_split_by_token_input, + image_task_draw_ocr_input, + image_task_draw_semantic_segmentation_input, + json_task_marshal_input, + json_task_unmarshal_input, text_task_convert_to_text_input, - helper, + text_task_split_by_token_input, ) @@ -36,7 +37,7 @@ def __init__( component_type, definition_name, inp ) - super().__init__(name, component_type, component) + super().__init__(name, component) class JSONOperator(Component): @@ -57,7 +58,7 @@ def __init__( component_type, definition_name, inp ) - super().__init__(name, component_type, component) + super().__init__(name, component) class ImageOperator(Component): @@ -82,7 +83,7 @@ def __init__( component_type, definition_name, inp ) - super().__init__(name, component_type, component) + super().__init__(name, component) class TextOperator(Component): @@ -103,4 +104,4 @@ def __init__( component_type, definition_name, inp ) - super().__init__(name, component_type, component) + super().__init__(name, component) diff --git a/instill/resources/trigger.py b/instill/resources/trigger.py index 5ae7350..372998d 100644 --- a/instill/resources/trigger.py +++ b/instill/resources/trigger.py @@ -1,6 +1,6 @@ # pylint: disable=no-member,wrong-import-position -from typing import List from dataclasses import dataclass +from typing import List import instill.protogen.vdp.pipeline.v1beta.pipeline_pb2 as pipeline_pb @@ -27,23 +27,28 @@ def __init__( request_fields: List[TriggerByRequestRequestFields], response_fields: List[TriggerByRequestResponseFields], ) -> None: - t = pipeline_pb.TriggerByRequest + req = {} - for f in request_fields: - req[f.key] = pipeline_pb.TriggerByRequest.RequestField( - title=f.title, - description=f.description, - instill_format=f.format, + for req_f in request_fields: + req[req_f.key] = pipeline_pb.TriggerByRequest.RequestField( + title=req_f.title, + description=req_f.description, + instill_format=req_f.format, ) resp = {} - for f in response_fields: - resp[f.key] = pipeline_pb.TriggerByRequest.ResponseField( - title=f.title, - description=f.description, - value=f.value, + for resp_f in response_fields: + resp[resp_f.key] = pipeline_pb.TriggerByRequest.ResponseField( + title=resp_f.title, + description=resp_f.description, + value=resp_f.value, + ) + + t = pipeline_pb.Trigger( + trigger_by_request=pipeline_pb.TriggerByRequest( + request_fields=req, + response_fields=resp, ) - t.request_fields = req - t.response_fields = resp + ) self.t = t