diff --git a/python/lib/sift_py/file_attachment/__init__.py b/python/lib/sift_py/file_attachment/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/python/lib/sift_py/file_attachment/_internal/__init__.py b/python/lib/sift_py/file_attachment/_internal/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/python/lib/sift_py/file_attachment/_internal/upload.py b/python/lib/sift_py/file_attachment/_internal/upload.py new file mode 100644 index 00000000..a85e6c69 --- /dev/null +++ b/python/lib/sift_py/file_attachment/_internal/upload.py @@ -0,0 +1,109 @@ +from typing import Any, Dict, Optional, Tuple +from requests_toolbelt import MultipartEncoder +from pathlib import Path +from urllib.parse import urljoin, urlparse + +from sift_py._internal.convert.json import to_json +from sift_py.file_attachment.entity import Entity +from sift_py.file_attachment.metadata import Metadata +from sift_py.rest import SiftRestConfig + +import mimetypes +import requests + + +class UploadService: + UPLOAD_PATH = "/api/v0/remote-files/upload" + UPLOAD_BULK_PATH = "/api/v0/remote-files/upload:bulk" + + _upload_uri: str + _upload_bulk_uri: str + _apikey: str + + def __init__(self, restconf: SiftRestConfig): + base_uri = self.__class__._compute_uri(restconf) + self._upload_uri = urljoin(base_uri, self.UPLOAD_PATH) + self._upload_bulk_uri = urljoin(base_uri, self.UPLOAD_BULK_PATH) + self._apikey = restconf["apikey"] + + def upload_attachment( + self, + path: str, + entity: Entity, + metadata: Metadata, + description: Optional[str] = None, + organization_id: Optional[str] = None, + ) -> str: + posix_path = Path(path) + + if not posix_path.is_file(): + raise Exception(f"Provided path, '{path}', must be a regular file") + + file_name, mimetype, content_encoding = self.__class__._mime_and_content_type_from_path( + posix_path + ) + + if not mimetype: + raise Exception(f"The MIME-type of '{posix_path}' could not be computed") + + with open(path, "rb") as file: + form_fields: Dict[str, Any] = { + "entityId": entity.entity_id, + "entityType": entity.entity_type.value, + "metadata": to_json(metadata), + } + + if content_encoding: + form_fields["file"] = ( + file_name, + file, + mimetype, + { + "Content-Encoding": content_encoding, + }, + ) + else: + form_fields["file"] = (file_name, file, mimetype) + + if organization_id: + form_fields["organizationId"] = organization_id + + if description: + form_fields["description"] = description + + form_data = MultipartEncoder(fields=form_fields) + + # https://github.com/requests/toolbelt/issues/312 + # Issue above is reason for the type ignoring + response = requests.post( + url=self._upload_uri, + data=form_data, # type: ignore + headers={ + "Authorization": f"Bearer {self._apikey}", + "Content-Type": form_data.content_type, + }, + ) + + if response.status_code != 200: + raise Exception(response.text) + + return response.json().get("remoteFile").get("remoteFileId") + + @staticmethod + def _mime_and_content_type_from_path(path: Path) -> Tuple[str, Optional[str], Optional[str]]: + file_name = path.name + mime, encoding = mimetypes.guess_type(path) + return file_name, mime, encoding + + @classmethod + def _compute_uri(cls, restconf: SiftRestConfig) -> str: + uri = restconf["uri"] + parsed_uri = urlparse(uri) + + if parsed_uri.scheme != "": + raise Exception(f"The URL scheme '{parsed_uri.scheme}' should not be included") + + if restconf.get("use_ssl", True): + return f"https://{uri}" + + return f"http://{uri}" diff --git a/python/lib/sift_py/file_attachment/entity.py b/python/lib/sift_py/file_attachment/entity.py new file mode 100644 index 00000000..7fe39d94 --- /dev/null +++ b/python/lib/sift_py/file_attachment/entity.py @@ -0,0 +1,21 @@ +""" +Entities represent things that files can be attached to. +""" + +from __future__ import annotations +from enum import Enum + + +class Entity: + entity_id: str + entity_type: EntityType + + def __init__(self, entity_id: str, entity_type: EntityType): + self.entity_id = entity_id + self.entity_type = entity_type + + +class EntityType(Enum): + RUN = "runs" + ANNOTATION = "annotations" + ANNOTATION_LOG = "annotation_logs" diff --git a/python/lib/sift_py/file_attachment/metadata.py b/python/lib/sift_py/file_attachment/metadata.py new file mode 100644 index 00000000..525c02c1 --- /dev/null +++ b/python/lib/sift_py/file_attachment/metadata.py @@ -0,0 +1,73 @@ +from __future__ import annotations +from typing import Any, Type +from typing_extensions import Self +from sift.remote_files.v1.remote_files_pb2 import ( + ImageMetadata as ImageMetadataPb, + VideoMetadata as VideoMetadataPb, +) +from sift_py._internal.convert.protobuf import AsProtobuf +from sift_py._internal.convert.json import AsJson + + +class Metadata(AsJson): ... + + +class VideoMetadata(AsProtobuf, Metadata): + width: int + height: int + duration_seconds: float + + def __init__(self, width: int, height: int, duration_seconds: float): + self.width = width + self.height = height + self.duration_seconds = duration_seconds + + def as_pb(self, klass: Type[VideoMetadataPb]) -> VideoMetadataPb: + return klass( + width=self.width, + height=self.height, + duration_seconds=self.duration_seconds, + ) + + @classmethod + def from_pb(cls, message: VideoMetadataPb) -> Self: + return cls( + width=message.width, + height=message.height, + duration_seconds=message.duration_seconds, + ) + + def as_json(self) -> Any: + return { + "height": self.height, + "width": self.width, + "duration_seconds": self.duration_seconds, + } + + +class ImageMetadata(AsProtobuf, Metadata): + width: int + height: int + + def __init__(self, width: int, height: int): + self.width = width + self.height = height + + def as_pb(self, klass: Type[ImageMetadataPb]) -> ImageMetadataPb: + return klass( + width=self.width, + height=self.height, + ) + + @classmethod + def from_pb(cls, message: ImageMetadataPb) -> Self: + return cls( + width=message.width, + height=message.height, + ) + + def as_json(self) -> Any: + return { + "height": self.height, + "width": self.width, + } diff --git a/python/lib/sift_py/file_attachment/service.py b/python/lib/sift_py/file_attachment/service.py new file mode 100644 index 00000000..9e029067 --- /dev/null +++ b/python/lib/sift_py/file_attachment/service.py @@ -0,0 +1,40 @@ +from typing import Optional, cast +from sift.remote_files.v1.remote_files_pb2_grpc import RemoteFileServiceStub +from sift_py.file_attachment._internal.upload import UploadService +from sift_py.file_attachment.entity import Entity +from sift_py.file_attachment.metadata import Metadata +from sift_py.grpc.transport import SiftChannel +from sift.remote_files.v1.remote_files_pb2 import ( + RemoteFile, + GetRemoteFileRequest, + GetRemoteFileResponse, +) +from sift_py.rest import SiftRestConfig + + +class FileAttachmentService: + _remote_file_service_stub: RemoteFileServiceStub + _upload_service: UploadService + + def __init__(self, channel: SiftChannel, restconf: SiftRestConfig): + self._remote_file_service_stub = RemoteFileServiceStub(channel) + self._upload_service = UploadService(restconf) + + def upload_attachment( + self, + path: str, + entity: Entity, + metadata: Metadata, + description: Optional[str] = None, + organization_id: Optional[str] = None, + ) -> RemoteFile: + remote_file_id = self._upload_service.upload_attachment( + path, + entity, + metadata, + description, + organization_id, + ) + req = GetRemoteFileRequest(remote_file_id=remote_file_id) + res = cast(GetRemoteFileResponse, self._remote_file_service_stub.GetRemoteFile(req)) + return res.remote_file diff --git a/python/lib/sift_py/rest.py b/python/lib/sift_py/rest.py new file mode 100644 index 00000000..9157c1ab --- /dev/null +++ b/python/lib/sift_py/rest.py @@ -0,0 +1,15 @@ +from typing import TypedDict +from typing_extensions import NotRequired + + +class SiftRestConfig(TypedDict): + """ + Config class used to to interact with services that use Sift's REST API.`. + - `uri`: The URI of Sift's REST API. The scheme portion of the URI i.e. `https://` should be ommitted. + - `apikey`: User-generated API key generated via the Sift application. + - `use_ssl`: INTERNAL USE. Meant to be used for local development. + """ + + uri: str + apikey: str + use_ssl: NotRequired[bool] diff --git a/python/pyproject.toml b/python/pyproject.toml index 541eeb65..ccdfdc24 100644 --- a/python/pyproject.toml +++ b/python/pyproject.toml @@ -77,6 +77,11 @@ module = "grpc" ignore_missing_imports = true ignore_errors = true +[[tool.mypy.overrides]] +module = "grpc.aio" +ignore_missing_imports = true +ignore_errors = true + [[tool.mypy.overrides]] module = "requests_toolbelt" ignore_missing_imports = true