diff --git a/.gitignore b/.gitignore index fd4789e7..56d53b2f 100644 --- a/.gitignore +++ b/.gitignore @@ -6,10 +6,12 @@ examples/python/annotations/build/**/* examples/python/annotations/dist/**/* examples/rust/annotations/target/**/* examples/rust/ingestion_with_config/target/**/* - rust/target/**/* rust/protos/**/* go/protos/**/* python/protos/**/* .DS_Store + +python/build +*.egg-info/ diff --git a/makefile b/makefile index e39b043b..71b61549 100644 --- a/makefile +++ b/makefile @@ -1,3 +1,5 @@ +SHELL := /bin/zsh + gen: bash scripts/gen.sh diff --git a/python/gen/__init__.py b/python/lib/__init__.py similarity index 100% rename from python/gen/__init__.py rename to python/lib/__init__.py diff --git a/python/gen/google/__init__.py b/python/lib/google/__init__.py similarity index 100% rename from python/gen/google/__init__.py rename to python/lib/google/__init__.py diff --git a/python/gen/google/api/__init__.py b/python/lib/google/api/__init__.py similarity index 100% rename from python/gen/google/api/__init__.py rename to python/lib/google/api/__init__.py diff --git a/python/gen/google/api/annotations_pb2.py b/python/lib/google/api/annotations_pb2.py similarity index 100% rename from python/gen/google/api/annotations_pb2.py rename to python/lib/google/api/annotations_pb2.py diff --git a/python/gen/google/api/annotations_pb2.pyi b/python/lib/google/api/annotations_pb2.pyi similarity index 100% rename from python/gen/google/api/annotations_pb2.pyi rename to python/lib/google/api/annotations_pb2.pyi diff --git a/python/gen/google/api/annotations_pb2_grpc.py b/python/lib/google/api/annotations_pb2_grpc.py similarity index 100% rename from python/gen/google/api/annotations_pb2_grpc.py rename to python/lib/google/api/annotations_pb2_grpc.py diff --git a/python/gen/google/api/annotations_pb2_grpc.pyi b/python/lib/google/api/annotations_pb2_grpc.pyi similarity index 100% rename from python/gen/google/api/annotations_pb2_grpc.pyi rename to python/lib/google/api/annotations_pb2_grpc.pyi diff --git a/python/gen/google/api/field_behavior_pb2.py b/python/lib/google/api/field_behavior_pb2.py similarity index 100% rename from python/gen/google/api/field_behavior_pb2.py rename to python/lib/google/api/field_behavior_pb2.py diff --git a/python/gen/google/api/field_behavior_pb2.pyi b/python/lib/google/api/field_behavior_pb2.pyi similarity index 100% rename from python/gen/google/api/field_behavior_pb2.pyi rename to python/lib/google/api/field_behavior_pb2.pyi diff --git a/python/gen/google/api/field_behavior_pb2_grpc.py b/python/lib/google/api/field_behavior_pb2_grpc.py similarity index 100% rename from python/gen/google/api/field_behavior_pb2_grpc.py rename to python/lib/google/api/field_behavior_pb2_grpc.py diff --git a/python/gen/google/api/field_behavior_pb2_grpc.pyi b/python/lib/google/api/field_behavior_pb2_grpc.pyi similarity index 100% rename from python/gen/google/api/field_behavior_pb2_grpc.pyi rename to python/lib/google/api/field_behavior_pb2_grpc.pyi diff --git a/python/gen/google/api/http_pb2.py b/python/lib/google/api/http_pb2.py similarity index 100% rename from python/gen/google/api/http_pb2.py rename to python/lib/google/api/http_pb2.py diff --git a/python/gen/google/api/http_pb2.pyi b/python/lib/google/api/http_pb2.pyi similarity index 100% rename from python/gen/google/api/http_pb2.pyi rename to python/lib/google/api/http_pb2.pyi diff --git a/python/gen/google/api/http_pb2_grpc.py b/python/lib/google/api/http_pb2_grpc.py similarity index 100% rename from python/gen/google/api/http_pb2_grpc.py rename to python/lib/google/api/http_pb2_grpc.py diff --git a/python/gen/google/api/http_pb2_grpc.pyi b/python/lib/google/api/http_pb2_grpc.pyi similarity index 100% rename from python/gen/google/api/http_pb2_grpc.pyi rename to python/lib/google/api/http_pb2_grpc.pyi diff --git a/python/gen/protoc_gen_openapiv2/__init__.py b/python/lib/protoc_gen_openapiv2/__init__.py similarity index 100% rename from python/gen/protoc_gen_openapiv2/__init__.py rename to python/lib/protoc_gen_openapiv2/__init__.py diff --git a/python/gen/protoc_gen_openapiv2/options/__init__.py b/python/lib/protoc_gen_openapiv2/options/__init__.py similarity index 100% rename from python/gen/protoc_gen_openapiv2/options/__init__.py rename to python/lib/protoc_gen_openapiv2/options/__init__.py diff --git a/python/gen/protoc_gen_openapiv2/options/annotations_pb2.py b/python/lib/protoc_gen_openapiv2/options/annotations_pb2.py similarity index 100% rename from python/gen/protoc_gen_openapiv2/options/annotations_pb2.py rename to python/lib/protoc_gen_openapiv2/options/annotations_pb2.py diff --git a/python/gen/protoc_gen_openapiv2/options/annotations_pb2.pyi b/python/lib/protoc_gen_openapiv2/options/annotations_pb2.pyi similarity index 100% rename from python/gen/protoc_gen_openapiv2/options/annotations_pb2.pyi rename to python/lib/protoc_gen_openapiv2/options/annotations_pb2.pyi diff --git a/python/gen/protoc_gen_openapiv2/options/annotations_pb2_grpc.py b/python/lib/protoc_gen_openapiv2/options/annotations_pb2_grpc.py similarity index 100% rename from python/gen/protoc_gen_openapiv2/options/annotations_pb2_grpc.py rename to python/lib/protoc_gen_openapiv2/options/annotations_pb2_grpc.py diff --git a/python/gen/protoc_gen_openapiv2/options/annotations_pb2_grpc.pyi b/python/lib/protoc_gen_openapiv2/options/annotations_pb2_grpc.pyi similarity index 100% rename from python/gen/protoc_gen_openapiv2/options/annotations_pb2_grpc.pyi rename to python/lib/protoc_gen_openapiv2/options/annotations_pb2_grpc.pyi diff --git a/python/gen/protoc_gen_openapiv2/options/openapiv2_pb2.py b/python/lib/protoc_gen_openapiv2/options/openapiv2_pb2.py similarity index 100% rename from python/gen/protoc_gen_openapiv2/options/openapiv2_pb2.py rename to python/lib/protoc_gen_openapiv2/options/openapiv2_pb2.py diff --git a/python/gen/protoc_gen_openapiv2/options/openapiv2_pb2.pyi b/python/lib/protoc_gen_openapiv2/options/openapiv2_pb2.pyi similarity index 100% rename from python/gen/protoc_gen_openapiv2/options/openapiv2_pb2.pyi rename to python/lib/protoc_gen_openapiv2/options/openapiv2_pb2.pyi diff --git a/python/gen/protoc_gen_openapiv2/options/openapiv2_pb2_grpc.py b/python/lib/protoc_gen_openapiv2/options/openapiv2_pb2_grpc.py similarity index 100% rename from python/gen/protoc_gen_openapiv2/options/openapiv2_pb2_grpc.py rename to python/lib/protoc_gen_openapiv2/options/openapiv2_pb2_grpc.py diff --git a/python/gen/protoc_gen_openapiv2/options/openapiv2_pb2_grpc.pyi b/python/lib/protoc_gen_openapiv2/options/openapiv2_pb2_grpc.pyi similarity index 100% rename from python/gen/protoc_gen_openapiv2/options/openapiv2_pb2_grpc.pyi rename to python/lib/protoc_gen_openapiv2/options/openapiv2_pb2_grpc.pyi diff --git a/python/gen/sift/__init__.py b/python/lib/sift/__init__.py similarity index 100% rename from python/gen/sift/__init__.py rename to python/lib/sift/__init__.py diff --git a/python/gen/sift/annotation_logs/__init__.py b/python/lib/sift/annotation_logs/__init__.py similarity index 100% rename from python/gen/sift/annotation_logs/__init__.py rename to python/lib/sift/annotation_logs/__init__.py diff --git a/python/gen/sift/annotation_logs/v1/__init__.py b/python/lib/sift/annotation_logs/v1/__init__.py similarity index 100% rename from python/gen/sift/annotation_logs/v1/__init__.py rename to python/lib/sift/annotation_logs/v1/__init__.py diff --git a/python/gen/sift/annotation_logs/v1/annotation_logs_pb2.py b/python/lib/sift/annotation_logs/v1/annotation_logs_pb2.py similarity index 100% rename from python/gen/sift/annotation_logs/v1/annotation_logs_pb2.py rename to python/lib/sift/annotation_logs/v1/annotation_logs_pb2.py diff --git a/python/gen/sift/annotation_logs/v1/annotation_logs_pb2.pyi b/python/lib/sift/annotation_logs/v1/annotation_logs_pb2.pyi similarity index 100% rename from python/gen/sift/annotation_logs/v1/annotation_logs_pb2.pyi rename to python/lib/sift/annotation_logs/v1/annotation_logs_pb2.pyi diff --git a/python/gen/sift/annotation_logs/v1/annotation_logs_pb2_grpc.py b/python/lib/sift/annotation_logs/v1/annotation_logs_pb2_grpc.py similarity index 100% rename from python/gen/sift/annotation_logs/v1/annotation_logs_pb2_grpc.py rename to python/lib/sift/annotation_logs/v1/annotation_logs_pb2_grpc.py diff --git a/python/gen/sift/annotation_logs/v1/annotation_logs_pb2_grpc.pyi b/python/lib/sift/annotation_logs/v1/annotation_logs_pb2_grpc.pyi similarity index 100% rename from python/gen/sift/annotation_logs/v1/annotation_logs_pb2_grpc.pyi rename to python/lib/sift/annotation_logs/v1/annotation_logs_pb2_grpc.pyi diff --git a/python/gen/sift/annotations/__init__.py b/python/lib/sift/annotations/__init__.py similarity index 100% rename from python/gen/sift/annotations/__init__.py rename to python/lib/sift/annotations/__init__.py diff --git a/python/gen/sift/annotations/v1/__init__.py b/python/lib/sift/annotations/v1/__init__.py similarity index 100% rename from python/gen/sift/annotations/v1/__init__.py rename to python/lib/sift/annotations/v1/__init__.py diff --git a/python/gen/sift/annotations/v1/annotations_pb2.py b/python/lib/sift/annotations/v1/annotations_pb2.py similarity index 100% rename from python/gen/sift/annotations/v1/annotations_pb2.py rename to python/lib/sift/annotations/v1/annotations_pb2.py diff --git a/python/gen/sift/annotations/v1/annotations_pb2.pyi b/python/lib/sift/annotations/v1/annotations_pb2.pyi similarity index 100% rename from python/gen/sift/annotations/v1/annotations_pb2.pyi rename to python/lib/sift/annotations/v1/annotations_pb2.pyi diff --git a/python/gen/sift/annotations/v1/annotations_pb2_grpc.py b/python/lib/sift/annotations/v1/annotations_pb2_grpc.py similarity index 100% rename from python/gen/sift/annotations/v1/annotations_pb2_grpc.py rename to python/lib/sift/annotations/v1/annotations_pb2_grpc.py diff --git a/python/gen/sift/annotations/v1/annotations_pb2_grpc.pyi b/python/lib/sift/annotations/v1/annotations_pb2_grpc.pyi similarity index 100% rename from python/gen/sift/annotations/v1/annotations_pb2_grpc.pyi rename to python/lib/sift/annotations/v1/annotations_pb2_grpc.pyi diff --git a/python/gen/sift/assets/__init__.py b/python/lib/sift/assets/__init__.py similarity index 100% rename from python/gen/sift/assets/__init__.py rename to python/lib/sift/assets/__init__.py diff --git a/python/gen/sift/assets/v1/__init__.py b/python/lib/sift/assets/v1/__init__.py similarity index 100% rename from python/gen/sift/assets/v1/__init__.py rename to python/lib/sift/assets/v1/__init__.py diff --git a/python/gen/sift/assets/v1/assets_pb2.py b/python/lib/sift/assets/v1/assets_pb2.py similarity index 100% rename from python/gen/sift/assets/v1/assets_pb2.py rename to python/lib/sift/assets/v1/assets_pb2.py diff --git a/python/gen/sift/assets/v1/assets_pb2.pyi b/python/lib/sift/assets/v1/assets_pb2.pyi similarity index 100% rename from python/gen/sift/assets/v1/assets_pb2.pyi rename to python/lib/sift/assets/v1/assets_pb2.pyi diff --git a/python/gen/sift/assets/v1/assets_pb2_grpc.py b/python/lib/sift/assets/v1/assets_pb2_grpc.py similarity index 100% rename from python/gen/sift/assets/v1/assets_pb2_grpc.py rename to python/lib/sift/assets/v1/assets_pb2_grpc.py diff --git a/python/gen/sift/assets/v1/assets_pb2_grpc.pyi b/python/lib/sift/assets/v1/assets_pb2_grpc.pyi similarity index 100% rename from python/gen/sift/assets/v1/assets_pb2_grpc.pyi rename to python/lib/sift/assets/v1/assets_pb2_grpc.pyi diff --git a/python/gen/sift/calculated_channels/__init__.py b/python/lib/sift/calculated_channels/__init__.py similarity index 100% rename from python/gen/sift/calculated_channels/__init__.py rename to python/lib/sift/calculated_channels/__init__.py diff --git a/python/gen/sift/calculated_channels/v1/__init__.py b/python/lib/sift/calculated_channels/v1/__init__.py similarity index 100% rename from python/gen/sift/calculated_channels/v1/__init__.py rename to python/lib/sift/calculated_channels/v1/__init__.py diff --git a/python/gen/sift/calculated_channels/v1/calculated_channels_pb2.py b/python/lib/sift/calculated_channels/v1/calculated_channels_pb2.py similarity index 100% rename from python/gen/sift/calculated_channels/v1/calculated_channels_pb2.py rename to python/lib/sift/calculated_channels/v1/calculated_channels_pb2.py diff --git a/python/gen/sift/calculated_channels/v1/calculated_channels_pb2.pyi b/python/lib/sift/calculated_channels/v1/calculated_channels_pb2.pyi similarity index 100% rename from python/gen/sift/calculated_channels/v1/calculated_channels_pb2.pyi rename to python/lib/sift/calculated_channels/v1/calculated_channels_pb2.pyi diff --git a/python/gen/sift/calculated_channels/v1/calculated_channels_pb2_grpc.py b/python/lib/sift/calculated_channels/v1/calculated_channels_pb2_grpc.py similarity index 100% rename from python/gen/sift/calculated_channels/v1/calculated_channels_pb2_grpc.py rename to python/lib/sift/calculated_channels/v1/calculated_channels_pb2_grpc.py diff --git a/python/gen/sift/calculated_channels/v1/calculated_channels_pb2_grpc.pyi b/python/lib/sift/calculated_channels/v1/calculated_channels_pb2_grpc.pyi similarity index 100% rename from python/gen/sift/calculated_channels/v1/calculated_channels_pb2_grpc.pyi rename to python/lib/sift/calculated_channels/v1/calculated_channels_pb2_grpc.pyi diff --git a/python/gen/sift/channel_schemas/__init__.py b/python/lib/sift/channel_schemas/__init__.py similarity index 100% rename from python/gen/sift/channel_schemas/__init__.py rename to python/lib/sift/channel_schemas/__init__.py diff --git a/python/gen/sift/channel_schemas/v1/__init__.py b/python/lib/sift/channel_schemas/v1/__init__.py similarity index 100% rename from python/gen/sift/channel_schemas/v1/__init__.py rename to python/lib/sift/channel_schemas/v1/__init__.py diff --git a/python/gen/sift/channel_schemas/v1/channel_schemas_pb2.py b/python/lib/sift/channel_schemas/v1/channel_schemas_pb2.py similarity index 100% rename from python/gen/sift/channel_schemas/v1/channel_schemas_pb2.py rename to python/lib/sift/channel_schemas/v1/channel_schemas_pb2.py diff --git a/python/gen/sift/channel_schemas/v1/channel_schemas_pb2.pyi b/python/lib/sift/channel_schemas/v1/channel_schemas_pb2.pyi similarity index 100% rename from python/gen/sift/channel_schemas/v1/channel_schemas_pb2.pyi rename to python/lib/sift/channel_schemas/v1/channel_schemas_pb2.pyi diff --git a/python/gen/sift/channel_schemas/v1/channel_schemas_pb2_grpc.py b/python/lib/sift/channel_schemas/v1/channel_schemas_pb2_grpc.py similarity index 100% rename from python/gen/sift/channel_schemas/v1/channel_schemas_pb2_grpc.py rename to python/lib/sift/channel_schemas/v1/channel_schemas_pb2_grpc.py diff --git a/python/gen/sift/channel_schemas/v1/channel_schemas_pb2_grpc.pyi b/python/lib/sift/channel_schemas/v1/channel_schemas_pb2_grpc.pyi similarity index 100% rename from python/gen/sift/channel_schemas/v1/channel_schemas_pb2_grpc.pyi rename to python/lib/sift/channel_schemas/v1/channel_schemas_pb2_grpc.pyi diff --git a/python/gen/sift/channels/__init__.py b/python/lib/sift/channels/__init__.py similarity index 100% rename from python/gen/sift/channels/__init__.py rename to python/lib/sift/channels/__init__.py diff --git a/python/gen/sift/channels/v2/__init__.py b/python/lib/sift/channels/v2/__init__.py similarity index 100% rename from python/gen/sift/channels/v2/__init__.py rename to python/lib/sift/channels/v2/__init__.py diff --git a/python/gen/sift/channels/v2/channels_pb2.py b/python/lib/sift/channels/v2/channels_pb2.py similarity index 100% rename from python/gen/sift/channels/v2/channels_pb2.py rename to python/lib/sift/channels/v2/channels_pb2.py diff --git a/python/gen/sift/channels/v2/channels_pb2.pyi b/python/lib/sift/channels/v2/channels_pb2.pyi similarity index 100% rename from python/gen/sift/channels/v2/channels_pb2.pyi rename to python/lib/sift/channels/v2/channels_pb2.pyi diff --git a/python/gen/sift/channels/v2/channels_pb2_grpc.py b/python/lib/sift/channels/v2/channels_pb2_grpc.py similarity index 100% rename from python/gen/sift/channels/v2/channels_pb2_grpc.py rename to python/lib/sift/channels/v2/channels_pb2_grpc.py diff --git a/python/gen/sift/channels/v2/channels_pb2_grpc.pyi b/python/lib/sift/channels/v2/channels_pb2_grpc.pyi similarity index 100% rename from python/gen/sift/channels/v2/channels_pb2_grpc.pyi rename to python/lib/sift/channels/v2/channels_pb2_grpc.pyi diff --git a/python/gen/sift/common/__init__.py b/python/lib/sift/common/__init__.py similarity index 100% rename from python/gen/sift/common/__init__.py rename to python/lib/sift/common/__init__.py diff --git a/python/gen/sift/common/type/__init__.py b/python/lib/sift/common/type/__init__.py similarity index 100% rename from python/gen/sift/common/type/__init__.py rename to python/lib/sift/common/type/__init__.py diff --git a/python/gen/sift/common/type/v1/__init__.py b/python/lib/sift/common/type/v1/__init__.py similarity index 100% rename from python/gen/sift/common/type/v1/__init__.py rename to python/lib/sift/common/type/v1/__init__.py diff --git a/python/gen/sift/common/type/v1/channel_bit_field_element_pb2.py b/python/lib/sift/common/type/v1/channel_bit_field_element_pb2.py similarity index 100% rename from python/gen/sift/common/type/v1/channel_bit_field_element_pb2.py rename to python/lib/sift/common/type/v1/channel_bit_field_element_pb2.py diff --git a/python/gen/sift/common/type/v1/channel_bit_field_element_pb2.pyi b/python/lib/sift/common/type/v1/channel_bit_field_element_pb2.pyi similarity index 100% rename from python/gen/sift/common/type/v1/channel_bit_field_element_pb2.pyi rename to python/lib/sift/common/type/v1/channel_bit_field_element_pb2.pyi diff --git a/python/gen/sift/common/type/v1/channel_bit_field_element_pb2_grpc.py b/python/lib/sift/common/type/v1/channel_bit_field_element_pb2_grpc.py similarity index 100% rename from python/gen/sift/common/type/v1/channel_bit_field_element_pb2_grpc.py rename to python/lib/sift/common/type/v1/channel_bit_field_element_pb2_grpc.py diff --git a/python/gen/sift/common/type/v1/channel_bit_field_element_pb2_grpc.pyi b/python/lib/sift/common/type/v1/channel_bit_field_element_pb2_grpc.pyi similarity index 100% rename from python/gen/sift/common/type/v1/channel_bit_field_element_pb2_grpc.pyi rename to python/lib/sift/common/type/v1/channel_bit_field_element_pb2_grpc.pyi diff --git a/python/gen/sift/common/type/v1/channel_data_type_pb2.py b/python/lib/sift/common/type/v1/channel_data_type_pb2.py similarity index 100% rename from python/gen/sift/common/type/v1/channel_data_type_pb2.py rename to python/lib/sift/common/type/v1/channel_data_type_pb2.py diff --git a/python/gen/sift/common/type/v1/channel_data_type_pb2.pyi b/python/lib/sift/common/type/v1/channel_data_type_pb2.pyi similarity index 100% rename from python/gen/sift/common/type/v1/channel_data_type_pb2.pyi rename to python/lib/sift/common/type/v1/channel_data_type_pb2.pyi diff --git a/python/gen/sift/common/type/v1/channel_data_type_pb2_grpc.py b/python/lib/sift/common/type/v1/channel_data_type_pb2_grpc.py similarity index 100% rename from python/gen/sift/common/type/v1/channel_data_type_pb2_grpc.py rename to python/lib/sift/common/type/v1/channel_data_type_pb2_grpc.py diff --git a/python/gen/sift/common/type/v1/channel_data_type_pb2_grpc.pyi b/python/lib/sift/common/type/v1/channel_data_type_pb2_grpc.pyi similarity index 100% rename from python/gen/sift/common/type/v1/channel_data_type_pb2_grpc.pyi rename to python/lib/sift/common/type/v1/channel_data_type_pb2_grpc.pyi diff --git a/python/gen/sift/common/type/v1/channel_enum_type_pb2.py b/python/lib/sift/common/type/v1/channel_enum_type_pb2.py similarity index 100% rename from python/gen/sift/common/type/v1/channel_enum_type_pb2.py rename to python/lib/sift/common/type/v1/channel_enum_type_pb2.py diff --git a/python/gen/sift/common/type/v1/channel_enum_type_pb2.pyi b/python/lib/sift/common/type/v1/channel_enum_type_pb2.pyi similarity index 100% rename from python/gen/sift/common/type/v1/channel_enum_type_pb2.pyi rename to python/lib/sift/common/type/v1/channel_enum_type_pb2.pyi diff --git a/python/gen/sift/common/type/v1/channel_enum_type_pb2_grpc.py b/python/lib/sift/common/type/v1/channel_enum_type_pb2_grpc.py similarity index 100% rename from python/gen/sift/common/type/v1/channel_enum_type_pb2_grpc.py rename to python/lib/sift/common/type/v1/channel_enum_type_pb2_grpc.py diff --git a/python/gen/sift/common/type/v1/channel_enum_type_pb2_grpc.pyi b/python/lib/sift/common/type/v1/channel_enum_type_pb2_grpc.pyi similarity index 100% rename from python/gen/sift/common/type/v1/channel_enum_type_pb2_grpc.pyi rename to python/lib/sift/common/type/v1/channel_enum_type_pb2_grpc.pyi diff --git a/python/gen/sift/common/type/v1/organization_pb2.py b/python/lib/sift/common/type/v1/organization_pb2.py similarity index 100% rename from python/gen/sift/common/type/v1/organization_pb2.py rename to python/lib/sift/common/type/v1/organization_pb2.py diff --git a/python/gen/sift/common/type/v1/organization_pb2.pyi b/python/lib/sift/common/type/v1/organization_pb2.pyi similarity index 100% rename from python/gen/sift/common/type/v1/organization_pb2.pyi rename to python/lib/sift/common/type/v1/organization_pb2.pyi diff --git a/python/gen/sift/common/type/v1/organization_pb2_grpc.py b/python/lib/sift/common/type/v1/organization_pb2_grpc.py similarity index 100% rename from python/gen/sift/common/type/v1/organization_pb2_grpc.py rename to python/lib/sift/common/type/v1/organization_pb2_grpc.py diff --git a/python/gen/sift/common/type/v1/organization_pb2_grpc.pyi b/python/lib/sift/common/type/v1/organization_pb2_grpc.pyi similarity index 100% rename from python/gen/sift/common/type/v1/organization_pb2_grpc.pyi rename to python/lib/sift/common/type/v1/organization_pb2_grpc.pyi diff --git a/python/gen/sift/common/type/v1/user_pb2.py b/python/lib/sift/common/type/v1/user_pb2.py similarity index 100% rename from python/gen/sift/common/type/v1/user_pb2.py rename to python/lib/sift/common/type/v1/user_pb2.py diff --git a/python/gen/sift/common/type/v1/user_pb2.pyi b/python/lib/sift/common/type/v1/user_pb2.pyi similarity index 100% rename from python/gen/sift/common/type/v1/user_pb2.pyi rename to python/lib/sift/common/type/v1/user_pb2.pyi diff --git a/python/gen/sift/common/type/v1/user_pb2_grpc.py b/python/lib/sift/common/type/v1/user_pb2_grpc.py similarity index 100% rename from python/gen/sift/common/type/v1/user_pb2_grpc.py rename to python/lib/sift/common/type/v1/user_pb2_grpc.py diff --git a/python/gen/sift/common/type/v1/user_pb2_grpc.pyi b/python/lib/sift/common/type/v1/user_pb2_grpc.pyi similarity index 100% rename from python/gen/sift/common/type/v1/user_pb2_grpc.pyi rename to python/lib/sift/common/type/v1/user_pb2_grpc.pyi diff --git a/python/gen/sift/data/__init__.py b/python/lib/sift/data/__init__.py similarity index 100% rename from python/gen/sift/data/__init__.py rename to python/lib/sift/data/__init__.py diff --git a/python/gen/sift/data/v1/__init__.py b/python/lib/sift/data/v1/__init__.py similarity index 100% rename from python/gen/sift/data/v1/__init__.py rename to python/lib/sift/data/v1/__init__.py diff --git a/python/gen/sift/data/v1/data_pb2.py b/python/lib/sift/data/v1/data_pb2.py similarity index 100% rename from python/gen/sift/data/v1/data_pb2.py rename to python/lib/sift/data/v1/data_pb2.py diff --git a/python/gen/sift/data/v1/data_pb2.pyi b/python/lib/sift/data/v1/data_pb2.pyi similarity index 100% rename from python/gen/sift/data/v1/data_pb2.pyi rename to python/lib/sift/data/v1/data_pb2.pyi diff --git a/python/gen/sift/data/v1/data_pb2_grpc.py b/python/lib/sift/data/v1/data_pb2_grpc.py similarity index 100% rename from python/gen/sift/data/v1/data_pb2_grpc.py rename to python/lib/sift/data/v1/data_pb2_grpc.py diff --git a/python/gen/sift/data/v1/data_pb2_grpc.pyi b/python/lib/sift/data/v1/data_pb2_grpc.pyi similarity index 100% rename from python/gen/sift/data/v1/data_pb2_grpc.pyi rename to python/lib/sift/data/v1/data_pb2_grpc.pyi diff --git a/python/gen/sift/ingest/__init__.py b/python/lib/sift/ingest/__init__.py similarity index 100% rename from python/gen/sift/ingest/__init__.py rename to python/lib/sift/ingest/__init__.py diff --git a/python/gen/sift/ingest/v1/__init__.py b/python/lib/sift/ingest/v1/__init__.py similarity index 100% rename from python/gen/sift/ingest/v1/__init__.py rename to python/lib/sift/ingest/v1/__init__.py diff --git a/python/gen/sift/ingest/v1/ingest_pb2.py b/python/lib/sift/ingest/v1/ingest_pb2.py similarity index 100% rename from python/gen/sift/ingest/v1/ingest_pb2.py rename to python/lib/sift/ingest/v1/ingest_pb2.py diff --git a/python/gen/sift/ingest/v1/ingest_pb2.pyi b/python/lib/sift/ingest/v1/ingest_pb2.pyi similarity index 100% rename from python/gen/sift/ingest/v1/ingest_pb2.pyi rename to python/lib/sift/ingest/v1/ingest_pb2.pyi diff --git a/python/gen/sift/ingest/v1/ingest_pb2_grpc.py b/python/lib/sift/ingest/v1/ingest_pb2_grpc.py similarity index 100% rename from python/gen/sift/ingest/v1/ingest_pb2_grpc.py rename to python/lib/sift/ingest/v1/ingest_pb2_grpc.py diff --git a/python/gen/sift/ingest/v1/ingest_pb2_grpc.pyi b/python/lib/sift/ingest/v1/ingest_pb2_grpc.pyi similarity index 100% rename from python/gen/sift/ingest/v1/ingest_pb2_grpc.pyi rename to python/lib/sift/ingest/v1/ingest_pb2_grpc.pyi diff --git a/python/gen/sift/ingestion_configs/__init__.py b/python/lib/sift/ingestion_configs/__init__.py similarity index 100% rename from python/gen/sift/ingestion_configs/__init__.py rename to python/lib/sift/ingestion_configs/__init__.py diff --git a/python/gen/sift/ingestion_configs/v1/__init__.py b/python/lib/sift/ingestion_configs/v1/__init__.py similarity index 100% rename from python/gen/sift/ingestion_configs/v1/__init__.py rename to python/lib/sift/ingestion_configs/v1/__init__.py diff --git a/python/gen/sift/ingestion_configs/v1/ingestion_configs_pb2.py b/python/lib/sift/ingestion_configs/v1/ingestion_configs_pb2.py similarity index 100% rename from python/gen/sift/ingestion_configs/v1/ingestion_configs_pb2.py rename to python/lib/sift/ingestion_configs/v1/ingestion_configs_pb2.py diff --git a/python/gen/sift/ingestion_configs/v1/ingestion_configs_pb2.pyi b/python/lib/sift/ingestion_configs/v1/ingestion_configs_pb2.pyi similarity index 100% rename from python/gen/sift/ingestion_configs/v1/ingestion_configs_pb2.pyi rename to python/lib/sift/ingestion_configs/v1/ingestion_configs_pb2.pyi diff --git a/python/gen/sift/ingestion_configs/v1/ingestion_configs_pb2_grpc.py b/python/lib/sift/ingestion_configs/v1/ingestion_configs_pb2_grpc.py similarity index 100% rename from python/gen/sift/ingestion_configs/v1/ingestion_configs_pb2_grpc.py rename to python/lib/sift/ingestion_configs/v1/ingestion_configs_pb2_grpc.py diff --git a/python/gen/sift/ingestion_configs/v1/ingestion_configs_pb2_grpc.pyi b/python/lib/sift/ingestion_configs/v1/ingestion_configs_pb2_grpc.pyi similarity index 100% rename from python/gen/sift/ingestion_configs/v1/ingestion_configs_pb2_grpc.pyi rename to python/lib/sift/ingestion_configs/v1/ingestion_configs_pb2_grpc.pyi diff --git a/python/gen/sift/notifications/__init__.py b/python/lib/sift/notifications/__init__.py similarity index 100% rename from python/gen/sift/notifications/__init__.py rename to python/lib/sift/notifications/__init__.py diff --git a/python/gen/sift/notifications/v1/__init__.py b/python/lib/sift/notifications/v1/__init__.py similarity index 100% rename from python/gen/sift/notifications/v1/__init__.py rename to python/lib/sift/notifications/v1/__init__.py diff --git a/python/gen/sift/notifications/v1/notifications_pb2.py b/python/lib/sift/notifications/v1/notifications_pb2.py similarity index 100% rename from python/gen/sift/notifications/v1/notifications_pb2.py rename to python/lib/sift/notifications/v1/notifications_pb2.py diff --git a/python/gen/sift/notifications/v1/notifications_pb2.pyi b/python/lib/sift/notifications/v1/notifications_pb2.pyi similarity index 100% rename from python/gen/sift/notifications/v1/notifications_pb2.pyi rename to python/lib/sift/notifications/v1/notifications_pb2.pyi diff --git a/python/gen/sift/notifications/v1/notifications_pb2_grpc.py b/python/lib/sift/notifications/v1/notifications_pb2_grpc.py similarity index 100% rename from python/gen/sift/notifications/v1/notifications_pb2_grpc.py rename to python/lib/sift/notifications/v1/notifications_pb2_grpc.py diff --git a/python/gen/sift/notifications/v1/notifications_pb2_grpc.pyi b/python/lib/sift/notifications/v1/notifications_pb2_grpc.pyi similarity index 100% rename from python/gen/sift/notifications/v1/notifications_pb2_grpc.pyi rename to python/lib/sift/notifications/v1/notifications_pb2_grpc.pyi diff --git a/python/gen/sift/rules/__init__.py b/python/lib/sift/rules/__init__.py similarity index 100% rename from python/gen/sift/rules/__init__.py rename to python/lib/sift/rules/__init__.py diff --git a/python/gen/sift/rules/v1/__init__.py b/python/lib/sift/rules/v1/__init__.py similarity index 100% rename from python/gen/sift/rules/v1/__init__.py rename to python/lib/sift/rules/v1/__init__.py diff --git a/python/gen/sift/rules/v1/rules_pb2.py b/python/lib/sift/rules/v1/rules_pb2.py similarity index 100% rename from python/gen/sift/rules/v1/rules_pb2.py rename to python/lib/sift/rules/v1/rules_pb2.py diff --git a/python/gen/sift/rules/v1/rules_pb2.pyi b/python/lib/sift/rules/v1/rules_pb2.pyi similarity index 100% rename from python/gen/sift/rules/v1/rules_pb2.pyi rename to python/lib/sift/rules/v1/rules_pb2.pyi diff --git a/python/gen/sift/rules/v1/rules_pb2_grpc.py b/python/lib/sift/rules/v1/rules_pb2_grpc.py similarity index 100% rename from python/gen/sift/rules/v1/rules_pb2_grpc.py rename to python/lib/sift/rules/v1/rules_pb2_grpc.py diff --git a/python/gen/sift/rules/v1/rules_pb2_grpc.pyi b/python/lib/sift/rules/v1/rules_pb2_grpc.pyi similarity index 100% rename from python/gen/sift/rules/v1/rules_pb2_grpc.pyi rename to python/lib/sift/rules/v1/rules_pb2_grpc.pyi diff --git a/python/gen/sift/runs/__init__.py b/python/lib/sift/runs/__init__.py similarity index 100% rename from python/gen/sift/runs/__init__.py rename to python/lib/sift/runs/__init__.py diff --git a/python/gen/sift/runs/v2/__init__.py b/python/lib/sift/runs/v2/__init__.py similarity index 100% rename from python/gen/sift/runs/v2/__init__.py rename to python/lib/sift/runs/v2/__init__.py diff --git a/python/gen/sift/runs/v2/runs_pb2.py b/python/lib/sift/runs/v2/runs_pb2.py similarity index 100% rename from python/gen/sift/runs/v2/runs_pb2.py rename to python/lib/sift/runs/v2/runs_pb2.py diff --git a/python/gen/sift/runs/v2/runs_pb2.pyi b/python/lib/sift/runs/v2/runs_pb2.pyi similarity index 100% rename from python/gen/sift/runs/v2/runs_pb2.pyi rename to python/lib/sift/runs/v2/runs_pb2.pyi diff --git a/python/gen/sift/runs/v2/runs_pb2_grpc.py b/python/lib/sift/runs/v2/runs_pb2_grpc.py similarity index 100% rename from python/gen/sift/runs/v2/runs_pb2_grpc.py rename to python/lib/sift/runs/v2/runs_pb2_grpc.py diff --git a/python/gen/sift/runs/v2/runs_pb2_grpc.pyi b/python/lib/sift/runs/v2/runs_pb2_grpc.pyi similarity index 100% rename from python/gen/sift/runs/v2/runs_pb2_grpc.pyi rename to python/lib/sift/runs/v2/runs_pb2_grpc.pyi diff --git a/python/gen/sift/tags/__init__.py b/python/lib/sift/tags/__init__.py similarity index 100% rename from python/gen/sift/tags/__init__.py rename to python/lib/sift/tags/__init__.py diff --git a/python/gen/sift/tags/v1/__init__.py b/python/lib/sift/tags/v1/__init__.py similarity index 100% rename from python/gen/sift/tags/v1/__init__.py rename to python/lib/sift/tags/v1/__init__.py diff --git a/python/gen/sift/tags/v1/tags_pb2.py b/python/lib/sift/tags/v1/tags_pb2.py similarity index 100% rename from python/gen/sift/tags/v1/tags_pb2.py rename to python/lib/sift/tags/v1/tags_pb2.py diff --git a/python/gen/sift/tags/v1/tags_pb2.pyi b/python/lib/sift/tags/v1/tags_pb2.pyi similarity index 100% rename from python/gen/sift/tags/v1/tags_pb2.pyi rename to python/lib/sift/tags/v1/tags_pb2.pyi diff --git a/python/gen/sift/tags/v1/tags_pb2_grpc.py b/python/lib/sift/tags/v1/tags_pb2_grpc.py similarity index 100% rename from python/gen/sift/tags/v1/tags_pb2_grpc.py rename to python/lib/sift/tags/v1/tags_pb2_grpc.py diff --git a/python/gen/sift/tags/v1/tags_pb2_grpc.pyi b/python/lib/sift/tags/v1/tags_pb2_grpc.pyi similarity index 100% rename from python/gen/sift/tags/v1/tags_pb2_grpc.pyi rename to python/lib/sift/tags/v1/tags_pb2_grpc.pyi diff --git a/python/gen/sift/users/__init__.py b/python/lib/sift/users/__init__.py similarity index 100% rename from python/gen/sift/users/__init__.py rename to python/lib/sift/users/__init__.py diff --git a/python/gen/sift/users/v2/__init__.py b/python/lib/sift/users/v2/__init__.py similarity index 100% rename from python/gen/sift/users/v2/__init__.py rename to python/lib/sift/users/v2/__init__.py diff --git a/python/gen/sift/users/v2/users_pb2.py b/python/lib/sift/users/v2/users_pb2.py similarity index 100% rename from python/gen/sift/users/v2/users_pb2.py rename to python/lib/sift/users/v2/users_pb2.py diff --git a/python/gen/sift/users/v2/users_pb2.pyi b/python/lib/sift/users/v2/users_pb2.pyi similarity index 100% rename from python/gen/sift/users/v2/users_pb2.pyi rename to python/lib/sift/users/v2/users_pb2.pyi diff --git a/python/gen/sift/users/v2/users_pb2_grpc.py b/python/lib/sift/users/v2/users_pb2_grpc.py similarity index 100% rename from python/gen/sift/users/v2/users_pb2_grpc.py rename to python/lib/sift/users/v2/users_pb2_grpc.py diff --git a/python/gen/sift/users/v2/users_pb2_grpc.pyi b/python/lib/sift/users/v2/users_pb2_grpc.pyi similarity index 100% rename from python/gen/sift/users/v2/users_pb2_grpc.pyi rename to python/lib/sift/users/v2/users_pb2_grpc.pyi diff --git a/python/lib/sift_internal/__init__.py b/python/lib/sift_internal/__init__.py new file mode 100644 index 00000000..bb0726cd --- /dev/null +++ b/python/lib/sift_internal/__init__.py @@ -0,0 +1,5 @@ +""" +INTERNAL MODULE + +This module is for internal use only. +""" diff --git a/python/lib/sift_internal/convert/__init__.py b/python/lib/sift_internal/convert/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/python/lib/sift_internal/convert/protobuf.py b/python/lib/sift_internal/convert/protobuf.py new file mode 100644 index 00000000..390a0b0f --- /dev/null +++ b/python/lib/sift_internal/convert/protobuf.py @@ -0,0 +1,40 @@ +from abc import ABC, abstractmethod +from google.protobuf.message import Message +from typing import cast, Type, TypeVar + +ProtobufMessage = Message + + +class AsProtobuf(ABC): + """ + Flexible abstract class to create classes that can be converted into different protobuf + targets. All conversion logic goes inside of `as_pb` and should do a runtime check for + class name, module name, and/or things of the like using the `klass` argument available + to generate the appropriate protobuf target. + + The `as_pb` method should rarely be used directly since it returns the super-type. Prefer + to use the `cast_pb` function to convert sub-types of `AsProtobuf` to the concrete protobuf type. + """ + + @abstractmethod + def as_pb(self, klass: Type[Message]) -> Message: + pass + + +T = TypeVar("T", bound=ProtobufMessage) + + +def try_convert_pb(val: AsProtobuf, target_type: Type[T]) -> T: + """ + Utility to convert sub-types of `AsProtobuf` to its concrete protobuf type. + Will raise a `TypeError` if the underlying type of `val` is not `target_type`. + """ + + pb_value = val.as_pb(target_type) + + if isinstance(pb_value, target_type): + return cast(target_type, pb_value) + else: + raise TypeError( + f"Expected `val` to be a '{target_type.__module__}.{target_type.__name__}' but it is a '{val.__module__}.{val.__class__.__name__}'." + ) diff --git a/python/lib/sift_internal/types.py b/python/lib/sift_internal/types.py new file mode 100644 index 00000000..760a2c3a --- /dev/null +++ b/python/lib/sift_internal/types.py @@ -0,0 +1,15 @@ +from typing import Any, Optional, Type, TypeVar + +T = TypeVar("T") +def any_as(val: Any, target_type: Type[T]) -> Optional[T]: + """ + Tries to cast `val` into `target_type` otherwise returns `None`. + """ + + if val is None: + return None + + if isinstance(val, target_type): + return val + else: + return None diff --git a/python/lib/sift_py/__init__.py b/python/lib/sift_py/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/python/lib/sift_py/grpc/__init__.py b/python/lib/sift_py/grpc/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/python/lib/sift_py/grpc/interceptors.py b/python/lib/sift_py/grpc/interceptors.py new file mode 100644 index 00000000..3e7eb358 --- /dev/null +++ b/python/lib/sift_py/grpc/interceptors.py @@ -0,0 +1,43 @@ +""" +Internal Module: This module contains implementation details and is not intended for external use. + +This module is concerned with defining interceptors for unary and streaming RPCs. Any sub-class +of `ClientInterceptor` will be invoked for all types of RPCs: unary-unary, unary-stream, stream-unary, +and stream-stream. To create interceptors for particular kinds of RPCs you'll need to create a sub-class +for the particular types of interceptors found in the base `grpc` module. +""" + +from __future__ import annotations +from grpc_interceptor import ClientInterceptor, ClientCallDetails +from typing import Any, Callable, cast, List, Tuple + +import grpc + +Metadata = List[Tuple[str, str]] + + +class MetadataInterceptor(ClientInterceptor): + """ + Interceptor to add metadata to all unary and streaming RPCs + """ + + def __init__(self, metadata: Metadata): + self.metadata = metadata + + def intercept( + self, + method: Callable, + request_or_iterator: Any, + call_details: grpc.ClientCallDetails, + ): + call_details = cast(ClientCallDetails, call_details) + new_details = ClientCallDetails( + call_details.method, + call_details.timeout, + self.metadata, + call_details.credentials, + call_details.wait_for_ready, + call_details.compression, + ) + + return method(request_or_iterator, new_details) diff --git a/python/lib/sift_py/grpc/transport.py b/python/lib/sift_py/grpc/transport.py new file mode 100644 index 00000000..c38c0861 --- /dev/null +++ b/python/lib/sift_py/grpc/transport.py @@ -0,0 +1,67 @@ +""" +This module is concerned with creating a gRPC transport channel specifically for +interacting with Sift's gRPC API. the `use_sift_channel` method creates said channel +and should generally be used within a with-block for correct resource management. +""" + +from __future__ import annotations +from grpc_interceptor import ClientInterceptor +from .interceptors import Metadata, MetadataInterceptor +from typing import List, TypedDict + +import grpc + +SiftChannel = grpc.Channel + + +def use_sift_channel(config: SiftChannelConfig) -> SiftChannel: + """ + Returns an intercepted channel that is meant to be used across all services that + make RPCs to Sift's API. It is highly encouraged to use this within a with-block + for correct resouce clean-up as to ensure no long-lived idle channels. + """ + credentials = grpc.ssl_channel_credentials() + channel = grpc.secure_channel(config["uri"], credentials) + interceptors = _compute_sift_interceptors(config) + return grpc.intercept_channel(channel, *interceptors) + + +def use_insecure_sift_channel(config: SiftChannelConfig) -> SiftChannel: + """ + FOR DEVELOPMENT PURPOSES ONLY + """ + channel = grpc.insecure_channel(config["uri"]) + interceptors = _compute_sift_interceptors(config) + return grpc.intercept_channel(channel, *interceptors) + + +def _compute_sift_interceptors(config: SiftChannelConfig) -> List[ClientInterceptor]: + """ + Initialized all interceptors here. + """ + return [ + _metadata_interceptor(config), + ] + + +def _metadata_interceptor(config: SiftChannelConfig) -> ClientInterceptor: + """ + Any new metadata goes here. + """ + metadata: Metadata = [ + ("authorization", f"Bearer {config["apikey"]}"), + ] + return MetadataInterceptor(metadata) + + +class SiftChannelConfig(TypedDict): + """ + Config class used to instantiate a `SiftChannel` via `use_sift_channel`. + + Attributes: + uri: The URI of Sift's gRPC API. The scheme portion of the URI i.e. `https://` should be ommitted. + apikey: User-generated API key generated via the Sift application. + """ + + uri: str + apikey: str diff --git a/python/lib/sift_py/ingestion/__init__.py b/python/lib/sift_py/ingestion/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/python/lib/sift_py/ingestion/channel.py b/python/lib/sift_py/ingestion/channel.py new file mode 100644 index 00000000..290a63a0 --- /dev/null +++ b/python/lib/sift_py/ingestion/channel.py @@ -0,0 +1,209 @@ +from __future__ import annotations +from google.protobuf.empty_pb2 import Empty +from sift_internal.convert.protobuf import AsProtobuf, ProtobufMessage, try_convert_pb +from enum import Enum +from sift.common.type.v1.channel_enum_type_pb2 import ChannelEnumType as ChannelEnumTypePb +from sift.common.type.v1.channel_bit_field_element_pb2 import ( + ChannelBitFieldElement as ChannelBitFieldElementPb, +) +from sift.ingest.v1.ingest_pb2 import IngestWithConfigDataChannelValue +from sift.ingestion_configs.v1.ingestion_configs_pb2 import ( + ChannelConfig as ChannelConfigPb, +) +from typing import List, NotRequired, Optional, Type, TypedDict + +import sift.common.type.v1.channel_data_type_pb2 as channel_pb + + +class ChannelValue(TypedDict): + """ + Represents a fully qualified data point for a channel + """ + + channel_name: str + component: NotRequired[str] + value: IngestWithConfigDataChannelValue + + +class ChannelConfig(AsProtobuf): + """ + A description for a channel + """ + + name: str + data_type: ChannelDataType + description: Optional[str] + unit: Optional[str] + component: Optional[str] + bit_field_elements: List[ChannelBitFieldElement] + enum_types: List[ChannelEnumType] + + def __init__( + self, + name: str, + data_type: ChannelDataType, + description: Optional[str] = None, + unit: Optional[str] = None, + component: Optional[str] = None, + bit_field_elements: List[ChannelBitFieldElement] = [], + enum_types: List[ChannelEnumType] = [], + ): + self.name = name + self.data_type = data_type + self.description = description + self.unit = unit + self.component = component + self.bit_field_elements = bit_field_elements + self.enum_types = enum_types + + def as_pb(self, klass: Type[ProtobufMessage]) -> ProtobufMessage: + return ChannelConfigPb( + name=self.name, + component=self.component or "", + unit=self.unit or "", + description=self.description or "", + data_type=self.data_type.value, + enum_types=[try_convert_pb(etype, ChannelEnumTypePb) for etype in self.enum_types], + bit_field_elements=[ + try_convert_pb(el, ChannelBitFieldElementPb) for el in self.bit_field_elements + ], + ) + + +class ChannelBitFieldElement(AsProtobuf): + name: str + index: int + bit_count: int + + def __init__(self, name: str, index: int, bit_count: int): + self.name = name + self.index = index + self.bit_count = bit_count + + def as_pb(self, klass: Type[ProtobufMessage]) -> ProtobufMessage: + return ChannelBitFieldElementPb( + name=self.name, + index=self.index, + bit_count=self.bit_count, + ) + + +class ChannelEnumType(AsProtobuf): + name: str + key: int + + def __init__(self, name: str, key: int): + self.name = name + self.key = key + + def as_pb(self, klass: Type[ProtobufMessage]) -> ProtobufMessage: + return ChannelEnumTypePb(name=self.name, key=self.key) + + +class ChannelDataType(Enum): + """ + Utility enum class to simplify working with channel data-types generated from protobuf + """ + + DOUBLE = channel_pb.CHANNEL_DATA_TYPE_DOUBLE + STRING = channel_pb.CHANNEL_DATA_TYPE_STRING + ENUM = channel_pb.CHANNEL_DATA_TYPE_ENUM + BIT_FIELD = channel_pb.CHANNEL_DATA_TYPE_BIT_FIELD + BOOL = channel_pb.CHANNEL_DATA_TYPE_BOOL + FLOAT = channel_pb.CHANNEL_DATA_TYPE_FLOAT + INT_32 = channel_pb.CHANNEL_DATA_TYPE_INT_32 + INT_64 = channel_pb.CHANNEL_DATA_TYPE_INT_64 + UINT_32 = channel_pb.CHANNEL_DATA_TYPE_UINT_32 + UINT_64 = channel_pb.CHANNEL_DATA_TYPE_UINT_64 + + @classmethod + def from_str(cls, val: str) -> Optional["ChannelDataType"]: + if val == "CHANNEL_DATA_TYPE_DOUBLE": + return cls.DOUBLE + elif val == "CHANNEL_DATA_TYPE_STRING": + return cls.STRING + elif val == "CHANNEL_DATA_TYPE_ENUM": + return cls.ENUM + elif val == "CHANNEL_DATA_TYPE_BIT_FIELD": + return cls.BIT_FIELD + elif val == "CHANNEL_DATA_TYPE_BOOL": + return cls.BOOL + elif val == "CHANNEL_DATA_TYPE_FLOAT": + return cls.FLOAT + elif val == "CHANNEL_DATA_TYPE_INT_32": + return cls.INT_32 + elif val == "CHANNEL_DATA_TYPE_INT_64": + return cls.INT_64 + elif val == "CHANNEL_DATA_TYPE_UINT_32": + return cls.UINT_32 + elif val == "CHANNEL_DATA_TYPE_UINT_64": + return cls.UINT_64 + + return None + + +def string_value(val: str) -> IngestWithConfigDataChannelValue: + return IngestWithConfigDataChannelValue(string=val) + + +def double_value(val: float) -> IngestWithConfigDataChannelValue: + return IngestWithConfigDataChannelValue(double=val) + + +def float_value(val: float) -> IngestWithConfigDataChannelValue: + return IngestWithConfigDataChannelValue(float=val) + + +def bool_value(val: bool) -> IngestWithConfigDataChannelValue: + return IngestWithConfigDataChannelValue(bool=val) + + +def int32_value(val: int) -> IngestWithConfigDataChannelValue: + return IngestWithConfigDataChannelValue(int32=val) + + +def uint32_value(val: int) -> IngestWithConfigDataChannelValue: + return IngestWithConfigDataChannelValue(uint32=val) + + +def int64_value(val: int) -> IngestWithConfigDataChannelValue: + return IngestWithConfigDataChannelValue(int64=val) + + +def uint64_value(val: int) -> IngestWithConfigDataChannelValue: + return IngestWithConfigDataChannelValue(uint64=val) + + +def bit_field_value(val: bytes) -> IngestWithConfigDataChannelValue: + return IngestWithConfigDataChannelValue(bit_field=val) + + +def enum_value(val: int) -> IngestWithConfigDataChannelValue: + return IngestWithConfigDataChannelValue(enum=val) + + +def empty_value() -> IngestWithConfigDataChannelValue: + return IngestWithConfigDataChannelValue(empty=Empty()) + + +def is_data_type(val: IngestWithConfigDataChannelValue, target_type: ChannelDataType) -> bool: + if target_type == ChannelDataType.DOUBLE: + return val.HasField("double") + elif target_type == ChannelDataType.STRING: + return val.HasField("string") + elif target_type == ChannelDataType.ENUM: + return val.HasField("enum") + elif target_type == ChannelDataType.BIT_FIELD: + return val.HasField("bit_field") + elif target_type == ChannelDataType.BOOL: + return val.HasField("bool") + elif target_type == ChannelDataType.FLOAT: + return val.HasField("float") + elif target_type == ChannelDataType.INT_32: + return val.HasField("int32") + elif target_type == ChannelDataType.INT_64: + return val.HasField("int64") + elif target_type == ChannelDataType.UINT_32: + return val.HasField("uint32") + elif target_type == ChannelDataType.UINT_64: + return val.HasField("uint64") diff --git a/python/lib/sift_py/ingestion/config.py b/python/lib/sift_py/ingestion/config.py new file mode 100644 index 00000000..c71d5a11 --- /dev/null +++ b/python/lib/sift_py/ingestion/config.py @@ -0,0 +1,193 @@ +from __future__ import annotations +from .channel import ChannelDataType, ChannelBitFieldElement, ChannelEnumType +from .error import YamlConfigError +from .flow import ChannelConfig, FlowConfig +from pathlib import Path +from sift_internal.types import any_as +from typing import Any, Dict, List, Optional + +import yaml + + +class TelemetryConfig: + """ + Configurations necessary to start ingestion. + + Attributes: + asset_name: The name of the asset that you wish to telemeter data for. + ingestion_client_key: An arbitrary string completely chosen by the user to uniquely identify + this ingestion configuration. It should be unique with respect to your + organization. + + flows: The list of `FlowConfig`. A single flow can specify a single channel value + or a set of channel values, with each value belonging to a different channel. Channels + that send data at the same frequency and time should be in the same flow. + + organization_id: ID of your organization in Sift. This field is only required if your user + belongs to multiple organizations + """ + + asset_name: str + ingestion_client_key: str + organization_id: Optional[str] + flows: List[FlowConfig] + + def __init__( + self, + asset_name: str, + ingestion_client_key: str, + organization_id: Optional[str] = None, + flows: List[FlowConfig] = [], + ): + self.asset_name = asset_name + self.ingestion_client_key = ingestion_client_key + self.organization_id = organization_id + self.flows = flows + + +def try_load_from_yaml(config_fs_path: Path) -> TelemetryConfig: + """ + Loads in YAML config file and deserializes it into an instance of `TelemetryConfig`. If + the YAML config has any malformed or missing properties than a `YamlConfigError` is raised. + """ + + suffix = config_fs_path.suffix + if suffix != ".yaml" and suffix != ".yml": + raise YamlConfigError(f"Unsupported file-type '{suffix}', expected YAML.") + + with open(config_fs_path, "r") as file: + config: Dict[Any, Any] = yaml.safe_load(file) + + asset_name = any_as(config.get("asset_name"), str) + if asset_name is None or len(asset_name) == 0: + raise YamlConfigError("Expected a non-blank string for top-level 'asset_name' property") + + ingestion_client_key = any_as(config.get("ingestion_client_key"), str) + if ingestion_client_key is None or len(ingestion_client_key) == 0: + raise YamlConfigError( + "Expected a non-blank string top-level 'ingestion_client_key' property" + ) + + organization_id = any_as(config.get("organization_id"), str) + + raw_flows = any_as(config.get("flows"), list) + if raw_flows is None: + raise YamlConfigError("Expected 'flows' to be a list property") + + return TelemetryConfig( + asset_name=asset_name, + ingestion_client_key=ingestion_client_key, + organization_id=organization_id, + flows=_deserialize_flows_from_yaml(raw_flows), + ) + + +def _deserialize_flows_from_yaml(raw_flow_configs: List[Dict]) -> List[FlowConfig]: + flow_configs = [] + + for raw_flow_config in raw_flow_configs: + flow_name = any_as(raw_flow_config.get("name"), str) + if flow_name is None or len(flow_name) == 0: + raise YamlConfigError("Expected flow to have a non-blank 'name' property") + + raw_channel_configs = any_as(raw_flow_config.get("channels"), list) + if raw_channel_configs is None: + raise YamlConfigError("Expected 'channels' to be a list property") + + flow_config = FlowConfig( + name=flow_name, channels=_deserialize_channels_from_yaml(raw_channel_configs) + ) + + flow_configs.append(flow_config) + + return flow_configs + + +def _deserialize_channels_from_yaml(raw_channel_configs: List[Dict]) -> List[ChannelConfig]: + channel_configs = [] + + for raw_channel_config in raw_channel_configs: + channel_name = any_as(raw_channel_config.get("name"), str) + if channel_name is None or len(channel_name) == 0: + raise YamlConfigError("Expected channel to have a non-blank 'name' property") + + channel_data_type_str = any_as(raw_channel_config.get("data_type"), str) + if channel_data_type_str is None or len(channel_data_type_str) == 0: + raise YamlConfigError("Missing property for 'flows.channel.data_type' property") + + channel_data_type = ChannelDataType.from_str(channel_data_type_str) + if channel_data_type is None: + raise YamlConfigError("Invalid property for 'flows.channel.data_type' property") + + description = any_as(raw_channel_config.get("description"), str) + unit = any_as(raw_channel_config.get("unit"), str) + component = any_as(raw_channel_config.get("component"), str) + + bit_field_elements = [] + raw_bit_field_elements = any_as(raw_channel_config.get("bit_field_elements"), list) + if raw_bit_field_elements is not None: + for element in raw_bit_field_elements: + el = _deserialize_bit_field_element_from_yaml(element) + bit_field_elements.append(el) + + enum_types = [] + raw_enum_types = any_as(raw_channel_config.get("enum_types"), list) + if raw_enum_types is not None: + for enum_type in raw_enum_types: + etype = _deserialize_enum_type_from_yaml(enum_type) + enum_types.append(etype) + + channel_config = ChannelConfig( + name=channel_name, + data_type=channel_data_type, + description=description, + unit=unit, + component=component, + bit_field_elements=bit_field_elements, + enum_types=enum_types, + ) + + channel_configs.append(channel_config) + + return channel_configs + + +def _deserialize_bit_field_element_from_yaml(bit_field_element: Dict) -> ChannelBitFieldElement: + name = any_as(bit_field_element.get("name"), str) + if name is None or len(name) == 0: + raise YamlConfigError( + "Expected a non-blank value for 'flows.channels.bit_field_element.name'" + ) + + index = any_as(bit_field_element.get("index"), int) + if index is None: + raise YamlConfigError( + "Expected an integer value for 'flows.channels.bit_field_element.index'" + ) + + bit_count = any_as(bit_field_element.get("bit_count"), int) + if bit_count is None: + raise YamlConfigError( + "Expected an integer value for 'flows.channels.bit_field_element.bit_count'" + ) + + return ChannelBitFieldElement( + name=name, + index=index, + bit_count=bit_count, + ) + + +def _deserialize_enum_type_from_yaml(enum_type: Any) -> ChannelEnumType: + name = any_as(enum_type.get("name"), str) + if name is None or len(name) == 0: + raise YamlConfigError("Expected a non-blank value for 'flows.channels.enum_types.name'") + + key = any_as(enum_type.get("key"), int) + if key is None: + raise YamlConfigError("Expected an integer value for 'flows.channels.enum_types.key'") + + return ChannelEnumType( + name=name, + key=key, + ) diff --git a/python/lib/sift_py/ingestion/error.py b/python/lib/sift_py/ingestion/error.py new file mode 100644 index 00000000..09383916 --- /dev/null +++ b/python/lib/sift_py/ingestion/error.py @@ -0,0 +1,14 @@ +""" +Errors specific to the `sift_py` ingestion module. +""" + + +class YamlConfigError(Exception): + """ + Used when the YAML config has missing or invalid properties. + """ + + message: str + + def __init__(self, message: str): + super().__init__(message) diff --git a/python/lib/sift_py/ingestion/flow.py b/python/lib/sift_py/ingestion/flow.py new file mode 100644 index 00000000..75414344 --- /dev/null +++ b/python/lib/sift_py/ingestion/flow.py @@ -0,0 +1,58 @@ +from __future__ import annotations +from .channel import ChannelConfig +from sift_internal.convert.protobuf import try_convert_pb, AsProtobuf, ProtobufMessage +from sift.ingestion_configs.v1.ingestion_configs_pb2 import ( + ChannelConfig as ChannelConfigPb, + FlowConfig as FlowConfigPb, +) +from typing import Dict, List, Optional, Type + + +class FlowConfig(AsProtobuf): + """ + Describes a flow which is a set of channels whose values are often + ingested together. + + The `channel_by_fqn` attribute is a mapping of a channel's fully-qualified name + to the index of the `ChannelConfig` instance as it appears in the `channels` attribute. + """ + + name: str + channels: List[ChannelConfig] + channel_by_fqn: Dict[str, int] + + def __init__(self, name: str, channels: List[ChannelConfig]): + self.name = name + self.channels = channels + self.channel_by_fqn = { + self.__class__.compute_fqn(c.name, c.component): i for i, c in enumerate(channels) + } + + def get_channel(self, name: str, component: Optional[str] = "") -> Optional[ChannelConfig]: + """ + Retrieves a `ChannelConfig` by its fully qualified name. Returns `None` if it cannot be found. + """ + fqn = self.__class__.compute_fqn(name, component) + index = self.channel_by_fqn[fqn] + + try: + return self.channels[index] + except IndexError: + return None + + def as_pb(self, klass: Type[ProtobufMessage]) -> ProtobufMessage: + return FlowConfigPb( + name=self.name, + channels=[try_convert_pb(conf, ChannelConfigPb) for conf in self.channels], + ) + + @staticmethod + def compute_fqn(name: str, component: Optional[str]) -> str: + """ + The fully-qualified channel name of a channel called 'voltage' is simply `voltage'. The + fully qualified name of a channel called 'temperature' of component 'motor' is a `motor.temperature'. + """ + if component is None or len(component) == "": + return name + else: + return f"{component}.{name}" diff --git a/python/lib/sift_py/ingestion/ingestion_impl/__init__.py b/python/lib/sift_py/ingestion/ingestion_impl/__init__.py new file mode 100644 index 00000000..8dfc9116 --- /dev/null +++ b/python/lib/sift_py/ingestion/ingestion_impl/__init__.py @@ -0,0 +1,6 @@ +""" +INTERNAL MODULE + +This module contains implementation details that isn't meant to be used directly. +APIs in this module are garaunteed to not be stable so proceed at your own risk. +""" diff --git a/python/lib/sift_py/ingestion/ingestion_impl/ingest.py b/python/lib/sift_py/ingestion/ingestion_impl/ingest.py new file mode 100644 index 00000000..0a1ae616 --- /dev/null +++ b/python/lib/sift_py/ingestion/ingestion_impl/ingest.py @@ -0,0 +1,163 @@ +from __future__ import annotations +from ..channel import ChannelValue, is_data_type, empty_value +from ..flow import FlowConfig +from .ingestion_config import get_ingestion_config_by_client_key, create_ingestion_config +from ..config import TelemetryConfig +from ...grpc.transport import SiftChannel +from sift.ingestion_configs.v1.ingestion_configs_pb2 import IngestionConfig +from sift.ingest.v1.ingest_pb2 import ( + IngestWithConfigDataChannelValue, + IngestWithConfigDataStreamRequest, +) +from sift.ingest.v1.ingest_pb2_grpc import IngestServiceStub +from sift.runs.v2.runs_pb2 import CreateRunRequest, CreateRunResponse +from sift.runs.v2.runs_pb2_grpc import RunServiceStub +from google.protobuf.timestamp_pb2 import Timestamp +from typing import cast, Dict, List, Optional +from datetime import datetime + + +class IngestionServiceImpl: + transport_channel: SiftChannel + ingestion_config: IngestionConfig + asset_name: str + + # TODO: Multiple flows can have the same name if their channel configs differ... + flow_configs_by_name: Dict[str, FlowConfig] + + run_id: Optional[str] + organization_id: Optional[str] + end_stream_on_error: bool + + def __init__( + self, + channel: SiftChannel, + config: TelemetryConfig, + run_id: Optional[str] = None, + end_stream_on_error: bool = False, + ): + # TODO: Handle case where new Flows are added to an existing ingestion config + ingestion_config = get_ingestion_config_by_client_key(channel, config.ingestion_client_key) + + if ingestion_config is not None: + self.ingestion_config = ingestion_config + else: + self.ingestion_config = create_ingestion_config( + channel, + config.asset_name, + config.flows, + config.ingestion_client_key, + config.organization_id, + ) + + self.asset_name = config.asset_name + self.flow_configs_by_name = {flow.name: flow for flow in config.flows} + self.transport_channel = channel + self.run_id = run_id + self.organization_id = config.organization_id + self.end_stream_on_error = end_stream_on_error + + def ingest(self, *requests: IngestWithConfigDataStreamRequest): + # TODO: Add logic to re-establish connection if channel has been closed due to idle timeout + + svc = IngestServiceStub(self.transport_channel) + svc.IngestWithConfigDataStream(iter(requests)) + + def start_run( + self, + channel: SiftChannel, + run_name: str, + description: Optional[str] = None, + organization_id: Optional[str] = None, + tags: Optional[List[str]] = None, + ): + svc = RunServiceStub(channel) + req = CreateRunRequest( + name=run_name, + description=description or "", + organization_id=organization_id or "", + tags=tags, + ) + res = cast(CreateRunResponse, svc.CreateRun(req)) + self.run_id = res.run.run_id + + def end_run(self): + # TODO: Should hit the stop run endpoint + self.run_id = None + + def try_create_ingestion_request( + self, + flow_name: str, + timestamp: datetime, + channel_values: List[ChannelValue], + ) -> IngestWithConfigDataStreamRequest: + flow_config = self.flow_configs_by_name.get(flow_name) + + if flow_config is None: + raise ValueError(f"A flow config of name '{flow_name}' could not be found.") + + channel_values_by_fqn: Dict[str, ChannelValue] = {} + + for channel_value in channel_values: + name = channel_value["channel_name"] + component = channel_value.get("component") + fqn = FlowConfig.compute_fqn(name, component) + + if channel_values_by_fqn.get(fqn, None) is None: + channel_values_by_fqn[fqn] = channel_value + else: + raise ValueError(f"Encountered multiple values for {fqn}") + + values: List[IngestWithConfigDataChannelValue] = [] + + for channel in flow_config.channels: + fqn = FlowConfig.compute_fqn(channel.name, channel.component) + channel_value = channel_values_by_fqn.pop(fqn, None) + + if channel_value is None: + values.append(empty_value()) + continue + + value = channel_value["value"] + + if is_data_type(value, channel.data_type): + values.append(value) + else: + raise ValueError( + f"Expected value for `{channel.name}` to be a '{channel.data_type}'." + ) + + if len(channel_values_by_fqn) > 0: + unexpected_channels = [name for name in channel_values_by_fqn.keys()] + raise ValueError( + f"Unexpected channels for flow '{flow_name}' or 'component' field missing for channel: {unexpected_channels}" + ) + + if timestamp.tzname() != "UTC": + raise ValueError( + f"Expected 'timestamp' to be in UTC but it is in {timestamp.tzname()}." + ) + + timestamp_pb = Timestamp() + timestamp_pb.FromDatetime(timestamp) + + return self.create_ingestion_request(flow_name, timestamp, values) + + def create_ingestion_request( + self, + flow_name: str, + timestamp: datetime, + channel_values: List[IngestWithConfigDataChannelValue], + ) -> IngestWithConfigDataStreamRequest: + timestamp_pb = Timestamp() + timestamp_pb.FromDatetime(timestamp) + + return IngestWithConfigDataStreamRequest( + ingestion_config_id=self.ingestion_config.ingestion_config_id, + flow=flow_name, + timestamp=timestamp_pb, + channel_values=channel_values, + run_id=self.run_id or "", + organization_id=self.organization_id or "", + end_stream_on_validation_error=self.end_stream_on_error, + ) diff --git a/python/lib/sift_py/ingestion/ingestion_impl/ingestion_config.py b/python/lib/sift_py/ingestion/ingestion_impl/ingestion_config.py new file mode 100644 index 00000000..137ef43e --- /dev/null +++ b/python/lib/sift_py/ingestion/ingestion_impl/ingestion_config.py @@ -0,0 +1,62 @@ +""" +Internal module: This module contains implementation details that are not meant to be +used by consumers of this library and are not garaunteed to be stable. +""" + +from ...grpc.transport import SiftChannel +from ..flow import FlowConfig +from ...convert.protobuf import try_convert_pb +from sift.ingestion_configs.v1.ingestion_configs_pb2 import ( + IngestionConfig, + CreateIngestionConfigRequest, + CreateIngestionConfigResponse, + ListIngestionConfigsRequest, + ListIngestionConfigsResponse, + FlowConfig as FlowConfigPb, +) +from sift.ingestion_configs.v1.ingestion_configs_pb2_grpc import IngestionConfigServiceStub +from typing import cast, List, Optional + + +def get_ingestion_config_by_client_key( + channel: SiftChannel, + client_key: str, +) -> Optional[IngestionConfig]: + """ + Returns `None` if no ingestion config can be matched with the provided `client_key` + """ + + svc = IngestionConfigServiceStub(channel) + req = ListIngestionConfigsRequest( + filter=f'client_key=="{client_key}"', + page_token="", + page_size=1, + ) + res = cast(ListIngestionConfigsResponse, svc.ListIngestionConfigs(req)) + + if len(res.ingestion_configs) == 0: + return None + else: + return res.ingestion_configs[0] + + +def create_ingestion_config( + channel: SiftChannel, + asset_name: str, + flows: List[FlowConfig], + client_key: str, + organization_id: Optional[str], +) -> IngestionConfig: + """ + Creates a new ingestion config + """ + + svc = IngestionConfigServiceStub(channel) + req = CreateIngestionConfigRequest( + asset_name=asset_name, + client_key=client_key, + organization_id=organization_id or "", + flows=[try_convert_pb(flow, FlowConfigPb) for flow in flows], + ) + res = cast(CreateIngestionConfigResponse, svc.CreateIngestionConfig(req)) + return res.ingestion_config diff --git a/python/lib/sift_py/ingestion/service.py b/python/lib/sift_py/ingestion/service.py new file mode 100644 index 00000000..6fd00007 --- /dev/null +++ b/python/lib/sift_py/ingestion/service.py @@ -0,0 +1,112 @@ +from __future__ import annotations +from ..grpc.transport import SiftChannel +from .config import TelemetryConfig +from ..ingestion.flow import FlowConfig +from .channel import ChannelValue +from sift.ingest.v1.ingest_pb2 import ( + IngestWithConfigDataChannelValue, + IngestWithConfigDataStreamRequest, +) +from sift.ingestion_configs.v1.ingestion_configs_pb2 import IngestionConfig +from typing import Dict, List, Optional +from .ingestion_impl.ingest import IngestionServiceImpl +from datetime import datetime + + +class IngestionService(IngestionServiceImpl): + """ + A fully configured service that, when instantiated, is ready to start ingesting data. + """ + + transport_channel: SiftChannel + ingestion_config: IngestionConfig + asset_name: str + flow_configs_by_name: Dict[str, FlowConfig] + run_id: Optional[str] + organization_id: Optional[str] + end_stream_on_error: bool + + def __init__( + self, + channel: SiftChannel, + config: TelemetryConfig, + run_id: Optional[str] = None, + end_stream_on_error: bool = False, + ): + super().__init__(channel, config, run_id, end_stream_on_error) + + def ingest(self, *requests: IngestWithConfigDataStreamRequest): + """ + This method performs the actual data ingestion given a list of data ingestion requests. + """ + super().ingest(*requests) + + def start_run( + self, + channel: SiftChannel, + run_name: str, + description: Optional[str] = None, + organization_id: Optional[str] = None, + tags: Optional[List[str]] = None, + ): + """ + Create a run to use as part of the call to `ingest`. + """ + super().start_run(channel, run_name, description, organization_id, tags) + + def end_run(self): + """ + End the current run if any and don't include it in subsequent calls to `ingest`. + """ + super().end_run() + + def try_create_ingestion_request( + self, + flow_name: str, + timestamp: datetime, + channel_values: List[ChannelValue], + ) -> IngestWithConfigDataStreamRequest: + """ + Creates an `IngestWithConfigDataStreamRequest`, i.e. a flow, given a `flow_name` and a + list of `ChannelValue` objects. Channels that appear in the flow config but not in the + `channel_value_by_channel_name` will be assigned an empty value. + + This function will perform validation checks to ensure that the values provided in the dictionary; this + includes: + - Making sure the flow exists + - Making sure that the there are no unexpected channels provided for the given flow + - Making sure the channel value is the expected type + - Making sure that the timestamp is in UTC + - Making sure channels that belong to a component have the 'component' field for the channel value + + If any of the above validations fail then a `ValueError` will be raised. + + If for performance reasons you'd prefer to skip the validation checks, or perhaps you did the + validations on your own, prefer to use `create_ingestion_request`. Any errors that occur during + ingestion will be handled by the Sift API. + """ + return super().try_create_ingestion_request(flow_name, timestamp, channel_values) + + def create_ingestion_request( + self, + flow_name: str, + timestamp: datetime, + channel_values: List[IngestWithConfigDataChannelValue], + ) -> IngestWithConfigDataStreamRequest: + """ + Unlike `try_create_ingestion_request`, this skips argument validations. Useful for when user has already done their own + argument validation or if they require low-latency execution time client-side. + + If there are errors that occur during ingestion and the `end_stream_on_error` attribute is set to `False`, + the data ingestion stream will skip over them and errors instead will be produced asynchronously and become + available in the UI application in the errors page. If `end_stream_on_error` is set to `True`, then the + data ingestion stream will be terminated if an error is encountered during ingestion. + + These are some things to look out for when using this method instead of `try_create_ingestion_request`: + - Values in `channel_values` must appear in the same order its corresponding channel appears in the flow config + associated with the `flow_name`. + - The length of `channel_values` is expected to match the length of the channel configs list of the flow config + associated with `flow_name`. `google.protobuf.empty_pb2.Empty` may be used if you require empty values. + - The `timestamp` must be in UTC. + """ + return super().create_ingestion_request(flow_name, timestamp, channel_values) diff --git a/python/pyproject.toml b/python/pyproject.toml new file mode 100644 index 00000000..19fc34b3 --- /dev/null +++ b/python/pyproject.toml @@ -0,0 +1,33 @@ +[project] +name = "sift_py" +version = "0.1" +description = "Python client library for the Sift API" +dependencies = [ + # https://grpc.github.io/grpc/python/ + "grpcio==1.64.1", + # https://googleapis.dev/python/protobuf/latest/ + "protobuf==5.27.1", + # https://grpc-interceptor.readthedocs.io + "grpc-interceptor==0.15.4", + # https://pyyaml.org/wiki/PyYAMLDocumentation + "PyYAML==6.0.1", + "types-PyYAML==6.0.12.20240311", +] + +[project.optional-dependencies] +development = [ + "black", # formatter +] + +[build-system] +requires = ["setuptools"] +build-backend = "setuptools.build_meta" + +[tool.setuptools.packages.find] +where = ["gen", "lib"] + +# The formatter +[tool.black] +line-length = 100 +target-version = ['py38'] +include = '\.pyi?$' diff --git a/python/scripts/dev b/python/scripts/dev new file mode 100755 index 00000000..a1b8290c --- /dev/null +++ b/python/scripts/dev @@ -0,0 +1,63 @@ +#!/usr/bin/env bash + +usage() { + cat<