diff --git a/README.md b/README.md index c12110b..76fef75 100644 --- a/README.md +++ b/README.md @@ -94,7 +94,7 @@ user = IngestRequest( schema_name="users", typ=0, old=None, - new=Record(values=[Value(int_value=1), Value(string_value="vivek")]), + new=[Value(int_value=1), Value(string_value="vivek")], seq_no=1 ) ingestor.ingest(user) diff --git a/protos/common.proto b/protos/common.proto index 6d951a4..71039ce 100644 --- a/protos/common.proto +++ b/protos/common.proto @@ -54,12 +54,8 @@ message CountResponse { // Request for `OnEvent`. message OnEventRequest { - // The event type to subscribe to. - dozer.types.EventType type = 1; - // The name of the endpoint to subscribe to. - string endpoint = 2; - // JSON filter string. - optional string filter = 3; + // The endpoints to subscribe to. Key is the endpoint name, value is the filter. + map endpoints = 1; } // Request for `getFields`. @@ -81,7 +77,7 @@ message QueryResponse { // The list of field definitions. repeated dozer.types.FieldDefinition fields = 1; // The list of record data. - repeated dozer.types.RecordWithId records = 2; + repeated dozer.types.Record records = 2; } // Request for `getEndpoints`. diff --git a/protos/ingest.proto b/protos/ingest.proto index f5c840f..5050a21 100644 --- a/protos/ingest.proto +++ b/protos/ingest.proto @@ -25,9 +25,9 @@ message IngestRequest { // The operation type. dozer.types.OperationType typ = 2; // Old record data, only applicable for UPDATE type. - optional dozer.types.Record old = 3; + repeated dozer.types.Value old = 3; // New record data. - dozer.types.Record new = 4; + repeated dozer.types.Value new = 4; uint32 seq_no = 5; } diff --git a/protos/types.proto b/protos/types.proto index 86cc124..a6c0ce1 100644 --- a/protos/types.proto +++ b/protos/types.proto @@ -13,6 +13,14 @@ enum EventType { DELETE_ONLY = 3; // Only DELETE events. } +// Event filter. +message EventFilter { + // The event type to subscribe to. + dozer.types.EventType type = 1; + // JSON filter string. + optional string filter = 3; +} + // The event types. enum OperationType { INSERT = 0; // INSERT operation. @@ -28,8 +36,6 @@ message Operation { optional Record old = 2; // New record data. Record new = 3; - // New record id, only applicable for INSERT type. - optional uint64 new_id = 4; // Name of the endpoint that this event is from. string endpoint_name = 5; } @@ -38,16 +44,10 @@ message Operation { message Record { // The list of field values. repeated Value values = 1; + // The record id in cache. + uint64 id = 2; // Records with same primary key will have increasing version. - uint32 version = 2; -} - -// A record with its id in cache. -message RecordWithId { - // The record id. - uint64 id = 1; - // The record data. - Record record = 2; + uint32 version = 3; } // Supported data types in Dozer. @@ -126,3 +126,14 @@ message Value { google.protobuf.Value json_value = 14; // JSON type. }; } +message SchemasResponse { + map schemas = 1; + map errors = 2; +} + +message Schema { + // The list of indexes of the keys that are used as the primary index. + repeated int32 primary_index = 1; + // The list of field definitions. + repeated FieldDefinition fields = 2; +} \ No newline at end of file diff --git a/pydozer/api.py b/pydozer/api.py index a144ebc..c6da439 100644 --- a/pydozer/api.py +++ b/pydozer/api.py @@ -6,6 +6,7 @@ from pydozer.common_pb2_grpc import CommonGrpcServiceStub from pydozer.common_pb2 import QueryRequest, OnEventRequest, QueryResponse, CountResponse from pydozer.health_pb2 import HealthCheckRequest, HealthCheckResponse +from pydozer.types_pb2 import EventFilter DOZER_API_URL = os.getenv("DOZER_API_URL", "0.0.0.0:50051") @@ -109,7 +110,9 @@ def on_event(self, request={}): Args: request (OnEventRequest): Optionally accepts a filter """ - _req = OnEventRequest(endpoint=self.endpoint) + _req = OnEventRequest(endpoints={ + self.endpoint: EventFilter() + }) for key, value in request.items(): setattr(_req, key, value) diff --git a/pydozer/auth_pb2_grpc.py b/pydozer/auth_pb2_grpc.py index bb32328..c79f030 100644 --- a/pydozer/auth_pb2_grpc.py +++ b/pydozer/auth_pb2_grpc.py @@ -2,7 +2,7 @@ """Client and server classes corresponding to protobuf-defined services.""" import grpc -import pydozer.auth_pb2 as auth__pb2 +import auth_pb2 as auth__pb2 class AuthGrpcServiceStub(object): diff --git a/pydozer/common_pb2.py b/pydozer/common_pb2.py index 16b1b6d..7ee4086 100644 --- a/pydozer/common_pb2.py +++ b/pydozer/common_pb2.py @@ -11,32 +11,36 @@ _sym_db = _symbol_database.Default() -import pydozer.types_pb2 as types__pb2 +import types_pb2 as types__pb2 -DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(b'\n\x0c\x63ommon.proto\x12\x0c\x64ozer.common\x1a\x0btypes.proto\">\n\x0cQueryRequest\x12\x10\n\x08\x65ndpoint\x18\x01 \x01(\t\x12\x12\n\x05query\x18\x02 \x01(\tH\x00\x88\x01\x01\x42\x08\n\x06_query\"\x1e\n\rCountResponse\x12\r\n\x05\x63ount\x18\x01 \x01(\x04\"h\n\x0eOnEventRequest\x12$\n\x04type\x18\x01 \x01(\x0e\x32\x16.dozer.types.EventType\x12\x10\n\x08\x65ndpoint\x18\x02 \x01(\t\x12\x13\n\x06\x66ilter\x18\x03 \x01(\tH\x00\x88\x01\x01\x42\t\n\x07_filter\"$\n\x10GetFieldsRequest\x12\x10\n\x08\x65ndpoint\x18\x01 \x01(\t\"X\n\x11GetFieldsResponse\x12\x15\n\rprimary_index\x18\x01 \x03(\x05\x12,\n\x06\x66ields\x18\x02 \x03(\x0b\x32\x1c.dozer.types.FieldDefinition\"i\n\rQueryResponse\x12,\n\x06\x66ields\x18\x01 \x03(\x0b\x32\x1c.dozer.types.FieldDefinition\x12*\n\x07records\x18\x02 \x03(\x0b\x32\x19.dozer.types.RecordWithId\"\x15\n\x13GetEndpointsRequest\")\n\x14GetEndpointsResponse\x12\x11\n\tendpoints\x18\x01 \x03(\t2\xff\x02\n\x11\x43ommonGrpcService\x12@\n\x05\x63ount\x12\x1a.dozer.common.QueryRequest\x1a\x1b.dozer.common.CountResponse\x12@\n\x05query\x12\x1a.dozer.common.QueryRequest\x1a\x1b.dozer.common.QueryResponse\x12\x41\n\x07OnEvent\x12\x1c.dozer.common.OnEventRequest\x1a\x16.dozer.types.Operation0\x01\x12U\n\x0cgetEndpoints\x12!.dozer.common.GetEndpointsRequest\x1a\".dozer.common.GetEndpointsResponse\x12L\n\tgetFields\x12\x1e.dozer.common.GetFieldsRequest\x1a\x1f.dozer.common.GetFieldsResponseb\x06proto3') +DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(b'\n\x0c\x63ommon.proto\x12\x0c\x64ozer.common\x1a\x0btypes.proto\">\n\x0cQueryRequest\x12\x10\n\x08\x65ndpoint\x18\x01 \x01(\t\x12\x12\n\x05query\x18\x02 \x01(\tH\x00\x88\x01\x01\x42\x08\n\x06_query\"\x1e\n\rCountResponse\x12\r\n\x05\x63ount\x18\x01 \x01(\x04\"\x9c\x01\n\x0eOnEventRequest\x12>\n\tendpoints\x18\x01 \x03(\x0b\x32+.dozer.common.OnEventRequest.EndpointsEntry\x1aJ\n\x0e\x45ndpointsEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\'\n\x05value\x18\x02 \x01(\x0b\x32\x18.dozer.types.EventFilter:\x02\x38\x01\"$\n\x10GetFieldsRequest\x12\x10\n\x08\x65ndpoint\x18\x01 \x01(\t\"X\n\x11GetFieldsResponse\x12\x15\n\rprimary_index\x18\x01 \x03(\x05\x12,\n\x06\x66ields\x18\x02 \x03(\x0b\x32\x1c.dozer.types.FieldDefinition\"c\n\rQueryResponse\x12,\n\x06\x66ields\x18\x01 \x03(\x0b\x32\x1c.dozer.types.FieldDefinition\x12$\n\x07records\x18\x02 \x03(\x0b\x32\x13.dozer.types.Record\"\x15\n\x13GetEndpointsRequest\")\n\x14GetEndpointsResponse\x12\x11\n\tendpoints\x18\x01 \x03(\t2\xff\x02\n\x11\x43ommonGrpcService\x12@\n\x05\x63ount\x12\x1a.dozer.common.QueryRequest\x1a\x1b.dozer.common.CountResponse\x12@\n\x05query\x12\x1a.dozer.common.QueryRequest\x1a\x1b.dozer.common.QueryResponse\x12\x41\n\x07OnEvent\x12\x1c.dozer.common.OnEventRequest\x1a\x16.dozer.types.Operation0\x01\x12U\n\x0cgetEndpoints\x12!.dozer.common.GetEndpointsRequest\x1a\".dozer.common.GetEndpointsResponse\x12L\n\tgetFields\x12\x1e.dozer.common.GetFieldsRequest\x1a\x1f.dozer.common.GetFieldsResponseb\x06proto3') _builder.BuildMessageAndEnumDescriptors(DESCRIPTOR, globals()) _builder.BuildTopDescriptorsAndMessages(DESCRIPTOR, 'common_pb2', globals()) if _descriptor._USE_C_DESCRIPTORS == False: DESCRIPTOR._options = None + _ONEVENTREQUEST_ENDPOINTSENTRY._options = None + _ONEVENTREQUEST_ENDPOINTSENTRY._serialized_options = b'8\001' _QUERYREQUEST._serialized_start=43 _QUERYREQUEST._serialized_end=105 _COUNTRESPONSE._serialized_start=107 _COUNTRESPONSE._serialized_end=137 - _ONEVENTREQUEST._serialized_start=139 - _ONEVENTREQUEST._serialized_end=243 - _GETFIELDSREQUEST._serialized_start=245 - _GETFIELDSREQUEST._serialized_end=281 - _GETFIELDSRESPONSE._serialized_start=283 - _GETFIELDSRESPONSE._serialized_end=371 - _QUERYRESPONSE._serialized_start=373 - _QUERYRESPONSE._serialized_end=478 - _GETENDPOINTSREQUEST._serialized_start=480 - _GETENDPOINTSREQUEST._serialized_end=501 - _GETENDPOINTSRESPONSE._serialized_start=503 - _GETENDPOINTSRESPONSE._serialized_end=544 - _COMMONGRPCSERVICE._serialized_start=547 - _COMMONGRPCSERVICE._serialized_end=930 + _ONEVENTREQUEST._serialized_start=140 + _ONEVENTREQUEST._serialized_end=296 + _ONEVENTREQUEST_ENDPOINTSENTRY._serialized_start=222 + _ONEVENTREQUEST_ENDPOINTSENTRY._serialized_end=296 + _GETFIELDSREQUEST._serialized_start=298 + _GETFIELDSREQUEST._serialized_end=334 + _GETFIELDSRESPONSE._serialized_start=336 + _GETFIELDSRESPONSE._serialized_end=424 + _QUERYRESPONSE._serialized_start=426 + _QUERYRESPONSE._serialized_end=525 + _GETENDPOINTSREQUEST._serialized_start=527 + _GETENDPOINTSREQUEST._serialized_end=548 + _GETENDPOINTSRESPONSE._serialized_start=550 + _GETENDPOINTSRESPONSE._serialized_end=591 + _COMMONGRPCSERVICE._serialized_start=594 + _COMMONGRPCSERVICE._serialized_end=977 # @@protoc_insertion_point(module_scope) diff --git a/pydozer/common_pb2_grpc.py b/pydozer/common_pb2_grpc.py index a559a78..3353b13 100644 --- a/pydozer/common_pb2_grpc.py +++ b/pydozer/common_pb2_grpc.py @@ -2,8 +2,8 @@ """Client and server classes corresponding to protobuf-defined services.""" import grpc -import pydozer.common_pb2 as common__pb2 -import pydozer.types_pb2 as types__pb2 +import common_pb2 as common__pb2 +import types_pb2 as types__pb2 class CommonGrpcServiceStub(object): diff --git a/pydozer/health_pb2_grpc.py b/pydozer/health_pb2_grpc.py index b09efdf..8474399 100644 --- a/pydozer/health_pb2_grpc.py +++ b/pydozer/health_pb2_grpc.py @@ -2,7 +2,7 @@ """Client and server classes corresponding to protobuf-defined services.""" import grpc -import pydozer.health_pb2 as health__pb2 +import health_pb2 as health__pb2 class HealthGrpcServiceStub(object): diff --git a/pydozer/helper.py b/pydozer/helper.py index c8c3905..2ab1f52 100644 --- a/pydozer/helper.py +++ b/pydozer/helper.py @@ -124,19 +124,18 @@ def map_value(col, typ) -> Value: return Value() -def map_record(schema_name: str, row: list, types, idx) -> Record: +def map_record(schema_name: str, row: list, types, idx) -> IngestRequest: values = [] assert len(row) == len(types), "Row and types must be the same length" for i in range(len(row)): val = map_value(row[i], types[i]) values.append(val) - rec = Record(values=values, version=1) return IngestRequest( schema_name=schema_name, typ=0, old=None, - new=rec, + new=values, seq_no=idx ) diff --git a/pydozer/ingest.py b/pydozer/ingest.py index 6ad4294..310ae8a 100644 --- a/pydozer/ingest.py +++ b/pydozer/ingest.py @@ -41,7 +41,7 @@ def ingest_raw(self, request) -> IngestResponse: schema_name="users", typ=0, old=None, - new=Record(values=[Value(int_value=1), Value(string_value="vivek")]), + new=[Value(int_value=1), Value(string_value="vivek")], seq_no=1 ) ingestor.ingest(user) diff --git a/pydozer/ingest_pb2.py b/pydozer/ingest_pb2.py index 85aacaf..1a8507c 100644 --- a/pydozer/ingest_pb2.py +++ b/pydozer/ingest_pb2.py @@ -11,10 +11,10 @@ _sym_db = _symbol_database.Default() -import pydozer.types_pb2 as types__pb2 +import types_pb2 as types__pb2 -DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(b'\n\x0cingest.proto\x12\x0c\x64ozer.ingest\x1a\x0btypes.proto\"\xae\x01\n\rIngestRequest\x12\x13\n\x0bschema_name\x18\x01 \x01(\t\x12\'\n\x03typ\x18\x02 \x01(\x0e\x32\x1a.dozer.types.OperationType\x12%\n\x03old\x18\x03 \x01(\x0b\x32\x13.dozer.types.RecordH\x00\x88\x01\x01\x12 \n\x03new\x18\x04 \x01(\x0b\x32\x13.dozer.types.Record\x12\x0e\n\x06seq_no\x18\x05 \x01(\rB\x06\n\x04_old\" \n\x0eIngestResponse\x12\x0e\n\x06seq_no\x18\x01 \x01(\r\"\xdb\x01\n\x12IngestArrowRequest\x12\x13\n\x0bschema_name\x18\x01 \x01(\t\x12\x0f\n\x07records\x18\x02 \x01(\x0c\x12\x0e\n\x06seq_no\x18\x03 \x01(\r\x12@\n\x08metadata\x18\x04 \x03(\x0b\x32..dozer.ingest.IngestArrowRequest.MetadataEntry\x1aM\n\rMetadataEntry\x12\x0b\n\x03key\x18\x01 \x01(\r\x12+\n\x05value\x18\x02 \x01(\x0b\x32\x1c.dozer.ingest.IngestMetadata:\x02\x38\x01\"K\n\x0eIngestMetadata\x12(\n\x03typ\x18\x01 \x01(\x0e\x32\x1b.dozer.ingest.OperationType\x12\x0f\n\x07version\x18\x02 \x01(\r*3\n\rOperationType\x12\n\n\x06INSERT\x10\x00\x12\n\n\x06\x44\x45LETE\x10\x01\x12\n\n\x06UPDATE\x10\x02\x32\xcb\x02\n\rIngestService\x12\x43\n\x06ingest\x12\x1b.dozer.ingest.IngestRequest\x1a\x1c.dozer.ingest.IngestResponse\x12L\n\ringest_stream\x12\x1b.dozer.ingest.IngestRequest\x1a\x1c.dozer.ingest.IngestResponse(\x01\x12N\n\x0cingest_arrow\x12 .dozer.ingest.IngestArrowRequest\x1a\x1c.dozer.ingest.IngestResponse\x12W\n\x13ingest_arrow_stream\x12 .dozer.ingest.IngestArrowRequest\x1a\x1c.dozer.ingest.IngestResponse(\x01\x62\x06proto3') +DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(b'\n\x0cingest.proto\x12\x0c\x64ozer.ingest\x1a\x0btypes.proto\"\x9f\x01\n\rIngestRequest\x12\x13\n\x0bschema_name\x18\x01 \x01(\t\x12\'\n\x03typ\x18\x02 \x01(\x0e\x32\x1a.dozer.types.OperationType\x12\x1f\n\x03old\x18\x03 \x03(\x0b\x32\x12.dozer.types.Value\x12\x1f\n\x03new\x18\x04 \x03(\x0b\x32\x12.dozer.types.Value\x12\x0e\n\x06seq_no\x18\x05 \x01(\r\" \n\x0eIngestResponse\x12\x0e\n\x06seq_no\x18\x01 \x01(\r\"\xdb\x01\n\x12IngestArrowRequest\x12\x13\n\x0bschema_name\x18\x01 \x01(\t\x12\x0f\n\x07records\x18\x02 \x01(\x0c\x12\x0e\n\x06seq_no\x18\x03 \x01(\r\x12@\n\x08metadata\x18\x04 \x03(\x0b\x32..dozer.ingest.IngestArrowRequest.MetadataEntry\x1aM\n\rMetadataEntry\x12\x0b\n\x03key\x18\x01 \x01(\r\x12+\n\x05value\x18\x02 \x01(\x0b\x32\x1c.dozer.ingest.IngestMetadata:\x02\x38\x01\"K\n\x0eIngestMetadata\x12(\n\x03typ\x18\x01 \x01(\x0e\x32\x1b.dozer.ingest.OperationType\x12\x0f\n\x07version\x18\x02 \x01(\r*3\n\rOperationType\x12\n\n\x06INSERT\x10\x00\x12\n\n\x06\x44\x45LETE\x10\x01\x12\n\n\x06UPDATE\x10\x02\x32\xcb\x02\n\rIngestService\x12\x43\n\x06ingest\x12\x1b.dozer.ingest.IngestRequest\x1a\x1c.dozer.ingest.IngestResponse\x12L\n\ringest_stream\x12\x1b.dozer.ingest.IngestRequest\x1a\x1c.dozer.ingest.IngestResponse(\x01\x12N\n\x0cingest_arrow\x12 .dozer.ingest.IngestArrowRequest\x1a\x1c.dozer.ingest.IngestResponse\x12W\n\x13ingest_arrow_stream\x12 .dozer.ingest.IngestArrowRequest\x1a\x1c.dozer.ingest.IngestResponse(\x01\x62\x06proto3') _builder.BuildMessageAndEnumDescriptors(DESCRIPTOR, globals()) _builder.BuildTopDescriptorsAndMessages(DESCRIPTOR, 'ingest_pb2', globals()) @@ -23,18 +23,18 @@ DESCRIPTOR._options = None _INGESTARROWREQUEST_METADATAENTRY._options = None _INGESTARROWREQUEST_METADATAENTRY._serialized_options = b'8\001' - _OPERATIONTYPE._serialized_start=553 - _OPERATIONTYPE._serialized_end=604 + _OPERATIONTYPE._serialized_start=538 + _OPERATIONTYPE._serialized_end=589 _INGESTREQUEST._serialized_start=44 - _INGESTREQUEST._serialized_end=218 - _INGESTRESPONSE._serialized_start=220 - _INGESTRESPONSE._serialized_end=252 - _INGESTARROWREQUEST._serialized_start=255 - _INGESTARROWREQUEST._serialized_end=474 - _INGESTARROWREQUEST_METADATAENTRY._serialized_start=397 - _INGESTARROWREQUEST_METADATAENTRY._serialized_end=474 - _INGESTMETADATA._serialized_start=476 - _INGESTMETADATA._serialized_end=551 - _INGESTSERVICE._serialized_start=607 - _INGESTSERVICE._serialized_end=938 + _INGESTREQUEST._serialized_end=203 + _INGESTRESPONSE._serialized_start=205 + _INGESTRESPONSE._serialized_end=237 + _INGESTARROWREQUEST._serialized_start=240 + _INGESTARROWREQUEST._serialized_end=459 + _INGESTARROWREQUEST_METADATAENTRY._serialized_start=382 + _INGESTARROWREQUEST_METADATAENTRY._serialized_end=459 + _INGESTMETADATA._serialized_start=461 + _INGESTMETADATA._serialized_end=536 + _INGESTSERVICE._serialized_start=592 + _INGESTSERVICE._serialized_end=923 # @@protoc_insertion_point(module_scope) diff --git a/pydozer/ingest_pb2_grpc.py b/pydozer/ingest_pb2_grpc.py index 1e46ccf..c24d826 100644 --- a/pydozer/ingest_pb2_grpc.py +++ b/pydozer/ingest_pb2_grpc.py @@ -2,7 +2,7 @@ """Client and server classes corresponding to protobuf-defined services.""" import grpc -import pydozer.ingest_pb2 as ingest__pb2 +import ingest_pb2 as ingest__pb2 class IngestServiceStub(object): diff --git a/pydozer/types_pb2.py b/pydozer/types_pb2.py index 67e4814..65dcde2 100644 --- a/pydozer/types_pb2.py +++ b/pydozer/types_pb2.py @@ -15,25 +15,29 @@ from google.protobuf import struct_pb2 as google_dot_protobuf_dot_struct__pb2 -DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(b'\n\x0btypes.proto\x12\x0b\x64ozer.types\x1a\x1fgoogle/protobuf/timestamp.proto\x1a\x1cgoogle/protobuf/struct.proto\"\xbc\x01\n\tOperation\x12\'\n\x03typ\x18\x01 \x01(\x0e\x32\x1a.dozer.types.OperationType\x12%\n\x03old\x18\x02 \x01(\x0b\x32\x13.dozer.types.RecordH\x00\x88\x01\x01\x12 \n\x03new\x18\x03 \x01(\x0b\x32\x13.dozer.types.Record\x12\x13\n\x06new_id\x18\x04 \x01(\x04H\x01\x88\x01\x01\x12\x15\n\rendpoint_name\x18\x05 \x01(\tB\x06\n\x04_oldB\t\n\x07_new_id\"=\n\x06Record\x12\"\n\x06values\x18\x01 \x03(\x0b\x32\x12.dozer.types.Value\x12\x0f\n\x07version\x18\x02 \x01(\r\"?\n\x0cRecordWithId\x12\n\n\x02id\x18\x01 \x01(\x04\x12#\n\x06record\x18\x02 \x01(\x0b\x32\x13.dozer.types.Record\"u\n\x0bSchemaEvent\x12\x10\n\x08\x65ndpoint\x18\x01 \x01(\t\x12\x0f\n\x07version\x18\x02 \x01(\x04\x12\x15\n\rprimary_index\x18\x03 \x03(\x05\x12,\n\x06\x66ields\x18\x04 \x03(\x0b\x32\x1c.dozer.types.FieldDefinition\"Q\n\x0f\x46ieldDefinition\x12\x1e\n\x03typ\x18\x01 \x01(\x0e\x32\x11.dozer.types.Type\x12\x0c\n\x04name\x18\x02 \x01(\t\x12\x10\n\x08nullable\x18\x03 \x01(\x08\"!\n\tPointType\x12\t\n\x01x\x18\x01 \x01(\x01\x12\t\n\x01y\x18\x02 \x01(\x01\"0\n\x0c\x44urationType\x12\r\n\x05value\x18\x01 \x01(\t\x12\x11\n\ttime_unit\x18\x02 \x01(\t\"S\n\x0bRustDecimal\x12\r\n\x05scale\x18\x01 \x01(\r\x12\n\n\x02lo\x18\x02 \x01(\r\x12\x0b\n\x03mid\x18\x03 \x01(\r\x12\n\n\x02hi\x18\x04 \x01(\r\x12\x10\n\x08negative\x18\x05 \x01(\x08\"\xdc\x03\n\x05Value\x12\x14\n\nuint_value\x18\x01 \x01(\x04H\x00\x12\x18\n\x0euint_128_value\x18\x02 \x01(\tH\x00\x12\x13\n\tint_value\x18\x03 \x01(\x03H\x00\x12\x17\n\rint_128_value\x18\x04 \x01(\tH\x00\x12\x15\n\x0b\x66loat_value\x18\x05 \x01(\x01H\x00\x12\x14\n\nbool_value\x18\x06 \x01(\x08H\x00\x12\x16\n\x0cstring_value\x18\x07 \x01(\tH\x00\x12\x15\n\x0b\x62ytes_value\x18\x08 \x01(\x0cH\x00\x12\x31\n\rdecimal_value\x18\t \x01(\x0b\x32\x18.dozer.types.RustDecimalH\x00\x12\x35\n\x0ftimestamp_value\x18\n \x01(\x0b\x32\x1a.google.protobuf.TimestampH\x00\x12\x14\n\ndate_value\x18\x0b \x01(\tH\x00\x12-\n\x0bpoint_value\x18\x0c \x01(\x0b\x32\x16.dozer.types.PointTypeH\x00\x12\x33\n\x0e\x64uration_value\x18\r \x01(\x0b\x32\x19.dozer.types.DurationTypeH\x00\x12,\n\njson_value\x18\x0e \x01(\x0b\x32\x16.google.protobuf.ValueH\x00\x42\x07\n\x05value*G\n\tEventType\x12\x07\n\x03\x41LL\x10\x00\x12\x0f\n\x0bINSERT_ONLY\x10\x01\x12\x0f\n\x0bUPDATE_ONLY\x10\x02\x12\x0f\n\x0b\x44\x45LETE_ONLY\x10\x03*3\n\rOperationType\x12\n\n\x06INSERT\x10\x00\x12\n\n\x06\x44\x45LETE\x10\x01\x12\n\n\x06UPDATE\x10\x02*\xb0\x01\n\x04Type\x12\x08\n\x04UInt\x10\x00\x12\x08\n\x04U128\x10\x01\x12\x07\n\x03Int\x10\x02\x12\x08\n\x04I128\x10\x03\x12\t\n\x05\x46loat\x10\x04\x12\x0b\n\x07\x42oolean\x10\x05\x12\n\n\x06String\x10\x06\x12\x08\n\x04Text\x10\x07\x12\n\n\x06\x42inary\x10\x08\x12\x0b\n\x07\x44\x65\x63imal\x10\t\x12\r\n\tTimestamp\x10\n\x12\x08\n\x04\x44\x61te\x10\x0b\x12\x08\n\x04Json\x10\x0c\x12\t\n\x05Point\x10\r\x12\x0c\n\x08\x44uration\x10\x0e\x62\x06proto3') +DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(b'\n\x0btypes.proto\x12\x0b\x64ozer.types\x1a\x1fgoogle/protobuf/timestamp.proto\x1a\x1cgoogle/protobuf/struct.proto\"S\n\x0b\x45ventFilter\x12$\n\x04type\x18\x01 \x01(\x0e\x32\x16.dozer.types.EventType\x12\x13\n\x06\x66ilter\x18\x03 \x01(\tH\x00\x88\x01\x01\x42\t\n\x07_filter\"\x9c\x01\n\tOperation\x12\'\n\x03typ\x18\x01 \x01(\x0e\x32\x1a.dozer.types.OperationType\x12%\n\x03old\x18\x02 \x01(\x0b\x32\x13.dozer.types.RecordH\x00\x88\x01\x01\x12 \n\x03new\x18\x03 \x01(\x0b\x32\x13.dozer.types.Record\x12\x15\n\rendpoint_name\x18\x05 \x01(\tB\x06\n\x04_old\"I\n\x06Record\x12\"\n\x06values\x18\x01 \x03(\x0b\x32\x12.dozer.types.Value\x12\n\n\x02id\x18\x02 \x01(\x04\x12\x0f\n\x07version\x18\x03 \x01(\r\"u\n\x0bSchemaEvent\x12\x10\n\x08\x65ndpoint\x18\x01 \x01(\t\x12\x0f\n\x07version\x18\x02 \x01(\x04\x12\x15\n\rprimary_index\x18\x03 \x03(\x05\x12,\n\x06\x66ields\x18\x04 \x03(\x0b\x32\x1c.dozer.types.FieldDefinition\"Q\n\x0f\x46ieldDefinition\x12\x1e\n\x03typ\x18\x01 \x01(\x0e\x32\x11.dozer.types.Type\x12\x0c\n\x04name\x18\x02 \x01(\t\x12\x10\n\x08nullable\x18\x03 \x01(\x08\"!\n\tPointType\x12\t\n\x01x\x18\x01 \x01(\x01\x12\t\n\x01y\x18\x02 \x01(\x01\"0\n\x0c\x44urationType\x12\r\n\x05value\x18\x01 \x01(\t\x12\x11\n\ttime_unit\x18\x02 \x01(\t\"S\n\x0bRustDecimal\x12\r\n\x05scale\x18\x01 \x01(\r\x12\n\n\x02lo\x18\x02 \x01(\r\x12\x0b\n\x03mid\x18\x03 \x01(\r\x12\n\n\x02hi\x18\x04 \x01(\r\x12\x10\n\x08negative\x18\x05 \x01(\x08\"\xdc\x03\n\x05Value\x12\x14\n\nuint_value\x18\x01 \x01(\x04H\x00\x12\x18\n\x0euint_128_value\x18\x02 \x01(\tH\x00\x12\x13\n\tint_value\x18\x03 \x01(\x03H\x00\x12\x17\n\rint_128_value\x18\x04 \x01(\tH\x00\x12\x15\n\x0b\x66loat_value\x18\x05 \x01(\x01H\x00\x12\x14\n\nbool_value\x18\x06 \x01(\x08H\x00\x12\x16\n\x0cstring_value\x18\x07 \x01(\tH\x00\x12\x15\n\x0b\x62ytes_value\x18\x08 \x01(\x0cH\x00\x12\x31\n\rdecimal_value\x18\t \x01(\x0b\x32\x18.dozer.types.RustDecimalH\x00\x12\x35\n\x0ftimestamp_value\x18\n \x01(\x0b\x32\x1a.google.protobuf.TimestampH\x00\x12\x14\n\ndate_value\x18\x0b \x01(\tH\x00\x12-\n\x0bpoint_value\x18\x0c \x01(\x0b\x32\x16.dozer.types.PointTypeH\x00\x12\x33\n\x0e\x64uration_value\x18\r \x01(\x0b\x32\x19.dozer.types.DurationTypeH\x00\x12,\n\njson_value\x18\x0e \x01(\x0b\x32\x16.google.protobuf.ValueH\x00\x42\x07\n\x05value\"\xfb\x01\n\x0fSchemasResponse\x12:\n\x07schemas\x18\x01 \x03(\x0b\x32).dozer.types.SchemasResponse.SchemasEntry\x12\x38\n\x06\x65rrors\x18\x02 \x03(\x0b\x32(.dozer.types.SchemasResponse.ErrorsEntry\x1a\x43\n\x0cSchemasEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\"\n\x05value\x18\x02 \x01(\x0b\x32\x13.dozer.types.Schema:\x02\x38\x01\x1a-\n\x0b\x45rrorsEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\r\n\x05value\x18\x02 \x01(\t:\x02\x38\x01\"M\n\x06Schema\x12\x15\n\rprimary_index\x18\x01 \x03(\x05\x12,\n\x06\x66ields\x18\x02 \x03(\x0b\x32\x1c.dozer.types.FieldDefinition*G\n\tEventType\x12\x07\n\x03\x41LL\x10\x00\x12\x0f\n\x0bINSERT_ONLY\x10\x01\x12\x0f\n\x0bUPDATE_ONLY\x10\x02\x12\x0f\n\x0b\x44\x45LETE_ONLY\x10\x03*3\n\rOperationType\x12\n\n\x06INSERT\x10\x00\x12\n\n\x06\x44\x45LETE\x10\x01\x12\n\n\x06UPDATE\x10\x02*\xb0\x01\n\x04Type\x12\x08\n\x04UInt\x10\x00\x12\x08\n\x04U128\x10\x01\x12\x07\n\x03Int\x10\x02\x12\x08\n\x04I128\x10\x03\x12\t\n\x05\x46loat\x10\x04\x12\x0b\n\x07\x42oolean\x10\x05\x12\n\n\x06String\x10\x06\x12\x08\n\x04Text\x10\x07\x12\n\n\x06\x42inary\x10\x08\x12\x0b\n\x07\x44\x65\x63imal\x10\t\x12\r\n\tTimestamp\x10\n\x12\x08\n\x04\x44\x61te\x10\x0b\x12\x08\n\x04Json\x10\x0c\x12\t\n\x05Point\x10\r\x12\x0c\n\x08\x44uration\x10\x0e\x62\x06proto3') _builder.BuildMessageAndEnumDescriptors(DESCRIPTOR, globals()) _builder.BuildTopDescriptorsAndMessages(DESCRIPTOR, 'types_pb2', globals()) if _descriptor._USE_C_DESCRIPTORS == False: DESCRIPTOR._options = None - _EVENTTYPE._serialized_start=1261 - _EVENTTYPE._serialized_end=1332 - _OPERATIONTYPE._serialized_start=1334 - _OPERATIONTYPE._serialized_end=1385 - _TYPE._serialized_start=1388 - _TYPE._serialized_end=1564 - _OPERATION._serialized_start=92 - _OPERATION._serialized_end=280 - _RECORD._serialized_start=282 - _RECORD._serialized_end=343 - _RECORDWITHID._serialized_start=345 - _RECORDWITHID._serialized_end=408 + _SCHEMASRESPONSE_SCHEMASENTRY._options = None + _SCHEMASRESPONSE_SCHEMASENTRY._serialized_options = b'8\001' + _SCHEMASRESPONSE_ERRORSENTRY._options = None + _SCHEMASRESPONSE_ERRORSENTRY._serialized_options = b'8\001' + _EVENTTYPE._serialized_start=1594 + _EVENTTYPE._serialized_end=1665 + _OPERATIONTYPE._serialized_start=1667 + _OPERATIONTYPE._serialized_end=1718 + _TYPE._serialized_start=1721 + _TYPE._serialized_end=1897 + _EVENTFILTER._serialized_start=91 + _EVENTFILTER._serialized_end=174 + _OPERATION._serialized_start=177 + _OPERATION._serialized_end=333 + _RECORD._serialized_start=335 + _RECORD._serialized_end=408 _SCHEMAEVENT._serialized_start=410 _SCHEMAEVENT._serialized_end=527 _FIELDDEFINITION._serialized_start=529 @@ -46,4 +50,12 @@ _RUSTDECIMAL._serialized_end=780 _VALUE._serialized_start=783 _VALUE._serialized_end=1259 + _SCHEMASRESPONSE._serialized_start=1262 + _SCHEMASRESPONSE._serialized_end=1513 + _SCHEMASRESPONSE_SCHEMASENTRY._serialized_start=1399 + _SCHEMASRESPONSE_SCHEMASENTRY._serialized_end=1466 + _SCHEMASRESPONSE_ERRORSENTRY._serialized_start=1468 + _SCHEMASRESPONSE_ERRORSENTRY._serialized_end=1513 + _SCHEMA._serialized_start=1515 + _SCHEMA._serialized_end=1592 # @@protoc_insertion_point(module_scope) diff --git a/pyproject.toml b/pyproject.toml index df3f759..42c6854 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "pydozer" -version = "0.1.35" +version = "0.2.1" description = "Python wrapper for the Dozer API" authors = ["Dozer Team "] license = "Apache-2.0" diff --git a/tests/test_dozer.py b/tests/test_dozer.py index 465f211..cb5cecb 100644 --- a/tests/test_dozer.py +++ b/tests/test_dozer.py @@ -43,8 +43,7 @@ def test_ingest(ingestion_client: IngestClient): schema_name="users", typ=0, old=None, - new=Record(values=[Value(int_value=1), - Value(string_value="superman")]), + new=[Value(int_value=1), Value(string_value="superman")], seq_no=1 ) res = ingestion_client.ingest_raw(user) @@ -71,8 +70,7 @@ def test_ingest_query(ingestion_client: IngestClient, api_client: ApiClient): schema_name="users", typ=0, old=None, - new=Record(values=[Value(int_value=1), - Value(string_value="superman")]), + new=[Value(int_value=1), Value(string_value="superman")], seq_no=1 ) res = ingestion_client.ingest_raw(user)