Merge pull request Aiven-Open#986 from Aiven-Open/keejon/fix-protobuf…

…-optionals

fix: proto3 optionals from base64 encoded proto descriptors

src/karapace/protobuf/serialization.py

@@ -93,17 +93,31 @@ def _deserialize_msg(msgtype: Any) -> MessageElement:
     for nested_enum in msgtype.enum_type:
         nested_types.append(_deserialize_enum(nested_enum))
 
-    one_ofs: list[OneOfElement] = [OneOfElement(oneof.name) for oneof in msgtype.oneof_decl]
+    one_ofs: list[OneOfElement | None] = [OneOfElement(oneof.name) for oneof in msgtype.oneof_decl]
 
     for f in msgtype.field:
         sf = _deserialize_field(f)
-        if f.HasField("oneof_index"):
+        is_oneof = f.HasField("oneof_index")
+        is_proto3_optional = f.HasField("oneof_index") and f.HasField("proto3_optional") and f.proto3_optional
+        if is_proto3_optional:
+            # Every proto3 optional field is placed into a one-field oneof, called a "synthetic" oneof,
+            # as it was not present in the source .proto file.
+            # This will make sure that we don't interpret those optionals as oneof.
+            one_ofs[f.oneof_index] = None
+            fields.append(sf)
+        elif is_oneof:
             one_ofs[f.oneof_index].fields.append(sf)
         else:
             fields.append(sf)
 
+    one_ofs_filtered: list[OneOfElement] = [oneof for oneof in one_ofs if oneof is not None]
     return MessageElement(
-        DEFAULT_LOCATION, msgtype.name, nested_types=nested_types, reserveds=reserveds, fields=fields, one_ofs=one_ofs
+        DEFAULT_LOCATION,
+        msgtype.name,
+        nested_types=nested_types,
+        reserveds=reserveds,
+        fields=fields,
+        one_ofs=one_ofs_filtered,
     )
 
 

tests/schemas/protobuf.py

@@ -261,3 +261,21 @@
     "lzdGVyLk1ldGFkYXRhEhYKDmNvbXBhbnlfbnVtYmVyGAIgASgJGhYKCE1ldGFkYXRhEgoK"
     "AmlkGAEgASgJYgZwcm90bzM="
 )
+
+schema_protobuf_optionals_bin = (
+    "Cgp0ZXN0LnByb3RvIqYBCgpEaW1lbnNpb25zEhEKBHNpemUYASABKAFIAIgBARISCgV3aWR0aBgCIAEoAUgBiAEBEhMKBmhlaWdodBgDIAEo"
+    + "AUgCiAEBEhMKBmxlbmd0aBgEIAEoAUgDiAEBEhMKBndlaWdodBgFIAEoAUgEiAEBQgcKBV9zaXplQggKBl93aWR0aEIJCgdfaGVpZ2h0Qg"
+    + "kKB19sZW5ndGhCCQoHX3dlaWdodGIGcHJvdG8z"
+)
+
+schema_protobuf_optionals = """\
+syntax = "proto3";
+
+message Dimensions {
+  optional double size = 1;
+  optional double width = 2;
+  optional double height = 3;
+  optional double length = 4;
+  optional double weight = 5;
+}
+"""

tests/unit/test_protobuf_binary_serialization.py

@@ -16,6 +16,8 @@
     schema_protobuf_nested_message4_bin_protoc,
     schema_protobuf_oneof,
     schema_protobuf_oneof_bin,
+    schema_protobuf_optionals,
+    schema_protobuf_optionals_bin,
     schema_protobuf_order_after,
     schema_protobuf_order_after_bin,
     schema_protobuf_plain,
@@ -89,6 +91,7 @@
         (schema_protobuf_references, schema_protobuf_references_bin),
         (schema_protobuf_references2, schema_protobuf_references2_bin),
         (schema_protobuf_complex, schema_protobuf_complex_bin),
+        (schema_protobuf_optionals, schema_protobuf_optionals_bin),
     ],
 )
 def test_schema_deserialize(schema_plain, schema_serialized):
@@ -125,6 +128,7 @@ def test_protoc_serialized_schema_deserialize(schema_plain, schema_serialized):
         schema_protobuf_references,
         schema_protobuf_references2,
         schema_protobuf_complex,
+        schema_protobuf_optionals,
     ],
 )
 def test_simple_schema_serialize(schema):