Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

feat: implement fine grained serialization and deserialization control #1365

Open
wants to merge 4 commits into
base: master
Choose a base branch
from
+626 −466
Open

feat: implement fine grained serialization and deserialization control #1365

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
eed6b4e
feat: implement fine grained serialization and deserialization control
ekazakas Dec 13, 2024
93ea851
Merge branch 'master' into feat-serde-hints
ekazakas Dec 16, 2024
b88dbd1
Organize code
ekazakas Dec 16, 2024
0f283ae
cleanup
ekazakas Dec 16, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion examples/avro_generic_consumer_example/avro_generic_consumer_example.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -97,7 +97,7 @@ func main() {
switch e := ev.(type) {
case *kafka.Message:
value := User{}
err := deser.DeserializeInto(*e.TopicPartition.Topic, e.Value, &value)
err := deser.DeserializeInto(*e.TopicPartition.Topic, e.Value, &value, serde.NewDeserializeHint())
if err != nil {
fmt.Printf("Failed to deserialize payload: %s\n", err)
} else {
Expand Down
2 changes: 1 addition & 1 deletion examples/avro_generic_producer_example/avro_generic_producer_example.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -71,7 +71,7 @@ func main() {
FavoriteNumber: 42,
FavoriteColor: "blue",
}
payload, err := ser.Serialize(topic, &value)
payload, err := ser.Serialize(topic, &value, serde.NewSerializeHint())
if err != nil {
fmt.Printf("Failed to serialize payload: %s\n", err)
os.Exit(1)
Expand Down
2 changes: 1 addition & 1 deletion examples/avro_specific_consumer_example/avro_specific_consumer_example.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -97,7 +97,7 @@ func main() {
switch e := ev.(type) {
case *kafka.Message:
value := User{}
err := deser.DeserializeInto(*e.TopicPartition.Topic, e.Value, &value)
err := deser.DeserializeInto(*e.TopicPartition.Topic, e.Value, &value, serde.NewDeserializeHint())
if err != nil {
fmt.Printf("Failed to deserialize payload: %s\n", err)
} else {
Expand Down
2 changes: 1 addition & 1 deletion examples/avro_specific_producer_example/avro_specific_producer_example.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -71,7 +71,7 @@ func main() {
Favorite_number: 42,
Favorite_color: "blue",
}
payload, err := ser.Serialize(topic, &value)
payload, err := ser.Serialize(topic, &value, serde.NewSerializeHint())
if err != nil {
fmt.Printf("Failed to serialize payload: %s\n", err)
os.Exit(1)
Expand Down
2 changes: 1 addition & 1 deletion examples/avrov2_consumer_encryption_example/avrov2_consumer_encryption_example.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -111,7 +111,7 @@ func main() {
switch e := ev.(type) {
case *kafka.Message:
value := User{}
err := deser.DeserializeInto(*e.TopicPartition.Topic, e.Value, &value)
err := deser.DeserializeInto(*e.TopicPartition.Topic, e.Value, &value, serde.NewDeserializeHint())
if err != nil {
fmt.Printf("Failed to deserialize payload: %s\n", err)
} else {
Expand Down
2 changes: 1 addition & 1 deletion examples/avrov2_consumer_example/avrov2_consumer_example.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -97,7 +97,7 @@ func main() {
switch e := ev.(type) {
case *kafka.Message:
value := User{}
err := deser.DeserializeInto(*e.TopicPartition.Topic, e.Value, &value)
err := deser.DeserializeInto(*e.TopicPartition.Topic, e.Value, &value, serde.NewDeserializeHint())
if err != nil {
fmt.Printf("Failed to deserialize payload: %s\n", err)
} else {
Expand Down
10 changes: 5 additions & 5 deletions examples/avrov2_consumer_migration_example/avrov2_consumer_migration_example.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -71,13 +71,13 @@ func main() {
os.Exit(1)
}

deserConfig := avrov2.NewDeserializerConfig()
deserConfig.UseLatestWithMetadata = map[string]string{
deser, err := avrov2.NewDeserializer(client, serde.ValueSerde, avrov2.NewDeserializerConfig())

deserializeHint := serde.NewDeserializeHint()
deserializeHint.UseLatestWithMetadata = map[string]string{
"application.major.version": "2",
}

deser, err := avrov2.NewDeserializer(client, serde.ValueSerde, deserConfig)

if err != nil {
fmt.Printf("Failed to create deserializer: %s\n", err)
os.Exit(1)
Expand Down Expand Up @@ -106,7 +106,7 @@ func main() {
switch e := ev.(type) {
case *kafka.Message:
value := User{}
err := deser.DeserializeInto(*e.TopicPartition.Topic, e.Value, &value)
err := deser.DeserializeInto(*e.TopicPartition.Topic, e.Value, &value, deserializeHint)
if err != nil {
fmt.Printf("Failed to deserialize payload: %s\n", err)
} else {
Expand Down
8 changes: 5 additions & 3 deletions examples/avrov2_producer_encryption_example/avrov2_producer_encryption_example.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -114,7 +114,6 @@ func main() {

serConfig := avrov2.NewSerializerConfig()
serConfig.AutoRegisterSchemas = false
serConfig.UseLatestVersion = true
// KMS properties can be passed as follows
//serConfig.RuleConfig = map[string]string{
// "secret.access.key": "xxx",
Expand All @@ -123,6 +122,9 @@ func main() {

ser, err := avrov2.NewSerializer(client, serde.ValueSerde, serConfig)

serializeHint := serde.NewSerializeHint()
serializeHint.UseLatestVersion = true

if err != nil {
fmt.Printf("Failed to create serializer: %s\n", err)
os.Exit(1)
Expand All @@ -137,7 +139,7 @@ func main() {
FavoriteNumber: 42,
FavoriteColor: "blue",
}
payload, err := ser.Serialize(topic, &value)
payload, err := ser.Serialize(topic, &value, serializeHint)
if err != nil {
fmt.Printf("Failed to serialize payload: %s\n", err)
os.Exit(1)
Expand All @@ -158,7 +160,7 @@ func main() {
FavoriteNumber: 42,
FavoriteColor: "blue",
}
payload, err = ser.Serialize(topic, &value)
payload, err = ser.Serialize(topic, &value, serializeHint)
if err != nil {
fmt.Printf("Failed to serialize payload: %s\n", err)
os.Exit(1)
Expand Down
2 changes: 1 addition & 1 deletion examples/avrov2_producer_example/avrov2_producer_example.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -71,7 +71,7 @@ func main() {
FavoriteNumber: 42,
FavoriteColor: "blue",
}
payload, err := ser.Serialize(topic, &value)
payload, err := ser.Serialize(topic, &value, serde.NewSerializeHint())
if err != nil {
fmt.Printf("Failed to serialize payload: %s\n", err)
os.Exit(1)
Expand Down
13 changes: 7 additions & 6 deletions examples/avrov2_producer_migration_example/avrov2_producer_migration_example.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -134,17 +134,18 @@ func main() {

serConfig := avrov2.NewSerializerConfig()
serConfig.AutoRegisterSchemas = false
serConfig.UseLatestWithMetadata = map[string]string{
"application.major.version": "1",
}

ser, err := avrov2.NewSerializer(client, serde.ValueSerde, serConfig)

if err != nil {
fmt.Printf("Failed to create serializer: %s\n", err)
os.Exit(1)
}

serializeHint := serde.NewSerializeHint()
serializeHint.UseLatestWithMetadata = map[string]string{
"application.major.version": "1",
}

// Optional delivery channel, if not specified the Producer object's
// .Events channel is used.
deliveryChan := make(chan kafka.Event)
Expand All @@ -154,7 +155,7 @@ func main() {
FavoriteNumber: 42,
FavoriteColor: "blue",
}
payload, err := ser.Serialize(topic, &value)
payload, err := ser.Serialize(topic, &value, serializeHint)
if err != nil {
fmt.Printf("Failed to serialize payload: %s\n", err)
os.Exit(1)
Expand All @@ -175,7 +176,7 @@ func main() {
FavoriteNumber: 42,
FavoriteColor: "blue",
}
payload, err = ser.Serialize(topic, &value)
payload, err = ser.Serialize(topic, &value, serializeHint)
if err != nil {
fmt.Printf("Failed to serialize payload: %s\n", err)
os.Exit(1)
Expand Down
4 changes: 2 additions & 2 deletions examples/confluent_cloud_example/confluent_cloud_example.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -118,7 +118,7 @@ func main() {
FavoriteColor: "blue",
}

payload, err := ser.Serialize(topic, &value)
payload, err := ser.Serialize(topic, &value, serde.NewSerializeHint())
if err != nil {
fmt.Printf("Failed to serialize payload: %s\n", err)
os.Exit(1)
Expand Down Expand Up @@ -171,7 +171,7 @@ func main() {
message, err := consumer.ReadMessage(100 * time.Millisecond)
if err == nil {
received := User{}
err := deser.DeserializeInto(*message.TopicPartition.Topic, message.Value, &received)
err := deser.DeserializeInto(*message.TopicPartition.Topic, message.Value, &received, serde.NewDeserializeHint())
if err != nil {
fmt.Printf("Failed to deserialize payload: %s\n", err)
} else {
Expand Down
1 change: 1 addition & 0 deletions examples/docker_aws_lambda_example/go.sum
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -53,6 +53,7 @@ github.com/confluentinc/confluent-kafka-go/v2 v2.4.0 h1:NbOku86JJlsRJPJKE0snNsz6
github.com/confluentinc/confluent-kafka-go/v2 v2.4.0/go.mod h1:E1dEQy50ZLfqs7T9luxz0rLxaeFZJZE92XvApJOr/Rk=
github.com/confluentinc/confluent-kafka-go/v2 v2.5.0/go.mod h1:Hyo+IIQ/tmsfkOcRP8T6VlSeOW3T33v0Me8Xvq4u90Y=
github.com/confluentinc/confluent-kafka-go/v2 v2.5.3/go.mod h1:QxYLPRKR1MVlkXCCjzjjrpXb0VyFNfVaZXi0obZykJ0=
github.com/confluentinc/confluent-kafka-go/v2 v2.6.1/go.mod h1:hScqtFIGUI1wqHIgM3mjoqEou4VweGGGX7dMpcUKves=
github.com/containerd/console v1.0.3 h1:lIr7SlA5PxZyMV30bDW0MGbiOPXwc63yRuCP0ARubLw=
github.com/containerd/console v1.0.3/go.mod h1:7LqA/THxQ86k76b8c/EMSiaJ3h1eZkMkXar0TQ1gf3U=
github.com/containerd/containerd v1.7.12 h1:+KQsnv4VnzyxWcfO9mlxxELaoztsDEjOuCMPAuPqgU0=
Expand Down
2 changes: 1 addition & 1 deletion examples/json_consumer_example/json_consumer_example.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -97,7 +97,7 @@ func main() {
switch e := ev.(type) {
case *kafka.Message:
value := User{}
err := deser.DeserializeInto(*e.TopicPartition.Topic, e.Value, &value)
err := deser.DeserializeInto(*e.TopicPartition.Topic, e.Value, &value, serde.NewDeserializeHint())
if err != nil {
fmt.Printf("Failed to deserialize payload: %s\n", err)
} else {
Expand Down
7 changes: 4 additions & 3 deletions examples/json_producer_encryption_example/json_producer_encryption_example.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -116,20 +116,21 @@ func main() {

serConfig := jsonschema.NewSerializerConfig()
serConfig.AutoRegisterSchemas = false
serConfig.UseLatestVersion = true
// KMS properties can be passed as follows
//serConfig.RuleConfig = map[string]string{
// "secret.access.key": "xxx",
// "access.key,id": "xxx",
//}

ser, err := jsonschema.NewSerializer(client, serde.ValueSerde, serConfig)

if err != nil {
fmt.Printf("Failed to create serializer: %s\n", err)
os.Exit(1)
}

serializeHint := serde.NewSerializeHint()
serializeHint.UseLatestVersion = true

// Optional delivery channel, if not specified the Producer object's
// .Events channel is used.
deliveryChan := make(chan kafka.Event)
Expand All @@ -139,7 +140,7 @@ func main() {
FavoriteNumber: 42,
FavoriteColor: "blue",
}
payload, err := ser.Serialize(topic, &value)
payload, err := ser.Serialize(topic, &value, serializeHint)
if err != nil {
fmt.Printf("Failed to serialize payload: %s\n", err)
os.Exit(1)
Expand Down
2 changes: 1 addition & 1 deletion examples/json_producer_example/json_producer_example.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -71,7 +71,7 @@ func main() {
FavoriteNumber: 42,
FavoriteColor: "blue",
}
payload, err := ser.Serialize(topic, &value)
payload, err := ser.Serialize(topic, &value, serde.NewSerializeHint())
if err != nil {
fmt.Printf("Failed to serialize payload: %s\n", err)
os.Exit(1)
Expand Down
2 changes: 1 addition & 1 deletion examples/protobuf_consumer_encryption_example/protobuf_consumer_encryption_example.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -116,7 +116,7 @@ func main() {

switch e := ev.(type) {
case *kafka.Message:
value, err := deser.Deserialize(*e.TopicPartition.Topic, e.Value)
value, err := deser.Deserialize(*e.TopicPartition.Topic, e.Value, serde.NewDeserializeHint())
if err != nil {
fmt.Printf("Failed to deserialize payload: %s\n", err)
} else {
Expand Down
2 changes: 1 addition & 1 deletion examples/protobuf_consumer_example/protobuf_consumer_example.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -101,7 +101,7 @@ func main() {

switch e := ev.(type) {
case *kafka.Message:
value, err := deser.Deserialize(*e.TopicPartition.Topic, e.Value)
value, err := deser.Deserialize(*e.TopicPartition.Topic, e.Value, serde.NewDeserializeHint())
if err != nil {
fmt.Printf("Failed to deserialize payload: %s\n", err)
} else {
Expand Down
7 changes: 4 additions & 3 deletions examples/protobuf_producer_encryption_example/protobuf_producer_encryption_example.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -117,20 +117,21 @@ message User {

serConfig := protobuf.NewSerializerConfig()
serConfig.AutoRegisterSchemas = false
serConfig.UseLatestVersion = true
// KMS properties can be passed as follows
//serConfig.RuleConfig = map[string]string{
// "secret.access.key": "xxx",
// "access.key,id": "xxx",
//}

ser, err := protobuf.NewSerializer(client, serde.ValueSerde, serConfig)

if err != nil {
fmt.Printf("Failed to create serializer: %s\n", err)
os.Exit(1)
}

serializeHint := serde.NewSerializeHint()
serializeHint.UseLatestVersion = true

// Optional delivery channel, if not specified the Producer object's
// .Events channel is used.
deliveryChan := make(chan kafka.Event)
Expand All @@ -141,7 +142,7 @@ message User {
FavoriteNumber: 42,
FavoriteColor: "blue",
}
payload, err := ser.Serialize(topic, &value)
payload, err := ser.Serialize(topic, &value, serializeHint)
if err != nil {
fmt.Printf("Failed to serialize payload: %s\n", err)
os.Exit(1)
Expand Down
2 changes: 1 addition & 1 deletion examples/protobuf_producer_example/protobuf_producer_example.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -71,7 +71,7 @@ func main() {
FavoriteNumber: 42,
FavoriteColor: "blue",
}
payload, err := ser.Serialize(topic, &value)
payload, err := ser.Serialize(topic, &value, serde.NewSerializeHint())
if err != nil {
fmt.Printf("Failed to serialize payload: %s\n", err)
os.Exit(1)
Expand Down
8 changes: 4 additions & 4 deletions schemaregistry/serde/avro/avro_generic.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -51,7 +51,7 @@ func NewGenericSerializer(client schemaregistry.Client, serdeType serde.Type, co
}

// Serialize implements serialization of generic Avro data
func (s *GenericSerializer) Serialize(topic string, msg interface{}) ([]byte, error) {
func (s *GenericSerializer) Serialize(topic string, msg interface{}, hint *serde.SerializeHint) ([]byte, error) {
if msg == nil {
return nil, nil
}
Expand All @@ -67,7 +67,7 @@ func (s *GenericSerializer) Serialize(topic string, msg interface{}) ([]byte, er
info := schemaregistry.SchemaInfo{
Schema: avroType.String(),
}
id, err := s.GetID(topic, msg, &info)
id, err := s.GetID(topic, msg, &info, hint)
if err != nil {
return nil, err
}
Expand All @@ -93,7 +93,7 @@ func NewGenericDeserializer(client schemaregistry.Client, serdeType serde.Type,
}

// Deserialize implements deserialization of generic Avro data
func (s *GenericDeserializer) Deserialize(topic string, payload []byte) (interface{}, error) {
func (s *GenericDeserializer) Deserialize(topic string, payload []byte, _ *serde.DeserializeHint) (interface{}, error) {
if payload == nil {
return nil, nil
}
Expand All @@ -118,7 +118,7 @@ func (s *GenericDeserializer) Deserialize(topic string, payload []byte) (interfa
}

// DeserializeInto implements deserialization of generic Avro data to the given object
func (s *GenericDeserializer) DeserializeInto(topic string, payload []byte, msg interface{}) error {
func (s *GenericDeserializer) DeserializeInto(topic string, payload []byte, msg interface{}, _ *serde.DeserializeHint) error {
if payload == nil {
return nil
}
Expand Down
Loading