diff --git a/.github/workflows/go.yml b/.github/workflows/go.yml new file mode 100644 index 0000000..62700ab --- /dev/null +++ b/.github/workflows/go.yml @@ -0,0 +1,60 @@ +name: Go + +on: + push: + branches: [main] + tags: ['v*'] + pull_request: + branches: ['*'] + +permissions: + contents: read + +jobs: + build: + runs-on: ubuntu-latest + strategy: + matrix: + go: ["1.21.x", "1.22.x"] + include: + - go: 1.22.x + steps: + - name: Checkout code + uses: actions/checkout@v4 + + - name: Setup Go + uses: actions/setup-go@v5 + with: + go-version: ${{ matrix.go }} + cache-dependency-path: '**/go.sum' + + - name: Download Dependencies + run: | + go mod download + + - name: Test + run: make cover + + lint: + name: Lint + runs-on: ubuntu-latest + + steps: + - uses: actions/checkout@v4 + name: Check out repository + - uses: actions/setup-go@v5 + name: Set up Go + with: + go-version: 1.22.x + cache: false # managed by golangci-lint + + - uses: golangci/golangci-lint-action@v6 + name: Install golangci-lint + with: + version: latest + # Hack: Use the official action to download, but not run. + # make lint below will handle actually running the linter. + args: --help + + - run: make lint + name: Lint \ No newline at end of file diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..c4193ab --- /dev/null +++ b/.gitignore @@ -0,0 +1,48 @@ +# Compiled Object files, Static and Dynamic libs (Shared Objects) +*.o +*.a +*.so + +# Folders +_obj +_test +vendor + +# Architecture specific extensions/prefixes +*.[568vq] +[568vq].out + +*.cgo1.go +*.cgo2.c +_cgo_defun.c +_cgo_gotypes.go +_cgo_export.* + +_testmain.go + +*.exe +*.test +*.prof +*.out +#*.jar + +config/local/config.json + +sms +main +.DS_Store +debug +appConfig.local.json +settings.json +liquibase.properties +*.sample.sql + +**/obj +**/bin + +# JetBrains +.idea/ +/.run/ + + +coverage.txt diff --git a/.golangci.yml b/.golangci.yml new file mode 100644 index 0000000..31a7ca0 --- /dev/null +++ b/.golangci.yml @@ -0,0 +1,69 @@ +run: + skip-dirs: + - docs + - datadog + - kustomize + skip-files: + - 'wire_gen.go' + tests: false +linters-settings: + errcheck: + check-type-assertions: true + check-blank: true + gci: + sections: + - standard + - default + gosimple: + go: '1.17' + govet: + check-shadowing: true + settings: + printf: + funcs: + - (gitlab.zgtools.net/devex/archetypes/gomods/zlog.Logger).Debug + - (gitlab.zgtools.net/devex/archetypes/gomods/zlog.Logger).Info + - (gitlab.zgtools.net/devex/archetypes/gomods/zlog.Logger).Warn + - (gitlab.zgtools.net/devex/archetypes/gomods/zlog.Logger).Error + depguard: + rules: + Main: + files: + - $all + - "!$test" + deny: + - github.com/satori/go.uuid: Prefer "github.com/google/uuid" + disable-all: true + enable: + - asciicheck + - bidichk + - bodyclose + - cyclop + - decorder + - depguard + - deadcode + - dupl + - errcheck + - errchkjson + - errname + - errorlint + - exportloopref + - gci + - gocognit + - goconst + - gocritic + - gocyclo + - gofmt + - gosimple + - govet + - ineffassign + - nolintlint + - prealloc + - staticcheck + - structcheck + - typecheck + - unconvert + - unparam + - unused + - varcheck + - whitespace diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..4bb136a --- /dev/null +++ b/Dockerfile @@ -0,0 +1,15 @@ +FROM golang:1.22 AS build + +ENV CGO_ENABLED=0 + +WORKDIR /go/src/zfmt +COPY . . + +RUN go install -v ./... +RUN go build -o zfmt +FROM debian + +COPY --from=build /go/src/zfmt / + + +ENTRYPOINT ["/zfmt"] \ No newline at end of file diff --git a/Makefile b/Makefile new file mode 100644 index 0000000..4d67cb3 --- /dev/null +++ b/Makefile @@ -0,0 +1,18 @@ +# Directories containing independent Go modules. +MODULE_DIRS = . +LOCAL_GOLANGCI_VERSION=$(shell golangci-lint --version) +REMOTE_GOLANGCI_VERSION=1.56.2 + +.PHONY: cover +cover: + go test -v ./... -count=1 -coverprofile=coverage.txt -covermode atomic + +.PHONY: lint +lint: golangci-lint + +.PHONY: golangci-lint +golangci-lint: + @$(foreach mod,$(MODULE_DIRS), \ + (cd $(mod) && \ + echo "[lint] golangci-lint: $(mod)" && \ + golangci-lint run --path-prefix $(mod) ./...) &&) true diff --git a/README.md b/README.md new file mode 100644 index 0000000..0e83ee1 --- /dev/null +++ b/README.md @@ -0,0 +1,27 @@ +# zfmt + +[![coverage report](https://gitlab.zgtools.net/devex/archetypes/gomods/zfmt/badges/main/coverage.svg)](https://gitlab.zgtools.net/devex/archetypes/gomods/zfmt/-/commits/main) + +[![pipeline status](https://gitlab.zgtools.net/devex/archetypes/gomods/zfmt/badges/main/pipeline.svg)](https://gitlab.zgtools.net/devex/archetypes/gomods/zfmt/-/commits/main) + + +A go library which contains a number of useful implementations of the Formatter interface, +an interface which contains Marshall and Unmarshall methods. + +## Dependencies + +### Gogen Avro + +Install [Gogen Avro](https://github.com/actgardner/gogen-avro) a utility for generating go code from avro schemas (used for testdata) + +``` +go get github.com/actgardner/gogen-avro/v7/cmd/... +``` + + +### Migration Guide + +1. #### V0 to V1 Migration + + * Enum `ProtoFmt`(`proto`) renamed to `ProtoBase64Fmt`(`proto_base64`). This is intended for SQS use only while the newly introduced `ProtoRawFmt`(`proto_raw`) is intended for most other use cases. + * Enum `ProtoSchemaFmt`(`proto_schema`) renamed to `ProtoSchemaDeprecatedFmt`(`proto_schema_deprecated`). The proto schema is deprecated because it doesn't work properly with the confluent schema registry. Use the `ProtoRawFmt` or `ProtoBase64Fmt` instead. \ No newline at end of file diff --git a/avro.go b/avro.go new file mode 100644 index 0000000..7959297 --- /dev/null +++ b/avro.go @@ -0,0 +1,58 @@ +package zfmt + +import ( + "bytes" + "fmt" + + "github.com/actgardner/gogen-avro/v10/compiler" + "github.com/actgardner/gogen-avro/v10/soe" + "github.com/actgardner/gogen-avro/v10/vm" + "github.com/actgardner/gogen-avro/v10/vm/types" + heetch "github.com/heetch/avro" + heetchtypegen "github.com/heetch/avro/avrotypegen" +) + +// GeneratedAvroRecord combines interfaces that make Encoding/Decoding possible +// for gogen-avro struct +type GeneratedAvroRecord interface { + soe.AvroRecord + types.Field + Schema() string +} + +type AvroFormatter struct{} + +func (p *AvroFormatter) Marshall(v any) ([]byte, error) { + switch m := v.(type) { + case soe.AvroRecord: + buf := &bytes.Buffer{} + err := m.Serialize(buf) + return buf.Bytes(), err + case heetchtypegen.AvroRecord: + b, _, err := heetch.Marshal(v) + return b, err + default: + return nil, fmt.Errorf("%T, avro formatter supports only gogen-avro or heetch avro messages", v) + } +} + +func (p *AvroFormatter) Unmarshal(b []byte, v any) error { + switch m := v.(type) { + case GeneratedAvroRecord: + r := bytes.NewReader(b) + deser, err := compiler.CompileSchemaBytes([]byte(m.Schema()), []byte(m.Schema())) + if err != nil { + return err + } + return vm.Eval(r, deser, m) + case heetchtypegen.AvroRecord: + t, err := heetch.ParseType(m.AvroRecord().Schema) + if err != nil { + return err + } + _, err = heetch.Unmarshal(b, v, t) + return err + default: + return fmt.Errorf("%T, avro formatter supports only gogen-avro or heetch avro messages", v) + } +} diff --git a/formatter.go b/formatter.go new file mode 100644 index 0000000..bbf2ebe --- /dev/null +++ b/formatter.go @@ -0,0 +1,61 @@ +package zfmt + +import ( + err "fmt" +) + +// FormatterType defines a formatter +type FormatterType string + +const ( + // JSONFmt indicates json formatter + JSONFmt FormatterType = "json" + // ProtoRawFmt indicates protocol buffer formatter. Does not Base64 encode/decode. + ProtoRawFmt FormatterType = "proto_raw" + // ProtoBase64Fmt indicates protocol buffer formatter. Base64 encodes/decodes the message. Only intended for use with SQS. + ProtoBase64Fmt FormatterType = "proto_base64" + // ProtoJSONFmt indicates usage protojson formatter (which should be used for json formatted proto messages). + ProtoJSONFmt FormatterType = "proto_json" + // StringFmt indicates string formatter + StringFmt FormatterType = "string" + // AvroFmt indicates apache avro formatter + AvroFmt FormatterType = "avro" + // AvroSchemaFmt indicates apache avro formatter with schemaID encoded + AvroSchemaFmt FormatterType = "avro_schema" + // JSONSchemaFmt indicates json formatter with schemaID encoded + JSONSchemaFmt FormatterType = "json_schema" + // ProtoSchemaDeprecatedFmt indicates proto formatter with schemaID encoded - deprecated because it doesn't work properly. + ProtoSchemaDeprecatedFmt FormatterType = "proto_schema_deprecated" +) + +// Formatter allows the user to extend formatting capability to unsupported data types +type Formatter interface { + Marshall(v any) ([]byte, error) + Unmarshal(data []byte, v any) error +} + +// GetFormatter returns supported formatter from its name +func GetFormatter(fmt FormatterType, schemaID int) (Formatter, error) { + switch fmt { + case StringFmt: + return &StringFormatter{}, nil + case JSONFmt: + return &JSONFormatter{}, nil + case ProtoJSONFmt: + return &ProtoJSONFormatter{}, nil + case ProtoBase64Fmt: + return &ProtobufBase64Formatter{}, nil + case ProtoRawFmt: + return &ProtobufRawFormatter{}, nil + case AvroFmt: + return &AvroFormatter{}, nil + case AvroSchemaFmt: + return &SchematizedAvroFormatter{SchemaID: schemaID}, nil + case JSONSchemaFmt: + return &SchematizedJSONFormatter{SchemaID: schemaID}, nil + case ProtoSchemaDeprecatedFmt: + return &SchematizedProtoFormatterDeprecated{SchemaID: schemaID}, nil + default: + return nil, err.Errorf("unsupported formatter %s", fmt) + } +} diff --git a/formatter_test.go b/formatter_test.go new file mode 100644 index 0000000..d110612 --- /dev/null +++ b/formatter_test.go @@ -0,0 +1,88 @@ +package zfmt + +import ( + "reflect" + "testing" +) + +func TestGetFormatter(t *testing.T) { + type args struct { + fmt FormatterType + schemaID int + } + tests := []struct { + name string + args args + want Formatter + wantErr bool + }{ + { + name: "json", + args: args{ + fmt: "json", + schemaID: 0, + }, + want: &JSONFormatter{}, + }, + { + name: "proto_base64", + args: args{ + fmt: "proto_base64", + schemaID: 0, + }, + want: &ProtobufBase64Formatter{}, + }, + { + name: "proto_raw", + args: args{ + fmt: "proto_raw", + schemaID: 111, + }, + want: &ProtobufRawFormatter{}, + }, + { + name: "proto_json", + args: args{ + fmt: "proto_json", + schemaID: 111, + }, + want: &ProtoJSONFormatter{}, + }, + { + name: "avro", + args: args{ + fmt: "avro", + schemaID: 0, + }, + want: &AvroFormatter{}, + }, + { + name: "json_schema", + args: args{ + fmt: "json_schema", + schemaID: 123, + }, + want: &SchematizedJSONFormatter{SchemaID: 123}, + }, + { + name: "avro_schema", + args: args{ + fmt: "avro_schema", + schemaID: 789, + }, + want: &SchematizedAvroFormatter{SchemaID: 789}, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got, err := GetFormatter(tt.args.fmt, tt.args.schemaID) + if (err != nil) != tt.wantErr { + t.Errorf("GetFormatter() error = %v, wantErr %v", err, tt.wantErr) + return + } + if !reflect.DeepEqual(got, tt.want) { + t.Errorf("GetFormatter() got = %v, want %v", got, tt.want) + } + }) + } +} diff --git a/go.mod b/go.mod new file mode 100644 index 0000000..b1cc74e --- /dev/null +++ b/go.mod @@ -0,0 +1,25 @@ +module gitlab.zgtools.net/devex/archetypes/gomods/zfmt + +go 1.22 + +toolchain go1.22.1 + +require ( + github.com/actgardner/gogen-avro/v10 v10.2.1 + github.com/golang/protobuf v1.5.4 + github.com/google/go-cmp v0.6.0 + github.com/heetch/avro v0.4.5 + github.com/stretchr/testify v1.9.0 + google.golang.org/protobuf v1.34.2 +) + +require ( + github.com/davecgh/go-spew v1.1.1 // indirect + github.com/golang/snappy v0.0.4 // indirect + github.com/google/uuid v1.6.0 // indirect + github.com/kr/pretty v0.3.1 // indirect + github.com/pmezard/go-difflib v1.0.0 // indirect + github.com/rogpeppe/go-internal v1.10.0 // indirect + gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c // indirect + gopkg.in/yaml.v3 v3.0.1 // indirect +) diff --git a/go.sum b/go.sum new file mode 100644 index 0000000..1c53ccb --- /dev/null +++ b/go.sum @@ -0,0 +1,44 @@ +github.com/actgardner/gogen-avro/v10 v10.2.1 h1:z3pOGblRjAJCYpkIJ8CmbMJdksi4rAhaygw0dyXZ930= +github.com/actgardner/gogen-avro/v10 v10.2.1/go.mod h1:QUhjeHPchheYmMDni/Nx7VB0RsT/ee8YIgGY/xpEQgQ= +github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= +github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/frankban/quicktest v1.14.0 h1:+cqqvzZV87b4adx/5ayVOaYZ2CrvM4ejQvUdBzPPUss= +github.com/frankban/quicktest v1.14.0/go.mod h1:NeW+ay9A/U67EYXNFA1nPE8e/tnQv/09mUdL/ijj8og= +github.com/golang/protobuf v1.5.0 h1:LUVKkCeviFUMKqHa4tXIIij/lbhnMbP7Fn5wKdKkRh4= +github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk= +github.com/golang/protobuf v1.5.4 h1:i7eJL8qZTpSEXOPTxNKhASYpMn+8e5Q6AdndVa1dWek= +github.com/golang/protobuf v1.5.4/go.mod h1:lnTiLA8Wa4RWRcIUkrtSVa5nRhsEGBg48fD6rSs7xps= +github.com/golang/snappy v0.0.4 h1:yAGX7huGHXlcLOEtBnF4w7FQwA26wojNCwOYAEhLjQM= +github.com/golang/snappy v0.0.4/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= +github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI= +github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= +github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= +github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/heetch/avro v0.4.5 h1:BSnj4wEeUG1IjMTm9/tBwQnV3euuIVa1mRWHnm1t8VU= +github.com/heetch/avro v0.4.5/go.mod h1:gxf9GnbjTXmWmqxhdNbAMcZCjpye7RV5r9t3Q0dL6ws= +github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= +github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= +github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= +github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= +github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= +github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= +github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= +github.com/pkg/diff v0.0.0-20210226163009-20ebb0f2a09e/go.mod h1:pJLUxLENpZxwdsKMEsNbx1VGcRFpLqf3715MtcvvzbA= +github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/rogpeppe/go-internal v1.9.0/go.mod h1:WtVeX8xhTBvf0smdhujwtBcq4Qrzq/fJaraNFVN+nFs= +github.com/rogpeppe/go-internal v1.10.0 h1:TMyTOH3F/DB16zRVcYyreMH6GnZZrwQVAoYjRBZyWFQ= +github.com/rogpeppe/go-internal v1.10.0/go.mod h1:UQnix2H7Ngw/k4C5ijL5+65zddjncjaFoBhdsK/akog= +github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg= +github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= +golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= +google.golang.org/protobuf v1.34.2 h1:6xV6lTsCfpGD21XK49h7MhtcApnLqkfYgPcdHftf6hg= +google.golang.org/protobuf v1.34.2/go.mod h1:qYOHts0dSfpeUzUFpOMr/WGzszTmLH+DiWniOlNbLDw= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= +gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/json.go b/json.go new file mode 100644 index 0000000..e588ed2 --- /dev/null +++ b/json.go @@ -0,0 +1,14 @@ +package zfmt + +import "encoding/json" + +// JSONFormatter encodes/decodes go struct to json format +type JSONFormatter struct{} + +func (j *JSONFormatter) Marshall(v any) ([]byte, error) { + return json.Marshal(v) +} + +func (j *JSONFormatter) Unmarshal(b []byte, v any) error { + return json.Unmarshal(b, v) +} diff --git a/protobase64.go b/protobase64.go new file mode 100644 index 0000000..7309867 --- /dev/null +++ b/protobase64.go @@ -0,0 +1,59 @@ +package zfmt + +import ( + "encoding/base64" + "fmt" + + //nolint:staticcheck // Older zillow libs have generated code which uses this deprecated package. To maintain backwards compatability with them, the older proto serialization lib should be maintained + v1proto "github.com/golang/protobuf/proto" + v2proto "google.golang.org/protobuf/proto" +) + +// ProtobufBase64Formatter implements formatter interface for both protobuf v1 and v2 messages. Intended for use with SQS +type ProtobufBase64Formatter struct{} + +// Marshall as proto and then base64 encode (useful for technologies like SQS which limit the character set) +func (p *ProtobufBase64Formatter) Marshall(v any) ([]byte, error) { + switch m := v.(type) { + case v1proto.Message: + b, err := v1proto.Marshal(m) + if err != nil { + return nil, err + } + return []byte(base64.StdEncoding.EncodeToString(b)), nil + case v2proto.Message: + b, err := v2proto.Marshal(m) + if err != nil { + return nil, err + } + return []byte(base64.StdEncoding.EncodeToString(b)), nil + default: + return nil, fmt.Errorf("%T, proto base64 formatter can only be used with proto messages", v) + } +} + +// Unmarshal with base64 decoding +func (p *ProtobufBase64Formatter) Unmarshal(b []byte, v any) error { + switch m := v.(type) { + case v1proto.Message: + raw, err := base64.StdEncoding.DecodeString(string(b)) + if err != nil { + return err + } + if err := v1proto.Unmarshal(raw, m); err != nil { + return err + } + return nil + case v2proto.Message: + raw, err := base64.StdEncoding.DecodeString(string(b)) + if err != nil { + return err + } + if err = v2proto.Unmarshal(raw, m); err != nil { + return err + } + return nil + default: + return fmt.Errorf("%T, proto base64 formatter can only be used with proto messages", v) + } +} diff --git a/protobase64_test.go b/protobase64_test.go new file mode 100644 index 0000000..a9679e6 --- /dev/null +++ b/protobase64_test.go @@ -0,0 +1,43 @@ +package zfmt + +import ( + "testing" + + "gitlab.zgtools.net/devex/archetypes/gomods/zfmt/testdata/example" +) + +func TestProtoBase64Formatter_MarshallUnmarshall(t *testing.T) { + ein := example.ExampleDef{ + Allowed: "happy", + Disallowed: 2, + Example: &example.ExampleDef_Foo{ + Foo: &example.Foo{ + Name: "sad", + }, + }, + } + + fmtr := ProtobufBase64Formatter{} + + b, err := fmtr.Marshall(&ein) + if err != nil { + t.Fatal(err) + } + + eout := &example.ExampleDef{} + err = fmtr.Unmarshal(b, eout) + if err != nil { + t.Fatal(err) + } + if ein.Allowed != eout.Allowed { + t.Error("Not allowed") + } + + if e, ok := eout.Example.(*example.ExampleDef_Foo); ok { + if e.Foo.Name != "sad" { + t.Error("foo sad") + } + } else { + t.Error("example not foo") + } +} diff --git a/protojson.go b/protojson.go new file mode 100644 index 0000000..d36f638 --- /dev/null +++ b/protojson.go @@ -0,0 +1,29 @@ +package zfmt + +import ( + "fmt" + + "google.golang.org/protobuf/encoding/protojson" + "google.golang.org/protobuf/proto" +) + +// ProtoJSONFormatter encodes/decodes proto go struct to json format +type ProtoJSONFormatter struct{} + +func (j *ProtoJSONFormatter) Marshall(v any) ([]byte, error) { + + if m, ok := v.(proto.Message); ok { + return protojson.Marshal(m) + } + + return nil, fmt.Errorf("%T, protojson formatter can only be used with proto messages", v) +} + +func (j *ProtoJSONFormatter) Unmarshal(b []byte, v any) error { + if m, ok := v.(proto.Message); ok { + unmarshaller := protojson.UnmarshalOptions{DiscardUnknown: true} + return unmarshaller.Unmarshal(b, m) + } + + return fmt.Errorf("%T, protojson formatter can only be used with proto messages", v) +} diff --git a/protojson_test.go b/protojson_test.go new file mode 100644 index 0000000..4a21ded --- /dev/null +++ b/protojson_test.go @@ -0,0 +1,55 @@ +package zfmt + +import ( + "testing" + + "github.com/stretchr/testify/require" + "gitlab.zgtools.net/devex/archetypes/gomods/zfmt/testdata/example" +) + +func TestProtoJSONFormatter_MarshallUnmarshall(t *testing.T) { + ein := example.ExampleDef{ + Allowed: "happy", + Disallowed: 2, + Example: &example.ExampleDef_Foo{ + Foo: &example.Foo{ + Name: "sad", + }, + }, + } + + fmtr := ProtoJSONFormatter{} + + b, err := fmtr.Marshall(&ein) + if err != nil { + t.Fatal(err) + } + + eout := &example.ExampleDef{} + err = fmtr.Unmarshal(b, eout) + if err != nil { + t.Fatal(err) + } + if ein.Allowed != eout.Allowed { + t.Error("Not allowed") + } + + if e, ok := eout.Example.(*example.ExampleDef_Foo); ok { + if e.Foo.Name != "sad" { + t.Error("foo sad") + } + } else { + t.Error("example not foo") + } +} + +func TestProtoJSONFormatter_UnmarshallWithUnknown(t *testing.T) { + data := "{\n \"allowed\": \"happy\",\n \"disallowed\": 2,\n \"MyName\": \"Stewart\"\n}" + + fmtr := ProtoJSONFormatter{} + eout := &example.ExampleDef{} + err := fmtr.Unmarshal([]byte(data), eout) + require.NoError(t, err) + require.Equal(t, "happy", eout.Allowed) + require.Equal(t, int32(2), eout.Disallowed) +} diff --git a/protoraw.go b/protoraw.go new file mode 100644 index 0000000..15e5ecb --- /dev/null +++ b/protoraw.go @@ -0,0 +1,50 @@ +package zfmt + +import ( + "fmt" + + //nolint:staticcheck // Older zillow libs have generated code which uses this deprecated package. To maintain backwards compatability with them, the older proto serialization lib should be maintained + v1proto "github.com/golang/protobuf/proto" + v2proto "google.golang.org/protobuf/proto" +) + +// ProtobufRawFormatter implements formatter interface for both protobuf v1 and v2 messages. Does not base64 encode. +type ProtobufRawFormatter struct{} + +// Marshall encodes the data as a proto binary +func (p *ProtobufRawFormatter) Marshall(v any) ([]byte, error) { + switch m := v.(type) { + case v1proto.Message: + b, err := v1proto.Marshal(m) + if err != nil { + return nil, err + } + return b, nil + case v2proto.Message: + b, err := v2proto.Marshal(m) + if err != nil { + return nil, err + } + return b, nil + default: + return nil, fmt.Errorf("%T, protoraw formatter can only be used with proto messages", v) + } +} + +// Unmarshal accepts proto binary and hydrates a proto generated struct +func (p *ProtobufRawFormatter) Unmarshal(b []byte, v any) error { + switch m := v.(type) { + case v1proto.Message: + if err := v1proto.Unmarshal(b, m); err != nil { + return err + } + return nil + case v2proto.Message: + if err := v2proto.Unmarshal(b, m); err != nil { + return err + } + return nil + default: + return fmt.Errorf("%T, protoraw formatter can only be used with proto messages", v) + } +} diff --git a/protoraw_test.go b/protoraw_test.go new file mode 100644 index 0000000..93fddfa --- /dev/null +++ b/protoraw_test.go @@ -0,0 +1,43 @@ +package zfmt + +import ( + "testing" + + "gitlab.zgtools.net/devex/archetypes/gomods/zfmt/testdata/example" +) + +func TestProtoRawFormatter_MarshallUnmarshall(t *testing.T) { + ein := example.ExampleDef{ + Allowed: "happy", + Disallowed: 2, + Example: &example.ExampleDef_Foo{ + Foo: &example.Foo{ + Name: "sad", + }, + }, + } + + fmtr := ProtobufRawFormatter{} + + b, err := fmtr.Marshall(&ein) + if err != nil { + t.Fatal(err) + } + + eout := &example.ExampleDef{} + err = fmtr.Unmarshal(b, eout) + if err != nil { + t.Fatal(err) + } + if ein.Allowed != eout.Allowed { + t.Error("Not allowed") + } + + if e, ok := eout.Example.(*example.ExampleDef_Foo); ok { + if e.Foo.Name != "sad" { + t.Error("foo sad") + } + } else { + t.Error("example not foo") + } +} diff --git a/schematizedavro.go b/schematizedavro.go new file mode 100644 index 0000000..f649601 --- /dev/null +++ b/schematizedavro.go @@ -0,0 +1,60 @@ +package zfmt + +import ( + "bytes" + "encoding/binary" + "errors" + "fmt" +) + +// SchematizedAvroFormatter follows the Confluent Wire Format https://docs.confluent.io/current/schema-registry/serdes-develop/index.html#wire-format +type SchematizedAvroFormatter struct { + formatter AvroFormatter + SchemaID int +} + +// Marshall converts input into avro binary data with schema ID attached +func (p *SchematizedAvroFormatter) Marshall(v any) ([]byte, error) { + return marshall(&p.formatter, p.SchemaID, v) +} + +// Unmarshal fills avro binary data into provided interface v and validates the schema ID +func (p *SchematizedAvroFormatter) Unmarshal(b []byte, v any) error { + return unmarshal(&p.formatter, p.SchemaID, b, v) +} + +// marshall converts input into binary data with schema ID also encoded via the wire format +func marshall(fmtter Formatter, schemaID int, v any) ([]byte, error) { + data, err := fmtter.Marshall(v) + if err != nil { + return nil, err + } + var body bytes.Buffer + // version, always 0 + body.WriteByte(0) + + // 4 byte for schema ID in BigEndian + schemaIDByte := make([]byte, 4) + binary.BigEndian.PutUint32(schemaIDByte, uint32(schemaID)) + body.Write(schemaIDByte) + + // the content + body.Write(data) + return body.Bytes(), nil +} + +// Unmarshal fills binary data into provided interface v and validates the schema ID +func unmarshal(fmtter Formatter, schemaID int, b []byte, v any) error { + if len(b) < 5 { + return errors.New("message does not contain schema") + } + schemaIDBin := b[1:5] + id := int(binary.BigEndian.Uint32(schemaIDBin)) + // for default schema (id == 0), it is implied that the user does not care about ID and attempt to unmarshal at their own risk + // This often happens when the topic is guaranteed to have one data type and the user would like + // to bypass schema validation while still conforming to the confluent wire format. + if schemaID != 0 && schemaID != id { + return fmt.Errorf("schema IDs do not match, expect %d, got %d", schemaID, id) + } + return fmtter.Unmarshal(b[5:], v) +} diff --git a/schematizedavro_test.go b/schematizedavro_test.go new file mode 100644 index 0000000..773ae59 --- /dev/null +++ b/schematizedavro_test.go @@ -0,0 +1,400 @@ +package zfmt + +import ( + "bytes" + "reflect" + "testing" + + "gitlab.zgtools.net/devex/archetypes/gomods/zfmt/testdata/heetch" + + av "gitlab.zgtools.net/devex/archetypes/gomods/zfmt/testdata" +) + +func TestSchematizedAvroFormatter_Marshall(t *testing.T) { + type fields struct { + avroFmt AvroFormatter + SchemaID int + } + type args struct { + v any + } + + tests := []struct { + name string + fields fields + args args + wantErr bool + }{ + { + name: "accept reference type of an avro object, default formatter should be useful", + fields: fields{}, + args: args{ + v: &av.DemoSchema{ + IntField: 123, + DoubleField: 123.4, + StringField: "12345", + BoolField: true, + BytesField: []byte("123456"), + }, + }, + wantErr: false, + }, + { + name: "accept reference type of an avro object, with avro formatter and schemaID", + fields: fields{ + avroFmt: AvroFormatter{}, + SchemaID: 99, + }, + args: args{ + v: &av.DemoSchema{ + IntField: 123, + DoubleField: 123.4, + StringField: "12345", + BoolField: true, + BytesField: []byte("123456"), + }, + }, + wantErr: false, + }, + { + name: "accept heetch avrorecord type", + fields: fields{ + avroFmt: AvroFormatter{}, + SchemaID: 99, + }, + args: args{ + v: heetch.DemoSchema{ + IntField: 123, + DoubleField: 123.4, + StringField: "12345", + BoolField: true, + BytesField: []byte("123456"), + }, + }, + wantErr: false, + }, + { + name: "accept value type of an avro object", + fields: fields{ + avroFmt: AvroFormatter{}, + SchemaID: 99, + }, + args: args{ + v: av.DemoSchema{ + IntField: 123, + DoubleField: 123.4, + StringField: "12345", + BoolField: true, + BytesField: []byte("123456"), + }, + }, + wantErr: false, + }, + { + name: "do not accept random type", + fields: fields{ + avroFmt: AvroFormatter{}, + SchemaID: 99, + }, + args: args{v: "what?"}, + wantErr: true, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + p := &SchematizedAvroFormatter{ + formatter: tt.fields.avroFmt, + SchemaID: tt.fields.SchemaID, + } + _, err := p.Marshall(tt.args.v) + if (err != nil) != tt.wantErr { + t.Errorf("SchematizedAvroFormatter.Marshall() error = %v, wantErr %v", err, tt.wantErr) + return + } + }) + } +} + +func TestSchematizedAvroFormatter_UnmarshalToInvalidAvroType(t *testing.T) { + fmtter := &SchematizedAvroFormatter{SchemaID: 99} + input := &av.DemoSchema{ + IntField: 123, + DoubleField: 123.4, + StringField: "12345", + BoolField: true, + BytesField: []byte("123456"), + } + data, err := fmtter.Marshall(input) + if err != nil { + t.Errorf("should not have error marshalling %v", err) + } + var output bytes.Buffer + err = fmtter.Unmarshal(data, &output) + if err == nil { + t.Errorf("should get error because output type is not an avro object") + } +} + +func TestSchematizedAvroFormatter_UnmarshalNonSchematizedAvro(t *testing.T) { + binAvro := []byte{} + var output bytes.Buffer + outFmtter := &SchematizedAvroFormatter{} + err := outFmtter.Unmarshal(binAvro, &output) + if err == nil { + t.Errorf("should get error because input does not contain schema") + } +} + +func TestSchematizedAvroFormatter_UnmarshalValidAvroWithSchemaID(t *testing.T) { + type testCase struct { + Name string + Input any + Expected any + Output any + } + + testCases := []testCase{ + { + Name: "gogen-avro with with schemaID", + Input: &av.DemoSchema{ + IntField: 123, + DoubleField: 123.4, + StringField: "12345", + BoolField: true, + BytesField: []byte("123456"), + }, + Expected: &av.DemoSchema{ + IntField: 123, + DoubleField: 123.4, + StringField: "12345", + BoolField: true, + BytesField: []byte("123456"), + }, + Output: &av.DemoSchema{}, + }, + { + Name: "heetch avro with with schemaID", + Input: heetch.DemoSchema{ + IntField: 123, + DoubleField: 123.4, + StringField: "12345", + BoolField: true, + BytesField: []byte("123456"), + }, + Expected: &heetch.DemoSchema{ + IntField: 123, + DoubleField: 123.4, + StringField: "12345", + BoolField: true, + BytesField: []byte("123456"), + }, + Output: &heetch.DemoSchema{}, + }, + } + + for _, tc := range testCases { + t.Run(tc.Name, func(t *testing.T) { + fmtter := &SchematizedAvroFormatter{SchemaID: 99} + input := tc.Input + data, err := fmtter.Marshall(input) + if err != nil { + t.Errorf("should not have error marshalling %v", err) + } + output := tc.Output + err = fmtter.Unmarshal(data, output) + if err != nil { + t.Errorf("should not have error marshalling %v", err) + } + if !reflect.DeepEqual(tc.Expected, output) { + t.Errorf("data should match, want=%v, got=%v", input, output) + } + }) + } +} + +func TestSchematizedAvroFormatter_UnmarshalValidAvroWithNoSchemaID(t *testing.T) { + type testCase struct { + Name string + Input any + Expected any + Output any + } + + testCases := []testCase{ + { + Name: "gogen-avro with no schemaID", + Input: &av.DemoSchema{ + IntField: 123, + DoubleField: 123.4, + StringField: "12345", + BoolField: true, + BytesField: []byte("123456"), + }, + Expected: &av.DemoSchema{ + IntField: 123, + DoubleField: 123.4, + StringField: "12345", + BoolField: true, + BytesField: []byte("123456"), + }, + Output: &av.DemoSchema{}, + }, + { + Name: "heetch avro with no schemaID", + Input: heetch.DemoSchema{ + IntField: 123, + DoubleField: 123.4, + StringField: "12345", + BoolField: true, + BytesField: []byte("123456"), + }, + Expected: &heetch.DemoSchema{ + IntField: 123, + DoubleField: 123.4, + StringField: "12345", + BoolField: true, + BytesField: []byte("123456"), + }, + Output: &heetch.DemoSchema{}, + }, + } + + for _, tc := range testCases { + t.Run(tc.Name, func(t *testing.T) { + inFmtter := &SchematizedAvroFormatter{SchemaID: 99} + input := tc.Input + data, err := inFmtter.Marshall(input) + if err != nil { + t.Errorf("should not have error marshalling %v", err) + } + + // when schemaID is unset, as long as the data is unmarshallable, we don't throw error + outFmtter := &SchematizedAvroFormatter{} + output := tc.Output + err = outFmtter.Unmarshal(data, output) + if err != nil { + t.Errorf("should not have error marshalling %v", err) + } + if !reflect.DeepEqual(tc.Expected, output) { + t.Errorf("data should match, want=%v, got=%v", input, output) + } + }) + } + +} + +func TestSchematizedAvroFormatter_UnmarshalValidAvroWithWrongSchemaID(t *testing.T) { + type testCase struct { + Name string + Input any + Output any + } + + testCases := []testCase{ + { + Name: "gogen-avro with wrong schemaID", + Input: &av.DemoSchema{ + IntField: 123, + DoubleField: 123.4, + StringField: "12345", + BoolField: true, + BytesField: []byte("123456"), + }, + Output: &av.DemoSchema{}, + }, + { + Name: "heetch avro with wrong schemaID", + Input: heetch.DemoSchema{ + IntField: 123, + DoubleField: 123.4, + StringField: "12345", + BoolField: true, + BytesField: []byte("123456"), + }, + Output: &heetch.DemoSchema{}, + }, + } + + for _, tc := range testCases { + t.Run(tc.Name, func(t *testing.T) { + inFmtter := &SchematizedAvroFormatter{SchemaID: 99} + data, err := inFmtter.Marshall(tc.Input) + if err != nil { + t.Errorf("should not have error marshalling %v", err) + } + + outFmtter := &SchematizedAvroFormatter{SchemaID: 100} + err = outFmtter.Unmarshal(data, tc.Output) + if err == nil { + t.Error("should have error unmarshalling due to incorrect schema ID") + } + }) + } +} + +func TestSchematizedAvroFormatter_Equivalency(t *testing.T) { + type testCase struct { + Name string + Input any + Expected any + Output any + } + + testCases := []testCase{ + { + Name: "marshal gogen-avro unmarshal heetch avro", + Input: &av.DemoSchema{ + IntField: 123, + DoubleField: 123.4, + StringField: "12345", + BoolField: true, + BytesField: []byte("123456"), + }, + Expected: &heetch.DemoSchema{ + IntField: 123, + DoubleField: 123.4, + StringField: "12345", + BoolField: true, + BytesField: []byte("123456"), + }, + Output: &heetch.DemoSchema{}, + }, + { + Name: "marshall heetch unmarshall gogen-avro", + Input: heetch.DemoSchema{ + IntField: 123, + DoubleField: 123.4, + StringField: "12345", + BoolField: true, + BytesField: []byte("123456"), + }, + Expected: &av.DemoSchema{ + IntField: 123, + DoubleField: 123.4, + StringField: "12345", + BoolField: true, + BytesField: []byte("123456"), + }, + Output: &av.DemoSchema{}, + }, + } + + for _, tc := range testCases { + t.Run(tc.Name, func(t *testing.T) { + fmtter := &SchematizedAvroFormatter{SchemaID: 99} + input := tc.Input + data, err := fmtter.Marshall(input) + if err != nil { + t.Errorf("should not have error marshalling %v", err) + } + output := tc.Output + err = fmtter.Unmarshal(data, output) + if err != nil { + t.Errorf("should not have error marshalling %v", err) + } + if !reflect.DeepEqual(tc.Expected, output) { + t.Errorf("data should match, want=%v, got=%v", input, output) + } + }) + } +} diff --git a/schematizedjson.go b/schematizedjson.go new file mode 100644 index 0000000..ffc8c2e --- /dev/null +++ b/schematizedjson.go @@ -0,0 +1,17 @@ +package zfmt + +// SchematizedJSONFormatter follows the Confluent Wire Format https://docs.confluent.io/current/schema-registry/serdes-develop/index.html#wire-format +type SchematizedJSONFormatter struct { + formatter JSONFormatter + SchemaID int +} + +// Marshall converts input into avro binary data with schema ID attached +func (p *SchematizedJSONFormatter) Marshall(v any) ([]byte, error) { + return marshall(&p.formatter, p.SchemaID, v) +} + +// Unmarshal fills avro binary data into provided interface v and validates the schema ID +func (p *SchematizedJSONFormatter) Unmarshal(b []byte, v any) error { + return unmarshal(&p.formatter, p.SchemaID, b, v) +} diff --git a/schematizedjson_test.go b/schematizedjson_test.go new file mode 100644 index 0000000..4be10d0 --- /dev/null +++ b/schematizedjson_test.go @@ -0,0 +1,114 @@ +package zfmt + +import ( + "testing" + + "github.com/google/go-cmp/cmp" + + av "gitlab.zgtools.net/devex/archetypes/gomods/zfmt/testdata" +) + +func TestSchematizedJsonFormatter_Marshall(t *testing.T) { + type fields struct { + protobufFormatter JSONFormatter + SchemaID int + } + type args struct { + v any + } + tests := []struct { + name string + fields fields + args args + wantErr bool + }{ + { + name: "accept reference type of an object, default formatter should be useful", + fields: fields{}, + args: args{ + v: &av.ExampleJson{ + Id: "hello", + }, + }, + wantErr: false, + }, + { + name: "accept reference type of an object, with avro formatter and schemaID", + fields: fields{ + protobufFormatter: JSONFormatter{}, + SchemaID: 99, + }, + args: args{ + v: &av.ExampleJson{ + Id: "hello", + }, + }, + wantErr: false, + }, + { + name: "accept value type of an object, with avro formatter and schemaID", + fields: fields{ + protobufFormatter: JSONFormatter{}, + SchemaID: 99, + }, + args: args{ + v: av.ExampleJson{ + Id: "hello", + }, + }, + wantErr: false, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + p := &SchematizedJSONFormatter{ + formatter: tt.fields.protobufFormatter, + SchemaID: tt.fields.SchemaID, + } + _, err := p.Marshall(tt.args.v) + if (err != nil) != tt.wantErr { + t.Errorf("SchematizedProtoFormatter.Marshall() error = %v, wantErr %v", err, tt.wantErr) + return + } + }) + } +} + +func TestSchematizedJsonFormmater_Unmarshall_PointerType(t *testing.T) { + fmtter := SchematizedJSONFormatter{ + formatter: JSONFormatter{}, + SchemaID: 123, + } + expected := av.ExampleJson{Id: "what"} + b, err := fmtter.Marshall(&expected) + if err != nil { + t.Fatal(err) + } + reslt := av.ExampleJson{} + if err := fmtter.Unmarshal(b, &reslt); err != nil { + t.Fatal(err) + } + diff := cmp.Diff(&reslt, &expected) + if diff != "" { + t.Fatal(diff) + } +} +func TestSchematizedJsonFormmater_Unmarshalll_ValueType(t *testing.T) { + fmtter := SchematizedJSONFormatter{ + formatter: JSONFormatter{}, + SchemaID: 123, + } + expected := av.ExampleJson{Id: "what"} + b, err := fmtter.Marshall(expected) + if err != nil { + t.Fatal(err) + } + reslt := av.ExampleJson{} + if err := fmtter.Unmarshal(b, &reslt); err != nil { + t.Fatal(err) + } + diff := cmp.Diff(&reslt, &expected) + if diff != "" { + t.Fatal(diff) + } +} diff --git a/schematizedproto_deprecated.go b/schematizedproto_deprecated.go new file mode 100644 index 0000000..62d5b73 --- /dev/null +++ b/schematizedproto_deprecated.go @@ -0,0 +1,17 @@ +package zfmt + +// SchematizedProtoFormatterDeprecated follows the Confluent Wire Format https://docs.confluent.io/current/schema-registry/serdes-develop/index.html#wire-format +type SchematizedProtoFormatterDeprecated struct { + formatter ProtobufBase64Formatter + SchemaID int +} + +// Marshall converts input into avro binary data with schema ID attached +func (p *SchematizedProtoFormatterDeprecated) Marshall(v any) ([]byte, error) { + return marshall(&p.formatter, p.SchemaID, v) +} + +// Unmarshal fills avro binary data into provided interface v and validates the schema ID +func (p *SchematizedProtoFormatterDeprecated) Unmarshal(b []byte, v any) error { + return unmarshal(&p.formatter, p.SchemaID, b, v) +} diff --git a/schematizedproto_deprecated_test.go b/schematizedproto_deprecated_test.go new file mode 100644 index 0000000..aa17979 --- /dev/null +++ b/schematizedproto_deprecated_test.go @@ -0,0 +1,107 @@ +package zfmt + +import ( + "testing" + + "github.com/google/go-cmp/cmp" + "github.com/google/go-cmp/cmp/cmpopts" + example2 "gitlab.zgtools.net/devex/archetypes/gomods/zfmt/testdata/example" +) + +func TestSchematizedProtoDeprecatedFormatter_Marshall(t *testing.T) { + type fields struct { + protobufFormatter ProtobufBase64Formatter + SchemaID int + } + type args struct { + v any + } + tests := []struct { + name string + fields fields + args args + wantErr bool + }{ + { + name: "accept reference type of an proto object, default formatter should be useful", + fields: fields{}, + args: args{ + v: &example2.ExampleDef{ + Allowed: "!23", + Disallowed: 7, + }, + }, + wantErr: false, + }, + { + name: "accept reference type of an proto object, with avro formatter and schemaID", + fields: fields{ + protobufFormatter: ProtobufBase64Formatter{}, + SchemaID: 99, + }, + args: args{ + v: &example2.ExampleDef{ + Allowed: "!23", + Disallowed: 7, + }, + }, + wantErr: false, + }, + { + name: "do not accept value type of a proto object", + fields: fields{ + protobufFormatter: ProtobufBase64Formatter{}, + SchemaID: 99, + }, + args: args{ + v: example2.ExampleDef{ + Allowed: "!23", + Disallowed: 7, + }, + }, + wantErr: true, + }, + { + name: "do not accept random type", + fields: fields{ + protobufFormatter: ProtobufBase64Formatter{}, + SchemaID: 99, + }, + args: args{v: "what?"}, + wantErr: true, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + p := &SchematizedProtoFormatterDeprecated{ + formatter: tt.fields.protobufFormatter, + SchemaID: tt.fields.SchemaID, + } + _, err := p.Marshall(tt.args.v) + if (err != nil) != tt.wantErr { + t.Errorf("SchematizedProtoFormatter.Marshall() error = %v, wantErr %v", err, tt.wantErr) + return + } + }) + } +} + +func TestSchematizedProtoDeprecatedFormmater_Unmarshall(t *testing.T) { + fmtter := SchematizedProtoFormatterDeprecated{ + formatter: ProtobufBase64Formatter{}, + SchemaID: 123, + } + expected := example2.ExampleDef{Allowed: "1243", Disallowed: 987} + b, err := fmtter.Marshall(&expected) + if err != nil { + t.Fatal(err) + } + reslt := example2.ExampleDef{} + if err := fmtter.Unmarshal(b, &reslt); err != nil { + t.Fatal(err) + } + diff := cmp.Diff(&reslt, &expected, cmpopts.IgnoreUnexported(example2.ExampleDef{})) + if diff != "" { + t.Fatal(diff) + } +} diff --git a/string.go b/string.go new file mode 100644 index 0000000..6a3c4fa --- /dev/null +++ b/string.go @@ -0,0 +1,34 @@ +package zfmt + +import ( + "fmt" + "io" +) + +// StringFormatter ... +type StringFormatter struct{} + +// Marshall ... +func (f *StringFormatter) Marshall(i any) ([]byte, error) { + switch v := i.(type) { + case string: + return []byte(v), nil + case []byte: + return v, nil + case io.Reader: + return io.ReadAll(v) + default: + return nil, fmt.Errorf("cannot convert to string %T", i) + } +} + +func (f *StringFormatter) Unmarshal(b []byte, i any) error { + // since string is immutable, we rely on io.Writer + switch v := i.(type) { + case io.Writer: + _, err := v.Write(b) + return err + default: + return fmt.Errorf("cannot convert from binary %T", i) + } +} diff --git a/string_test.go b/string_test.go new file mode 100644 index 0000000..3a90548 --- /dev/null +++ b/string_test.go @@ -0,0 +1,102 @@ +package zfmt + +import ( + "bytes" + "reflect" + "testing" +) + +func TestStringFormatter_Marshall(t *testing.T) { + type args struct { + i any + } + tests := []struct { + name string + f *StringFormatter + args args + want []byte + wantErr bool + }{ + { + name: "string", + f: &StringFormatter{}, + args: args{i: "test"}, + want: []byte("test"), + wantErr: false, + }, + { + name: "byte array", + f: &StringFormatter{}, + args: args{i: []byte("test")}, + want: []byte("test"), + wantErr: false, + }, + { + name: "bytes buffer", + f: &StringFormatter{}, + args: args{i: bytes.NewBufferString("test")}, + want: []byte("test"), + wantErr: false, + }, + { + name: "invalid type", + f: &StringFormatter{}, + args: args{i: 123}, + want: nil, + wantErr: true, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + f := &StringFormatter{} + got, err := f.Marshall(tt.args.i) + if (err != nil) != tt.wantErr { + t.Errorf("StringFormatter.Marshall() error = %v, wantErr %v", err, tt.wantErr) + return + } + if !reflect.DeepEqual(got, tt.want) { + t.Errorf("StringFormatter.Marshall() = %v, want %v", got, tt.want) + } + }) + } +} + +func TestStringFormatter_Unmarshal(t *testing.T) { + type args struct { + b []byte + i any + } + tests := []struct { + name string + f *StringFormatter + args args + wantErr bool + }{ + { + name: "string is immutable so that doesn't work", + f: &StringFormatter{}, + args: args{ + b: []byte("test"), + i: func(str string) *string { return &str }(""), + }, + wantErr: true, + }, + { + name: "supply io.Writer", + f: &StringFormatter{}, + args: args{ + b: []byte("test"), + i: new(bytes.Buffer), + }, + wantErr: false, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + f := &StringFormatter{} + if err := f.Unmarshal(tt.args.b, tt.args.i); (err != nil) != tt.wantErr { + t.Errorf("StringFormatter.Unmarshal() error = %v, wantErr %v", err, tt.wantErr) + } + }) + } +} diff --git a/testdata/bytes.go b/testdata/bytes.go new file mode 100644 index 0000000..9b79da9 --- /dev/null +++ b/testdata/bytes.go @@ -0,0 +1,90 @@ +// Code generated by github.com/actgardner/gogen-avro/v10. DO NOT EDIT. +/* + * SOURCE: + * example.avsc + */ +package avro + +import ( + "encoding/json" + + "github.com/actgardner/gogen-avro/v10/util" + "github.com/actgardner/gogen-avro/v10/vm/types" +) + +type Bytes []byte + +func (b *Bytes) UnmarshalJSON(data []byte) error { + var s string + if err := json.Unmarshal(data, &s); err != nil { + return err + } + *b = util.DecodeByteString(s) + return nil +} + +func (b Bytes) MarshalJSON() ([]byte, error) { + return []byte(util.EncodeByteString(b)), nil +} + +type BytesWrapper struct { + Target *Bytes +} + +func (b BytesWrapper) SetBoolean(v bool) { + panic("Unable to assign bytes to bytes field") +} + +func (b BytesWrapper) SetInt(v int32) { + panic("Unable to assign int to bytes field") +} + +func (b BytesWrapper) SetLong(v int64) { + panic("Unable to assign long to bytes field") +} + +func (b BytesWrapper) SetFloat(v float32) { + panic("Unable to assign float to bytes field") +} + +func (b BytesWrapper) SetDouble(v float64) { + panic("Unable to assign double to bytes field") +} + +func (b BytesWrapper) SetUnionElem(v int64) { + panic("Unable to assign union elem to bytes field") +} + +func (b BytesWrapper) SetBytes(v []byte) { + *(b.Target) = v +} + +func (b BytesWrapper) SetString(v string) { + *(b.Target) = []byte(v) +} + +func (b BytesWrapper) Get(i int) types.Field { + panic("Unable to get field from bytes field") +} + +func (b BytesWrapper) SetDefault(i int) { + panic("Unable to set default on bytes field") +} + +func (b BytesWrapper) AppendMap(key string) types.Field { + panic("Unable to append map key to from bytes field") +} + +func (b BytesWrapper) AppendArray() types.Field { + panic("Unable to append array element to from bytes field") +} + +func (b BytesWrapper) NullField(int) { + panic("Unable to null field in bytes field") +} + +func (b BytesWrapper) HintSize(int) { + panic("Unable to hint size in bytes field") +} + +func (b BytesWrapper) Finalize() {} diff --git a/testdata/demo_schema.go b/testdata/demo_schema.go new file mode 100644 index 0000000..1ee60c5 --- /dev/null +++ b/testdata/demo_schema.go @@ -0,0 +1,264 @@ +// Code generated by github.com/actgardner/gogen-avro/v10. DO NOT EDIT. +/* + * SOURCE: + * example.avsc + */ +package avro + +import ( + "encoding/json" + "fmt" + "io" + + "github.com/actgardner/gogen-avro/v10/compiler" + "github.com/actgardner/gogen-avro/v10/vm" + "github.com/actgardner/gogen-avro/v10/vm/types" +) + +var _ = fmt.Printf + +type DemoSchema struct { + IntField int32 `json:"IntField"` + + DoubleField float64 `json:"DoubleField"` + + StringField string `json:"StringField"` + + BoolField bool `json:"BoolField"` + + BytesField Bytes `json:"BytesField"` +} + +const DemoSchemaAvroCRC64Fingerprint = "\xc4V\xa9\x04ʛf\xad" + +func NewDemoSchema() DemoSchema { + r := DemoSchema{} + return r +} + +func DeserializeDemoSchema(r io.Reader) (DemoSchema, error) { + t := NewDemoSchema() + deser, err := compiler.CompileSchemaBytes([]byte(t.Schema()), []byte(t.Schema())) + if err != nil { + return t, err + } + + err = vm.Eval(r, deser, &t) + return t, err +} + +func DeserializeDemoSchemaFromSchema(r io.Reader, schema string) (DemoSchema, error) { + t := NewDemoSchema() + + deser, err := compiler.CompileSchemaBytes([]byte(schema), []byte(t.Schema())) + if err != nil { + return t, err + } + + err = vm.Eval(r, deser, &t) + return t, err +} + +func writeDemoSchema(r DemoSchema, w io.Writer) error { + var err error + err = vm.WriteInt(r.IntField, w) + if err != nil { + return err + } + err = vm.WriteDouble(r.DoubleField, w) + if err != nil { + return err + } + err = vm.WriteString(r.StringField, w) + if err != nil { + return err + } + err = vm.WriteBool(r.BoolField, w) + if err != nil { + return err + } + err = vm.WriteBytes(r.BytesField, w) + if err != nil { + return err + } + return err +} + +func (r DemoSchema) Serialize(w io.Writer) error { + return writeDemoSchema(r, w) +} + +func (r DemoSchema) Schema() string { + return "{\"fields\":[{\"name\":\"IntField\",\"type\":\"int\"},{\"name\":\"DoubleField\",\"type\":\"double\"},{\"name\":\"StringField\",\"type\":\"string\"},{\"name\":\"BoolField\",\"type\":\"boolean\"},{\"name\":\"BytesField\",\"type\":\"bytes\"}],\"name\":\"DemoSchema\",\"type\":\"record\"}" +} + +func (r DemoSchema) SchemaName() string { + return "DemoSchema" +} + +func (_ DemoSchema) SetBoolean(v bool) { panic("Unsupported operation") } +func (_ DemoSchema) SetInt(v int32) { panic("Unsupported operation") } +func (_ DemoSchema) SetLong(v int64) { panic("Unsupported operation") } +func (_ DemoSchema) SetFloat(v float32) { panic("Unsupported operation") } +func (_ DemoSchema) SetDouble(v float64) { panic("Unsupported operation") } +func (_ DemoSchema) SetBytes(v []byte) { panic("Unsupported operation") } +func (_ DemoSchema) SetString(v string) { panic("Unsupported operation") } +func (_ DemoSchema) SetUnionElem(v int64) { panic("Unsupported operation") } + +func (r *DemoSchema) Get(i int) types.Field { + switch i { + case 0: + w := types.Int{Target: &r.IntField} + + return w + + case 1: + w := types.Double{Target: &r.DoubleField} + + return w + + case 2: + w := types.String{Target: &r.StringField} + + return w + + case 3: + w := types.Boolean{Target: &r.BoolField} + + return w + + case 4: + w := BytesWrapper{Target: &r.BytesField} + + return w + + } + panic("Unknown field index") +} + +func (r *DemoSchema) SetDefault(i int) { + switch i { + } + panic("Unknown field index") +} + +func (r *DemoSchema) NullField(i int) { + switch i { + } + panic("Not a nullable field index") +} + +func (_ DemoSchema) AppendMap(key string) types.Field { panic("Unsupported operation") } +func (_ DemoSchema) AppendArray() types.Field { panic("Unsupported operation") } +func (_ DemoSchema) HintSize(int) { panic("Unsupported operation") } +func (_ DemoSchema) Finalize() {} + +func (_ DemoSchema) AvroCRC64Fingerprint() []byte { + return []byte(DemoSchemaAvroCRC64Fingerprint) +} + +func (r DemoSchema) MarshalJSON() ([]byte, error) { + var err error + output := make(map[string]json.RawMessage) + output["IntField"], err = json.Marshal(r.IntField) + if err != nil { + return nil, err + } + output["DoubleField"], err = json.Marshal(r.DoubleField) + if err != nil { + return nil, err + } + output["StringField"], err = json.Marshal(r.StringField) + if err != nil { + return nil, err + } + output["BoolField"], err = json.Marshal(r.BoolField) + if err != nil { + return nil, err + } + output["BytesField"], err = json.Marshal(r.BytesField) + if err != nil { + return nil, err + } + return json.Marshal(output) +} + +func (r *DemoSchema) UnmarshalJSON(data []byte) error { + var fields map[string]json.RawMessage + if err := json.Unmarshal(data, &fields); err != nil { + return err + } + + var val json.RawMessage + val = func() json.RawMessage { + if v, ok := fields["IntField"]; ok { + return v + } + return nil + }() + + if val != nil { + if err := json.Unmarshal([]byte(val), &r.IntField); err != nil { + return err + } + } else { + return fmt.Errorf("no value specified for IntField") + } + val = func() json.RawMessage { + if v, ok := fields["DoubleField"]; ok { + return v + } + return nil + }() + + if val != nil { + if err := json.Unmarshal([]byte(val), &r.DoubleField); err != nil { + return err + } + } else { + return fmt.Errorf("no value specified for DoubleField") + } + val = func() json.RawMessage { + if v, ok := fields["StringField"]; ok { + return v + } + return nil + }() + + if val != nil { + if err := json.Unmarshal([]byte(val), &r.StringField); err != nil { + return err + } + } else { + return fmt.Errorf("no value specified for StringField") + } + val = func() json.RawMessage { + if v, ok := fields["BoolField"]; ok { + return v + } + return nil + }() + + if val != nil { + if err := json.Unmarshal([]byte(val), &r.BoolField); err != nil { + return err + } + } else { + return fmt.Errorf("no value specified for BoolField") + } + val = func() json.RawMessage { + if v, ok := fields["BytesField"]; ok { + return v + } + return nil + }() + + if val != nil { + if err := json.Unmarshal([]byte(val), &r.BytesField); err != nil { + return err + } + } else { + return fmt.Errorf("no value specified for BytesField") + } + return nil +} diff --git a/testdata/demo_schema_container.go b/testdata/demo_schema_container.go new file mode 100644 index 0000000..c5d5a8a --- /dev/null +++ b/testdata/demo_schema_container.go @@ -0,0 +1,49 @@ +// Code generated by github.com/actgardner/gogen-avro/v10. DO NOT EDIT. +/* + * SOURCE: + * example.avsc + */ +package avro + +import ( + "io" + + "github.com/actgardner/gogen-avro/v10/compiler" + "github.com/actgardner/gogen-avro/v10/container" + "github.com/actgardner/gogen-avro/v10/vm" +) + +func NewDemoSchemaWriter(writer io.Writer, codec container.Codec, recordsPerBlock int64) (*container.Writer, error) { + str := NewDemoSchema() + return container.NewWriter(writer, codec, recordsPerBlock, str.Schema()) +} + +// container reader +type DemoSchemaReader struct { + r io.Reader + p *vm.Program +} + +func NewDemoSchemaReader(r io.Reader) (*DemoSchemaReader, error) { + containerReader, err := container.NewReader(r) + if err != nil { + return nil, err + } + + t := NewDemoSchema() + deser, err := compiler.CompileSchemaBytes([]byte(containerReader.AvroContainerSchema()), []byte(t.Schema())) + if err != nil { + return nil, err + } + + return &DemoSchemaReader{ + r: containerReader, + p: deser, + }, nil +} + +func (r DemoSchemaReader) Read() (DemoSchema, error) { + t := NewDemoSchema() + err := vm.Eval(r.r, r.p, &t) + return t, err +} diff --git a/testdata/example.avsc b/testdata/example.avsc new file mode 100644 index 0000000..7125155 --- /dev/null +++ b/testdata/example.avsc @@ -0,0 +1,12 @@ +{ + "type": "record", + "name": "DemoSchema", + "fields": [ + {"name": "IntField", "type": "int"}, + {"name": "DoubleField", "type": "double"}, + {"name": "StringField", "type": "string"}, + {"name": "BoolField", "type": "boolean"}, + {"name": "BytesField", "type": "bytes"} + ] + +} \ No newline at end of file diff --git a/testdata/example.proto b/testdata/example.proto new file mode 100644 index 0000000..22bd091 --- /dev/null +++ b/testdata/example.proto @@ -0,0 +1,17 @@ +syntax = "proto3"; + +package tagging; + +option go_package = "./example"; + +message ExampleDef { + string allowed = 1; + int32 disallowed = 2; + oneof example{ + Foo foo = 3; + Bar bar = 4; + } +} + +message Foo{string name = 1;} +message Bar{string name = 1;} \ No newline at end of file diff --git a/testdata/example/example.pb.go b/testdata/example/example.pb.go new file mode 100644 index 0000000..338c3c1 --- /dev/null +++ b/testdata/example/example.pb.go @@ -0,0 +1,328 @@ +// Code generated by protoc-gen-go. DO NOT EDIT. +// versions: +// protoc-gen-go v1.30.0 +// protoc v3.21.12 +// source: example.proto + +package example + +import ( + protoreflect "google.golang.org/protobuf/reflect/protoreflect" + protoimpl "google.golang.org/protobuf/runtime/protoimpl" + reflect "reflect" + sync "sync" +) + +const ( + // Verify that this generated code is sufficiently up-to-date. + _ = protoimpl.EnforceVersion(20 - protoimpl.MinVersion) + // Verify that runtime/protoimpl is sufficiently up-to-date. + _ = protoimpl.EnforceVersion(protoimpl.MaxVersion - 20) +) + +type ExampleDef struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + Allowed string `protobuf:"bytes,1,opt,name=allowed,proto3" json:"allowed,omitempty"` + Disallowed int32 `protobuf:"varint,2,opt,name=disallowed,proto3" json:"disallowed,omitempty"` + // Types that are assignable to Example: + // + // *ExampleDef_Foo + // *ExampleDef_Bar + Example isExampleDef_Example `protobuf_oneof:"example"` +} + +func (x *ExampleDef) Reset() { + *x = ExampleDef{} + if protoimpl.UnsafeEnabled { + mi := &file_example_proto_msgTypes[0] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *ExampleDef) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*ExampleDef) ProtoMessage() {} + +func (x *ExampleDef) ProtoReflect() protoreflect.Message { + mi := &file_example_proto_msgTypes[0] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use ExampleDef.ProtoReflect.Descriptor instead. +func (*ExampleDef) Descriptor() ([]byte, []int) { + return file_example_proto_rawDescGZIP(), []int{0} +} + +func (x *ExampleDef) GetAllowed() string { + if x != nil { + return x.Allowed + } + return "" +} + +func (x *ExampleDef) GetDisallowed() int32 { + if x != nil { + return x.Disallowed + } + return 0 +} + +func (m *ExampleDef) GetExample() isExampleDef_Example { + if m != nil { + return m.Example + } + return nil +} + +func (x *ExampleDef) GetFoo() *Foo { + if x, ok := x.GetExample().(*ExampleDef_Foo); ok { + return x.Foo + } + return nil +} + +func (x *ExampleDef) GetBar() *Bar { + if x, ok := x.GetExample().(*ExampleDef_Bar); ok { + return x.Bar + } + return nil +} + +type isExampleDef_Example interface { + isExampleDef_Example() +} + +type ExampleDef_Foo struct { + Foo *Foo `protobuf:"bytes,3,opt,name=foo,proto3,oneof"` +} + +type ExampleDef_Bar struct { + Bar *Bar `protobuf:"bytes,4,opt,name=bar,proto3,oneof"` +} + +func (*ExampleDef_Foo) isExampleDef_Example() {} + +func (*ExampleDef_Bar) isExampleDef_Example() {} + +type Foo struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + Name string `protobuf:"bytes,1,opt,name=name,proto3" json:"name,omitempty"` +} + +func (x *Foo) Reset() { + *x = Foo{} + if protoimpl.UnsafeEnabled { + mi := &file_example_proto_msgTypes[1] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *Foo) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*Foo) ProtoMessage() {} + +func (x *Foo) ProtoReflect() protoreflect.Message { + mi := &file_example_proto_msgTypes[1] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use Foo.ProtoReflect.Descriptor instead. +func (*Foo) Descriptor() ([]byte, []int) { + return file_example_proto_rawDescGZIP(), []int{1} +} + +func (x *Foo) GetName() string { + if x != nil { + return x.Name + } + return "" +} + +type Bar struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + Name string `protobuf:"bytes,1,opt,name=name,proto3" json:"name,omitempty"` +} + +func (x *Bar) Reset() { + *x = Bar{} + if protoimpl.UnsafeEnabled { + mi := &file_example_proto_msgTypes[2] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *Bar) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*Bar) ProtoMessage() {} + +func (x *Bar) ProtoReflect() protoreflect.Message { + mi := &file_example_proto_msgTypes[2] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use Bar.ProtoReflect.Descriptor instead. +func (*Bar) Descriptor() ([]byte, []int) { + return file_example_proto_rawDescGZIP(), []int{2} +} + +func (x *Bar) GetName() string { + if x != nil { + return x.Name + } + return "" +} + +var File_example_proto protoreflect.FileDescriptor + +var file_example_proto_rawDesc = []byte{ + 0x0a, 0x0d, 0x65, 0x78, 0x61, 0x6d, 0x70, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x12, + 0x07, 0x74, 0x61, 0x67, 0x67, 0x69, 0x6e, 0x67, 0x22, 0x95, 0x01, 0x0a, 0x0a, 0x45, 0x78, 0x61, + 0x6d, 0x70, 0x6c, 0x65, 0x44, 0x65, 0x66, 0x12, 0x18, 0x0a, 0x07, 0x61, 0x6c, 0x6c, 0x6f, 0x77, + 0x65, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x07, 0x61, 0x6c, 0x6c, 0x6f, 0x77, 0x65, + 0x64, 0x12, 0x1e, 0x0a, 0x0a, 0x64, 0x69, 0x73, 0x61, 0x6c, 0x6c, 0x6f, 0x77, 0x65, 0x64, 0x18, + 0x02, 0x20, 0x01, 0x28, 0x05, 0x52, 0x0a, 0x64, 0x69, 0x73, 0x61, 0x6c, 0x6c, 0x6f, 0x77, 0x65, + 0x64, 0x12, 0x20, 0x0a, 0x03, 0x66, 0x6f, 0x6f, 0x18, 0x03, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x0c, + 0x2e, 0x74, 0x61, 0x67, 0x67, 0x69, 0x6e, 0x67, 0x2e, 0x46, 0x6f, 0x6f, 0x48, 0x00, 0x52, 0x03, + 0x66, 0x6f, 0x6f, 0x12, 0x20, 0x0a, 0x03, 0x62, 0x61, 0x72, 0x18, 0x04, 0x20, 0x01, 0x28, 0x0b, + 0x32, 0x0c, 0x2e, 0x74, 0x61, 0x67, 0x67, 0x69, 0x6e, 0x67, 0x2e, 0x42, 0x61, 0x72, 0x48, 0x00, + 0x52, 0x03, 0x62, 0x61, 0x72, 0x42, 0x09, 0x0a, 0x07, 0x65, 0x78, 0x61, 0x6d, 0x70, 0x6c, 0x65, + 0x22, 0x19, 0x0a, 0x03, 0x46, 0x6f, 0x6f, 0x12, 0x12, 0x0a, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x18, + 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x22, 0x19, 0x0a, 0x03, 0x42, + 0x61, 0x72, 0x12, 0x12, 0x0a, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, + 0x52, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x42, 0x0b, 0x5a, 0x09, 0x2e, 0x2f, 0x65, 0x78, 0x61, 0x6d, + 0x70, 0x6c, 0x65, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33, +} + +var ( + file_example_proto_rawDescOnce sync.Once + file_example_proto_rawDescData = file_example_proto_rawDesc +) + +func file_example_proto_rawDescGZIP() []byte { + file_example_proto_rawDescOnce.Do(func() { + file_example_proto_rawDescData = protoimpl.X.CompressGZIP(file_example_proto_rawDescData) + }) + return file_example_proto_rawDescData +} + +var file_example_proto_msgTypes = make([]protoimpl.MessageInfo, 3) +var file_example_proto_goTypes = []any{ + (*ExampleDef)(nil), // 0: tagging.ExampleDef + (*Foo)(nil), // 1: tagging.Foo + (*Bar)(nil), // 2: tagging.Bar +} +var file_example_proto_depIdxs = []int32{ + 1, // 0: tagging.ExampleDef.foo:type_name -> tagging.Foo + 2, // 1: tagging.ExampleDef.bar:type_name -> tagging.Bar + 2, // [2:2] is the sub-list for method output_type + 2, // [2:2] is the sub-list for method input_type + 2, // [2:2] is the sub-list for extension type_name + 2, // [2:2] is the sub-list for extension extendee + 0, // [0:2] is the sub-list for field type_name +} + +func init() { file_example_proto_init() } +func file_example_proto_init() { + if File_example_proto != nil { + return + } + if !protoimpl.UnsafeEnabled { + file_example_proto_msgTypes[0].Exporter = func(v any, i int) any { + switch v := v.(*ExampleDef); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + file_example_proto_msgTypes[1].Exporter = func(v any, i int) any { + switch v := v.(*Foo); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + file_example_proto_msgTypes[2].Exporter = func(v any, i int) any { + switch v := v.(*Bar); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + } + file_example_proto_msgTypes[0].OneofWrappers = []any{ + (*ExampleDef_Foo)(nil), + (*ExampleDef_Bar)(nil), + } + type x struct{} + out := protoimpl.TypeBuilder{ + File: protoimpl.DescBuilder{ + GoPackagePath: reflect.TypeOf(x{}).PkgPath(), + RawDescriptor: file_example_proto_rawDesc, + NumEnums: 0, + NumMessages: 3, + NumExtensions: 0, + NumServices: 0, + }, + GoTypes: file_example_proto_goTypes, + DependencyIndexes: file_example_proto_depIdxs, + MessageInfos: file_example_proto_msgTypes, + }.Build() + File_example_proto = out.File + file_example_proto_rawDesc = nil + file_example_proto_goTypes = nil + file_example_proto_depIdxs = nil +} diff --git a/testdata/example_json.go b/testdata/example_json.go new file mode 100644 index 0000000..2ada915 --- /dev/null +++ b/testdata/example_json.go @@ -0,0 +1,5 @@ +package avro + +type ExampleJson struct { + Id string +} diff --git a/testdata/generator.go b/testdata/generator.go new file mode 100644 index 0000000..76ad615 --- /dev/null +++ b/testdata/generator.go @@ -0,0 +1,5 @@ +package avro + +//go:generate $GOPATH/bin/gogen-avro -containers . ./example.avsc +//go:generate protoc --proto_path=. --go_out=./ ./example.proto +//go:generate $GOPATH/bin/avrogo -p heetch -d ./heetch ./example.avsc diff --git a/testdata/heetch/example_gen.go b/testdata/heetch/example_gen.go new file mode 100644 index 0000000..028b0c0 --- /dev/null +++ b/testdata/heetch/example_gen.go @@ -0,0 +1,29 @@ +// Code generated by avrogen. DO NOT EDIT. + +package heetch + +import ( + "github.com/heetch/avro/avrotypegen" +) + +type DemoSchema struct { + IntField int + DoubleField float64 + StringField string + BoolField bool + BytesField []byte +} + +// AvroRecord implements the avro.AvroRecord interface. +func (DemoSchema) AvroRecord() avrotypegen.RecordInfo { + return avrotypegen.RecordInfo{ + Schema: `{"fields":[{"name":"IntField","type":"int"},{"name":"DoubleField","type":"double"},{"name":"StringField","type":"string"},{"name":"BoolField","type":"boolean"},{"name":"BytesField","type":"bytes"}],"name":"DemoSchema","type":"record"}`, + Required: []bool{ + 0: true, + 1: true, + 2: true, + 3: true, + 4: true, + }, + } +}