From 04865f91d2aa6190b8380a84cda1ce83f8537fbb Mon Sep 17 00:00:00 2001 From: stewartboyd119 Date: Mon, 22 Jul 2024 13:28:50 -0700 Subject: [PATCH] Port from gitlab (#1) # Port from gitlab 1. Copy from gitlab 2. Port gitlab-ci -> github workflows 3. Added test coverage and linting --- .github/workflows/go.yml | 60 +++++ .gitignore | 48 ++++ .golangci.yml | 69 +++++ Dockerfile | 15 ++ Makefile | 18 ++ README.md | 27 ++ avro.go | 58 ++++ formatter.go | 61 +++++ formatter_test.go | 88 ++++++ go.mod | 25 ++ go.sum | 44 +++ json.go | 14 + protobase64.go | 59 ++++ protobase64_test.go | 43 +++ protojson.go | 29 ++ protojson_test.go | 55 ++++ protoraw.go | 50 ++++ protoraw_test.go | 43 +++ schematizedavro.go | 60 +++++ schematizedavro_test.go | 400 ++++++++++++++++++++++++++++ schematizedjson.go | 17 ++ schematizedjson_test.go | 114 ++++++++ schematizedproto_deprecated.go | 17 ++ schematizedproto_deprecated_test.go | 107 ++++++++ string.go | 34 +++ string_test.go | 102 +++++++ testdata/bytes.go | 90 +++++++ testdata/demo_schema.go | 264 ++++++++++++++++++ testdata/demo_schema_container.go | 49 ++++ testdata/example.avsc | 12 + testdata/example.proto | 17 ++ testdata/example/example.pb.go | 328 +++++++++++++++++++++++ testdata/example_json.go | 5 + testdata/generator.go | 5 + testdata/heetch/example_gen.go | 29 ++ 35 files changed, 2456 insertions(+) create mode 100644 .github/workflows/go.yml create mode 100644 .gitignore create mode 100644 .golangci.yml create mode 100644 Dockerfile create mode 100644 Makefile create mode 100644 README.md create mode 100644 avro.go create mode 100644 formatter.go create mode 100644 formatter_test.go create mode 100644 go.mod create mode 100644 go.sum create mode 100644 json.go create mode 100644 protobase64.go create mode 100644 protobase64_test.go create mode 100644 protojson.go create mode 100644 protojson_test.go create mode 100644 protoraw.go create mode 100644 protoraw_test.go create mode 100644 schematizedavro.go create mode 100644 schematizedavro_test.go create mode 100644 schematizedjson.go create mode 100644 schematizedjson_test.go create mode 100644 schematizedproto_deprecated.go create mode 100644 schematizedproto_deprecated_test.go create mode 100644 string.go create mode 100644 string_test.go create mode 100644 testdata/bytes.go create mode 100644 testdata/demo_schema.go create mode 100644 testdata/demo_schema_container.go create mode 100644 testdata/example.avsc create mode 100644 testdata/example.proto create mode 100644 testdata/example/example.pb.go create mode 100644 testdata/example_json.go create mode 100644 testdata/generator.go create mode 100644 testdata/heetch/example_gen.go diff --git a/.github/workflows/go.yml b/.github/workflows/go.yml new file mode 100644 index 0000000..62700ab --- /dev/null +++ b/.github/workflows/go.yml @@ -0,0 +1,60 @@ +name: Go + +on: + push: + branches: [main] + tags: ['v*'] + pull_request: + branches: ['*'] + +permissions: + contents: read + +jobs: + build: + runs-on: ubuntu-latest + strategy: + matrix: + go: ["1.21.x", "1.22.x"] + include: + - go: 1.22.x + steps: + - name: Checkout code + uses: actions/checkout@v4 + + - name: Setup Go + uses: actions/setup-go@v5 + with: + go-version: ${{ matrix.go }} + cache-dependency-path: '**/go.sum' + + - name: Download Dependencies + run: | + go mod download + + - name: Test + run: make cover + + lint: + name: Lint + runs-on: ubuntu-latest + + steps: + - uses: actions/checkout@v4 + name: Check out repository + - uses: actions/setup-go@v5 + name: Set up Go + with: + go-version: 1.22.x + cache: false # managed by golangci-lint + + - uses: golangci/golangci-lint-action@v6 + name: Install golangci-lint + with: + version: latest + # Hack: Use the official action to download, but not run. + # make lint below will handle actually running the linter. + args: --help + + - run: make lint + name: Lint \ No newline at end of file diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..c4193ab --- /dev/null +++ b/.gitignore @@ -0,0 +1,48 @@ +# Compiled Object files, Static and Dynamic libs (Shared Objects) +*.o +*.a +*.so + +# Folders +_obj +_test +vendor + +# Architecture specific extensions/prefixes +*.[568vq] +[568vq].out + +*.cgo1.go +*.cgo2.c +_cgo_defun.c +_cgo_gotypes.go +_cgo_export.* + +_testmain.go + +*.exe +*.test +*.prof +*.out +#*.jar + +config/local/config.json + +sms +main +.DS_Store +debug +appConfig.local.json +settings.json +liquibase.properties +*.sample.sql + +**/obj +**/bin + +# JetBrains +.idea/ +/.run/ + + +coverage.txt diff --git a/.golangci.yml b/.golangci.yml new file mode 100644 index 0000000..31a7ca0 --- /dev/null +++ b/.golangci.yml @@ -0,0 +1,69 @@ +run: + skip-dirs: + - docs + - datadog + - kustomize + skip-files: + - 'wire_gen.go' + tests: false +linters-settings: + errcheck: + check-type-assertions: true + check-blank: true + gci: + sections: + - standard + - default + gosimple: + go: '1.17' + govet: + check-shadowing: true + settings: + printf: + funcs: + - (gitlab.zgtools.net/devex/archetypes/gomods/zlog.Logger).Debug + - (gitlab.zgtools.net/devex/archetypes/gomods/zlog.Logger).Info + - (gitlab.zgtools.net/devex/archetypes/gomods/zlog.Logger).Warn + - (gitlab.zgtools.net/devex/archetypes/gomods/zlog.Logger).Error + depguard: + rules: + Main: + files: + - $all + - "!$test" + deny: + - github.com/satori/go.uuid: Prefer "github.com/google/uuid" + disable-all: true + enable: + - asciicheck + - bidichk + - bodyclose + - cyclop + - decorder + - depguard + - deadcode + - dupl + - errcheck + - errchkjson + - errname + - errorlint + - exportloopref + - gci + - gocognit + - goconst + - gocritic + - gocyclo + - gofmt + - gosimple + - govet + - ineffassign + - nolintlint + - prealloc + - staticcheck + - structcheck + - typecheck + - unconvert + - unparam + - unused + - varcheck + - whitespace diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..4bb136a --- /dev/null +++ b/Dockerfile @@ -0,0 +1,15 @@ +FROM golang:1.22 AS build + +ENV CGO_ENABLED=0 + +WORKDIR /go/src/zfmt +COPY . . + +RUN go install -v ./... +RUN go build -o zfmt +FROM debian + +COPY --from=build /go/src/zfmt / + + +ENTRYPOINT ["/zfmt"] \ No newline at end of file diff --git a/Makefile b/Makefile new file mode 100644 index 0000000..4d67cb3 --- /dev/null +++ b/Makefile @@ -0,0 +1,18 @@ +# Directories containing independent Go modules. +MODULE_DIRS = . +LOCAL_GOLANGCI_VERSION=$(shell golangci-lint --version) +REMOTE_GOLANGCI_VERSION=1.56.2 + +.PHONY: cover +cover: + go test -v ./... -count=1 -coverprofile=coverage.txt -covermode atomic + +.PHONY: lint +lint: golangci-lint + +.PHONY: golangci-lint +golangci-lint: + @$(foreach mod,$(MODULE_DIRS), \ + (cd $(mod) && \ + echo "[lint] golangci-lint: $(mod)" && \ + golangci-lint run --path-prefix $(mod) ./...) &&) true diff --git a/README.md b/README.md new file mode 100644 index 0000000..0e83ee1 --- /dev/null +++ b/README.md @@ -0,0 +1,27 @@ +# zfmt + +[![coverage report](https://gitlab.zgtools.net/devex/archetypes/gomods/zfmt/badges/main/coverage.svg)](https://gitlab.zgtools.net/devex/archetypes/gomods/zfmt/-/commits/main) + +[![pipeline status](https://gitlab.zgtools.net/devex/archetypes/gomods/zfmt/badges/main/pipeline.svg)](https://gitlab.zgtools.net/devex/archetypes/gomods/zfmt/-/commits/main) + + +A go library which contains a number of useful implementations of the Formatter interface, +an interface which contains Marshall and Unmarshall methods. + +## Dependencies + +### Gogen Avro + +Install [Gogen Avro](https://github.com/actgardner/gogen-avro) a utility for generating go code from avro schemas (used for testdata) + +``` +go get github.com/actgardner/gogen-avro/v7/cmd/... +``` + + +### Migration Guide + +1. #### V0 to V1 Migration + + * Enum `ProtoFmt`(`proto`) renamed to `ProtoBase64Fmt`(`proto_base64`). This is intended for SQS use only while the newly introduced `ProtoRawFmt`(`proto_raw`) is intended for most other use cases. + * Enum `ProtoSchemaFmt`(`proto_schema`) renamed to `ProtoSchemaDeprecatedFmt`(`proto_schema_deprecated`). The proto schema is deprecated because it doesn't work properly with the confluent schema registry. Use the `ProtoRawFmt` or `ProtoBase64Fmt` instead. \ No newline at end of file diff --git a/avro.go b/avro.go new file mode 100644 index 0000000..7959297 --- /dev/null +++ b/avro.go @@ -0,0 +1,58 @@ +package zfmt + +import ( + "bytes" + "fmt" + + "github.com/actgardner/gogen-avro/v10/compiler" + "github.com/actgardner/gogen-avro/v10/soe" + "github.com/actgardner/gogen-avro/v10/vm" + "github.com/actgardner/gogen-avro/v10/vm/types" + heetch "github.com/heetch/avro" + heetchtypegen "github.com/heetch/avro/avrotypegen" +) + +// GeneratedAvroRecord combines interfaces that make Encoding/Decoding possible +// for gogen-avro struct +type GeneratedAvroRecord interface { + soe.AvroRecord + types.Field + Schema() string +} + +type AvroFormatter struct{} + +func (p *AvroFormatter) Marshall(v any) ([]byte, error) { + switch m := v.(type) { + case soe.AvroRecord: + buf := &bytes.Buffer{} + err := m.Serialize(buf) + return buf.Bytes(), err + case heetchtypegen.AvroRecord: + b, _, err := heetch.Marshal(v) + return b, err + default: + return nil, fmt.Errorf("%T, avro formatter supports only gogen-avro or heetch avro messages", v) + } +} + +func (p *AvroFormatter) Unmarshal(b []byte, v any) error { + switch m := v.(type) { + case GeneratedAvroRecord: + r := bytes.NewReader(b) + deser, err := compiler.CompileSchemaBytes([]byte(m.Schema()), []byte(m.Schema())) + if err != nil { + return err + } + return vm.Eval(r, deser, m) + case heetchtypegen.AvroRecord: + t, err := heetch.ParseType(m.AvroRecord().Schema) + if err != nil { + return err + } + _, err = heetch.Unmarshal(b, v, t) + return err + default: + return fmt.Errorf("%T, avro formatter supports only gogen-avro or heetch avro messages", v) + } +} diff --git a/formatter.go b/formatter.go new file mode 100644 index 0000000..bbf2ebe --- /dev/null +++ b/formatter.go @@ -0,0 +1,61 @@ +package zfmt + +import ( + err "fmt" +) + +// FormatterType defines a formatter +type FormatterType string + +const ( + // JSONFmt indicates json formatter + JSONFmt FormatterType = "json" + // ProtoRawFmt indicates protocol buffer formatter. Does not Base64 encode/decode. + ProtoRawFmt FormatterType = "proto_raw" + // ProtoBase64Fmt indicates protocol buffer formatter. Base64 encodes/decodes the message. Only intended for use with SQS. + ProtoBase64Fmt FormatterType = "proto_base64" + // ProtoJSONFmt indicates usage protojson formatter (which should be used for json formatted proto messages). + ProtoJSONFmt FormatterType = "proto_json" + // StringFmt indicates string formatter + StringFmt FormatterType = "string" + // AvroFmt indicates apache avro formatter + AvroFmt FormatterType = "avro" + // AvroSchemaFmt indicates apache avro formatter with schemaID encoded + AvroSchemaFmt FormatterType = "avro_schema" + // JSONSchemaFmt indicates json formatter with schemaID encoded + JSONSchemaFmt FormatterType = "json_schema" + // ProtoSchemaDeprecatedFmt indicates proto formatter with schemaID encoded - deprecated because it doesn't work properly. + ProtoSchemaDeprecatedFmt FormatterType = "proto_schema_deprecated" +) + +// Formatter allows the user to extend formatting capability to unsupported data types +type Formatter interface { + Marshall(v any) ([]byte, error) + Unmarshal(data []byte, v any) error +} + +// GetFormatter returns supported formatter from its name +func GetFormatter(fmt FormatterType, schemaID int) (Formatter, error) { + switch fmt { + case StringFmt: + return &StringFormatter{}, nil + case JSONFmt: + return &JSONFormatter{}, nil + case ProtoJSONFmt: + return &ProtoJSONFormatter{}, nil + case ProtoBase64Fmt: + return &ProtobufBase64Formatter{}, nil + case ProtoRawFmt: + return &ProtobufRawFormatter{}, nil + case AvroFmt: + return &AvroFormatter{}, nil + case AvroSchemaFmt: + return &SchematizedAvroFormatter{SchemaID: schemaID}, nil + case JSONSchemaFmt: + return &SchematizedJSONFormatter{SchemaID: schemaID}, nil + case ProtoSchemaDeprecatedFmt: + return &SchematizedProtoFormatterDeprecated{SchemaID: schemaID}, nil + default: + return nil, err.Errorf("unsupported formatter %s", fmt) + } +} diff --git a/formatter_test.go b/formatter_test.go new file mode 100644 index 0000000..d110612 --- /dev/null +++ b/formatter_test.go @@ -0,0 +1,88 @@ +package zfmt + +import ( + "reflect" + "testing" +) + +func TestGetFormatter(t *testing.T) { + type args struct { + fmt FormatterType + schemaID int + } + tests := []struct { + name string + args args + want Formatter + wantErr bool + }{ + { + name: "json", + args: args{ + fmt: "json", + schemaID: 0, + }, + want: &JSONFormatter{}, + }, + { + name: "proto_base64", + args: args{ + fmt: "proto_base64", + schemaID: 0, + }, + want: &ProtobufBase64Formatter{}, + }, + { + name: "proto_raw", + args: args{ + fmt: "proto_raw", + schemaID: 111, + }, + want: &ProtobufRawFormatter{}, + }, + { + name: "proto_json", + args: args{ + fmt: "proto_json", + schemaID: 111, + }, + want: &ProtoJSONFormatter{}, + }, + { + name: "avro", + args: args{ + fmt: "avro", + schemaID: 0, + }, + want: &AvroFormatter{}, + }, + { + name: "json_schema", + args: args{ + fmt: "json_schema", + schemaID: 123, + }, + want: &SchematizedJSONFormatter{SchemaID: 123}, + }, + { + name: "avro_schema", + args: args{ + fmt: "avro_schema", + schemaID: 789, + }, + want: &SchematizedAvroFormatter{SchemaID: 789}, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got, err := GetFormatter(tt.args.fmt, tt.args.schemaID) + if (err != nil) != tt.wantErr { + t.Errorf("GetFormatter() error = %v, wantErr %v", err, tt.wantErr) + return + } + if !reflect.DeepEqual(got, tt.want) { + t.Errorf("GetFormatter() got = %v, want %v", got, tt.want) + } + }) + } +} diff --git a/go.mod b/go.mod new file mode 100644 index 0000000..b1cc74e --- /dev/null +++ b/go.mod @@ -0,0 +1,25 @@ +module gitlab.zgtools.net/devex/archetypes/gomods/zfmt + +go 1.22 + +toolchain go1.22.1 + +require ( + github.com/actgardner/gogen-avro/v10 v10.2.1 + github.com/golang/protobuf v1.5.4 + github.com/google/go-cmp v0.6.0 + github.com/heetch/avro v0.4.5 + github.com/stretchr/testify v1.9.0 + google.golang.org/protobuf v1.34.2 +) + +require ( + github.com/davecgh/go-spew v1.1.1 // indirect + github.com/golang/snappy v0.0.4 // indirect + github.com/google/uuid v1.6.0 // indirect + github.com/kr/pretty v0.3.1 // indirect + github.com/pmezard/go-difflib v1.0.0 // indirect + github.com/rogpeppe/go-internal v1.10.0 // indirect + gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c // indirect + gopkg.in/yaml.v3 v3.0.1 // indirect +) diff --git a/go.sum b/go.sum new file mode 100644 index 0000000..1c53ccb --- /dev/null +++ b/go.sum @@ -0,0 +1,44 @@ +github.com/actgardner/gogen-avro/v10 v10.2.1 h1:z3pOGblRjAJCYpkIJ8CmbMJdksi4rAhaygw0dyXZ930= +github.com/actgardner/gogen-avro/v10 v10.2.1/go.mod h1:QUhjeHPchheYmMDni/Nx7VB0RsT/ee8YIgGY/xpEQgQ= +github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= +github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/frankban/quicktest v1.14.0 h1:+cqqvzZV87b4adx/5ayVOaYZ2CrvM4ejQvUdBzPPUss= +github.com/frankban/quicktest v1.14.0/go.mod h1:NeW+ay9A/U67EYXNFA1nPE8e/tnQv/09mUdL/ijj8og= +github.com/golang/protobuf v1.5.0 h1:LUVKkCeviFUMKqHa4tXIIij/lbhnMbP7Fn5wKdKkRh4= +github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk= +github.com/golang/protobuf v1.5.4 h1:i7eJL8qZTpSEXOPTxNKhASYpMn+8e5Q6AdndVa1dWek= +github.com/golang/protobuf v1.5.4/go.mod h1:lnTiLA8Wa4RWRcIUkrtSVa5nRhsEGBg48fD6rSs7xps= +github.com/golang/snappy v0.0.4 h1:yAGX7huGHXlcLOEtBnF4w7FQwA26wojNCwOYAEhLjQM= +github.com/golang/snappy v0.0.4/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= +github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI= +github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= +github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= +github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/heetch/avro v0.4.5 h1:BSnj4wEeUG1IjMTm9/tBwQnV3euuIVa1mRWHnm1t8VU= +github.com/heetch/avro v0.4.5/go.mod h1:gxf9GnbjTXmWmqxhdNbAMcZCjpye7RV5r9t3Q0dL6ws= +github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= +github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= +github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= +github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= +github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= +github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= +github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= +github.com/pkg/diff v0.0.0-20210226163009-20ebb0f2a09e/go.mod h1:pJLUxLENpZxwdsKMEsNbx1VGcRFpLqf3715MtcvvzbA= +github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/rogpeppe/go-internal v1.9.0/go.mod h1:WtVeX8xhTBvf0smdhujwtBcq4Qrzq/fJaraNFVN+nFs= +github.com/rogpeppe/go-internal v1.10.0 h1:TMyTOH3F/DB16zRVcYyreMH6GnZZrwQVAoYjRBZyWFQ= +github.com/rogpeppe/go-internal v1.10.0/go.mod h1:UQnix2H7Ngw/k4C5ijL5+65zddjncjaFoBhdsK/akog= +github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg= +github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= +golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= +google.golang.org/protobuf v1.34.2 h1:6xV6lTsCfpGD21XK49h7MhtcApnLqkfYgPcdHftf6hg= +google.golang.org/protobuf v1.34.2/go.mod h1:qYOHts0dSfpeUzUFpOMr/WGzszTmLH+DiWniOlNbLDw= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= +gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/json.go b/json.go new file mode 100644 index 0000000..e588ed2 --- /dev/null +++ b/json.go @@ -0,0 +1,14 @@ +package zfmt + +import "encoding/json" + +// JSONFormatter encodes/decodes go struct to json format +type JSONFormatter struct{} + +func (j *JSONFormatter) Marshall(v any) ([]byte, error) { + return json.Marshal(v) +} + +func (j *JSONFormatter) Unmarshal(b []byte, v any) error { + return json.Unmarshal(b, v) +} diff --git a/protobase64.go b/protobase64.go new file mode 100644 index 0000000..7309867 --- /dev/null +++ b/protobase64.go @@ -0,0 +1,59 @@ +package zfmt + +import ( + "encoding/base64" + "fmt" + + //nolint:staticcheck // Older zillow libs have generated code which uses this deprecated package. To maintain backwards compatability with them, the older proto serialization lib should be maintained + v1proto "github.com/golang/protobuf/proto" + v2proto "google.golang.org/protobuf/proto" +) + +// ProtobufBase64Formatter implements formatter interface for both protobuf v1 and v2 messages. Intended for use with SQS +type ProtobufBase64Formatter struct{} + +// Marshall as proto and then base64 encode (useful for technologies like SQS which limit the character set) +func (p *ProtobufBase64Formatter) Marshall(v any) ([]byte, error) { + switch m := v.(type) { + case v1proto.Message: + b, err := v1proto.Marshal(m) + if err != nil { + return nil, err + } + return []byte(base64.StdEncoding.EncodeToString(b)), nil + case v2proto.Message: + b, err := v2proto.Marshal(m) + if err != nil { + return nil, err + } + return []byte(base64.StdEncoding.EncodeToString(b)), nil + default: + return nil, fmt.Errorf("%T, proto base64 formatter can only be used with proto messages", v) + } +} + +// Unmarshal with base64 decoding +func (p *ProtobufBase64Formatter) Unmarshal(b []byte, v any) error { + switch m := v.(type) { + case v1proto.Message: + raw, err := base64.StdEncoding.DecodeString(string(b)) + if err != nil { + return err + } + if err := v1proto.Unmarshal(raw, m); err != nil { + return err + } + return nil + case v2proto.Message: + raw, err := base64.StdEncoding.DecodeString(string(b)) + if err != nil { + return err + } + if err = v2proto.Unmarshal(raw, m); err != nil { + return err + } + return nil + default: + return fmt.Errorf("%T, proto base64 formatter can only be used with proto messages", v) + } +} diff --git a/protobase64_test.go b/protobase64_test.go new file mode 100644 index 0000000..a9679e6 --- /dev/null +++ b/protobase64_test.go @@ -0,0 +1,43 @@ +package zfmt + +import ( + "testing" + + "gitlab.zgtools.net/devex/archetypes/gomods/zfmt/testdata/example" +) + +func TestProtoBase64Formatter_MarshallUnmarshall(t *testing.T) { + ein := example.ExampleDef{ + Allowed: "happy", + Disallowed: 2, + Example: &example.ExampleDef_Foo{ + Foo: &example.Foo{ + Name: "sad", + }, + }, + } + + fmtr := ProtobufBase64Formatter{} + + b, err := fmtr.Marshall(&ein) + if err != nil { + t.Fatal(err) + } + + eout := &example.ExampleDef{} + err = fmtr.Unmarshal(b, eout) + if err != nil { + t.Fatal(err) + } + if ein.Allowed != eout.Allowed { + t.Error("Not allowed") + } + + if e, ok := eout.Example.(*example.ExampleDef_Foo); ok { + if e.Foo.Name != "sad" { + t.Error("foo sad") + } + } else { + t.Error("example not foo") + } +} diff --git a/protojson.go b/protojson.go new file mode 100644 index 0000000..d36f638 --- /dev/null +++ b/protojson.go @@ -0,0 +1,29 @@ +package zfmt + +import ( + "fmt" + + "google.golang.org/protobuf/encoding/protojson" + "google.golang.org/protobuf/proto" +) + +// ProtoJSONFormatter encodes/decodes proto go struct to json format +type ProtoJSONFormatter struct{} + +func (j *ProtoJSONFormatter) Marshall(v any) ([]byte, error) { + + if m, ok := v.(proto.Message); ok { + return protojson.Marshal(m) + } + + return nil, fmt.Errorf("%T, protojson formatter can only be used with proto messages", v) +} + +func (j *ProtoJSONFormatter) Unmarshal(b []byte, v any) error { + if m, ok := v.(proto.Message); ok { + unmarshaller := protojson.UnmarshalOptions{DiscardUnknown: true} + return unmarshaller.Unmarshal(b, m) + } + + return fmt.Errorf("%T, protojson formatter can only be used with proto messages", v) +} diff --git a/protojson_test.go b/protojson_test.go new file mode 100644 index 0000000..4a21ded --- /dev/null +++ b/protojson_test.go @@ -0,0 +1,55 @@ +package zfmt + +import ( + "testing" + + "github.com/stretchr/testify/require" + "gitlab.zgtools.net/devex/archetypes/gomods/zfmt/testdata/example" +) + +func TestProtoJSONFormatter_MarshallUnmarshall(t *testing.T) { + ein := example.ExampleDef{ + Allowed: "happy", + Disallowed: 2, + Example: &example.ExampleDef_Foo{ + Foo: &example.Foo{ + Name: "sad", + }, + }, + } + + fmtr := ProtoJSONFormatter{} + + b, err := fmtr.Marshall(&ein) + if err != nil { + t.Fatal(err) + } + + eout := &example.ExampleDef{} + err = fmtr.Unmarshal(b, eout) + if err != nil { + t.Fatal(err) + } + if ein.Allowed != eout.Allowed { + t.Error("Not allowed") + } + + if e, ok := eout.Example.(*example.ExampleDef_Foo); ok { + if e.Foo.Name != "sad" { + t.Error("foo sad") + } + } else { + t.Error("example not foo") + } +} + +func TestProtoJSONFormatter_UnmarshallWithUnknown(t *testing.T) { + data := "{\n \"allowed\": \"happy\",\n \"disallowed\": 2,\n \"MyName\": \"Stewart\"\n}" + + fmtr := ProtoJSONFormatter{} + eout := &example.ExampleDef{} + err := fmtr.Unmarshal([]byte(data), eout) + require.NoError(t, err) + require.Equal(t, "happy", eout.Allowed) + require.Equal(t, int32(2), eout.Disallowed) +} diff --git a/protoraw.go b/protoraw.go new file mode 100644 index 0000000..15e5ecb --- /dev/null +++ b/protoraw.go @@ -0,0 +1,50 @@ +package zfmt + +import ( + "fmt" + + //nolint:staticcheck // Older zillow libs have generated code which uses this deprecated package. To maintain backwards compatability with them, the older proto serialization lib should be maintained + v1proto "github.com/golang/protobuf/proto" + v2proto "google.golang.org/protobuf/proto" +) + +// ProtobufRawFormatter implements formatter interface for both protobuf v1 and v2 messages. Does not base64 encode. +type ProtobufRawFormatter struct{} + +// Marshall encodes the data as a proto binary +func (p *ProtobufRawFormatter) Marshall(v any) ([]byte, error) { + switch m := v.(type) { + case v1proto.Message: + b, err := v1proto.Marshal(m) + if err != nil { + return nil, err + } + return b, nil + case v2proto.Message: + b, err := v2proto.Marshal(m) + if err != nil { + return nil, err + } + return b, nil + default: + return nil, fmt.Errorf("%T, protoraw formatter can only be used with proto messages", v) + } +} + +// Unmarshal accepts proto binary and hydrates a proto generated struct +func (p *ProtobufRawFormatter) Unmarshal(b []byte, v any) error { + switch m := v.(type) { + case v1proto.Message: + if err := v1proto.Unmarshal(b, m); err != nil { + return err + } + return nil + case v2proto.Message: + if err := v2proto.Unmarshal(b, m); err != nil { + return err + } + return nil + default: + return fmt.Errorf("%T, protoraw formatter can only be used with proto messages", v) + } +} diff --git a/protoraw_test.go b/protoraw_test.go new file mode 100644 index 0000000..93fddfa --- /dev/null +++ b/protoraw_test.go @@ -0,0 +1,43 @@ +package zfmt + +import ( + "testing" + + "gitlab.zgtools.net/devex/archetypes/gomods/zfmt/testdata/example" +) + +func TestProtoRawFormatter_MarshallUnmarshall(t *testing.T) { + ein := example.ExampleDef{ + Allowed: "happy", + Disallowed: 2, + Example: &example.ExampleDef_Foo{ + Foo: &example.Foo{ + Name: "sad", + }, + }, + } + + fmtr := ProtobufRawFormatter{} + + b, err := fmtr.Marshall(&ein) + if err != nil { + t.Fatal(err) + } + + eout := &example.ExampleDef{} + err = fmtr.Unmarshal(b, eout) + if err != nil { + t.Fatal(err) + } + if ein.Allowed != eout.Allowed { + t.Error("Not allowed") + } + + if e, ok := eout.Example.(*example.ExampleDef_Foo); ok { + if e.Foo.Name != "sad" { + t.Error("foo sad") + } + } else { + t.Error("example not foo") + } +} diff --git a/schematizedavro.go b/schematizedavro.go new file mode 100644 index 0000000..f649601 --- /dev/null +++ b/schematizedavro.go @@ -0,0 +1,60 @@ +package zfmt + +import ( + "bytes" + "encoding/binary" + "errors" + "fmt" +) + +// SchematizedAvroFormatter follows the Confluent Wire Format https://docs.confluent.io/current/schema-registry/serdes-develop/index.html#wire-format +type SchematizedAvroFormatter struct { + formatter AvroFormatter + SchemaID int +} + +// Marshall converts input into avro binary data with schema ID attached +func (p *SchematizedAvroFormatter) Marshall(v any) ([]byte, error) { + return marshall(&p.formatter, p.SchemaID, v) +} + +// Unmarshal fills avro binary data into provided interface v and validates the schema ID +func (p *SchematizedAvroFormatter) Unmarshal(b []byte, v any) error { + return unmarshal(&p.formatter, p.SchemaID, b, v) +} + +// marshall converts input into binary data with schema ID also encoded via the wire format +func marshall(fmtter Formatter, schemaID int, v any) ([]byte, error) { + data, err := fmtter.Marshall(v) + if err != nil { + return nil, err + } + var body bytes.Buffer + // version, always 0 + body.WriteByte(0) + + // 4 byte for schema ID in BigEndian + schemaIDByte := make([]byte, 4) + binary.BigEndian.PutUint32(schemaIDByte, uint32(schemaID)) + body.Write(schemaIDByte) + + // the content + body.Write(data) + return body.Bytes(), nil +} + +// Unmarshal fills binary data into provided interface v and validates the schema ID +func unmarshal(fmtter Formatter, schemaID int, b []byte, v any) error { + if len(b) < 5 { + return errors.New("message does not contain schema") + } + schemaIDBin := b[1:5] + id := int(binary.BigEndian.Uint32(schemaIDBin)) + // for default schema (id == 0), it is implied that the user does not care about ID and attempt to unmarshal at their own risk + // This often happens when the topic is guaranteed to have one data type and the user would like + // to bypass schema validation while still conforming to the confluent wire format. + if schemaID != 0 && schemaID != id { + return fmt.Errorf("schema IDs do not match, expect %d, got %d", schemaID, id) + } + return fmtter.Unmarshal(b[5:], v) +} diff --git a/schematizedavro_test.go b/schematizedavro_test.go new file mode 100644 index 0000000..773ae59 --- /dev/null +++ b/schematizedavro_test.go @@ -0,0 +1,400 @@ +package zfmt + +import ( + "bytes" + "reflect" + "testing" + + "gitlab.zgtools.net/devex/archetypes/gomods/zfmt/testdata/heetch" + + av "gitlab.zgtools.net/devex/archetypes/gomods/zfmt/testdata" +) + +func TestSchematizedAvroFormatter_Marshall(t *testing.T) { + type fields struct { + avroFmt AvroFormatter + SchemaID int + } + type args struct { + v any + } + + tests := []struct { + name string + fields fields + args args + wantErr bool + }{ + { + name: "accept reference type of an avro object, default formatter should be useful", + fields: fields{}, + args: args{ + v: &av.DemoSchema{ + IntField: 123, + DoubleField: 123.4, + StringField: "12345", + BoolField: true, + BytesField: []byte("123456"), + }, + }, + wantErr: false, + }, + { + name: "accept reference type of an avro object, with avro formatter and schemaID", + fields: fields{ + avroFmt: AvroFormatter{}, + SchemaID: 99, + }, + args: args{ + v: &av.DemoSchema{ + IntField: 123, + DoubleField: 123.4, + StringField: "12345", + BoolField: true, + BytesField: []byte("123456"), + }, + }, + wantErr: false, + }, + { + name: "accept heetch avrorecord type", + fields: fields{ + avroFmt: AvroFormatter{}, + SchemaID: 99, + }, + args: args{ + v: heetch.DemoSchema{ + IntField: 123, + DoubleField: 123.4, + StringField: "12345", + BoolField: true, + BytesField: []byte("123456"), + }, + }, + wantErr: false, + }, + { + name: "accept value type of an avro object", + fields: fields{ + avroFmt: AvroFormatter{}, + SchemaID: 99, + }, + args: args{ + v: av.DemoSchema{ + IntField: 123, + DoubleField: 123.4, + StringField: "12345", + BoolField: true, + BytesField: []byte("123456"), + }, + }, + wantErr: false, + }, + { + name: "do not accept random type", + fields: fields{ + avroFmt: AvroFormatter{}, + SchemaID: 99, + }, + args: args{v: "what?"}, + wantErr: true, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + p := &SchematizedAvroFormatter{ + formatter: tt.fields.avroFmt, + SchemaID: tt.fields.SchemaID, + } + _, err := p.Marshall(tt.args.v) + if (err != nil) != tt.wantErr { + t.Errorf("SchematizedAvroFormatter.Marshall() error = %v, wantErr %v", err, tt.wantErr) + return + } + }) + } +} + +func TestSchematizedAvroFormatter_UnmarshalToInvalidAvroType(t *testing.T) { + fmtter := &SchematizedAvroFormatter{SchemaID: 99} + input := &av.DemoSchema{ + IntField: 123, + DoubleField: 123.4, + StringField: "12345", + BoolField: true, + BytesField: []byte("123456"), + } + data, err := fmtter.Marshall(input) + if err != nil { + t.Errorf("should not have error marshalling %v", err) + } + var output bytes.Buffer + err = fmtter.Unmarshal(data, &output) + if err == nil { + t.Errorf("should get error because output type is not an avro object") + } +} + +func TestSchematizedAvroFormatter_UnmarshalNonSchematizedAvro(t *testing.T) { + binAvro := []byte{} + var output bytes.Buffer + outFmtter := &SchematizedAvroFormatter{} + err := outFmtter.Unmarshal(binAvro, &output) + if err == nil { + t.Errorf("should get error because input does not contain schema") + } +} + +func TestSchematizedAvroFormatter_UnmarshalValidAvroWithSchemaID(t *testing.T) { + type testCase struct { + Name string + Input any + Expected any + Output any + } + + testCases := []testCase{ + { + Name: "gogen-avro with with schemaID", + Input: &av.DemoSchema{ + IntField: 123, + DoubleField: 123.4, + StringField: "12345", + BoolField: true, + BytesField: []byte("123456"), + }, + Expected: &av.DemoSchema{ + IntField: 123, + DoubleField: 123.4, + StringField: "12345", + BoolField: true, + BytesField: []byte("123456"), + }, + Output: &av.DemoSchema{}, + }, + { + Name: "heetch avro with with schemaID", + Input: heetch.DemoSchema{ + IntField: 123, + DoubleField: 123.4, + StringField: "12345", + BoolField: true, + BytesField: []byte("123456"), + }, + Expected: &heetch.DemoSchema{ + IntField: 123, + DoubleField: 123.4, + StringField: "12345", + BoolField: true, + BytesField: []byte("123456"), + }, + Output: &heetch.DemoSchema{}, + }, + } + + for _, tc := range testCases { + t.Run(tc.Name, func(t *testing.T) { + fmtter := &SchematizedAvroFormatter{SchemaID: 99} + input := tc.Input + data, err := fmtter.Marshall(input) + if err != nil { + t.Errorf("should not have error marshalling %v", err) + } + output := tc.Output + err = fmtter.Unmarshal(data, output) + if err != nil { + t.Errorf("should not have error marshalling %v", err) + } + if !reflect.DeepEqual(tc.Expected, output) { + t.Errorf("data should match, want=%v, got=%v", input, output) + } + }) + } +} + +func TestSchematizedAvroFormatter_UnmarshalValidAvroWithNoSchemaID(t *testing.T) { + type testCase struct { + Name string + Input any + Expected any + Output any + } + + testCases := []testCase{ + { + Name: "gogen-avro with no schemaID", + Input: &av.DemoSchema{ + IntField: 123, + DoubleField: 123.4, + StringField: "12345", + BoolField: true, + BytesField: []byte("123456"), + }, + Expected: &av.DemoSchema{ + IntField: 123, + DoubleField: 123.4, + StringField: "12345", + BoolField: true, + BytesField: []byte("123456"), + }, + Output: &av.DemoSchema{}, + }, + { + Name: "heetch avro with no schemaID", + Input: heetch.DemoSchema{ + IntField: 123, + DoubleField: 123.4, + StringField: "12345", + BoolField: true, + BytesField: []byte("123456"), + }, + Expected: &heetch.DemoSchema{ + IntField: 123, + DoubleField: 123.4, + StringField: "12345", + BoolField: true, + BytesField: []byte("123456"), + }, + Output: &heetch.DemoSchema{}, + }, + } + + for _, tc := range testCases { + t.Run(tc.Name, func(t *testing.T) { + inFmtter := &SchematizedAvroFormatter{SchemaID: 99} + input := tc.Input + data, err := inFmtter.Marshall(input) + if err != nil { + t.Errorf("should not have error marshalling %v", err) + } + + // when schemaID is unset, as long as the data is unmarshallable, we don't throw error + outFmtter := &SchematizedAvroFormatter{} + output := tc.Output + err = outFmtter.Unmarshal(data, output) + if err != nil { + t.Errorf("should not have error marshalling %v", err) + } + if !reflect.DeepEqual(tc.Expected, output) { + t.Errorf("data should match, want=%v, got=%v", input, output) + } + }) + } + +} + +func TestSchematizedAvroFormatter_UnmarshalValidAvroWithWrongSchemaID(t *testing.T) { + type testCase struct { + Name string + Input any + Output any + } + + testCases := []testCase{ + { + Name: "gogen-avro with wrong schemaID", + Input: &av.DemoSchema{ + IntField: 123, + DoubleField: 123.4, + StringField: "12345", + BoolField: true, + BytesField: []byte("123456"), + }, + Output: &av.DemoSchema{}, + }, + { + Name: "heetch avro with wrong schemaID", + Input: heetch.DemoSchema{ + IntField: 123, + DoubleField: 123.4, + StringField: "12345", + BoolField: true, + BytesField: []byte("123456"), + }, + Output: &heetch.DemoSchema{}, + }, + } + + for _, tc := range testCases { + t.Run(tc.Name, func(t *testing.T) { + inFmtter := &SchematizedAvroFormatter{SchemaID: 99} + data, err := inFmtter.Marshall(tc.Input) + if err != nil { + t.Errorf("should not have error marshalling %v", err) + } + + outFmtter := &SchematizedAvroFormatter{SchemaID: 100} + err = outFmtter.Unmarshal(data, tc.Output) + if err == nil { + t.Error("should have error unmarshalling due to incorrect schema ID") + } + }) + } +} + +func TestSchematizedAvroFormatter_Equivalency(t *testing.T) { + type testCase struct { + Name string + Input any + Expected any + Output any + } + + testCases := []testCase{ + { + Name: "marshal gogen-avro unmarshal heetch avro", + Input: &av.DemoSchema{ + IntField: 123, + DoubleField: 123.4, + StringField: "12345", + BoolField: true, + BytesField: []byte("123456"), + }, + Expected: &heetch.DemoSchema{ + IntField: 123, + DoubleField: 123.4, + StringField: "12345", + BoolField: true, + BytesField: []byte("123456"), + }, + Output: &heetch.DemoSchema{}, + }, + { + Name: "marshall heetch unmarshall gogen-avro", + Input: heetch.DemoSchema{ + IntField: 123, + DoubleField: 123.4, + StringField: "12345", + BoolField: true, + BytesField: []byte("123456"), + }, + Expected: &av.DemoSchema{ + IntField: 123, + DoubleField: 123.4, + StringField: "12345", + BoolField: true, + BytesField: []byte("123456"), + }, + Output: &av.DemoSchema{}, + }, + } + + for _, tc := range testCases { + t.Run(tc.Name, func(t *testing.T) { + fmtter := &SchematizedAvroFormatter{SchemaID: 99} + input := tc.Input + data, err := fmtter.Marshall(input) + if err != nil { + t.Errorf("should not have error marshalling %v", err) + } + output := tc.Output + err = fmtter.Unmarshal(data, output) + if err != nil { + t.Errorf("should not have error marshalling %v", err) + } + if !reflect.DeepEqual(tc.Expected, output) { + t.Errorf("data should match, want=%v, got=%v", input, output) + } + }) + } +} diff --git a/schematizedjson.go b/schematizedjson.go new file mode 100644 index 0000000..ffc8c2e --- /dev/null +++ b/schematizedjson.go @@ -0,0 +1,17 @@ +package zfmt + +// SchematizedJSONFormatter follows the Confluent Wire Format https://docs.confluent.io/current/schema-registry/serdes-develop/index.html#wire-format +type SchematizedJSONFormatter struct { + formatter JSONFormatter + SchemaID int +} + +// Marshall converts input into avro binary data with schema ID attached +func (p *SchematizedJSONFormatter) Marshall(v any) ([]byte, error) { + return marshall(&p.formatter, p.SchemaID, v) +} + +// Unmarshal fills avro binary data into provided interface v and validates the schema ID +func (p *SchematizedJSONFormatter) Unmarshal(b []byte, v any) error { + return unmarshal(&p.formatter, p.SchemaID, b, v) +} diff --git a/schematizedjson_test.go b/schematizedjson_test.go new file mode 100644 index 0000000..4be10d0 --- /dev/null +++ b/schematizedjson_test.go @@ -0,0 +1,114 @@ +package zfmt + +import ( + "testing" + + "github.com/google/go-cmp/cmp" + + av "gitlab.zgtools.net/devex/archetypes/gomods/zfmt/testdata" +) + +func TestSchematizedJsonFormatter_Marshall(t *testing.T) { + type fields struct { + protobufFormatter JSONFormatter + SchemaID int + } + type args struct { + v any + } + tests := []struct { + name string + fields fields + args args + wantErr bool + }{ + { + name: "accept reference type of an object, default formatter should be useful", + fields: fields{}, + args: args{ + v: &av.ExampleJson{ + Id: "hello", + }, + }, + wantErr: false, + }, + { + name: "accept reference type of an object, with avro formatter and schemaID", + fields: fields{ + protobufFormatter: JSONFormatter{}, + SchemaID: 99, + }, + args: args{ + v: &av.ExampleJson{ + Id: "hello", + }, + }, + wantErr: false, + }, + { + name: "accept value type of an object, with avro formatter and schemaID", + fields: fields{ + protobufFormatter: JSONFormatter{}, + SchemaID: 99, + }, + args: args{ + v: av.ExampleJson{ + Id: "hello", + }, + }, + wantErr: false, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + p := &SchematizedJSONFormatter{ + formatter: tt.fields.protobufFormatter, + SchemaID: tt.fields.SchemaID, + } + _, err := p.Marshall(tt.args.v) + if (err != nil) != tt.wantErr { + t.Errorf("SchematizedProtoFormatter.Marshall() error = %v, wantErr %v", err, tt.wantErr) + return + } + }) + } +} + +func TestSchematizedJsonFormmater_Unmarshall_PointerType(t *testing.T) { + fmtter := SchematizedJSONFormatter{ + formatter: JSONFormatter{}, + SchemaID: 123, + } + expected := av.ExampleJson{Id: "what"} + b, err := fmtter.Marshall(&expected) + if err != nil { + t.Fatal(err) + } + reslt := av.ExampleJson{} + if err := fmtter.Unmarshal(b, &reslt); err != nil { + t.Fatal(err) + } + diff := cmp.Diff(&reslt, &expected) + if diff != "" { + t.Fatal(diff) + } +} +func TestSchematizedJsonFormmater_Unmarshalll_ValueType(t *testing.T) { + fmtter := SchematizedJSONFormatter{ + formatter: JSONFormatter{}, + SchemaID: 123, + } + expected := av.ExampleJson{Id: "what"} + b, err := fmtter.Marshall(expected) + if err != nil { + t.Fatal(err) + } + reslt := av.ExampleJson{} + if err := fmtter.Unmarshal(b, &reslt); err != nil { + t.Fatal(err) + } + diff := cmp.Diff(&reslt, &expected) + if diff != "" { + t.Fatal(diff) + } +} diff --git a/schematizedproto_deprecated.go b/schematizedproto_deprecated.go new file mode 100644 index 0000000..62d5b73 --- /dev/null +++ b/schematizedproto_deprecated.go @@ -0,0 +1,17 @@ +package zfmt + +// SchematizedProtoFormatterDeprecated follows the Confluent Wire Format https://docs.confluent.io/current/schema-registry/serdes-develop/index.html#wire-format +type SchematizedProtoFormatterDeprecated struct { + formatter ProtobufBase64Formatter + SchemaID int +} + +// Marshall converts input into avro binary data with schema ID attached +func (p *SchematizedProtoFormatterDeprecated) Marshall(v any) ([]byte, error) { + return marshall(&p.formatter, p.SchemaID, v) +} + +// Unmarshal fills avro binary data into provided interface v and validates the schema ID +func (p *SchematizedProtoFormatterDeprecated) Unmarshal(b []byte, v any) error { + return unmarshal(&p.formatter, p.SchemaID, b, v) +} diff --git a/schematizedproto_deprecated_test.go b/schematizedproto_deprecated_test.go new file mode 100644 index 0000000..aa17979 --- /dev/null +++ b/schematizedproto_deprecated_test.go @@ -0,0 +1,107 @@ +package zfmt + +import ( + "testing" + + "github.com/google/go-cmp/cmp" + "github.com/google/go-cmp/cmp/cmpopts" + example2 "gitlab.zgtools.net/devex/archetypes/gomods/zfmt/testdata/example" +) + +func TestSchematizedProtoDeprecatedFormatter_Marshall(t *testing.T) { + type fields struct { + protobufFormatter ProtobufBase64Formatter + SchemaID int + } + type args struct { + v any + } + tests := []struct { + name string + fields fields + args args + wantErr bool + }{ + { + name: "accept reference type of an proto object, default formatter should be useful", + fields: fields{}, + args: args{ + v: &example2.ExampleDef{ + Allowed: "!23", + Disallowed: 7, + }, + }, + wantErr: false, + }, + { + name: "accept reference type of an proto object, with avro formatter and schemaID", + fields: fields{ + protobufFormatter: ProtobufBase64Formatter{}, + SchemaID: 99, + }, + args: args{ + v: &example2.ExampleDef{ + Allowed: "!23", + Disallowed: 7, + }, + }, + wantErr: false, + }, + { + name: "do not accept value type of a proto object", + fields: fields{ + protobufFormatter: ProtobufBase64Formatter{}, + SchemaID: 99, + }, + args: args{ + v: example2.ExampleDef{ + Allowed: "!23", + Disallowed: 7, + }, + }, + wantErr: true, + }, + { + name: "do not accept random type", + fields: fields{ + protobufFormatter: ProtobufBase64Formatter{}, + SchemaID: 99, + }, + args: args{v: "what?"}, + wantErr: true, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + p := &SchematizedProtoFormatterDeprecated{ + formatter: tt.fields.protobufFormatter, + SchemaID: tt.fields.SchemaID, + } + _, err := p.Marshall(tt.args.v) + if (err != nil) != tt.wantErr { + t.Errorf("SchematizedProtoFormatter.Marshall() error = %v, wantErr %v", err, tt.wantErr) + return + } + }) + } +} + +func TestSchematizedProtoDeprecatedFormmater_Unmarshall(t *testing.T) { + fmtter := SchematizedProtoFormatterDeprecated{ + formatter: ProtobufBase64Formatter{}, + SchemaID: 123, + } + expected := example2.ExampleDef{Allowed: "1243", Disallowed: 987} + b, err := fmtter.Marshall(&expected) + if err != nil { + t.Fatal(err) + } + reslt := example2.ExampleDef{} + if err := fmtter.Unmarshal(b, &reslt); err != nil { + t.Fatal(err) + } + diff := cmp.Diff(&reslt, &expected, cmpopts.IgnoreUnexported(example2.ExampleDef{})) + if diff != "" { + t.Fatal(diff) + } +} diff --git a/string.go b/string.go new file mode 100644 index 0000000..6a3c4fa --- /dev/null +++ b/string.go @@ -0,0 +1,34 @@ +package zfmt + +import ( + "fmt" + "io" +) + +// StringFormatter ... +type StringFormatter struct{} + +// Marshall ... +func (f *StringFormatter) Marshall(i any) ([]byte, error) { + switch v := i.(type) { + case string: + return []byte(v), nil + case []byte: + return v, nil + case io.Reader: + return io.ReadAll(v) + default: + return nil, fmt.Errorf("cannot convert to string %T", i) + } +} + +func (f *StringFormatter) Unmarshal(b []byte, i any) error { + // since string is immutable, we rely on io.Writer + switch v := i.(type) { + case io.Writer: + _, err := v.Write(b) + return err + default: + return fmt.Errorf("cannot convert from binary %T", i) + } +} diff --git a/string_test.go b/string_test.go new file mode 100644 index 0000000..3a90548 --- /dev/null +++ b/string_test.go @@ -0,0 +1,102 @@ +package zfmt + +import ( + "bytes" + "reflect" + "testing" +) + +func TestStringFormatter_Marshall(t *testing.T) { + type args struct { + i any + } + tests := []struct { + name string + f *StringFormatter + args args + want []byte + wantErr bool + }{ + { + name: "string", + f: &StringFormatter{}, + args: args{i: "test"}, + want: []byte("test"), + wantErr: false, + }, + { + name: "byte array", + f: &StringFormatter{}, + args: args{i: []byte("test")}, + want: []byte("test"), + wantErr: false, + }, + { + name: "bytes buffer", + f: &StringFormatter{}, + args: args{i: bytes.NewBufferString("test")}, + want: []byte("test"), + wantErr: false, + }, + { + name: "invalid type", + f: &StringFormatter{}, + args: args{i: 123}, + want: nil, + wantErr: true, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + f := &StringFormatter{} + got, err := f.Marshall(tt.args.i) + if (err != nil) != tt.wantErr { + t.Errorf("StringFormatter.Marshall() error = %v, wantErr %v", err, tt.wantErr) + return + } + if !reflect.DeepEqual(got, tt.want) { + t.Errorf("StringFormatter.Marshall() = %v, want %v", got, tt.want) + } + }) + } +} + +func TestStringFormatter_Unmarshal(t *testing.T) { + type args struct { + b []byte + i any + } + tests := []struct { + name string + f *StringFormatter + args args + wantErr bool + }{ + { + name: "string is immutable so that doesn't work", + f: &StringFormatter{}, + args: args{ + b: []byte("test"), + i: func(str string) *string { return &str }(""), + }, + wantErr: true, + }, + { + name: "supply io.Writer", + f: &StringFormatter{}, + args: args{ + b: []byte("test"), + i: new(bytes.Buffer), + }, + wantErr: false, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + f := &StringFormatter{} + if err := f.Unmarshal(tt.args.b, tt.args.i); (err != nil) != tt.wantErr { + t.Errorf("StringFormatter.Unmarshal() error = %v, wantErr %v", err, tt.wantErr) + } + }) + } +} diff --git a/testdata/bytes.go b/testdata/bytes.go new file mode 100644 index 0000000..9b79da9 --- /dev/null +++ b/testdata/bytes.go @@ -0,0 +1,90 @@ +// Code generated by github.com/actgardner/gogen-avro/v10. DO NOT EDIT. +/* + * SOURCE: + * example.avsc + */ +package avro + +import ( + "encoding/json" + + "github.com/actgardner/gogen-avro/v10/util" + "github.com/actgardner/gogen-avro/v10/vm/types" +) + +type Bytes []byte + +func (b *Bytes) UnmarshalJSON(data []byte) error { + var s string + if err := json.Unmarshal(data, &s); err != nil { + return err + } + *b = util.DecodeByteString(s) + return nil +} + +func (b Bytes) MarshalJSON() ([]byte, error) { + return []byte(util.EncodeByteString(b)), nil +} + +type BytesWrapper struct { + Target *Bytes +} + +func (b BytesWrapper) SetBoolean(v bool) { + panic("Unable to assign bytes to bytes field") +} + +func (b BytesWrapper) SetInt(v int32) { + panic("Unable to assign int to bytes field") +} + +func (b BytesWrapper) SetLong(v int64) { + panic("Unable to assign long to bytes field") +} + +func (b BytesWrapper) SetFloat(v float32) { + panic("Unable to assign float to bytes field") +} + +func (b BytesWrapper) SetDouble(v float64) { + panic("Unable to assign double to bytes field") +} + +func (b BytesWrapper) SetUnionElem(v int64) { + panic("Unable to assign union elem to bytes field") +} + +func (b BytesWrapper) SetBytes(v []byte) { + *(b.Target) = v +} + +func (b BytesWrapper) SetString(v string) { + *(b.Target) = []byte(v) +} + +func (b BytesWrapper) Get(i int) types.Field { + panic("Unable to get field from bytes field") +} + +func (b BytesWrapper) SetDefault(i int) { + panic("Unable to set default on bytes field") +} + +func (b BytesWrapper) AppendMap(key string) types.Field { + panic("Unable to append map key to from bytes field") +} + +func (b BytesWrapper) AppendArray() types.Field { + panic("Unable to append array element to from bytes field") +} + +func (b BytesWrapper) NullField(int) { + panic("Unable to null field in bytes field") +} + +func (b BytesWrapper) HintSize(int) { + panic("Unable to hint size in bytes field") +} + +func (b BytesWrapper) Finalize() {} diff --git a/testdata/demo_schema.go b/testdata/demo_schema.go new file mode 100644 index 0000000..1ee60c5 --- /dev/null +++ b/testdata/demo_schema.go @@ -0,0 +1,264 @@ +// Code generated by github.com/actgardner/gogen-avro/v10. DO NOT EDIT. +/* + * SOURCE: + * example.avsc + */ +package avro + +import ( + "encoding/json" + "fmt" + "io" + + "github.com/actgardner/gogen-avro/v10/compiler" + "github.com/actgardner/gogen-avro/v10/vm" + "github.com/actgardner/gogen-avro/v10/vm/types" +) + +var _ = fmt.Printf + +type DemoSchema struct { + IntField int32 `json:"IntField"` + + DoubleField float64 `json:"DoubleField"` + + StringField string `json:"StringField"` + + BoolField bool `json:"BoolField"` + + BytesField Bytes `json:"BytesField"` +} + +const DemoSchemaAvroCRC64Fingerprint = "\xc4V\xa9\x04ʛf\xad" + +func NewDemoSchema() DemoSchema { + r := DemoSchema{} + return r +} + +func DeserializeDemoSchema(r io.Reader) (DemoSchema, error) { + t := NewDemoSchema() + deser, err := compiler.CompileSchemaBytes([]byte(t.Schema()), []byte(t.Schema())) + if err != nil { + return t, err + } + + err = vm.Eval(r, deser, &t) + return t, err +} + +func DeserializeDemoSchemaFromSchema(r io.Reader, schema string) (DemoSchema, error) { + t := NewDemoSchema() + + deser, err := compiler.CompileSchemaBytes([]byte(schema), []byte(t.Schema())) + if err != nil { + return t, err + } + + err = vm.Eval(r, deser, &t) + return t, err +} + +func writeDemoSchema(r DemoSchema, w io.Writer) error { + var err error + err = vm.WriteInt(r.IntField, w) + if err != nil { + return err + } + err = vm.WriteDouble(r.DoubleField, w) + if err != nil { + return err + } + err = vm.WriteString(r.StringField, w) + if err != nil { + return err + } + err = vm.WriteBool(r.BoolField, w) + if err != nil { + return err + } + err = vm.WriteBytes(r.BytesField, w) + if err != nil { + return err + } + return err +} + +func (r DemoSchema) Serialize(w io.Writer) error { + return writeDemoSchema(r, w) +} + +func (r DemoSchema) Schema() string { + return "{\"fields\":[{\"name\":\"IntField\",\"type\":\"int\"},{\"name\":\"DoubleField\",\"type\":\"double\"},{\"name\":\"StringField\",\"type\":\"string\"},{\"name\":\"BoolField\",\"type\":\"boolean\"},{\"name\":\"BytesField\",\"type\":\"bytes\"}],\"name\":\"DemoSchema\",\"type\":\"record\"}" +} + +func (r DemoSchema) SchemaName() string { + return "DemoSchema" +} + +func (_ DemoSchema) SetBoolean(v bool) { panic("Unsupported operation") } +func (_ DemoSchema) SetInt(v int32) { panic("Unsupported operation") } +func (_ DemoSchema) SetLong(v int64) { panic("Unsupported operation") } +func (_ DemoSchema) SetFloat(v float32) { panic("Unsupported operation") } +func (_ DemoSchema) SetDouble(v float64) { panic("Unsupported operation") } +func (_ DemoSchema) SetBytes(v []byte) { panic("Unsupported operation") } +func (_ DemoSchema) SetString(v string) { panic("Unsupported operation") } +func (_ DemoSchema) SetUnionElem(v int64) { panic("Unsupported operation") } + +func (r *DemoSchema) Get(i int) types.Field { + switch i { + case 0: + w := types.Int{Target: &r.IntField} + + return w + + case 1: + w := types.Double{Target: &r.DoubleField} + + return w + + case 2: + w := types.String{Target: &r.StringField} + + return w + + case 3: + w := types.Boolean{Target: &r.BoolField} + + return w + + case 4: + w := BytesWrapper{Target: &r.BytesField} + + return w + + } + panic("Unknown field index") +} + +func (r *DemoSchema) SetDefault(i int) { + switch i { + } + panic("Unknown field index") +} + +func (r *DemoSchema) NullField(i int) { + switch i { + } + panic("Not a nullable field index") +} + +func (_ DemoSchema) AppendMap(key string) types.Field { panic("Unsupported operation") } +func (_ DemoSchema) AppendArray() types.Field { panic("Unsupported operation") } +func (_ DemoSchema) HintSize(int) { panic("Unsupported operation") } +func (_ DemoSchema) Finalize() {} + +func (_ DemoSchema) AvroCRC64Fingerprint() []byte { + return []byte(DemoSchemaAvroCRC64Fingerprint) +} + +func (r DemoSchema) MarshalJSON() ([]byte, error) { + var err error + output := make(map[string]json.RawMessage) + output["IntField"], err = json.Marshal(r.IntField) + if err != nil { + return nil, err + } + output["DoubleField"], err = json.Marshal(r.DoubleField) + if err != nil { + return nil, err + } + output["StringField"], err = json.Marshal(r.StringField) + if err != nil { + return nil, err + } + output["BoolField"], err = json.Marshal(r.BoolField) + if err != nil { + return nil, err + } + output["BytesField"], err = json.Marshal(r.BytesField) + if err != nil { + return nil, err + } + return json.Marshal(output) +} + +func (r *DemoSchema) UnmarshalJSON(data []byte) error { + var fields map[string]json.RawMessage + if err := json.Unmarshal(data, &fields); err != nil { + return err + } + + var val json.RawMessage + val = func() json.RawMessage { + if v, ok := fields["IntField"]; ok { + return v + } + return nil + }() + + if val != nil { + if err := json.Unmarshal([]byte(val), &r.IntField); err != nil { + return err + } + } else { + return fmt.Errorf("no value specified for IntField") + } + val = func() json.RawMessage { + if v, ok := fields["DoubleField"]; ok { + return v + } + return nil + }() + + if val != nil { + if err := json.Unmarshal([]byte(val), &r.DoubleField); err != nil { + return err + } + } else { + return fmt.Errorf("no value specified for DoubleField") + } + val = func() json.RawMessage { + if v, ok := fields["StringField"]; ok { + return v + } + return nil + }() + + if val != nil { + if err := json.Unmarshal([]byte(val), &r.StringField); err != nil { + return err + } + } else { + return fmt.Errorf("no value specified for StringField") + } + val = func() json.RawMessage { + if v, ok := fields["BoolField"]; ok { + return v + } + return nil + }() + + if val != nil { + if err := json.Unmarshal([]byte(val), &r.BoolField); err != nil { + return err + } + } else { + return fmt.Errorf("no value specified for BoolField") + } + val = func() json.RawMessage { + if v, ok := fields["BytesField"]; ok { + return v + } + return nil + }() + + if val != nil { + if err := json.Unmarshal([]byte(val), &r.BytesField); err != nil { + return err + } + } else { + return fmt.Errorf("no value specified for BytesField") + } + return nil +} diff --git a/testdata/demo_schema_container.go b/testdata/demo_schema_container.go new file mode 100644 index 0000000..c5d5a8a --- /dev/null +++ b/testdata/demo_schema_container.go @@ -0,0 +1,49 @@ +// Code generated by github.com/actgardner/gogen-avro/v10. DO NOT EDIT. +/* + * SOURCE: + * example.avsc + */ +package avro + +import ( + "io" + + "github.com/actgardner/gogen-avro/v10/compiler" + "github.com/actgardner/gogen-avro/v10/container" + "github.com/actgardner/gogen-avro/v10/vm" +) + +func NewDemoSchemaWriter(writer io.Writer, codec container.Codec, recordsPerBlock int64) (*container.Writer, error) { + str := NewDemoSchema() + return container.NewWriter(writer, codec, recordsPerBlock, str.Schema()) +} + +// container reader +type DemoSchemaReader struct { + r io.Reader + p *vm.Program +} + +func NewDemoSchemaReader(r io.Reader) (*DemoSchemaReader, error) { + containerReader, err := container.NewReader(r) + if err != nil { + return nil, err + } + + t := NewDemoSchema() + deser, err := compiler.CompileSchemaBytes([]byte(containerReader.AvroContainerSchema()), []byte(t.Schema())) + if err != nil { + return nil, err + } + + return &DemoSchemaReader{ + r: containerReader, + p: deser, + }, nil +} + +func (r DemoSchemaReader) Read() (DemoSchema, error) { + t := NewDemoSchema() + err := vm.Eval(r.r, r.p, &t) + return t, err +} diff --git a/testdata/example.avsc b/testdata/example.avsc new file mode 100644 index 0000000..7125155 --- /dev/null +++ b/testdata/example.avsc @@ -0,0 +1,12 @@ +{ + "type": "record", + "name": "DemoSchema", + "fields": [ + {"name": "IntField", "type": "int"}, + {"name": "DoubleField", "type": "double"}, + {"name": "StringField", "type": "string"}, + {"name": "BoolField", "type": "boolean"}, + {"name": "BytesField", "type": "bytes"} + ] + +} \ No newline at end of file diff --git a/testdata/example.proto b/testdata/example.proto new file mode 100644 index 0000000..22bd091 --- /dev/null +++ b/testdata/example.proto @@ -0,0 +1,17 @@ +syntax = "proto3"; + +package tagging; + +option go_package = "./example"; + +message ExampleDef { + string allowed = 1; + int32 disallowed = 2; + oneof example{ + Foo foo = 3; + Bar bar = 4; + } +} + +message Foo{string name = 1;} +message Bar{string name = 1;} \ No newline at end of file diff --git a/testdata/example/example.pb.go b/testdata/example/example.pb.go new file mode 100644 index 0000000..338c3c1 --- /dev/null +++ b/testdata/example/example.pb.go @@ -0,0 +1,328 @@ +// Code generated by protoc-gen-go. DO NOT EDIT. +// versions: +// protoc-gen-go v1.30.0 +// protoc v3.21.12 +// source: example.proto + +package example + +import ( + protoreflect "google.golang.org/protobuf/reflect/protoreflect" + protoimpl "google.golang.org/protobuf/runtime/protoimpl" + reflect "reflect" + sync "sync" +) + +const ( + // Verify that this generated code is sufficiently up-to-date. + _ = protoimpl.EnforceVersion(20 - protoimpl.MinVersion) + // Verify that runtime/protoimpl is sufficiently up-to-date. + _ = protoimpl.EnforceVersion(protoimpl.MaxVersion - 20) +) + +type ExampleDef struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + Allowed string `protobuf:"bytes,1,opt,name=allowed,proto3" json:"allowed,omitempty"` + Disallowed int32 `protobuf:"varint,2,opt,name=disallowed,proto3" json:"disallowed,omitempty"` + // Types that are assignable to Example: + // + // *ExampleDef_Foo + // *ExampleDef_Bar + Example isExampleDef_Example `protobuf_oneof:"example"` +} + +func (x *ExampleDef) Reset() { + *x = ExampleDef{} + if protoimpl.UnsafeEnabled { + mi := &file_example_proto_msgTypes[0] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *ExampleDef) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*ExampleDef) ProtoMessage() {} + +func (x *ExampleDef) ProtoReflect() protoreflect.Message { + mi := &file_example_proto_msgTypes[0] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use ExampleDef.ProtoReflect.Descriptor instead. +func (*ExampleDef) Descriptor() ([]byte, []int) { + return file_example_proto_rawDescGZIP(), []int{0} +} + +func (x *ExampleDef) GetAllowed() string { + if x != nil { + return x.Allowed + } + return "" +} + +func (x *ExampleDef) GetDisallowed() int32 { + if x != nil { + return x.Disallowed + } + return 0 +} + +func (m *ExampleDef) GetExample() isExampleDef_Example { + if m != nil { + return m.Example + } + return nil +} + +func (x *ExampleDef) GetFoo() *Foo { + if x, ok := x.GetExample().(*ExampleDef_Foo); ok { + return x.Foo + } + return nil +} + +func (x *ExampleDef) GetBar() *Bar { + if x, ok := x.GetExample().(*ExampleDef_Bar); ok { + return x.Bar + } + return nil +} + +type isExampleDef_Example interface { + isExampleDef_Example() +} + +type ExampleDef_Foo struct { + Foo *Foo `protobuf:"bytes,3,opt,name=foo,proto3,oneof"` +} + +type ExampleDef_Bar struct { + Bar *Bar `protobuf:"bytes,4,opt,name=bar,proto3,oneof"` +} + +func (*ExampleDef_Foo) isExampleDef_Example() {} + +func (*ExampleDef_Bar) isExampleDef_Example() {} + +type Foo struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + Name string `protobuf:"bytes,1,opt,name=name,proto3" json:"name,omitempty"` +} + +func (x *Foo) Reset() { + *x = Foo{} + if protoimpl.UnsafeEnabled { + mi := &file_example_proto_msgTypes[1] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *Foo) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*Foo) ProtoMessage() {} + +func (x *Foo) ProtoReflect() protoreflect.Message { + mi := &file_example_proto_msgTypes[1] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use Foo.ProtoReflect.Descriptor instead. +func (*Foo) Descriptor() ([]byte, []int) { + return file_example_proto_rawDescGZIP(), []int{1} +} + +func (x *Foo) GetName() string { + if x != nil { + return x.Name + } + return "" +} + +type Bar struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + Name string `protobuf:"bytes,1,opt,name=name,proto3" json:"name,omitempty"` +} + +func (x *Bar) Reset() { + *x = Bar{} + if protoimpl.UnsafeEnabled { + mi := &file_example_proto_msgTypes[2] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *Bar) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*Bar) ProtoMessage() {} + +func (x *Bar) ProtoReflect() protoreflect.Message { + mi := &file_example_proto_msgTypes[2] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use Bar.ProtoReflect.Descriptor instead. +func (*Bar) Descriptor() ([]byte, []int) { + return file_example_proto_rawDescGZIP(), []int{2} +} + +func (x *Bar) GetName() string { + if x != nil { + return x.Name + } + return "" +} + +var File_example_proto protoreflect.FileDescriptor + +var file_example_proto_rawDesc = []byte{ + 0x0a, 0x0d, 0x65, 0x78, 0x61, 0x6d, 0x70, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x12, + 0x07, 0x74, 0x61, 0x67, 0x67, 0x69, 0x6e, 0x67, 0x22, 0x95, 0x01, 0x0a, 0x0a, 0x45, 0x78, 0x61, + 0x6d, 0x70, 0x6c, 0x65, 0x44, 0x65, 0x66, 0x12, 0x18, 0x0a, 0x07, 0x61, 0x6c, 0x6c, 0x6f, 0x77, + 0x65, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x07, 0x61, 0x6c, 0x6c, 0x6f, 0x77, 0x65, + 0x64, 0x12, 0x1e, 0x0a, 0x0a, 0x64, 0x69, 0x73, 0x61, 0x6c, 0x6c, 0x6f, 0x77, 0x65, 0x64, 0x18, + 0x02, 0x20, 0x01, 0x28, 0x05, 0x52, 0x0a, 0x64, 0x69, 0x73, 0x61, 0x6c, 0x6c, 0x6f, 0x77, 0x65, + 0x64, 0x12, 0x20, 0x0a, 0x03, 0x66, 0x6f, 0x6f, 0x18, 0x03, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x0c, + 0x2e, 0x74, 0x61, 0x67, 0x67, 0x69, 0x6e, 0x67, 0x2e, 0x46, 0x6f, 0x6f, 0x48, 0x00, 0x52, 0x03, + 0x66, 0x6f, 0x6f, 0x12, 0x20, 0x0a, 0x03, 0x62, 0x61, 0x72, 0x18, 0x04, 0x20, 0x01, 0x28, 0x0b, + 0x32, 0x0c, 0x2e, 0x74, 0x61, 0x67, 0x67, 0x69, 0x6e, 0x67, 0x2e, 0x42, 0x61, 0x72, 0x48, 0x00, + 0x52, 0x03, 0x62, 0x61, 0x72, 0x42, 0x09, 0x0a, 0x07, 0x65, 0x78, 0x61, 0x6d, 0x70, 0x6c, 0x65, + 0x22, 0x19, 0x0a, 0x03, 0x46, 0x6f, 0x6f, 0x12, 0x12, 0x0a, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x18, + 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x22, 0x19, 0x0a, 0x03, 0x42, + 0x61, 0x72, 0x12, 0x12, 0x0a, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, + 0x52, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x42, 0x0b, 0x5a, 0x09, 0x2e, 0x2f, 0x65, 0x78, 0x61, 0x6d, + 0x70, 0x6c, 0x65, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33, +} + +var ( + file_example_proto_rawDescOnce sync.Once + file_example_proto_rawDescData = file_example_proto_rawDesc +) + +func file_example_proto_rawDescGZIP() []byte { + file_example_proto_rawDescOnce.Do(func() { + file_example_proto_rawDescData = protoimpl.X.CompressGZIP(file_example_proto_rawDescData) + }) + return file_example_proto_rawDescData +} + +var file_example_proto_msgTypes = make([]protoimpl.MessageInfo, 3) +var file_example_proto_goTypes = []any{ + (*ExampleDef)(nil), // 0: tagging.ExampleDef + (*Foo)(nil), // 1: tagging.Foo + (*Bar)(nil), // 2: tagging.Bar +} +var file_example_proto_depIdxs = []int32{ + 1, // 0: tagging.ExampleDef.foo:type_name -> tagging.Foo + 2, // 1: tagging.ExampleDef.bar:type_name -> tagging.Bar + 2, // [2:2] is the sub-list for method output_type + 2, // [2:2] is the sub-list for method input_type + 2, // [2:2] is the sub-list for extension type_name + 2, // [2:2] is the sub-list for extension extendee + 0, // [0:2] is the sub-list for field type_name +} + +func init() { file_example_proto_init() } +func file_example_proto_init() { + if File_example_proto != nil { + return + } + if !protoimpl.UnsafeEnabled { + file_example_proto_msgTypes[0].Exporter = func(v any, i int) any { + switch v := v.(*ExampleDef); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + file_example_proto_msgTypes[1].Exporter = func(v any, i int) any { + switch v := v.(*Foo); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + file_example_proto_msgTypes[2].Exporter = func(v any, i int) any { + switch v := v.(*Bar); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + } + file_example_proto_msgTypes[0].OneofWrappers = []any{ + (*ExampleDef_Foo)(nil), + (*ExampleDef_Bar)(nil), + } + type x struct{} + out := protoimpl.TypeBuilder{ + File: protoimpl.DescBuilder{ + GoPackagePath: reflect.TypeOf(x{}).PkgPath(), + RawDescriptor: file_example_proto_rawDesc, + NumEnums: 0, + NumMessages: 3, + NumExtensions: 0, + NumServices: 0, + }, + GoTypes: file_example_proto_goTypes, + DependencyIndexes: file_example_proto_depIdxs, + MessageInfos: file_example_proto_msgTypes, + }.Build() + File_example_proto = out.File + file_example_proto_rawDesc = nil + file_example_proto_goTypes = nil + file_example_proto_depIdxs = nil +} diff --git a/testdata/example_json.go b/testdata/example_json.go new file mode 100644 index 0000000..2ada915 --- /dev/null +++ b/testdata/example_json.go @@ -0,0 +1,5 @@ +package avro + +type ExampleJson struct { + Id string +} diff --git a/testdata/generator.go b/testdata/generator.go new file mode 100644 index 0000000..76ad615 --- /dev/null +++ b/testdata/generator.go @@ -0,0 +1,5 @@ +package avro + +//go:generate $GOPATH/bin/gogen-avro -containers . ./example.avsc +//go:generate protoc --proto_path=. --go_out=./ ./example.proto +//go:generate $GOPATH/bin/avrogo -p heetch -d ./heetch ./example.avsc diff --git a/testdata/heetch/example_gen.go b/testdata/heetch/example_gen.go new file mode 100644 index 0000000..028b0c0 --- /dev/null +++ b/testdata/heetch/example_gen.go @@ -0,0 +1,29 @@ +// Code generated by avrogen. DO NOT EDIT. + +package heetch + +import ( + "github.com/heetch/avro/avrotypegen" +) + +type DemoSchema struct { + IntField int + DoubleField float64 + StringField string + BoolField bool + BytesField []byte +} + +// AvroRecord implements the avro.AvroRecord interface. +func (DemoSchema) AvroRecord() avrotypegen.RecordInfo { + return avrotypegen.RecordInfo{ + Schema: `{"fields":[{"name":"IntField","type":"int"},{"name":"DoubleField","type":"double"},{"name":"StringField","type":"string"},{"name":"BoolField","type":"boolean"},{"name":"BytesField","type":"bytes"}],"name":"DemoSchema","type":"record"}`, + Required: []bool{ + 0: true, + 1: true, + 2: true, + 3: true, + 4: true, + }, + } +}