From 4bb7e951a213f3b5fe1558178a3113930a1ade9e Mon Sep 17 00:00:00 2001 From: Gregory Oschwald Date: Tue, 18 Aug 2020 14:55:15 -0700 Subject: [PATCH 1/4] Add experimental Deserializer interface --- decoder.go | 127 ++++++++++++++++++++++++++++++++++++++++++++++++ deserializer.go | 30 ++++++++++++ reader.go | 5 ++ 3 files changed, 162 insertions(+) create mode 100644 deserializer.go diff --git a/decoder.go b/decoder.go index 2df2810..c19be9d 100644 --- a/decoder.go +++ b/decoder.go @@ -56,6 +56,18 @@ func (d *decoder) decode(offset uint, result reflect.Value, depth int) (uint, er return d.decodeFromType(typeNum, size, newOffset, result, depth+1) } +func (d *decoder) decodeToDeserializer(offset uint, dser Deserializer, depth int) (uint, error) { + if depth > maximumDataStructureDepth { + return 0, newInvalidDatabaseError("exceeded maximum data structure depth; database is likely corrupt") + } + typeNum, size, newOffset, err := d.decodeCtrlData(offset) + if err != nil { + return 0, err + } + + return d.decodeFromTypeToDeserializer(typeNum, size, newOffset, dser, depth+1) +} + func (d *decoder) decodeCtrlData(offset uint) (dataType, uint, uint, error) { newOffset := offset + 1 if offset >= uint(len(d.buffer)) { @@ -157,6 +169,68 @@ func (d *decoder) decodeFromType( } } +func (d *decoder) decodeFromTypeToDeserializer( + dtype dataType, + size uint, + offset uint, + dser Deserializer, + depth int, +) (uint, error) { + // For these types, size has a special meaning + switch dtype { + case _Bool: + v, offset := d.decodeBool(size, offset) + return offset, dser.Bool(v) + case _Map: + return d.decodeMapToDeserializer(size, offset, dser, depth) + case _Pointer: + pointer, newOffset, err := d.decodePointer(size, offset) + if err != nil { + return 0, err + } + _, err = d.decodeToDeserializer(pointer, dser, depth) + return newOffset, err + case _Slice: + return d.decodeSliceToDeserializer(size, offset, dser, depth) + } + + // For the remaining types, size is the byte size + if offset+size > uint(len(d.buffer)) { + return 0, newOffsetError() + } + switch dtype { + case _Bytes: + v, offset := d.decodeBytes(size, offset) + return offset, dser.Bytes(v) + case _Float32: + v, offset := d.decodeFloat32(size, offset) + return offset, dser.Float32(v) + case _Float64: + v, offset := d.decodeFloat64(size, offset) + return offset, dser.Float64(v) + case _Int32: + v, offset := d.decodeInt(size, offset) + return offset, dser.Int32(int32(v)) + case _String: + v, offset := d.decodeString(size, offset) + return offset, dser.String(v) + case _Uint16: + v, offset := d.decodeUint(size, offset) + return offset, dser.Uint16(uint16(v)) + case _Uint32: + v, offset := d.decodeUint(size, offset) + return offset, dser.Uint32(uint32(v)) + case _Uint64: + v, offset := d.decodeUint(size, offset) + return offset, dser.Uint64(v) + case _Uint128: + v, offset := d.decodeUint128(size, offset) + return offset, dser.Uint128(v) + default: + return 0, newInvalidDatabaseError("unknown type: %d", dtype) + } +} + func (d *decoder) unmarshalBool(size, offset uint, result reflect.Value) (uint, error) { if size > 1 { return 0, newInvalidDatabaseError("the MaxMind DB file's data section contains bad data (bool size of %v)", size) @@ -199,6 +273,7 @@ func (d *decoder) indirect(result reflect.Value) reflect.Value { if result.IsNil() { result.Set(reflect.New(result.Type().Elem())) } + result = result.Elem() } return result @@ -486,6 +561,35 @@ func (d *decoder) decodeMap( return offset, nil } +func (d *decoder) decodeMapToDeserializer( + size uint, + offset uint, + dser Deserializer, + depth int, +) (uint, error) { + err := dser.StartMap(size) + if err != nil { + return 0, err + } + for i := uint(0); i < size; i++ { + // TODO - implement key/value skipping? + offset, err = d.decodeToDeserializer(offset, dser, depth) + if err != nil { + return 0, err + } + + offset, err = d.decodeToDeserializer(offset, dser, depth) + if err != nil { + return 0, err + } + } + err = dser.End() + if err != nil { + return 0, err + } + return offset, nil +} + func (d *decoder) decodePointer( size uint, offset uint, @@ -538,6 +642,29 @@ func (d *decoder) decodeSlice( return offset, nil } +func (d *decoder) decodeSliceToDeserializer( + size uint, + offset uint, + dser Deserializer, + depth int, +) (uint, error) { + err := dser.StartSlice(size) + if err != nil { + return 0, err + } + for i := uint(0); i < size; i++ { + offset, err = d.decodeToDeserializer(offset, dser, depth) + if err != nil { + return 0, err + } + } + err = dser.End() + if err != nil { + return 0, err + } + return offset, nil +} + func (d *decoder) decodeString(size, offset uint) (string, uint) { newOffset := offset + size return string(d.buffer[offset:newOffset]), newOffset diff --git a/deserializer.go b/deserializer.go new file mode 100644 index 0000000..0428ebb --- /dev/null +++ b/deserializer.go @@ -0,0 +1,30 @@ +package maxminddb + +import "math/big" + +// Deserializer is an interface for a type that deserializes an MaxMind DB +// data record to some other type. This exists as an alternative to the +// standard reflection API. +// +// This is fundamentally different than the Unmarshaler interface that +// several packages provide. A Deserializer will generally create the +// final struct or value rather than unmarshaling to itself. +// +// This interface and the associated unmarshaling code is EXPERIMENTAL! +// It is not currently covered by any Semantic Versioning guarantees. +// Use at your own risk. +type Deserializer interface { + StartSlice(uint) error + StartMap(uint) error + End() error + String(string) error + Float64(float64) error + Bytes([]byte) error + Uint16(uint16) error + Uint32(uint32) error + Int32(int32) error + Uint64(uint64) error + Uint128(*big.Int) error + Bool(bool) error + Float32(float32) error +} diff --git a/reader.go b/reader.go index b825ada..210c73e 100644 --- a/reader.go +++ b/reader.go @@ -227,6 +227,11 @@ func (r *Reader) decode(offset uintptr, result interface{}) error { return errors.New("result param must be a pointer") } + if dser, ok := result.(Deserializer); ok { + _, err := r.decoder.decodeToDeserializer(uint(offset), dser, 0) + return err + } + _, err := r.decoder.decode(uint(offset), rv, 0) return err } From a126ec19368d7c8ea97fe3b521cb2318a7dcb215 Mon Sep 17 00:00:00 2001 From: Gregory Oschwald Date: Wed, 19 Aug 2020 09:34:47 -0700 Subject: [PATCH 2/4] Make deserializer interface private I'd like to avoid people using this for the time being. --- decoder.go | 8 ++++---- deserializer.go | 4 ++-- reader.go | 2 +- 3 files changed, 7 insertions(+), 7 deletions(-) diff --git a/decoder.go b/decoder.go index c19be9d..bf135af 100644 --- a/decoder.go +++ b/decoder.go @@ -56,7 +56,7 @@ func (d *decoder) decode(offset uint, result reflect.Value, depth int) (uint, er return d.decodeFromType(typeNum, size, newOffset, result, depth+1) } -func (d *decoder) decodeToDeserializer(offset uint, dser Deserializer, depth int) (uint, error) { +func (d *decoder) decodeToDeserializer(offset uint, dser deserializer, depth int) (uint, error) { if depth > maximumDataStructureDepth { return 0, newInvalidDatabaseError("exceeded maximum data structure depth; database is likely corrupt") } @@ -173,7 +173,7 @@ func (d *decoder) decodeFromTypeToDeserializer( dtype dataType, size uint, offset uint, - dser Deserializer, + dser deserializer, depth int, ) (uint, error) { // For these types, size has a special meaning @@ -564,7 +564,7 @@ func (d *decoder) decodeMap( func (d *decoder) decodeMapToDeserializer( size uint, offset uint, - dser Deserializer, + dser deserializer, depth int, ) (uint, error) { err := dser.StartMap(size) @@ -645,7 +645,7 @@ func (d *decoder) decodeSlice( func (d *decoder) decodeSliceToDeserializer( size uint, offset uint, - dser Deserializer, + dser deserializer, depth int, ) (uint, error) { err := dser.StartSlice(size) diff --git a/deserializer.go b/deserializer.go index 0428ebb..542c00e 100644 --- a/deserializer.go +++ b/deserializer.go @@ -2,7 +2,7 @@ package maxminddb import "math/big" -// Deserializer is an interface for a type that deserializes an MaxMind DB +// deserializer is an interface for a type that deserializes an MaxMind DB // data record to some other type. This exists as an alternative to the // standard reflection API. // @@ -13,7 +13,7 @@ import "math/big" // This interface and the associated unmarshaling code is EXPERIMENTAL! // It is not currently covered by any Semantic Versioning guarantees. // Use at your own risk. -type Deserializer interface { +type deserializer interface { StartSlice(uint) error StartMap(uint) error End() error diff --git a/reader.go b/reader.go index 210c73e..7b33ace 100644 --- a/reader.go +++ b/reader.go @@ -227,7 +227,7 @@ func (r *Reader) decode(offset uintptr, result interface{}) error { return errors.New("result param must be a pointer") } - if dser, ok := result.(Deserializer); ok { + if dser, ok := result.(deserializer); ok { _, err := r.decoder.decodeToDeserializer(uint(offset), dser, 0) return err } From b2a1addd159f584700632381fd88957f046105b4 Mon Sep 17 00:00:00 2001 From: Gregory Oschwald Date: Wed, 19 Aug 2020 09:42:26 -0700 Subject: [PATCH 3/4] Add ability to skip decoding in deserializer --- decoder.go | 8 ++++++++ deserializer.go | 5 +++-- 2 files changed, 11 insertions(+), 2 deletions(-) diff --git a/decoder.go b/decoder.go index bf135af..ebf01c3 100644 --- a/decoder.go +++ b/decoder.go @@ -65,6 +65,14 @@ func (d *decoder) decodeToDeserializer(offset uint, dser deserializer, depth int return 0, err } + skip, err := dser.ShouldSkip(uintptr(offset)) + if err != nil { + return 0, err + } + if skip { + return d.nextValueOffset(offset, 1) + } + return d.decodeFromTypeToDeserializer(typeNum, size, newOffset, dser, depth+1) } diff --git a/deserializer.go b/deserializer.go index 542c00e..c6dd68d 100644 --- a/deserializer.go +++ b/deserializer.go @@ -14,8 +14,9 @@ import "math/big" // It is not currently covered by any Semantic Versioning guarantees. // Use at your own risk. type deserializer interface { - StartSlice(uint) error - StartMap(uint) error + ShouldSkip(offset uintptr) (bool, error) + StartSlice(size uint) error + StartMap(size uint) error End() error String(string) error Float64(float64) error From 1f1e288ee3f9654de1cee1819c6f3b79b1ed488b Mon Sep 17 00:00:00 2001 From: Gregory Oschwald Date: Wed, 19 Aug 2020 12:11:25 -0700 Subject: [PATCH 4/4] Add basic test for deserializer --- deserializer.go deserializer_test.go | 119 +++++++++++++++++++++++++++ go.sum | 8 -- reader_test.go | 4 + 3 files changed, 123 insertions(+), 8 deletions(-) create mode 100644 deserializer.go deserializer_test.go diff --git a/deserializer.go deserializer_test.go b/deserializer.go deserializer_test.go new file mode 100644 index 0000000..8b15fee --- /dev/null +++ b/deserializer.go deserializer_test.go @@ -0,0 +1,119 @@ +package maxminddb + +import ( + "math/big" + "net" + "testing" + + "github.com/stretchr/testify/require" +) + +func TestDecodingToDeserializer(t *testing.T) { + reader, err := Open(testFile("MaxMind-DB-test-decoder.mmdb")) + require.NoError(t, err, "unexpected error while opening database: %v", err) + + dser := testDeserializer{} + err = reader.Lookup(net.ParseIP("::1.1.1.0"), &dser) + require.NoError(t, err, "unexpected error while doing lookup: %v", err) + + checkDecodingToInterface(t, dser.rv) +} + +type stackValue struct { + value interface{} + curNum int +} + +type testDeserializer struct { + stack []*stackValue + rv interface{} + key *string +} + +func (d *testDeserializer) ShouldSkip(offset uintptr) (bool, error) { + return false, nil +} + +func (d *testDeserializer) StartSlice(size uint) error { + return d.add(make([]interface{}, size)) +} + +func (d *testDeserializer) StartMap(size uint) error { + return d.add(map[string]interface{}{}) +} + +func (d *testDeserializer) End() error { + d.stack = d.stack[:len(d.stack)-1] + return nil +} + +func (d *testDeserializer) String(v string) error { + return d.add(v) +} + +func (d *testDeserializer) Float64(v float64) error { + return d.add(v) +} + +func (d *testDeserializer) Bytes(v []byte) error { + return d.add(v) +} + +func (d *testDeserializer) Uint16(v uint16) error { + return d.add(uint64(v)) +} + +func (d *testDeserializer) Uint32(v uint32) error { + return d.add(uint64(v)) +} + +func (d *testDeserializer) Int32(v int32) error { + return d.add(int(v)) +} + +func (d *testDeserializer) Uint64(v uint64) error { + return d.add(v) +} + +func (d *testDeserializer) Uint128(v *big.Int) error { + return d.add(v) +} + +func (d *testDeserializer) Bool(v bool) error { + return d.add(v) +} + +func (d *testDeserializer) Float32(v float32) error { + return d.add(v) +} + +func (d *testDeserializer) add(v interface{}) error { + if len(d.stack) == 0 { + d.rv = v + } else { + top := d.stack[len(d.stack)-1] + switch parent := top.value.(type) { + case map[string]interface{}: + if d.key == nil { + key := v.(string) + d.key = &key + } else { + parent[*d.key] = v + d.key = nil + } + + case []interface{}: + parent[top.curNum] = v + top.curNum++ + default: + } + } + + switch v := v.(type) { + case map[string]interface{}, []interface{}: + d.stack = append(d.stack, &stackValue{value: v}) + default: + } + + return nil +} diff --git a/go.sum b/go.sum index 45bf757..ae4f4e7 100644 --- a/go.sum +++ b/go.sum @@ -4,19 +4,11 @@ github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZb github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/stretchr/objx v0.1.0 h1:4G4v2dO3VZwixGIRoQ5Lfboy6nUhCyYzaqnIAPPhYs4= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= -github.com/stretchr/testify v1.4.0 h1:2E4SXV/wtOkTonXsotYi4li6zVWxYlZuYNCXe9XRJyk= -github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= -github.com/stretchr/testify v1.5.0 h1:DMOzIV76tmoDNE9pX6RSN0aDtCYeCg5VueieJaAo1uw= -github.com/stretchr/testify v1.5.0/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA= -github.com/stretchr/testify v1.5.1 h1:nOGnQDM7FYENwehXlg/kFVnos3rEvtKTjRvOWSzb6H4= -github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA= github.com/stretchr/testify v1.6.1 h1:hDPOHmpOpP40lSULcqw7IrRb/u7w6RpDC9399XyoNd0= github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= golang.org/x/sys v0.0.0-20191224085550-c709ea063b76 h1:Dho5nD6R3PcW2SH1or8vS0dszDaXRxIw55lBX7XiE5g= golang.org/x/sys v0.0.0-20191224085550-c709ea063b76/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= -gopkg.in/yaml.v2 v2.2.2 h1:ZCJp+EgiOT7lHqUV2J862kp8Qj64Jo6az82+3Td9dZw= -gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c h1:dUUwHk2QECo/6vqA44rthZ8ie2QXMNeKRTHCNY2nXvo= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/reader_test.go b/reader_test.go index 863462d..19031c2 100644 --- a/reader_test.go +++ b/reader_test.go @@ -203,6 +203,10 @@ func TestDecodingToInterface(t *testing.T) { err = reader.Lookup(net.ParseIP("::1.1.1.0"), &recordInterface) require.NoError(t, err, "unexpected error while doing lookup: %v", err) + checkDecodingToInterface(t, recordInterface) +} + +func checkDecodingToInterface(t *testing.T, recordInterface interface{}) { record := recordInterface.(map[string]interface{}) assert.Equal(t, []interface{}{uint64(1), uint64(2), uint64(3)}, record["array"]) assert.Equal(t, true, record["boolean"])