Skip to content

Commit

Permalink
feat(strmap): supports load from slice
Browse files Browse the repository at this point in the history
  • Loading branch information
xiaost committed Sep 17, 2024
1 parent ab10e41 commit 9832438
Show file tree
Hide file tree
Showing 2 changed files with 101 additions and 20 deletions.
84 changes: 69 additions & 15 deletions container/strmap/strmap.go
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,7 @@
package strmap

import (
"errors"
"fmt"
"math"
"sort"
Expand Down Expand Up @@ -49,28 +50,77 @@ type mapItem[V any] struct {
v V
}

// New creates StrMap from map[string]V
func New[V any](m map[string]V) *StrMap[V] {
// New creates a StrMap instance,
func New[V any]() *StrMap[V] {
return &StrMap[V]{seed: maphash.MakeSeed()}
}

// NewFromMap creates StrMap from map
func NewFromMap[V any](m map[string]V) *StrMap[V] {
ret := New[V]()
if err := ret.LoadFromMap(m); err != nil {
panic(err)
}
return ret
}

// NewFromSlice creates StrMap from slices, len(kk) must equal to len(vv)
func NewFromSlice[V any](kk []string, vv []V) *StrMap[V] {
ret := New[V]()
if err := ret.LoadFromSlice(kk, vv); err != nil {
panic(err)
}
return ret
}

// LoadFromMap resets StrMap and loads from map
func (p *StrMap[V]) LoadFromMap(m map[string]V) error {
kk := make([]string, 0, len(m))
vv := make([]V, 0, len(m))
for k, v := range m {
kk = append(kk, k)
vv = append(vv, v)
}
return p.LoadFromSlice(kk, vv)
}

// LoadFromSlice resets StrMap and loads from slices, len(kk) must equal to len(vv)
func (m *StrMap[V]) LoadFromSlice(kk []string, vv []V) error {
if len(kk) != len(vv) {
return errors.New("kv len not match")
}
m.data = m.data[:0]
m.items = m.items[:0]
m.hashtable = m.hashtable[:0]

sz := 0
for k := range m {
for _, k := range kk {
sz += len(k)
}
b := make([]byte, 0, sz)
if cap(m.data) < sz {
m.data = make([]byte, 0, sz)
}
if cap(m.items) < len(vv) {
m.items = make([]mapItem[V], 0, len(vv))
}

seed := maphash.MakeSeed()
items := make([]mapItem[V], 0, len(m))
for k, v := range m {
for i, k := range kk {
if len(k) > math.MaxUint32 {
// it doesn't make sense ...
panic("key too large")
return errors.New("key too large")
}
items = append(items, mapItem[V]{off: len(b), sz: uint32(len(k)), slot: uint32(maphash.String(seed, k)), v: v})
b = append(b, k...)
v := vv[i]
m.items = append(m.items,
mapItem[V]{
off: len(m.data),
sz: uint32(len(k)),
slot: uint32(maphash.String(m.seed, k)),
v: v,
})
m.data = append(m.data, k...)
}

ret := &StrMap[V]{data: b, items: items, seed: seed}
ret.makeHashtable()
return ret
m.makeHashtable()
return nil
}

// Len returns the size of map
Expand All @@ -93,7 +143,11 @@ func (x itemsBySlot[V]) Swap(i, j int) { x[i], x[j] = x[j], x[i] }

func (m *StrMap[V]) makeHashtable() {
slots := calcHashtableSlots(len(m.items))
m.hashtable = make([]int32, slots)
if cap(m.hashtable) < int(slots) {
m.hashtable = make([]int32, slots)
} else {
m.hashtable = m.hashtable[:slots]
}

// update `slot` of mapItem to fit the size of hashtable
for i := range m.items {
Expand Down
37 changes: 32 additions & 5 deletions container/strmap/strmap_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -41,7 +41,7 @@ func randStrings(m, n int) []string {
// newStdStrMap generates a map with uniq values
func newStdStrMap(ss []string) map[string]uint {
v := uint(1)
m := make(map[string]uint)
m := make(map[string]uint, len(ss))
for _, s := range ss {
_, ok := m[s]
if !ok {
Expand All @@ -55,7 +55,7 @@ func newStdStrMap(ss []string) map[string]uint {
func TestStrMap(t *testing.T) {
ss := randStrings(20, 100000)
m := newStdStrMap(ss)
sm := New(m)
sm := NewFromMap(m)
require.Equal(t, len(m), sm.Len())
for i, s := range ss {
v0 := m[s]
Expand All @@ -79,11 +79,38 @@ func TestStrMap(t *testing.T) {
func TestStrMapString(t *testing.T) {
ss := []string{"a", "b", "c"}
m := newStdStrMap(ss)
sm := New(m)
sm := NewFromMap(m)
t.Log(sm.String())
t.Log(sm.debugString())
}

func BenchmarkLoadFromMap(b *testing.B) {
sz := 50
n := 100000
ss := randStrings(sz, n)
m := newStdStrMap(ss)
p := New[uint]()
b.ResetTimer()
for i := 0; i < b.N; i++ {
p.LoadFromMap(m)
}
}

func BenchmarkLoadFromSlice(b *testing.B) {
sz := 50
n := 100000
kk := randStrings(sz, n)
vv := make([]int, n)
for i := range vv {
vv[i] = i
}
p := New[int]()
b.ResetTimer()
for i := 0; i < b.N; i++ {
p.LoadFromSlice(kk, vv)
}
}

func BenchmarkGet(b *testing.B) {
sizes := []int{20, 50, 100}
nn := []int{100000, 200000}
Expand All @@ -98,7 +125,7 @@ func BenchmarkGet(b *testing.B) {
}
})
b.Run(fmt.Sprintf("new-keysize_%d_n_%d", sz, n), func(b *testing.B) {
sm := New(m)
sm := NewFromMap(m)
b.ResetTimer()
for i := 0; i < b.N; i++ {
sm.Get(ss[i%len(ss)])
Expand All @@ -122,7 +149,7 @@ func BenchmarkGC(b *testing.B) {
}
})

sm := New(m)
sm := NewFromMap(m)
m = nil
runtime.GC()

Expand Down

0 comments on commit 9832438

Please sign in to comment.