From 07f45f351b3f3ecba0946226cd76801fa03ba7a9 Mon Sep 17 00:00:00 2001 From: Kyle Xiao Date: Fri, 13 Sep 2024 14:07:59 +0800 Subject: [PATCH] feat(hash): add xfnv --- go.mod | 1 + go.sum | 2 ++ hash/xfnv/xfnv.go | 73 ++++++++++++++++++++++++++++++++++++++ hash/xfnv/xfnv_test.go | 80 ++++++++++++++++++++++++++++++++++++++++++ 4 files changed, 156 insertions(+) create mode 100644 hash/xfnv/xfnv.go create mode 100644 hash/xfnv/xfnv_test.go diff --git a/go.mod b/go.mod index bb7777f..b0af1e1 100644 --- a/go.mod +++ b/go.mod @@ -12,6 +12,7 @@ require ( github.com/kr/pretty v0.1.0 // indirect github.com/pmezard/go-difflib v1.0.0 // indirect golang.org/x/net v0.24.0 // indirect + golang.org/x/sys v0.19.0 // indirect golang.org/x/text v0.14.0 // indirect gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect diff --git a/go.sum b/go.sum index ef6ea60..6bfd170 100644 --- a/go.sum +++ b/go.sum @@ -13,6 +13,8 @@ github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsT github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= golang.org/x/net v0.24.0 h1:1PcaxkF854Fu3+lvBIx5SYn9wRlBzzcnHZSiaFFAb0w= golang.org/x/net v0.24.0/go.mod h1:2Q7sJY5mzlzWjKtYUEXSlBWCdyaioyXzRB2RtU8KVE8= +golang.org/x/sys v0.19.0 h1:q5f1RH2jigJ1MoAWp2KTp3gm5zAGFUTarQZ5U386+4o= +golang.org/x/sys v0.19.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/text v0.14.0 h1:ScX5w1eTa3QqT8oi6+ziP7dTV1S2+ALU0bI+0zXKWiQ= golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= diff --git a/hash/xfnv/xfnv.go b/hash/xfnv/xfnv.go new file mode 100644 index 0000000..f3d1d33 --- /dev/null +++ b/hash/xfnv/xfnv.go @@ -0,0 +1,73 @@ +/* + * Copyright 2024 CloudWeGo Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// Package xfnv is modified and non-cross-platform compatible version of FNV-1a. +// +// It computes 8 bytes per round by converting bytes to uint64 directly +// as a result it doesn't generate the same result for diff cpu arch. +package xfnv + +import ( + "unsafe" +) + +const ( + fnvHashOffset64 = uint64(14695981039346656037) // fnv hash offset64 + fnvHashPrime64 = uint64(1099511628211) +) + +func strDataPtr(s string) unsafe.Pointer { + // for str, the Data ptr is always the 1st field + return *(*unsafe.Pointer)(unsafe.Pointer(&s)) +} + +func bytesDataPtr(b []byte) unsafe.Pointer { + // for []byte, the Data ptr is always the 1st field + return *(*unsafe.Pointer)(unsafe.Pointer(&b)) +} + +// Hash returns the hash of the given bytes +// +// DO NOT STORE the return value since it's NOT cross-platform compatible. +// It's designed for in-memory use. +func Hash(b []byte) uint64 { + return doHash(bytesDataPtr(b), len(b)) +} + +// HashStr returns the hash of the given string +// +// DO NOT STORE the return value since it's NOT cross-platform compatible. +// It's designed for in-memory use. +func HashStr(s string) uint64 { + return doHash(strDataPtr(s), len(s)) +} + +func doHash(p unsafe.Pointer, n int) uint64 { + h := fnvHashOffset64 + i := 0 + // 8 byte per round + for m := n >> 3; i < m; i++ { + h ^= *(*uint64)(unsafe.Add(p, i<<3)) // p[i*8] + h *= fnvHashPrime64 + } + // left 0-7 bytes + i = i << 3 + for ; i < n; i++ { + h ^= uint64(*(*byte)(unsafe.Add(p, i))) + h *= fnvHashPrime64 + } + return h +} diff --git a/hash/xfnv/xfnv_test.go b/hash/xfnv/xfnv_test.go new file mode 100644 index 0000000..31cb176 --- /dev/null +++ b/hash/xfnv/xfnv_test.go @@ -0,0 +1,80 @@ +/* + * Copyright 2024 CloudWeGo Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package xfnv + +import ( + "crypto/rand" + "fmt" + "hash/maphash" + "testing" + + "github.com/bytedance/gopkg/util/xxhash3" + "github.com/stretchr/testify/require" +) + +func TestHashStr(t *testing.T) { + require.Equal(t, HashStr("1234"), HashStr("1234")) + require.NotEqual(t, HashStr("12345"), HashStr("12346")) + require.Equal(t, HashStr("12345678"), HashStr("12345678")) + require.NotEqual(t, HashStr("123456789"), HashStr("123456788")) +} + +func BenchmarkHash(b *testing.B) { + sizes := []int{8, 16, 32, 64, 128, 512} + bb := make([][]byte, len(sizes)) + for i := range bb { + b := make([]byte, sizes[i]) + rand.Read(b) + bb[i] = b + } + b.ResetTimer() + for _, data := range bb { + b.Run(fmt.Sprintf("size-%d-xfnv", len(data)), func(b *testing.B) { + b.SetBytes(int64(len(data))) + for i := 0; i < b.N; i++ { + _ = Hash(data) + } + }) + } + + println("") + + for _, data := range bb { + b.Run(fmt.Sprintf("size-%d-xxhash3", len(data)), func(b *testing.B) { + b.SetBytes(int64(len(data))) + for i := 0; i < b.N; i++ { + _ = xxhash3.Hash(data) + } + }) + } + + println("") + + for _, data := range bb { + b.Run(fmt.Sprintf("size-%d-maphash", len(data)), func(b *testing.B) { + s := maphash.MakeSeed() + h := &maphash.Hash{} + h.SetSeed(s) + b.SetBytes(int64(len(data))) + for i := 0; i < b.N; i++ { + // use maphash.Bytes which is more fair to benchmark after go1.19 + // maphash.Bytes(s, data) + _, _ = h.Write(data) + } + }) + } +}