From bfdec40ce6d99593f517cbdd03721d26309cf7e4 Mon Sep 17 00:00:00 2001 From: xuzhaonan Date: Mon, 13 Jan 2025 14:08:51 +0800 Subject: [PATCH] feat: redis vectorstore --- components/indexer/redis/consts.go | 22 ++ components/indexer/redis/go.mod | 48 +++ components/indexer/redis/go.sum | 159 ++++++++++ components/indexer/redis/indexer.go | 302 +++++++++++++++++++ components/indexer/redis/indexer_test.go | 216 +++++++++++++ components/indexer/redis/utils.go | 34 +++ components/retriever/redis/consts.go | 28 ++ components/retriever/redis/go.mod | 48 +++ components/retriever/redis/go.sum | 159 ++++++++++ components/retriever/redis/options.go | 21 ++ components/retriever/redis/retriever.go | 255 ++++++++++++++++ components/retriever/redis/retriever_test.go | 259 ++++++++++++++++ components/retriever/redis/utils.go | 58 ++++ 13 files changed, 1609 insertions(+) create mode 100644 components/indexer/redis/consts.go create mode 100644 components/indexer/redis/go.mod create mode 100644 components/indexer/redis/go.sum create mode 100644 components/indexer/redis/indexer.go create mode 100644 components/indexer/redis/indexer_test.go create mode 100644 components/indexer/redis/utils.go create mode 100644 components/retriever/redis/consts.go create mode 100644 components/retriever/redis/go.mod create mode 100644 components/retriever/redis/go.sum create mode 100644 components/retriever/redis/options.go create mode 100644 components/retriever/redis/retriever.go create mode 100644 components/retriever/redis/retriever_test.go create mode 100644 components/retriever/redis/utils.go diff --git a/components/indexer/redis/consts.go b/components/indexer/redis/consts.go new file mode 100644 index 0000000..88fbff6 --- /dev/null +++ b/components/indexer/redis/consts.go @@ -0,0 +1,22 @@ +/* + * Copyright 2025 CloudWeGo Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package redis + +const ( + defaultReturnFieldContent = "content" + defaultReturnFieldVectorContent = "vector_content" +) diff --git a/components/indexer/redis/go.mod b/components/indexer/redis/go.mod new file mode 100644 index 0000000..385a435 --- /dev/null +++ b/components/indexer/redis/go.mod @@ -0,0 +1,48 @@ +module github.com/cloudwego/eino-ext/components/indexer/redis + +go 1.18 + +require ( + github.com/bytedance/mockey v1.2.13 + github.com/cloudwego/eino v0.3.6 + github.com/redis/go-redis/v9 v9.7.0 + github.com/smartystreets/goconvey v1.8.1 +) + +require ( + github.com/bytedance/sonic v1.12.2 // indirect + github.com/bytedance/sonic/loader v0.2.0 // indirect + github.com/cespare/xxhash/v2 v2.2.0 // indirect + github.com/cloudwego/base64x v0.1.4 // indirect + github.com/cloudwego/iasm v0.2.0 // indirect + github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f // indirect + github.com/dustin/go-humanize v1.0.1 // indirect + github.com/getkin/kin-openapi v0.118.0 // indirect + github.com/go-openapi/jsonpointer v0.19.5 // indirect + github.com/go-openapi/swag v0.19.5 // indirect + github.com/goph/emperror v0.17.2 // indirect + github.com/gopherjs/gopherjs v1.17.2 // indirect + github.com/invopop/yaml v0.1.0 // indirect + github.com/josharian/intern v1.0.0 // indirect + github.com/json-iterator/go v1.1.12 // indirect + github.com/jtolds/gls v4.20.0+incompatible // indirect + github.com/klauspost/cpuid/v2 v2.0.9 // indirect + github.com/mailru/easyjson v0.7.7 // indirect + github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect + github.com/modern-go/reflect2 v1.0.2 // indirect + github.com/mohae/deepcopy v0.0.0-20170929034955-c48cc78d4826 // indirect + github.com/nikolalohinski/gonja v1.5.3 // indirect + github.com/pelletier/go-toml/v2 v2.0.9 // indirect + github.com/perimeterx/marshmallow v1.1.4 // indirect + github.com/pkg/errors v0.9.1 // indirect + github.com/sirupsen/logrus v1.9.3 // indirect + github.com/slongfield/pyfmt v0.0.0-20220222012616-ea85ff4c361f // indirect + github.com/smarty/assertions v1.15.0 // indirect + github.com/twitchyliquid64/golang-asm v0.15.1 // indirect + github.com/yargevad/filepathx v1.0.0 // indirect + golang.org/x/arch v0.11.0 // indirect + golang.org/x/exp v0.0.0-20230713183714-613f0c0eb8a1 // indirect + golang.org/x/sys v0.26.0 // indirect + gopkg.in/yaml.v2 v2.4.0 // indirect + gopkg.in/yaml.v3 v3.0.1 // indirect +) diff --git a/components/indexer/redis/go.sum b/components/indexer/redis/go.sum new file mode 100644 index 0000000..bc6a972 --- /dev/null +++ b/components/indexer/redis/go.sum @@ -0,0 +1,159 @@ +github.com/airbrake/gobrake v3.6.1+incompatible/go.mod h1:wM4gu3Cn0W0K7GUuVWnlXZU11AGBXMILnrdOU8Kn00o= +github.com/bitly/go-simplejson v0.5.0/go.mod h1:cXHtHw4XUPsvGaxgjIAn8PhEWG9NfngEKAMDJEczWVA= +github.com/bmizerany/assert v0.0.0-20160611221934-b7ed37b82869/go.mod h1:Ekp36dRnpXw/yCqJaO+ZrUyxD+3VXMFFr56k5XYrpB4= +github.com/bsm/ginkgo/v2 v2.12.0 h1:Ny8MWAHyOepLGlLKYmXG4IEkioBysk6GpaRTLC8zwWs= +github.com/bsm/gomega v1.27.10 h1:yeMWxP2pV2fG3FgAODIY8EiRE3dy0aeFYt4l7wh6yKA= +github.com/bugsnag/bugsnag-go v1.4.0/go.mod h1:2oa8nejYd4cQ/b0hMIopN0lCRxU0bueqREvZLWFrtK8= +github.com/bugsnag/panicwrap v1.2.0/go.mod h1:D/8v3kj0zr8ZAKg1AQ6crr+5VwKN5eIywRkfhyM/+dE= +github.com/bytedance/mockey v1.2.13 h1:jokWZAm/pUEbD939Rhznz615MKUCZNuvCFQlJ2+ntoo= +github.com/bytedance/mockey v1.2.13/go.mod h1:1BPHF9sol5R1ud/+0VEHGQq/+i2lN+GTsr3O2Q9IENY= +github.com/bytedance/sonic v1.12.2 h1:oaMFuRTpMHYLpCntGca65YWt5ny+wAceDERTkT2L9lg= +github.com/bytedance/sonic v1.12.2/go.mod h1:B8Gt/XvtZ3Fqj+iSKMypzymZxw/FVwgIGKzMzT9r/rk= +github.com/bytedance/sonic/loader v0.1.1/go.mod h1:ncP89zfokxS5LZrJxl5z0UJcsk4M4yY2JpfqGeCtNLU= +github.com/bytedance/sonic/loader v0.2.0 h1:zNprn+lsIP06C/IqCHs3gPQIvnvpKbbxyXQP1iU4kWM= +github.com/bytedance/sonic/loader v0.2.0/go.mod h1:ncP89zfokxS5LZrJxl5z0UJcsk4M4yY2JpfqGeCtNLU= +github.com/certifi/gocertifi v0.0.0-20190105021004-abcd57078448/go.mod h1:GJKEexRPVJrBSOjoqN5VNOIKJ5Q3RViH6eu3puDRwx4= +github.com/cespare/xxhash/v2 v2.2.0 h1:DC2CZ1Ep5Y4k3ZQ899DldepgrayRUGE6BBZ/cd9Cj44= +github.com/cespare/xxhash/v2 v2.2.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= +github.com/cloudwego/base64x v0.1.4 h1:jwCgWpFanWmN8xoIUHa2rtzmkd5J2plF/dnLS6Xd/0Y= +github.com/cloudwego/base64x v0.1.4/go.mod h1:0zlkT4Wn5C6NdauXdJRhSKRlJvmclQ1hhJgA0rcu/8w= +github.com/cloudwego/eino v0.3.6 h1:3yfdKKxMVWefdOyGXHuqUMM5cc9iioijj2mpPsDZKIg= +github.com/cloudwego/eino v0.3.6/go.mod h1:+kmJimGEcKuSI6OKhet7kBedkm1WUZS3H1QRazxgWUo= +github.com/cloudwego/iasm v0.2.0 h1:1KNIy1I1H9hNNFEEH3DVnI4UujN+1zjpuk6gwHLTssg= +github.com/cloudwego/iasm v0.2.0/go.mod h1:8rXZaNYT2n95jn+zTI1sDr+IgcD2GVs0nlbbQPiEFhY= +github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f h1:lO4WD4F/rVNCu3HqELle0jiPLLBs70cWOduZpkS1E78= +github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f/go.mod h1:cuUVRXasLTGF7a8hSLbxyZXjz+1KgoB3wDUb6vlszIc= +github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY= +github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto= +github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= +github.com/getkin/kin-openapi v0.118.0 h1:z43njxPmJ7TaPpMSCQb7PN0dEYno4tyBPQcrFdHoLuM= +github.com/getkin/kin-openapi v0.118.0/go.mod h1:l5e9PaFUo9fyLJCPGQeXI2ML8c3P8BHOEV2VaAVf/pc= +github.com/getsentry/raven-go v0.2.0/go.mod h1:KungGk8q33+aIAZUIVWZDr2OfAEBsO49PX4NzFV5kcQ= +github.com/go-check/check v0.0.0-20180628173108-788fd7840127 h1:0gkP6mzaMqkmpcJYCFOLkIBwI7xFExG03bbkOkCvUPI= +github.com/go-openapi/jsonpointer v0.19.5 h1:gZr+CIYByUqjcgeLXnQu2gHYQC9o73G2XUeOFYEICuY= +github.com/go-openapi/jsonpointer v0.19.5/go.mod h1:Pl9vOtqEWErmShwVjC8pYs9cog34VGT37dQOVbmoatg= +github.com/go-openapi/swag v0.19.5 h1:lTz6Ys4CmqqCQmZPBlbQENR1/GucA2bzYTE12Pw4tFY= +github.com/go-openapi/swag v0.19.5/go.mod h1:POnQmlKehdgb5mhVOsnJFsivZCEZ/vjK9gh66Z9tfKk= +github.com/go-test/deep v1.0.8 h1:TDsG77qcSprGbC6vTN8OuXp5g+J+b5Pcguhf7Zt61VM= +github.com/go-test/deep v1.0.8/go.mod h1:5C2ZWiW0ErCdrYzpqxLbTX7MG14M9iiw8DgHncVwcsE= +github.com/gofrs/uuid v3.2.0+incompatible/go.mod h1:b2aQJv3Z4Fp6yNu3cdSllBxTCLRxnplIgP/c0N/04lM= +github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= +github.com/goph/emperror v0.17.2 h1:yLapQcmEsO0ipe9p5TaN22djm3OFV/TfM/fcYP0/J18= +github.com/goph/emperror v0.17.2/go.mod h1:+ZbQ+fUNO/6FNiUo0ujtMjhgad9Xa6fQL9KhH4LNHic= +github.com/gopherjs/gopherjs v1.17.2 h1:fQnZVsXk8uxXIStYb0N4bGk7jeyTalG/wsZjQ25dO0g= +github.com/gopherjs/gopherjs v1.17.2/go.mod h1:pRRIvn/QzFLrKfvEz3qUuEhtE/zLCWfreZ6J5gM2i+k= +github.com/gorilla/mux v1.8.0/go.mod h1:DVbg23sWSpFRCP0SfiEN6jmj59UnW/n46BH5rLB71So= +github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU= +github.com/invopop/yaml v0.1.0 h1:YW3WGUoJEXYfzWBjn00zIlrw7brGVD0fUKRYDPAPhrc= +github.com/invopop/yaml v0.1.0/go.mod h1:2XuRLgs/ouIrW3XNzuNj7J3Nvu/Dig5MXvbCEdiBN3Q= +github.com/josharian/intern v1.0.0 h1:vlS4z54oSdjm0bgjRigI+G1HpF+tI+9rE5LLzOg8HmY= +github.com/josharian/intern v1.0.0/go.mod h1:5DoeVV0s6jJacbCEi61lwdGj/aVlrQvzHFFd8Hwg//Y= +github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM= +github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo= +github.com/jtolds/gls v4.20.0+incompatible h1:xdiiI2gbIgH/gLH7ADydsJ1uDOEzR8yvV7C0MuV77Wo= +github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU= +github.com/kardianos/osext v0.0.0-20190222173326-2bc1f35cddc0/go.mod h1:1NbS8ALrpOvjt0rHPNLyCIeMtbizbir8U//inJ+zuB8= +github.com/klauspost/cpuid/v2 v2.0.9 h1:lgaqFMSdTdQYdZ04uHyN2d/eKdOMyi2YLSvlQIBFYa4= +github.com/klauspost/cpuid/v2 v2.0.9/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg= +github.com/knz/go-libedit v1.10.1/go.mod h1:MZTVkCWyz0oBc7JOWP3wNAzd002ZbM/5hgShxwh4x8M= +github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= +github.com/kr/pretty v0.1.0 h1:L/CwN0zerZDmRFUapSPitk6f+Q3+0za1rQkzVuMiMFI= +github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= +github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= +github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE= +github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= +github.com/mailru/easyjson v0.0.0-20190614124828-94de47d64c63/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= +github.com/mailru/easyjson v0.0.0-20190626092158-b2ccc519800e/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= +github.com/mailru/easyjson v0.7.7 h1:UGYAvKxe3sBsEDzO8ZeWOSlIQfWFlxbzLZe7hwFURr0= +github.com/mailru/easyjson v0.7.7/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJJLY9Nlc= +github.com/mattn/go-colorable v0.1.2 h1:/bC9yWikZXAL9uJdulbSfyVNIR3n3trXl+v8+1sx8mU= +github.com/mattn/go-isatty v0.0.8 h1:HLtExJ+uU2HOZ+wI0Tt5DtUDrx8yhUqDcp7fYERX4CE= +github.com/mgutz/ansi v0.0.0-20170206155736-9520e82c474b h1:j7+1HpAFS1zy5+Q4qx1fWh90gTKwiN4QCGoY9TWyyO4= +github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= +github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg= +github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= +github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9Gz0M= +github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk= +github.com/mohae/deepcopy v0.0.0-20170929034955-c48cc78d4826 h1:RWengNIwukTxcDr9M+97sNutRR1RKhG96O6jWumTTnw= +github.com/mohae/deepcopy v0.0.0-20170929034955-c48cc78d4826/go.mod h1:TaXosZuwdSHYgviHp1DAtfrULt5eUgsSMsZf+YrPgl8= +github.com/nikolalohinski/gonja v1.5.3 h1:GsA+EEaZDZPGJ8JtpeGN78jidhOlxeJROpqMT9fTj9c= +github.com/nikolalohinski/gonja v1.5.3/go.mod h1:RmjwxNiXAEqcq1HeK5SSMmqFJvKOfTfXhkJv6YBtPa4= +github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= +github.com/onsi/ginkgo v1.8.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= +github.com/onsi/gomega v1.5.0/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY= +github.com/pelletier/go-toml/v2 v2.0.9 h1:uH2qQXheeefCCkuBBSLi7jCiSmj3VRh2+Goq2N7Xxu0= +github.com/pelletier/go-toml/v2 v2.0.9/go.mod h1:tJU2Z3ZkXwnxa4DPO899bsyIoywizdUvyaeZurnPPDc= +github.com/perimeterx/marshmallow v1.1.4 h1:pZLDH9RjlLGGorbXhcaQLhfuV0pFMNfPO55FuFkxqLw= +github.com/perimeterx/marshmallow v1.1.4/go.mod h1:dsXbUu8CRzfYP5a87xpp0xq9S3u0Vchtcl8we9tYaXw= +github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= +github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/redis/go-redis/v9 v9.7.0 h1:HhLSs+B6O021gwzl+locl0zEDnyNkxMtf/Z3NNBMa9E= +github.com/redis/go-redis/v9 v9.7.0/go.mod h1:f6zhXITC7JUJIlPEiBOTXxJgPLdZcA93GewI7inzyWw= +github.com/rollbar/rollbar-go v1.0.2/go.mod h1:AcFs5f0I+c71bpHlXNNDbOWJiKwjFDtISeXco0L5PKQ= +github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= +github.com/sirupsen/logrus v1.9.3 h1:dueUQJ1C2q9oE3F7wvmSGAaVtTmUizReu6fjN8uqzbQ= +github.com/sirupsen/logrus v1.9.3/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ= +github.com/slongfield/pyfmt v0.0.0-20220222012616-ea85ff4c361f h1:Z2cODYsUxQPofhpYRMQVwWz4yUVpHF+vPi+eUdruUYI= +github.com/slongfield/pyfmt v0.0.0-20220222012616-ea85ff4c361f/go.mod h1:JqzWyvTuI2X4+9wOHmKSQCYxybB/8j6Ko43qVmXDuZg= +github.com/smarty/assertions v1.15.0 h1:cR//PqUBUiQRakZWqBiFFQ9wb8emQGDb0HeGdqGByCY= +github.com/smarty/assertions v1.15.0/go.mod h1:yABtdzeQs6l1brC900WlRNwj6ZR55d7B+E8C6HtKdec= +github.com/smartystreets/goconvey v1.8.1 h1:qGjIddxOk4grTu9JPOU31tVfq3cNdBlNa5sSznIX1xY= +github.com/smartystreets/goconvey v1.8.1/go.mod h1:+/u4qLyY6x1jReYOp7GOM2FSt8aP9CzCZL03bI28W60= +github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= +github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= +github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= +github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= +github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= +github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= +github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= +github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg= +github.com/twitchyliquid64/golang-asm v0.15.1 h1:SU5vSMR7hnwNxj24w34ZyCi/FmDZTkS4MhqMhdFk5YI= +github.com/twitchyliquid64/golang-asm v0.15.1/go.mod h1:a1lVb/DtPvCB8fslRZhAngC2+aY1QWCk3Cedj/Gdt08= +github.com/ugorji/go v1.2.7 h1:qYhyWUUd6WbiM+C6JZAUkIJt/1WrjzNHY9+KCIjVqTo= +github.com/ugorji/go v1.2.7/go.mod h1:nF9osbDWLy6bDVv/Rtoh6QgnvNDpmCalQV5urGCCS6M= +github.com/ugorji/go/codec v1.2.7 h1:YPXUKf7fYbp/y8xloBqZOw2qaVggbfwMlI8WM3wZUJ0= +github.com/ugorji/go/codec v1.2.7/go.mod h1:WGN1fab3R1fzQlVQTkfxVtIBhWDRqOviHU95kRgeqEY= +github.com/x-cray/logrus-prefixed-formatter v0.5.2 h1:00txxvfBM9muc0jiLIEAkAcIMJzfthRT6usrui8uGmg= +github.com/yargevad/filepathx v1.0.0 h1:SYcT+N3tYGi+NvazubCNlvgIPbzAk7i7y2dwg3I5FYc= +github.com/yargevad/filepathx v1.0.0/go.mod h1:BprfX/gpYNJHJfc35GjRRpVcwWXS89gGulUIU5tK3tA= +go.uber.org/mock v0.4.0 h1:VcM4ZOtdbR4f6VXfiOpwpVJDL6lCReaZ6mw31wqh7KU= +golang.org/x/arch v0.11.0 h1:KXV8WWKCXm6tRpLirl2szsO5j/oOODwZf4hATmGVNs4= +golang.org/x/arch v0.11.0/go.mod h1:FEVrYAQjsQXMVJ1nsMoVVXPZg6p2JE2mx8psSWTDQys= +golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= +golang.org/x/crypto v0.11.0 h1:6Ewdq3tDic1mg5xRO4milcWCfMVQhI4NkqWWvqejpuA= +golang.org/x/exp v0.0.0-20230713183714-613f0c0eb8a1 h1:MGwJjxBy0HJshjDNfLsYO8xppfqWlA5ZT9OhtUUhTNw= +golang.org/x/exp v0.0.0-20230713183714-613f0c0eb8a1/go.mod h1:FXUEEKJgO7OQYeo8N01OfiKP8RXMtf6e8aTskBGqWdc= +golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.26.0 h1:KHjCJyddX0LoSTb3J+vWpupP9p0oznkqVk/IfjymZbo= +golang.org/x/sys v0.26.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/term v0.10.0 h1:3R7pNqamzBraeqj/Tj8qt1aQ2HpmlC+Cx/qL/7hn4/c= +golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 h1:qIbj1fsPNlZgppZ+VLlY7N33q108Sa+fhmuc+sWQYwY= +gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys= +gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw= +gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= +gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= +gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gopkg.in/yaml.v3 v3.0.0/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +nullprogram.com/x/optparse v1.0.0/go.mod h1:KdyPE+Igbe0jQUrVfMqDMeJQIJZEuyV7pjYmp6pbG50= diff --git a/components/indexer/redis/indexer.go b/components/indexer/redis/indexer.go new file mode 100644 index 0000000..ee759b5 --- /dev/null +++ b/components/indexer/redis/indexer.go @@ -0,0 +1,302 @@ +/* + * Copyright 2025 CloudWeGo Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package redis + +import ( + "context" + "fmt" + + "github.com/cloudwego/eino/callbacks" + "github.com/cloudwego/eino/components" + "github.com/cloudwego/eino/components/embedding" + "github.com/cloudwego/eino/components/indexer" + "github.com/cloudwego/eino/schema" + "github.com/redis/go-redis/v9" +) + +type IndexerConfig struct { + // Client is a Redis client representing a pool of zero or more underlying connections. + // It's safe for concurrent use by multiple goroutines, which means is okay to pass + // an existed Client to create a new Indexer component. + Client *redis.Client + // KeyPrefix prefix for each key, hset key would be KeyPrefix+Hashes.Key. + // If not set, make sure each key from DocumentToHashes contains same prefix, for ft.Create requires. + // see: https://redis.io/docs/latest/develop/interact/search-and-query/advanced-concepts/vectors/#create-a-vector-index + KeyPrefix string + // DocumentToHashes supports customize key, field and value for redis hash. + // field2EmbeddingValue is field - text pairs, which text will be embedded, then field and embedding will join field2Value. + // field2Value is field - value pairs for hset. + // key is hash key, is okay to use document ID if it's unique. + // Eventually, command will look like: hset $(KeyPrefix+key) field_1 val_1 field_2 val_2 ... + // Default defaultDocumentToFields. + DocumentToHashes func(ctx context.Context, doc *schema.Document) (*Hashes, error) + // BatchSize controls embedding texts size. + // Default 10. + BatchSize int `json:"batch_size"` + // Embedding vectorization method for values need to be embedded from FieldValue. + Embedding embedding.Embedder +} + +type Hashes struct { + // Key redis hashes key + Key string + // Key redis hashes field - val pairs + Field2Value map[string]FieldValue +} + +type FieldValue struct { + // Value original Value + Value any + // EmbedKey if set, Value will be vectorized and saved to es. + // If Stringify method is provided, Embedding input text will be Stringify(Value). + // If Stringify method not set, retriever will try to assert Value as string. + EmbedKey string + // Stringify converts Value to string + Stringify func(val any) (string, error) +} + +type Indexer struct { + config *IndexerConfig +} + +func NewIndexer(ctx context.Context, config *IndexerConfig) (*Indexer, error) { + if config.Embedding == nil { + return nil, fmt.Errorf("[NewIndexer] embedding not provided for redis indexer") + } + + if config.Client == nil { + return nil, fmt.Errorf("[NewIndexer] redis client not provided") + } + + if config.DocumentToHashes == nil { + config.DocumentToHashes = defaultDocumentToFields + } + + if config.BatchSize == 0 { + config.BatchSize = 10 + } + + return &Indexer{ + config: config, + }, nil +} + +func (i *Indexer) Store(ctx context.Context, docs []*schema.Document, opts ...indexer.Option) (ids []string, err error) { + defer func() { + if err != nil { + callbacks.OnError(ctx, err) + } + }() + + options := indexer.GetCommonOptions(&indexer.Options{ + Embedding: i.config.Embedding, + }, opts...) + + ctx = callbacks.OnStart(ctx, &indexer.CallbackInput{Docs: docs}) + + if err = i.pipelineHSet(ctx, docs, options); err != nil { + return nil, err + } + + ids = make([]string, 0, len(docs)) + for _, doc := range docs { + // If you need hash key returned by FieldMapping, set doc.ID with key manually in DocumentToHashes. + ids = append(ids, doc.ID) + } + + callbacks.OnEnd(ctx, &indexer.CallbackOutput{IDs: ids}) + + return ids, nil +} + +func (i *Indexer) pipelineHSet(ctx context.Context, docs []*schema.Document, options *indexer.Options) (err error) { + emb := options.Embedding + pipeline := i.config.Client.Pipeline() + + var ( + tuples []tuple + texts []string + ) + + embAndAdd := func() error { + var vectors [][]float64 + + if len(texts) > 0 { + if emb == nil { + return fmt.Errorf("[pipelineHSet] embedding method not provided") + } + + vectors, err = emb.EmbedStrings(i.makeEmbeddingCtx(ctx, emb), texts) + if err != nil { + return fmt.Errorf("[pipelineHSet] embedding failed, %w", err) + } + + if len(vectors) != len(texts) { + return fmt.Errorf("[pipelineHSet] invalid vector length, expected=%d, got=%d", len(texts), len(vectors)) + } + } + + for _, t := range tuples { + fields := t.fields + for k, idx := range t.key2Idx { + fields[k] = vector2Bytes(vectors[idx]) + } + + pipeline.HSet(ctx, i.config.KeyPrefix+t.key, flatten(fields)...) + } + + tuples = tuples[:0] + texts = texts[:0] + + return nil + } + + for _, doc := range docs { + hashes, err := i.config.DocumentToHashes(ctx, doc) + if err != nil { + return err + } + + key := hashes.Key + field2Value := hashes.Field2Value + fields := make(map[string]any, len(field2Value)) + embSize := 0 + for k, v := range field2Value { + fields[k] = v.Value + if v.EmbedKey != "" { + embSize++ + } + } + + if embSize > i.config.BatchSize { + return fmt.Errorf("[pipelineHSet] embedding size over batch size, batch size=%d, got size=%d", + i.config.BatchSize, embSize) + } + + if len(texts)+embSize > i.config.BatchSize { + if err = embAndAdd(); err != nil { + return err + } + } + + key2Idx := make(map[string]int, embSize) + for k, v := range field2Value { + if v.EmbedKey != "" { + if _, found := fields[v.EmbedKey]; found { + return fmt.Errorf("[pipelineHSet] duplicate key for value and vector, field=%s", k) + } + + var text string + if v.Stringify != nil { + text, err = v.Stringify(v.Value) + if err != nil { + return err + } + } else { + var ok bool + text, ok = v.Value.(string) + if !ok { + return fmt.Errorf("[pipelineHSet] assert value as string failed, key=%s, emb_key=%s", k, v.EmbedKey) + } + } + + key2Idx[v.EmbedKey] = len(texts) + texts = append(texts, text) + } + } + + tuples = append(tuples, tuple{ + key: key, + fields: fields, + key2Idx: key2Idx, + }) + } + + if len(tuples) > 0 { + if err = embAndAdd(); err != nil { + return err + } + } + + if _, err = pipeline.Exec(ctx); err != nil { + return err + } + + return nil +} + +func (i *Indexer) makeEmbeddingCtx(ctx context.Context, emb embedding.Embedder) context.Context { + runInfo := &callbacks.RunInfo{ + Component: components.ComponentOfEmbedding, + } + + if embType, ok := components.GetType(emb); ok { + runInfo.Type = embType + } + + runInfo.Name = runInfo.Type + string(runInfo.Component) + + return callbacks.ReuseHandlers(ctx, runInfo) +} + +const typ = "Redis" + +func (i *Indexer) GetType() string { + return typ +} + +func (i *Indexer) IsCallbacksEnabled() bool { + return true +} + +func defaultDocumentToFields(ctx context.Context, doc *schema.Document) (*Hashes, error) { + if doc.ID == "" { + return nil, fmt.Errorf("[defaultFieldMapping] doc id not set") + } + + field2Value := map[string]FieldValue{ + defaultReturnFieldContent: { + Value: doc.Content, + EmbedKey: defaultReturnFieldVectorContent, + Stringify: nil, + }, + } + for k := range doc.MetaData { + field2Value[k] = FieldValue{ + Value: doc.MetaData[k], + } + } + + return &Hashes{ + Key: doc.ID, + Field2Value: field2Value, + }, nil +} + +type tuple struct { + key string + fields map[string]any + key2Idx map[string]int +} + +func flatten(fields map[string]any) []any { + r := make([]any, 0, len(fields)*2) + for k := range fields { + r = append(r, k, fields[k]) + } + return r +} diff --git a/components/indexer/redis/indexer_test.go b/components/indexer/redis/indexer_test.go new file mode 100644 index 0000000..a785287 --- /dev/null +++ b/components/indexer/redis/indexer_test.go @@ -0,0 +1,216 @@ +/* + * Copyright 2025 CloudWeGo Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package redis + +import ( + "context" + "fmt" + "testing" + + . "github.com/bytedance/mockey" + "github.com/cloudwego/eino/components/embedding" + "github.com/cloudwego/eino/components/indexer" + "github.com/cloudwego/eino/schema" + "github.com/redis/go-redis/v9" + "github.com/smartystreets/goconvey/convey" +) + +func TestPipelineHSet(t *testing.T) { + PatchConvey("test pipelineHSet", t, func() { + ctx := context.Background() + mockClient := &redis.Client{} + d1 := &schema.Document{ID: "1", Content: "asd"} + d2 := &schema.Document{ID: "2", Content: "qwe", MetaData: map[string]any{ + "mock_field_1": map[string]any{"extra_field_1": "asd"}, + "mock_field_2": int64(123), + }} + docs := []*schema.Document{d1, d2} + + PatchConvey("test DocumentToHashes failed", func() { + i := &Indexer{ + config: &IndexerConfig{ + Client: mockClient, + DocumentToHashes: func(ctx context.Context, doc *schema.Document) (*Hashes, error) { + return nil, fmt.Errorf("mock err") + }, + BatchSize: 10, + Embedding: nil, + }, + } + + convey.So(i.pipelineHSet(ctx, docs, &indexer.Options{ + Embedding: nil, + }), convey.ShouldBeError, fmt.Errorf("mock err")) + }) + + PatchConvey("test embSize > i.config.BatchSize", func() { + i := &Indexer{ + config: &IndexerConfig{ + Client: mockClient, + DocumentToHashes: func(ctx context.Context, doc *schema.Document) (*Hashes, error) { + return &Hashes{ + Key: doc.ID, + Field2Value: map[string]FieldValue{ + defaultReturnFieldContent: { + Value: doc.Content, + EmbedKey: defaultReturnFieldVectorContent, + }, + "mock_another_field": { + Value: doc.Content, + EmbedKey: "mock_another_vector_field", + }, + }, + }, nil + }, + BatchSize: 1, + Embedding: nil, + }, + } + + convey.So(i.pipelineHSet(ctx, docs, &indexer.Options{ + Embedding: nil, + }), convey.ShouldBeError, fmt.Errorf("[pipelineHSet] embedding size over batch size, batch size=%d, got size=%d", + i.config.BatchSize, 2)) + }) + + PatchConvey("test embedding not provided error", func() { + i := &Indexer{ + config: &IndexerConfig{ + Client: mockClient, + DocumentToHashes: defaultDocumentToFields, + BatchSize: 1, + Embedding: nil, + }, + } + + convey.So(i.pipelineHSet(ctx, docs, &indexer.Options{ + Embedding: nil, + }), convey.ShouldBeError, fmt.Errorf("[pipelineHSet] embedding method not provided")) + }) + + PatchConvey("test embedding failed", func() { + exp := fmt.Errorf("mock err") + i := &Indexer{ + config: &IndexerConfig{ + Client: mockClient, + DocumentToHashes: defaultDocumentToFields, + BatchSize: 1, + }, + } + + convey.So(i.pipelineHSet(ctx, docs, &indexer.Options{ + Embedding: &mockEmbedding{err: exp}, + }), convey.ShouldBeError, fmt.Errorf("[pipelineHSet] embedding failed, %w", exp)) + }) + + PatchConvey("test len(vectors) != len(texts)", func() { + i := &Indexer{ + config: &IndexerConfig{ + Client: mockClient, + DocumentToHashes: defaultDocumentToFields, + BatchSize: 1, + }, + } + + convey.So(i.pipelineHSet(ctx, docs, &indexer.Options{ + Embedding: &mockEmbedding{sizeForCall: []int{2}, dims: 1024}, + }), convey.ShouldBeError, fmt.Errorf("[pipelineHSet] invalid vector length, expected=1, got=2")) + }) + + PatchConvey("test success", func() { + args := make(map[string][]any) + pl := &redis.Pipeline{} + Mock(GetMethod(mockClient, "Pipeline")).Return(pl).Build() + Mock(GetMethod(pl, "HSet")).To(func(ctx context.Context, key string, values ...interface{}) *redis.IntCmd { + args[key] = values + return nil + }).Build() + Mock(GetMethod(pl, "Exec")).Return(nil, nil).Build() + + prefix := "test_prefix" + i := &Indexer{ + config: &IndexerConfig{ + Client: mockClient, + DocumentToHashes: defaultDocumentToFields, + KeyPrefix: prefix, + BatchSize: 1, + }, + } + + convey.So(i.pipelineHSet(ctx, docs, &indexer.Options{ + Embedding: &mockEmbedding{sizeForCall: []int{1, 1}, dims: 1024}, + }), convey.ShouldBeNil) + + slice := make([]float64, 1024) + for i := range slice { + slice[i] = 1.1 + } + + contains := func(doc *schema.Document) { + a := args[prefix+doc.ID] + convey.So(a, convey.ShouldNotBeNil) + f2v := make(map[string]any) + for i := 0; i < len(a); i += 2 { + f2v[a[i].(string)] = a[i+1] + } + for field, val := range f2v { + if field == defaultReturnFieldContent { + convey.So(val.(string), convey.ShouldEqual, doc.Content) + } else if field == defaultReturnFieldVectorContent { + convey.So(val.([]byte), convey.ShouldEqual, vector2Bytes(slice)) + } else { + val2, found := doc.MetaData[field] + convey.So(found, convey.ShouldBeTrue) + convey.So(val, convey.ShouldEqual, val2) + } + } + } + contains(d1) + contains(d2) + }) + }) +} + +type mockEmbedding struct { + err error + cnt int + sizeForCall []int + dims int +} + +func (m *mockEmbedding) EmbedStrings(ctx context.Context, texts []string, opts ...embedding.Option) ([][]float64, error) { + if m.cnt > len(m.sizeForCall) { + panic("unexpected") + } + + if m.err != nil { + return nil, m.err + } + + slice := make([]float64, m.dims) + for i := range slice { + slice[i] = 1.1 + } + + r := make([][]float64, m.sizeForCall[m.cnt]) + m.cnt++ + for i := range r { + r[i] = slice + } + + return r, nil +} diff --git a/components/indexer/redis/utils.go b/components/indexer/redis/utils.go new file mode 100644 index 0000000..1b05c60 --- /dev/null +++ b/components/indexer/redis/utils.go @@ -0,0 +1,34 @@ +/* + * Copyright 2025 CloudWeGo Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package redis + +import ( + "encoding/binary" + "math" +) + +func vector2Bytes(vector []float64) []byte { + float32Arr := make([]float32, len(vector)) + for i, v := range vector { + float32Arr[i] = float32(v) + } + bytes := make([]byte, len(float32Arr)*4) + for i, v := range float32Arr { + binary.LittleEndian.PutUint32(bytes[i*4:], math.Float32bits(v)) + } + return bytes +} diff --git a/components/retriever/redis/consts.go b/components/retriever/redis/consts.go new file mode 100644 index 0000000..cc39699 --- /dev/null +++ b/components/retriever/redis/consts.go @@ -0,0 +1,28 @@ +/* + * Copyright 2025 CloudWeGo Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package redis + +const ( + defaultReturnFieldContent = "content" + defaultReturnFieldVectorContent = "vector_content" + paramVector = "vector" + paramDistanceThreshold = "distance_threshold" + // SortByDistanceAttributeName is attribute name for ft search. + // Document fields should not contain this, or search won't process as expected. + // SortByDistanceAttributeName could also be one of the return fields. + SortByDistanceAttributeName = "distance" +) diff --git a/components/retriever/redis/go.mod b/components/retriever/redis/go.mod new file mode 100644 index 0000000..bc550df --- /dev/null +++ b/components/retriever/redis/go.mod @@ -0,0 +1,48 @@ +module github.com/cloudwego/eino-ext/components/retriever/redis + +go 1.18 + +require ( + github.com/bytedance/mockey v1.2.13 + github.com/cloudwego/eino v0.3.6 + github.com/redis/go-redis/v9 v9.7.0 + github.com/smartystreets/goconvey v1.8.1 +) + +require ( + github.com/bytedance/sonic v1.12.2 // indirect + github.com/bytedance/sonic/loader v0.2.0 // indirect + github.com/cespare/xxhash/v2 v2.2.0 // indirect + github.com/cloudwego/base64x v0.1.4 // indirect + github.com/cloudwego/iasm v0.2.0 // indirect + github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f // indirect + github.com/dustin/go-humanize v1.0.1 // indirect + github.com/getkin/kin-openapi v0.118.0 // indirect + github.com/go-openapi/jsonpointer v0.19.5 // indirect + github.com/go-openapi/swag v0.19.5 // indirect + github.com/goph/emperror v0.17.2 // indirect + github.com/gopherjs/gopherjs v1.17.2 // indirect + github.com/invopop/yaml v0.1.0 // indirect + github.com/josharian/intern v1.0.0 // indirect + github.com/json-iterator/go v1.1.12 // indirect + github.com/jtolds/gls v4.20.0+incompatible // indirect + github.com/klauspost/cpuid/v2 v2.0.9 // indirect + github.com/mailru/easyjson v0.7.7 // indirect + github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect + github.com/modern-go/reflect2 v1.0.2 // indirect + github.com/mohae/deepcopy v0.0.0-20170929034955-c48cc78d4826 // indirect + github.com/nikolalohinski/gonja v1.5.3 // indirect + github.com/pelletier/go-toml/v2 v2.0.9 // indirect + github.com/perimeterx/marshmallow v1.1.4 // indirect + github.com/pkg/errors v0.9.1 // indirect + github.com/sirupsen/logrus v1.9.3 // indirect + github.com/slongfield/pyfmt v0.0.0-20220222012616-ea85ff4c361f // indirect + github.com/smarty/assertions v1.15.0 // indirect + github.com/twitchyliquid64/golang-asm v0.15.1 // indirect + github.com/yargevad/filepathx v1.0.0 // indirect + golang.org/x/arch v0.11.0 // indirect + golang.org/x/exp v0.0.0-20230713183714-613f0c0eb8a1 // indirect + golang.org/x/sys v0.26.0 // indirect + gopkg.in/yaml.v2 v2.4.0 // indirect + gopkg.in/yaml.v3 v3.0.1 // indirect +) diff --git a/components/retriever/redis/go.sum b/components/retriever/redis/go.sum new file mode 100644 index 0000000..bc6a972 --- /dev/null +++ b/components/retriever/redis/go.sum @@ -0,0 +1,159 @@ +github.com/airbrake/gobrake v3.6.1+incompatible/go.mod h1:wM4gu3Cn0W0K7GUuVWnlXZU11AGBXMILnrdOU8Kn00o= +github.com/bitly/go-simplejson v0.5.0/go.mod h1:cXHtHw4XUPsvGaxgjIAn8PhEWG9NfngEKAMDJEczWVA= +github.com/bmizerany/assert v0.0.0-20160611221934-b7ed37b82869/go.mod h1:Ekp36dRnpXw/yCqJaO+ZrUyxD+3VXMFFr56k5XYrpB4= +github.com/bsm/ginkgo/v2 v2.12.0 h1:Ny8MWAHyOepLGlLKYmXG4IEkioBysk6GpaRTLC8zwWs= +github.com/bsm/gomega v1.27.10 h1:yeMWxP2pV2fG3FgAODIY8EiRE3dy0aeFYt4l7wh6yKA= +github.com/bugsnag/bugsnag-go v1.4.0/go.mod h1:2oa8nejYd4cQ/b0hMIopN0lCRxU0bueqREvZLWFrtK8= +github.com/bugsnag/panicwrap v1.2.0/go.mod h1:D/8v3kj0zr8ZAKg1AQ6crr+5VwKN5eIywRkfhyM/+dE= +github.com/bytedance/mockey v1.2.13 h1:jokWZAm/pUEbD939Rhznz615MKUCZNuvCFQlJ2+ntoo= +github.com/bytedance/mockey v1.2.13/go.mod h1:1BPHF9sol5R1ud/+0VEHGQq/+i2lN+GTsr3O2Q9IENY= +github.com/bytedance/sonic v1.12.2 h1:oaMFuRTpMHYLpCntGca65YWt5ny+wAceDERTkT2L9lg= +github.com/bytedance/sonic v1.12.2/go.mod h1:B8Gt/XvtZ3Fqj+iSKMypzymZxw/FVwgIGKzMzT9r/rk= +github.com/bytedance/sonic/loader v0.1.1/go.mod h1:ncP89zfokxS5LZrJxl5z0UJcsk4M4yY2JpfqGeCtNLU= +github.com/bytedance/sonic/loader v0.2.0 h1:zNprn+lsIP06C/IqCHs3gPQIvnvpKbbxyXQP1iU4kWM= +github.com/bytedance/sonic/loader v0.2.0/go.mod h1:ncP89zfokxS5LZrJxl5z0UJcsk4M4yY2JpfqGeCtNLU= +github.com/certifi/gocertifi v0.0.0-20190105021004-abcd57078448/go.mod h1:GJKEexRPVJrBSOjoqN5VNOIKJ5Q3RViH6eu3puDRwx4= +github.com/cespare/xxhash/v2 v2.2.0 h1:DC2CZ1Ep5Y4k3ZQ899DldepgrayRUGE6BBZ/cd9Cj44= +github.com/cespare/xxhash/v2 v2.2.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= +github.com/cloudwego/base64x v0.1.4 h1:jwCgWpFanWmN8xoIUHa2rtzmkd5J2plF/dnLS6Xd/0Y= +github.com/cloudwego/base64x v0.1.4/go.mod h1:0zlkT4Wn5C6NdauXdJRhSKRlJvmclQ1hhJgA0rcu/8w= +github.com/cloudwego/eino v0.3.6 h1:3yfdKKxMVWefdOyGXHuqUMM5cc9iioijj2mpPsDZKIg= +github.com/cloudwego/eino v0.3.6/go.mod h1:+kmJimGEcKuSI6OKhet7kBedkm1WUZS3H1QRazxgWUo= +github.com/cloudwego/iasm v0.2.0 h1:1KNIy1I1H9hNNFEEH3DVnI4UujN+1zjpuk6gwHLTssg= +github.com/cloudwego/iasm v0.2.0/go.mod h1:8rXZaNYT2n95jn+zTI1sDr+IgcD2GVs0nlbbQPiEFhY= +github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f h1:lO4WD4F/rVNCu3HqELle0jiPLLBs70cWOduZpkS1E78= +github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f/go.mod h1:cuUVRXasLTGF7a8hSLbxyZXjz+1KgoB3wDUb6vlszIc= +github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY= +github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto= +github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= +github.com/getkin/kin-openapi v0.118.0 h1:z43njxPmJ7TaPpMSCQb7PN0dEYno4tyBPQcrFdHoLuM= +github.com/getkin/kin-openapi v0.118.0/go.mod h1:l5e9PaFUo9fyLJCPGQeXI2ML8c3P8BHOEV2VaAVf/pc= +github.com/getsentry/raven-go v0.2.0/go.mod h1:KungGk8q33+aIAZUIVWZDr2OfAEBsO49PX4NzFV5kcQ= +github.com/go-check/check v0.0.0-20180628173108-788fd7840127 h1:0gkP6mzaMqkmpcJYCFOLkIBwI7xFExG03bbkOkCvUPI= +github.com/go-openapi/jsonpointer v0.19.5 h1:gZr+CIYByUqjcgeLXnQu2gHYQC9o73G2XUeOFYEICuY= +github.com/go-openapi/jsonpointer v0.19.5/go.mod h1:Pl9vOtqEWErmShwVjC8pYs9cog34VGT37dQOVbmoatg= +github.com/go-openapi/swag v0.19.5 h1:lTz6Ys4CmqqCQmZPBlbQENR1/GucA2bzYTE12Pw4tFY= +github.com/go-openapi/swag v0.19.5/go.mod h1:POnQmlKehdgb5mhVOsnJFsivZCEZ/vjK9gh66Z9tfKk= +github.com/go-test/deep v1.0.8 h1:TDsG77qcSprGbC6vTN8OuXp5g+J+b5Pcguhf7Zt61VM= +github.com/go-test/deep v1.0.8/go.mod h1:5C2ZWiW0ErCdrYzpqxLbTX7MG14M9iiw8DgHncVwcsE= +github.com/gofrs/uuid v3.2.0+incompatible/go.mod h1:b2aQJv3Z4Fp6yNu3cdSllBxTCLRxnplIgP/c0N/04lM= +github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= +github.com/goph/emperror v0.17.2 h1:yLapQcmEsO0ipe9p5TaN22djm3OFV/TfM/fcYP0/J18= +github.com/goph/emperror v0.17.2/go.mod h1:+ZbQ+fUNO/6FNiUo0ujtMjhgad9Xa6fQL9KhH4LNHic= +github.com/gopherjs/gopherjs v1.17.2 h1:fQnZVsXk8uxXIStYb0N4bGk7jeyTalG/wsZjQ25dO0g= +github.com/gopherjs/gopherjs v1.17.2/go.mod h1:pRRIvn/QzFLrKfvEz3qUuEhtE/zLCWfreZ6J5gM2i+k= +github.com/gorilla/mux v1.8.0/go.mod h1:DVbg23sWSpFRCP0SfiEN6jmj59UnW/n46BH5rLB71So= +github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU= +github.com/invopop/yaml v0.1.0 h1:YW3WGUoJEXYfzWBjn00zIlrw7brGVD0fUKRYDPAPhrc= +github.com/invopop/yaml v0.1.0/go.mod h1:2XuRLgs/ouIrW3XNzuNj7J3Nvu/Dig5MXvbCEdiBN3Q= +github.com/josharian/intern v1.0.0 h1:vlS4z54oSdjm0bgjRigI+G1HpF+tI+9rE5LLzOg8HmY= +github.com/josharian/intern v1.0.0/go.mod h1:5DoeVV0s6jJacbCEi61lwdGj/aVlrQvzHFFd8Hwg//Y= +github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM= +github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo= +github.com/jtolds/gls v4.20.0+incompatible h1:xdiiI2gbIgH/gLH7ADydsJ1uDOEzR8yvV7C0MuV77Wo= +github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU= +github.com/kardianos/osext v0.0.0-20190222173326-2bc1f35cddc0/go.mod h1:1NbS8ALrpOvjt0rHPNLyCIeMtbizbir8U//inJ+zuB8= +github.com/klauspost/cpuid/v2 v2.0.9 h1:lgaqFMSdTdQYdZ04uHyN2d/eKdOMyi2YLSvlQIBFYa4= +github.com/klauspost/cpuid/v2 v2.0.9/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg= +github.com/knz/go-libedit v1.10.1/go.mod h1:MZTVkCWyz0oBc7JOWP3wNAzd002ZbM/5hgShxwh4x8M= +github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= +github.com/kr/pretty v0.1.0 h1:L/CwN0zerZDmRFUapSPitk6f+Q3+0za1rQkzVuMiMFI= +github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= +github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= +github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE= +github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= +github.com/mailru/easyjson v0.0.0-20190614124828-94de47d64c63/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= +github.com/mailru/easyjson v0.0.0-20190626092158-b2ccc519800e/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= +github.com/mailru/easyjson v0.7.7 h1:UGYAvKxe3sBsEDzO8ZeWOSlIQfWFlxbzLZe7hwFURr0= +github.com/mailru/easyjson v0.7.7/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJJLY9Nlc= +github.com/mattn/go-colorable v0.1.2 h1:/bC9yWikZXAL9uJdulbSfyVNIR3n3trXl+v8+1sx8mU= +github.com/mattn/go-isatty v0.0.8 h1:HLtExJ+uU2HOZ+wI0Tt5DtUDrx8yhUqDcp7fYERX4CE= +github.com/mgutz/ansi v0.0.0-20170206155736-9520e82c474b h1:j7+1HpAFS1zy5+Q4qx1fWh90gTKwiN4QCGoY9TWyyO4= +github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= +github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg= +github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= +github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9Gz0M= +github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk= +github.com/mohae/deepcopy v0.0.0-20170929034955-c48cc78d4826 h1:RWengNIwukTxcDr9M+97sNutRR1RKhG96O6jWumTTnw= +github.com/mohae/deepcopy v0.0.0-20170929034955-c48cc78d4826/go.mod h1:TaXosZuwdSHYgviHp1DAtfrULt5eUgsSMsZf+YrPgl8= +github.com/nikolalohinski/gonja v1.5.3 h1:GsA+EEaZDZPGJ8JtpeGN78jidhOlxeJROpqMT9fTj9c= +github.com/nikolalohinski/gonja v1.5.3/go.mod h1:RmjwxNiXAEqcq1HeK5SSMmqFJvKOfTfXhkJv6YBtPa4= +github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= +github.com/onsi/ginkgo v1.8.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= +github.com/onsi/gomega v1.5.0/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY= +github.com/pelletier/go-toml/v2 v2.0.9 h1:uH2qQXheeefCCkuBBSLi7jCiSmj3VRh2+Goq2N7Xxu0= +github.com/pelletier/go-toml/v2 v2.0.9/go.mod h1:tJU2Z3ZkXwnxa4DPO899bsyIoywizdUvyaeZurnPPDc= +github.com/perimeterx/marshmallow v1.1.4 h1:pZLDH9RjlLGGorbXhcaQLhfuV0pFMNfPO55FuFkxqLw= +github.com/perimeterx/marshmallow v1.1.4/go.mod h1:dsXbUu8CRzfYP5a87xpp0xq9S3u0Vchtcl8we9tYaXw= +github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= +github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/redis/go-redis/v9 v9.7.0 h1:HhLSs+B6O021gwzl+locl0zEDnyNkxMtf/Z3NNBMa9E= +github.com/redis/go-redis/v9 v9.7.0/go.mod h1:f6zhXITC7JUJIlPEiBOTXxJgPLdZcA93GewI7inzyWw= +github.com/rollbar/rollbar-go v1.0.2/go.mod h1:AcFs5f0I+c71bpHlXNNDbOWJiKwjFDtISeXco0L5PKQ= +github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= +github.com/sirupsen/logrus v1.9.3 h1:dueUQJ1C2q9oE3F7wvmSGAaVtTmUizReu6fjN8uqzbQ= +github.com/sirupsen/logrus v1.9.3/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ= +github.com/slongfield/pyfmt v0.0.0-20220222012616-ea85ff4c361f h1:Z2cODYsUxQPofhpYRMQVwWz4yUVpHF+vPi+eUdruUYI= +github.com/slongfield/pyfmt v0.0.0-20220222012616-ea85ff4c361f/go.mod h1:JqzWyvTuI2X4+9wOHmKSQCYxybB/8j6Ko43qVmXDuZg= +github.com/smarty/assertions v1.15.0 h1:cR//PqUBUiQRakZWqBiFFQ9wb8emQGDb0HeGdqGByCY= +github.com/smarty/assertions v1.15.0/go.mod h1:yABtdzeQs6l1brC900WlRNwj6ZR55d7B+E8C6HtKdec= +github.com/smartystreets/goconvey v1.8.1 h1:qGjIddxOk4grTu9JPOU31tVfq3cNdBlNa5sSznIX1xY= +github.com/smartystreets/goconvey v1.8.1/go.mod h1:+/u4qLyY6x1jReYOp7GOM2FSt8aP9CzCZL03bI28W60= +github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= +github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= +github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= +github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= +github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= +github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= +github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= +github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg= +github.com/twitchyliquid64/golang-asm v0.15.1 h1:SU5vSMR7hnwNxj24w34ZyCi/FmDZTkS4MhqMhdFk5YI= +github.com/twitchyliquid64/golang-asm v0.15.1/go.mod h1:a1lVb/DtPvCB8fslRZhAngC2+aY1QWCk3Cedj/Gdt08= +github.com/ugorji/go v1.2.7 h1:qYhyWUUd6WbiM+C6JZAUkIJt/1WrjzNHY9+KCIjVqTo= +github.com/ugorji/go v1.2.7/go.mod h1:nF9osbDWLy6bDVv/Rtoh6QgnvNDpmCalQV5urGCCS6M= +github.com/ugorji/go/codec v1.2.7 h1:YPXUKf7fYbp/y8xloBqZOw2qaVggbfwMlI8WM3wZUJ0= +github.com/ugorji/go/codec v1.2.7/go.mod h1:WGN1fab3R1fzQlVQTkfxVtIBhWDRqOviHU95kRgeqEY= +github.com/x-cray/logrus-prefixed-formatter v0.5.2 h1:00txxvfBM9muc0jiLIEAkAcIMJzfthRT6usrui8uGmg= +github.com/yargevad/filepathx v1.0.0 h1:SYcT+N3tYGi+NvazubCNlvgIPbzAk7i7y2dwg3I5FYc= +github.com/yargevad/filepathx v1.0.0/go.mod h1:BprfX/gpYNJHJfc35GjRRpVcwWXS89gGulUIU5tK3tA= +go.uber.org/mock v0.4.0 h1:VcM4ZOtdbR4f6VXfiOpwpVJDL6lCReaZ6mw31wqh7KU= +golang.org/x/arch v0.11.0 h1:KXV8WWKCXm6tRpLirl2szsO5j/oOODwZf4hATmGVNs4= +golang.org/x/arch v0.11.0/go.mod h1:FEVrYAQjsQXMVJ1nsMoVVXPZg6p2JE2mx8psSWTDQys= +golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= +golang.org/x/crypto v0.11.0 h1:6Ewdq3tDic1mg5xRO4milcWCfMVQhI4NkqWWvqejpuA= +golang.org/x/exp v0.0.0-20230713183714-613f0c0eb8a1 h1:MGwJjxBy0HJshjDNfLsYO8xppfqWlA5ZT9OhtUUhTNw= +golang.org/x/exp v0.0.0-20230713183714-613f0c0eb8a1/go.mod h1:FXUEEKJgO7OQYeo8N01OfiKP8RXMtf6e8aTskBGqWdc= +golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.26.0 h1:KHjCJyddX0LoSTb3J+vWpupP9p0oznkqVk/IfjymZbo= +golang.org/x/sys v0.26.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/term v0.10.0 h1:3R7pNqamzBraeqj/Tj8qt1aQ2HpmlC+Cx/qL/7hn4/c= +golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 h1:qIbj1fsPNlZgppZ+VLlY7N33q108Sa+fhmuc+sWQYwY= +gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys= +gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw= +gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= +gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= +gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gopkg.in/yaml.v3 v3.0.0/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +nullprogram.com/x/optparse v1.0.0/go.mod h1:KdyPE+Igbe0jQUrVfMqDMeJQIJZEuyV7pjYmp6pbG50= diff --git a/components/retriever/redis/options.go b/components/retriever/redis/options.go new file mode 100644 index 0000000..07abc72 --- /dev/null +++ b/components/retriever/redis/options.go @@ -0,0 +1,21 @@ +/* + * Copyright 2025 CloudWeGo Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package redis + +type ImplOptions struct { + FilterQuery string +} diff --git a/components/retriever/redis/retriever.go b/components/retriever/redis/retriever.go new file mode 100644 index 0000000..e0ebd57 --- /dev/null +++ b/components/retriever/redis/retriever.go @@ -0,0 +1,255 @@ +/* + * Copyright 2025 CloudWeGo Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package redis + +import ( + "context" + "fmt" + + "github.com/cloudwego/eino/callbacks" + "github.com/cloudwego/eino/components" + "github.com/cloudwego/eino/components/embedding" + "github.com/cloudwego/eino/components/retriever" + "github.com/cloudwego/eino/schema" + "github.com/redis/go-redis/v9" +) + +type RetrieverConfig struct { + // Client is a Redis client representing a pool of zero or more underlying connections. + // It's safe for concurrent use by multiple goroutines, which means is okay to pass + // an existed Client to create a new Retriever component. + // ***NOTICE***: two client conn options should be configured correctly to enable ft.search. + // 1. Protocol: default is 3, here needs 2, or FT.Search result is raw. + // 2. UnstableResp3: default is false, here needs true. + Client *redis.Client + // Index name of index to search. + // see: https://redis.io/docs/latest/develop/interact/search-and-query/advanced-concepts/vectors/#create-a-vector-index + Index string + // VectorField vector field name in search query, correspond to FieldValue.EmbedKey from redis indexer. + // Default "vector_content" + VectorField string + // DistanceThreshold controls how to build search query. + // If DistanceThreshold is set, use vector range search. + // If DistanceThreshold is not set, use KNN vector search. + // Default is nil. + // Vector Range Queries: https://redis.io/docs/latest/develop/interact/search-and-query/advanced-concepts/vectors/#vector-range-queries + // KNN Vector Search: https://redis.io/docs/latest/develop/interact/search-and-query/advanced-concepts/vectors/#knn-vector-search + DistanceThreshold *float64 + // Dialect default 2. + // see: https://redis.io/docs/latest/develop/interact/search-and-query/advanced-concepts/dialects/ + Dialect int + // ReturnFields limits the attributes returned from the document. num is the number of attributes following the keyword. + // Default []string{"content", "vector_content"} + ReturnFields []string + // DocumentConverter converts retrieved raw document to eino Document, default defaultResultParser. + DocumentConverter func(ctx context.Context, doc redis.Document) (*schema.Document, error) + // TopK limits number of results given, default 5. + TopK int + // Embedding vectorization method for query. + Embedding embedding.Embedder +} + +type Retriever struct { + config *RetrieverConfig +} + +func NewRetriever(ctx context.Context, config *RetrieverConfig) (*Retriever, error) { + if config.Embedding == nil { + return nil, fmt.Errorf("[NewRetriever] embedding not provided for redis retriever") + } + + if config.Index == "" { + return nil, fmt.Errorf("[NewRetriever] redis index not provided") + } + + if config.Client == nil { + return nil, fmt.Errorf("[NewRetriever] redis client not provided") + } + + if config.Dialect < 2 { + // Support for vector search also was introduced in the 2.4 + config.Dialect = 2 + } + + if config.TopK == 0 { + config.TopK = 5 + } + + if config.VectorField == "" { + config.VectorField = defaultReturnFieldVectorContent + } + + if len(config.ReturnFields) == 0 { + config.ReturnFields = []string{ + defaultReturnFieldContent, + defaultReturnFieldVectorContent, + } + } + + if config.DocumentConverter == nil { + config.DocumentConverter = defaultResultParser(config.ReturnFields) + } + + return &Retriever{ + config: config, + }, nil +} + +func (r *Retriever) Retrieve(ctx context.Context, query string, opts ...retriever.Option) (docs []*schema.Document, err error) { + defer func() { + if err != nil { + callbacks.OnError(ctx, err) + } + }() + + co := retriever.GetCommonOptions(&retriever.Options{ + Index: &r.config.Index, + TopK: &r.config.TopK, + ScoreThreshold: r.config.DistanceThreshold, + Embedding: r.config.Embedding, + }, opts...) + io := retriever.GetImplSpecificOptions(&ImplOptions{}, opts...) + + ctx = callbacks.OnStart(ctx, &retriever.CallbackInput{ + Query: query, + TopK: *co.TopK, + Filter: io.FilterQuery, + ScoreThreshold: co.ScoreThreshold, + }) + + emb := co.Embedding + if emb == nil { + return nil, fmt.Errorf("[redis retriever] embedding not provided") + } + + vectors, err := emb.EmbedStrings(r.makeEmbeddingCtx(ctx, emb), []string{query}) + if err != nil { + return nil, err + } + + if len(vectors) != 1 { + return nil, fmt.Errorf("[redis retriever] invalid return length of vector, got=%d, expected=1", len(vectors)) + } + + params := map[string]any{ + paramVector: vector2Bytes(vectors[0]), + } + + var searchQuery string + if r.config.DistanceThreshold != nil { + params[paramDistanceThreshold] = dereferenceOrZero(r.config.DistanceThreshold) + baseQuery := fmt.Sprintf("@%s:[VECTOR_RANGE $%s $%s]", r.config.VectorField, paramDistanceThreshold, paramVector) + + if io.FilterQuery != "" { + baseQuery = io.FilterQuery + " " + baseQuery + } + + searchQuery = fmt.Sprintf("%s=>{$yield_distance_as: %s}", baseQuery, SortByDistanceAttributeName) + } else { + filter := "*" + if io.FilterQuery != "" { + filter = io.FilterQuery + } + + searchQuery = fmt.Sprintf("(%s)=>[KNN %d @%s $%s AS %s]", + filter, *co.TopK, r.config.VectorField, paramVector, SortByDistanceAttributeName) + } + + sr := make([]redis.FTSearchReturn, 0, len(r.config.ReturnFields)) + for _, field := range r.config.ReturnFields { + sr = append(sr, redis.FTSearchReturn{FieldName: field}) + } + + searchOptions := &redis.FTSearchOptions{ + Return: sr, + SortBy: []redis.FTSearchSortBy{{FieldName: SortByDistanceAttributeName, Asc: true}}, + Limit: *co.TopK, + DialectVersion: r.config.Dialect, + Params: params, + WithScores: false, + } + + cmd := r.config.Client.FTSearchWithArgs(ctx, *co.Index, searchQuery, searchOptions) + result, err := cmd.Result() // here required RESP protocol=2 + if err != nil { + return nil, err + } + + for _, raw := range result.Docs { + doc, err := r.config.DocumentConverter(ctx, raw) + if err != nil { + return nil, err + } + docs = append(docs, doc) + } + + callbacks.OnEnd(ctx, &retriever.CallbackOutput{Docs: docs}) + + return docs, nil + +} + +func (r *Retriever) makeEmbeddingCtx(ctx context.Context, emb embedding.Embedder) context.Context { + runInfo := &callbacks.RunInfo{ + Component: components.ComponentOfEmbedding, + } + + if embType, ok := components.GetType(emb); ok { + runInfo.Type = embType + } + + runInfo.Name = runInfo.Type + string(runInfo.Component) + + return callbacks.ReuseHandlers(ctx, runInfo) +} + +const typ = "Redis" + +func (r *Retriever) GetType() string { + return typ +} + +func (r *Retriever) IsCallbacksEnabled() bool { + return true +} + +func defaultResultParser(returnFields []string) func(ctx context.Context, doc redis.Document) (*schema.Document, error) { + return func(ctx context.Context, doc redis.Document) (*schema.Document, error) { + resp := &schema.Document{ + ID: doc.ID, + Content: "", + MetaData: map[string]any{}, + } + + for _, field := range returnFields { + val, found := doc.Fields[field] + if !found { + return nil, fmt.Errorf("[defaultResultParser] field=%s not found in doc, doc=%v", field, doc) + } + + if field == defaultReturnFieldContent { + resp.Content = val + } else if field == defaultReturnFieldVectorContent { + resp.WithDenseVector(Bytes2Vector([]byte(val))) + } else { + resp.MetaData[field] = val + } + } + + return resp, nil + } +} diff --git a/components/retriever/redis/retriever_test.go b/components/retriever/redis/retriever_test.go new file mode 100644 index 0000000..e8924ef --- /dev/null +++ b/components/retriever/redis/retriever_test.go @@ -0,0 +1,259 @@ +/* + * Copyright 2025 CloudWeGo Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package redis + +import ( + "context" + "fmt" + "testing" + + . "github.com/bytedance/mockey" + "github.com/cloudwego/eino/components/embedding" + "github.com/cloudwego/eino/schema" + "github.com/redis/go-redis/v9" + "github.com/smartystreets/goconvey/convey" +) + +func TestNewRetriever(t *testing.T) { + PatchConvey("test NewRetriever", t, func() { + ctx := context.Background() + mockClient := &redis.Client{} + + PatchConvey("test embedding not provided", func() { + r, err := NewRetriever(ctx, &RetrieverConfig{ + Client: mockClient, + Index: "asd", + Embedding: nil, + }) + convey.So(err, convey.ShouldBeError, fmt.Errorf("[NewRetriever] embedding not provided for redis retriever")) + convey.So(r, convey.ShouldBeNil) + }) + + PatchConvey("test index not provided", func() { + r, err := NewRetriever(ctx, &RetrieverConfig{ + Client: mockClient, + Index: "", + Embedding: &mockEmbedding{}, + }) + convey.So(err, convey.ShouldBeError, fmt.Errorf("[NewRetriever] redis index not provided")) + convey.So(r, convey.ShouldBeNil) + }) + + PatchConvey("test redis client not provided", func() { + r, err := NewRetriever(ctx, &RetrieverConfig{ + Client: nil, + Index: "asd", + Embedding: &mockEmbedding{}, + }) + convey.So(err, convey.ShouldBeError, fmt.Errorf("[NewRetriever] redis client not provided")) + convey.So(r, convey.ShouldBeNil) + }) + + PatchConvey("test success", func() { + r, err := NewRetriever(ctx, &RetrieverConfig{ + Client: mockClient, + Index: "asd", + Embedding: &mockEmbedding{}, + }) + convey.So(err, convey.ShouldBeNil) + convey.So(r, convey.ShouldNotBeNil) + }) + }) +} + +func TestRetrieve(t *testing.T) { + PatchConvey("test Retrieve", t, func() { + ctx := context.Background() + mockClient := redis.NewClient(&redis.Options{Addr: "123"}) + expv := make([]float64, 10) + for i := range expv { + expv[i] = 1.1 + } + d1 := &schema.Document{ID: "1", Content: "asd"} + d1.WithDenseVector(expv) + d2 := &schema.Document{ID: "2", Content: "qwe"} + d2.WithDenseVector(expv) + docs := []*schema.Document{d1, d2} + + PatchConvey("test Embedding not provided", func() { + r := &Retriever{config: &RetrieverConfig{Embedding: nil}} + resp, err := r.Retrieve(ctx, "test_query") + convey.So(err, convey.ShouldBeError, fmt.Errorf("[redis retriever] embedding not provided")) + convey.So(resp, convey.ShouldBeNil) + }) + + PatchConvey("test Embedding error", func() { + mockErr := fmt.Errorf("mock err") + r := &Retriever{config: &RetrieverConfig{Embedding: &mockEmbedding{err: mockErr}}} + resp, err := r.Retrieve(ctx, "test_query") + convey.So(err, convey.ShouldBeError, mockErr) + convey.So(resp, convey.ShouldBeNil) + }) + + PatchConvey("test vector size invalid", func() { + r := &Retriever{config: &RetrieverConfig{Embedding: &mockEmbedding{sizeForCall: []int{2}, dims: 10}}} + resp, err := r.Retrieve(ctx, "test_query") + convey.So(err, convey.ShouldBeError, fmt.Errorf("[redis retriever] invalid return length of vector, got=2, expected=1")) + convey.So(resp, convey.ShouldBeNil) + }) + + PatchConvey("test vector range query", func() { + dis := 10.2 + //origin := mockClient.FTSearchWithArgs + var ( + //cmd *redis.FTSearchCmd + mockCmd *redis.FTSearchCmd + ) + + //Mock(GetMethod(mockClient, "FTSearchWithArgs")).To( + // func(ctx context.Context, index string, query string, options *redis.FTSearchOptions) *redis.FTSearchCmd { + // cmd = origin(ctx, index, query, options) + // return mockCmd + // }).Origin(&origin).Build() + + Mock(GetMethod(mockClient, "FTSearchWithArgs")).Return(mockCmd).Build() + Mock(GetMethod(mockCmd, "Result")).Return(redis.FTSearchResult{ + Total: 2, + Docs: []redis.Document{ + { + ID: "1", + Fields: map[string]string{ + defaultReturnFieldContent: d1.Content, + defaultReturnFieldVectorContent: string(vector2Bytes(expv)), + }, + }, + { + ID: "2", + Fields: map[string]string{ + defaultReturnFieldContent: d2.Content, + defaultReturnFieldVectorContent: string(vector2Bytes(expv)), + }, + }, + }, + }, nil).Build() + + r, err := NewRetriever(ctx, &RetrieverConfig{ + Client: mockClient, + Index: "test_index", + DistanceThreshold: &dis, + Embedding: &mockEmbedding{sizeForCall: []int{1}, dims: 10}, + }) + convey.So(err, convey.ShouldBeNil) + resp, err := r.Retrieve(ctx, "test_query") + convey.So(err, convey.ShouldBeNil) + //s := "FT.SEARCH test_index @vector_content:[VECTOR_RANGE $distance_threshold $vector]=>{$yield_distance_as: distance} RETURN 2 content vector_content SORTBY distance ASC LIMIT 0 5" + //convey.So(strings.HasPrefix(cmd.String(), s), convey.ShouldBeTrue) + for i := range resp { + got := resp[i] + exp := docs[i] + convey.So(got.ID, convey.ShouldEqual, exp.ID) + convey.So(got.Content, convey.ShouldEqual, exp.Content) + convey.So(len(got.DenseVector()), convey.ShouldEqual, len(exp.DenseVector())) + for j, gf := range got.DenseVector() { + convey.So(gf, convey.ShouldAlmostEqual, exp.DenseVector()[j], 0.01) + } + } + }) + + PatchConvey("test knn vector search", func() { + //origin := mockClient.FTSearchWithArgs + var ( + //cmd *redis.FTSearchCmd + mockCmd *redis.FTSearchCmd + ) + + //Mock(GetMethod(mockClient, "FTSearchWithArgs")).To( + // func(ctx context.Context, index string, query string, options *redis.FTSearchOptions) *redis.FTSearchCmd { + // cmd = origin(ctx, index, query, options) + // return mockCmd + // }).Origin(&origin).Build() + + Mock(GetMethod(mockClient, "FTSearchWithArgs")).Return(mockCmd).Build() + Mock(GetMethod(mockCmd, "Result")).Return(redis.FTSearchResult{ + Total: 2, + Docs: []redis.Document{ + { + ID: "1", + Fields: map[string]string{ + defaultReturnFieldContent: d1.Content, + defaultReturnFieldVectorContent: string(vector2Bytes(expv)), + }, + }, + { + ID: "2", + Fields: map[string]string{ + defaultReturnFieldContent: d2.Content, + defaultReturnFieldVectorContent: string(vector2Bytes(expv)), + }, + }, + }, + }, nil).Build() + + r, err := NewRetriever(ctx, &RetrieverConfig{ + Client: mockClient, + Index: "test_index", + Embedding: &mockEmbedding{sizeForCall: []int{1}, dims: 10}, + }) + convey.So(err, convey.ShouldBeNil) + resp, err := r.Retrieve(ctx, "test_query") + convey.So(err, convey.ShouldBeNil) + //s := "FT.SEARCH test_index (*)=>[KNN 5 @vector_content $vector AS distance] RETURN 2 content vector_content SORTBY distance ASC LIMIT 0 5" + //convey.So(strings.HasPrefix(cmd.String(), s), convey.ShouldBeTrue) + for i := range resp { + got := resp[i] + exp := docs[i] + convey.So(got.ID, convey.ShouldEqual, exp.ID) + convey.So(got.Content, convey.ShouldEqual, exp.Content) + convey.So(len(got.DenseVector()), convey.ShouldEqual, len(exp.DenseVector())) + for j, gf := range got.DenseVector() { + convey.So(gf, convey.ShouldAlmostEqual, exp.DenseVector()[j], 0.01) + } + } + }) + + }) +} + +type mockEmbedding struct { + err error + cnt int + sizeForCall []int + dims int +} + +func (m *mockEmbedding) EmbedStrings(ctx context.Context, texts []string, opts ...embedding.Option) ([][]float64, error) { + if m.cnt > len(m.sizeForCall) { + panic("unexpected") + } + + if m.err != nil { + return nil, m.err + } + + slice := make([]float64, m.dims) + for i := range slice { + slice[i] = 1.1 + } + + r := make([][]float64, m.sizeForCall[m.cnt]) + m.cnt++ + for i := range r { + r[i] = slice + } + + return r, nil +} diff --git a/components/retriever/redis/utils.go b/components/retriever/redis/utils.go new file mode 100644 index 0000000..ff01c88 --- /dev/null +++ b/components/retriever/redis/utils.go @@ -0,0 +1,58 @@ +/* + * Copyright 2025 CloudWeGo Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package redis + +import ( + "encoding/binary" + "math" +) + +func Bytes2Vector(b []byte) []float64 { + n := len(b) / 4 + float32Arr := make([]float32, n) + for i := 0; i < n; i++ { + bits := binary.LittleEndian.Uint32(b[i*4 : (i+1)*4]) + float32Arr[i] = math.Float32frombits(bits) + } + + vector := make([]float64, n) + for i, v := range float32Arr { + vector[i] = float64(v) + } + return vector +} + +func vector2Bytes(vector []float64) []byte { + float32Arr := make([]float32, len(vector)) + for i, v := range vector { + float32Arr[i] = float32(v) + } + bytes := make([]byte, len(float32Arr)*4) + for i, v := range float32Arr { + binary.LittleEndian.PutUint32(bytes[i*4:], math.Float32bits(v)) + } + return bytes +} + +func dereferenceOrZero[T any](v *T) T { + if v == nil { + var t T + return t + } + + return *v +}