Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

cleaned Go support and added memory leak fix #667

Merged
merged 3 commits into from
Jul 29, 2024
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
31 changes: 16 additions & 15 deletions README_GO.rst
Original file line number Diff line number Diff line change
@@ -1,16 +1,15 @@
Install
-------

To install, you'll need Swig (tested with Swig 3.0.6 on OS X), and then just::
To install, you'll need Swig (tested with Swig 4.2.1 on Ubuntu 24.04), and then just::

swig -go -intgosize 64 -cgo -c++ src/annoygomodule.i
mkdir -p $GOPATH/src/annoyindex
cp src/annoygomodule_wrap.cxx src/annoyindex.go src/annoygomodule.h src/annoylib.h src/kissrandom.h test/annoy_test.go $GOPATH/src/annoyindex
cd $GOPATH/src/annoyindex
go mod init annoyindex
go get -t ...
mkdir -p $(go env GOPATH)/src/annoy
cp src/annoygomodule_wrap.cxx src/annoy.go src/annoygomodule.h src/annoylib.h src/kissrandom.h test/annoy_test.go $(go env GOPATH)/src/annoy
cd $(go env GOPATH)/src/annoy
go mod init github.com/spotify/annoy
go mod tidy
go test
go build

Background
----------
Expand All @@ -25,14 +24,15 @@ Go code example
package main

import (
"annoyindex"
"fmt"
"math/rand"

"github.com/spotify/annoy"
)

func main() {
f := 40
t := annoyindex.NewAnnoyIndexAngular(f)
t := annoy.NewAnnoyIndexAngular(f)
for i := 0; i < 1000; i++ {
item := make([]float32, 0, f)
for x:= 0; x < f; x++ {
Expand All @@ -43,14 +43,15 @@ Go code example
t.Build(10)
t.Save("test.ann")

annoyindex.DeleteAnnoyIndexAngular(t)
annoy.DeleteAnnoyIndexAngular(t)

t = annoyindex.NewAnnoyIndexAngular(f)
t = annoy.NewAnnoyIndexAngular(f)
t.Load("test.ann")

var result []int
t.GetNnsByItem(0, 1000, -1, &result)
fmt.Printf("%v\n", result)
result := annoyindex.NewAnnoyVectorInt()
defer result.Free()
t.GetNnsByItem(0, 1000, -1, result)
fmt.Printf("%v\n", result.ToSlice())

}

Expand All @@ -68,6 +69,6 @@ A simple test is supplied in test/annoy_test.go.
Discuss
-------

There might be some memory leaks. See [this issue](https://github.com/swig/swig/issues/2292).
Memroy leak in the previous versions has been fixed thanks to https://github.com/swig/swig/issues/2292. (memory leak fix is implemented in https://github.com/Rikanishu/annoy-go)

Go glue written by Taneli Leppä (@rosmo). You can contact me via email (see https://github.com/rosmo).
111 changes: 103 additions & 8 deletions src/annoygomodule.h
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,73 @@ using namespace Annoy;

namespace GoAnnoy {


class AnnoyVectorFloat {
protected:
float *ptr;
int len;

public:
~AnnoyVectorFloat() {
free(ptr);
};
float* ArrayPtr() {
return ptr;
};
int Len() {
return len;
};
float Get(int i) {
if (i >= len) {
return 0.0;
}
return ptr[i];
};
void fill_from_vector(vector<float>* v) {
if (ptr != NULL) {
free(ptr);
}
ptr = (float*) malloc(v->size() * sizeof(float));
for (int i = 0; i < v->size(); i++) {
ptr[i] = (float)(*v)[i];
}
len = v->size();
};
};

class AnnoyVectorInt {
protected:
int32_t *ptr;
int len;

public:
~AnnoyVectorInt() {
free(ptr);
};
int32_t* ArrayPtr() {
return ptr;
};
int Len() {
return len;
};
int32_t Get(int i) {
if (i >= len) {
return 0.0;
}
return ptr[i];
};
void fill_from_vector(vector<int32_t>* v) {
if (ptr != NULL) {
free(ptr);
}
ptr = (int32_t*) malloc(v->size() * sizeof(int32_t));
for (int i = 0; i < v->size(); i++) {
ptr[i] = (int32_t)(*v)[i];
}
len = v->size();
};
};

class AnnoyIndex {
protected:
::AnnoyIndexInterface<int32_t, float> *ptr;
Expand Down Expand Up @@ -39,17 +106,43 @@ class AnnoyIndex {
float getDistance(int i, int j) {
return ptr->get_distance(i, j);
};
void getNnsByItem(int item, int n, int search_k, vector<int32_t>* result, vector<float>* distances) {
void getNnsByItem(int item, int n, int search_k, AnnoyVectorInt* out_result, AnnoyVectorFloat* out_distances) {
vector<int32_t>* result = new vector<int32_t>();
vector<float>* distances = new vector<float>();

ptr->get_nns_by_item(item, n, search_k, result, distances);
};
void getNnsByVector(const float* w, int n, int search_k, vector<int32_t>* result, vector<float>* distances) {

out_result->fill_from_vector(result);
out_distances->fill_from_vector(distances);
delete result;
delete distances;
};
void getNnsByVector(const float* w, int n, int search_k, AnnoyVectorInt* out_result, AnnoyVectorFloat* out_distances) {
vector<int32_t>* result = new vector<int32_t>();
vector<float>* distances = new vector<float>();

ptr->get_nns_by_vector(w, n, search_k, result, distances);

out_result->fill_from_vector(result);
out_distances->fill_from_vector(distances);
delete result;
delete distances;
};
void getNnsByItem(int item, int n, int search_k, vector<int32_t>* result) {
void getNnsByItem(int item, int n, int search_k, AnnoyVectorInt* out_result) {
vector<int32_t>* result = new vector<int32_t>();

ptr->get_nns_by_item(item, n, search_k, result, NULL);

out_result->fill_from_vector(result);
delete result;
};
void getNnsByVector(const float* w, int n, int search_k, vector<int32_t>* result) {
void getNnsByVector(const float* w, int n, int search_k, AnnoyVectorInt* out_result) {
vector<int32_t>* result = new vector<int32_t>();

ptr->get_nns_by_vector(w, n, search_k, result, NULL);

out_result->fill_from_vector(result);
delete result;
};

int getNItems() {
Expand All @@ -58,9 +151,11 @@ class AnnoyIndex {
void verbose(bool v) {
ptr->verbose(v);
};
void getItem(int item, vector<float> *v) {
v->resize(this->f);
ptr->get_item(item, &v->front());
void getItem(int item, AnnoyVectorFloat *v) {
vector<float>* r = new vector<float>();
r->resize(this->f);
ptr->get_item(item, &r->front());
v->fill_from_vector(r);
};
bool onDiskBuild(const char* filename) {
return ptr->on_disk_build(filename);
Expand Down
144 changes: 97 additions & 47 deletions src/annoygomodule.i
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
%module annoyindex
%module annoy

namespace Annoy {}

Expand All @@ -9,6 +9,7 @@ namespace Annoy {}

// const float *
%typemap(gotype) (const float *) "[]float32"
%typemap(gotype) (int32_t) "int32"

%typemap(in) (const float *)
%{
Expand All @@ -21,76 +22,125 @@ namespace Annoy {}
$1 = &w[0];
%}

// vector<int32_t> *
%typemap(gotype) (vector<int32_t> *) "*[]int"

%typemap(in) (vector<int32_t> *)
%typemap(gotype) (const char *) "string"

%typemap(in) (const char *)
%{
$1 = new vector<int32_t>();
$1 = (char *)calloc((((_gostring_)$input).n + 1), sizeof(char));
strncpy($1, (((_gostring_)$input).p), ((_gostring_)$input).n);
%}

%typemap(freearg) (vector<int32_t> *)
%typemap(freearg) (const char *)
%{
delete $1;
free($1);
%}

%typemap(argout) (vector<int32_t> *)
%{
{
$input->len = $1->size();
$input->cap = $1->size();
$input->array = malloc($input->len * sizeof(intgo));
for (int i = 0; i < $1->size(); i++) {
((intgo *)$input->array)[i] = (intgo)(*$1)[i];

%ignore fill_from_vector;
%rename(X_RawAnnoyVectorInt) AnnoyVectorInt;
%rename(X_RawAnnoyVectorFloat) AnnoyVectorFloat;

%insert(go_wrapper) %{

type AnnoyVectorInt interface {
X_RawAnnoyVectorInt
ToSlice() []int32
Copy(in *[]int32)
InnerArray() []int32
Free()
}

func NewAnnoyVectorInt() AnnoyVectorInt {
vec := NewX_RawAnnoyVectorInt()
return vec.(SwigcptrX_RawAnnoyVectorInt)
}

func (p SwigcptrX_RawAnnoyVectorInt) ToSlice() []int32 {
var out []int32
p.Copy(&out)
return out
}

func (p SwigcptrX_RawAnnoyVectorInt) Copy(in *[]int32) {
out := *in
inner := p.InnerArray()
if cap(out) >= len(inner) {
if len(out) != len(inner) {
out = out[:len(inner)]
}
} else {
out = make([]int32, len(inner))
}
}
%}

copy(out, inner)
*in = out
}

// vector<float> *
%typemap(gotype) (vector<float> *) "*[]float32"
func (p SwigcptrX_RawAnnoyVectorInt) Free() {
DeleteX_RawAnnoyVectorInt(p)
}

%typemap(in) (vector<float> *)
%{
$1 = new vector<float>();
%}
func (p SwigcptrX_RawAnnoyVectorInt) InnerArray() []int32 {
length := p.Len()
ptr := unsafe.Pointer(p.ArrayPtr())
return ((*[1 << 30]int32)(ptr))[:length:length]
}

%typemap(freearg) (vector<float> *)
%{
delete $1;
%}

%typemap(argout) (vector<float> *)
%{
{
$input->len = $1->size();
$input->cap = $1->size();
$input->array = malloc($input->len * sizeof(float));
for (int i = 0; i < $1->size(); i++) {
((float *)$input->array)[i] = (float)(*$1)[i];
%insert(go_wrapper) %{

type AnnoyVectorFloat interface {
X_RawAnnoyVectorFloat
ToSlice() []float32
Copy(in *[]float32)
InnerArray() []float32
Free()
}

func NewAnnoyVectorFloat() AnnoyVectorFloat {
vec := NewX_RawAnnoyVectorFloat()
return vec.(SwigcptrX_RawAnnoyVectorFloat)
}

func (p SwigcptrX_RawAnnoyVectorFloat) ToSlice() []float32 {
var out []float32
p.Copy(&out)
return out
}

func (p SwigcptrX_RawAnnoyVectorFloat) Copy(in *[]float32) {
out := *in
inner := p.InnerArray()
if cap(out) >= len(inner) {
if len(out) != len(inner) {
out = out[:len(inner)]
}
} else {
out = make([]float32, len(inner))
}
}
%}

copy(out, inner)
*in = out
}

%typemap(gotype) (const char *) "string"
func (p SwigcptrX_RawAnnoyVectorFloat) Free() {
DeleteX_RawAnnoyVectorFloat(p)
}

%typemap(in) (const char *)
%{
$1 = (char *)calloc((((_gostring_)$input).n + 1), sizeof(char));
strncpy($1, (((_gostring_)$input).p), ((_gostring_)$input).n);
%}
func (p SwigcptrX_RawAnnoyVectorFloat) InnerArray() []float32 {
length := p.Len()
ptr := unsafe.Pointer(p.ArrayPtr())
return ((*[1 << 30]float32)(ptr))[:length:length]
}

%typemap(freearg) (const char *)
%{
free($1);
%}


/* Let's just grab the original header file here */
%include "annoygomodule.h"

%feature("notabstract") GoAnnoyIndexAngular;
%feature("notabstract") GoAnnoyIndexEuclidean;
%feature("notabstract") GoAnnoyIndexManhattan;
%feature("notabstract") GoAnnoyIndexDotProduct;
%feature("notabstract") GoAnnoyIndexDotProduct;
Loading
Loading