Skip to content

Commit

Permalink
Merge pull request #667 from AlisaLC/main
Browse files Browse the repository at this point in the history 
cleaned Go support and added memory leak fix
  • Loading branch information
@erikbern
erikbern authored Jul 29, 2024
2 parents 2be37c9 + 5455ad5 commit 8a7e82c
Show file tree
Hide file tree
Showing 4 changed files with 443 additions and 253 deletions.
31 changes: 16 additions & 15 deletions README_GO.rst
View file
Original file line number Diff line number Diff line change
@@ -1,16 +1,15 @@
Install
-------

To install, you'll need Swig (tested with Swig 3.0.6 on OS X), and then just::
To install, you'll need Swig (tested with Swig 4.2.1 on Ubuntu 24.04), and then just::

swig -go -intgosize 64 -cgo -c++ src/annoygomodule.i
mkdir -p $GOPATH/src/annoyindex
cp src/annoygomodule_wrap.cxx src/annoyindex.go src/annoygomodule.h src/annoylib.h src/kissrandom.h test/annoy_test.go $GOPATH/src/annoyindex
cd $GOPATH/src/annoyindex
go mod init annoyindex
go get -t ...
mkdir -p $(go env GOPATH)/src/annoy
cp src/annoygomodule_wrap.cxx src/annoy.go src/annoygomodule.h src/annoylib.h src/kissrandom.h test/annoy_test.go $(go env GOPATH)/src/annoy
cd $(go env GOPATH)/src/annoy
go mod init github.com/spotify/annoy
go mod tidy
go test
go build

Background
----------
Expand All @@ -25,14 +24,15 @@ Go code example
package main
import (
"annoyindex"
"fmt"
"math/rand"
"github.com/spotify/annoy"
)
func main() {
f := 40
t := annoyindex.NewAnnoyIndexAngular(f)
t := annoy.NewAnnoyIndexAngular(f)
for i := 0; i < 1000; i++ {
item := make([]float32, 0, f)
for x:= 0; x < f; x++ {
Expand All @@ -43,14 +43,15 @@ Go code example
t.Build(10)
t.Save("test.ann")
annoyindex.DeleteAnnoyIndexAngular(t)
annoy.DeleteAnnoyIndexAngular(t)
t = annoyindex.NewAnnoyIndexAngular(f)
t = annoy.NewAnnoyIndexAngular(f)
t.Load("test.ann")
var result []int
t.GetNnsByItem(0, 1000, -1, &result)
fmt.Printf("%v\n", result)
result := annoyindex.NewAnnoyVectorInt()
defer result.Free()
t.GetNnsByItem(0, 1000, -1, result)
fmt.Printf("%v\n", result.ToSlice())
}
Expand All @@ -68,6 +69,6 @@ A simple test is supplied in test/annoy_test.go.
Discuss
-------

There might be some memory leaks. See [this issue](https://github.com/swig/swig/issues/2292).
Memroy leak in the previous versions has been fixed thanks to https://github.com/swig/swig/issues/2292. (memory leak fix is implemented in https://github.com/Rikanishu/annoy-go)

Go glue written by Taneli Leppä (@rosmo). You can contact me via email (see https://github.com/rosmo).
111 changes: 103 additions & 8 deletions src/annoygomodule.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,73 @@ using namespace Annoy;

namespace GoAnnoy {


class AnnoyVectorFloat {
protected:
float *ptr;
int len;

public:
~AnnoyVectorFloat() {
free(ptr);
};
float* ArrayPtr() {
return ptr;
};
int Len() {
return len;
};
float Get(int i) {
if (i >= len) {
return 0.0;
}
return ptr[i];
};
void fill_from_vector(vector<float>* v) {
if (ptr != NULL) {
free(ptr);
}
ptr = (float*) malloc(v->size() * sizeof(float));
for (int i = 0; i < v->size(); i++) {
ptr[i] = (float)(*v)[i];
}
len = v->size();
};
};

class AnnoyVectorInt {
protected:
int32_t *ptr;
int len;

public:
~AnnoyVectorInt() {
free(ptr);
};
int32_t* ArrayPtr() {
return ptr;
};
int Len() {
return len;
};
int32_t Get(int i) {
if (i >= len) {
return 0.0;
}
return ptr[i];
};
void fill_from_vector(vector<int32_t>* v) {
if (ptr != NULL) {
free(ptr);
}
ptr = (int32_t*) malloc(v->size() * sizeof(int32_t));
for (int i = 0; i < v->size(); i++) {
ptr[i] = (int32_t)(*v)[i];
}
len = v->size();
};
};

class AnnoyIndex {
protected:
::AnnoyIndexInterface<int32_t, float> *ptr;
Expand Down Expand Up @@ -39,17 +106,43 @@ class AnnoyIndex {
float getDistance(int i, int j) {
return ptr->get_distance(i, j);
};
void getNnsByItem(int item, int n, int search_k, vector<int32_t>* result, vector<float>* distances) {
void getNnsByItem(int item, int n, int search_k, AnnoyVectorInt* out_result, AnnoyVectorFloat* out_distances) {
vector<int32_t>* result = new vector<int32_t>();
vector<float>* distances = new vector<float>();

ptr->get_nns_by_item(item, n, search_k, result, distances);
};
void getNnsByVector(const float* w, int n, int search_k, vector<int32_t>* result, vector<float>* distances) {

out_result->fill_from_vector(result);
out_distances->fill_from_vector(distances);
delete result;
delete distances;
};
void getNnsByVector(const float* w, int n, int search_k, AnnoyVectorInt* out_result, AnnoyVectorFloat* out_distances) {
vector<int32_t>* result = new vector<int32_t>();
vector<float>* distances = new vector<float>();

ptr->get_nns_by_vector(w, n, search_k, result, distances);

out_result->fill_from_vector(result);
out_distances->fill_from_vector(distances);
delete result;
delete distances;
};
void getNnsByItem(int item, int n, int search_k, vector<int32_t>* result) {
void getNnsByItem(int item, int n, int search_k, AnnoyVectorInt* out_result) {
vector<int32_t>* result = new vector<int32_t>();

ptr->get_nns_by_item(item, n, search_k, result, NULL);

out_result->fill_from_vector(result);
delete result;
};
void getNnsByVector(const float* w, int n, int search_k, vector<int32_t>* result) {
void getNnsByVector(const float* w, int n, int search_k, AnnoyVectorInt* out_result) {
vector<int32_t>* result = new vector<int32_t>();

ptr->get_nns_by_vector(w, n, search_k, result, NULL);

out_result->fill_from_vector(result);
delete result;
};

int getNItems() {
Expand All @@ -58,9 +151,11 @@ class AnnoyIndex {
void verbose(bool v) {
ptr->verbose(v);
};
void getItem(int item, vector<float> *v) {
v->resize(this->f);
ptr->get_item(item, &v->front());
void getItem(int item, AnnoyVectorFloat *v) {
vector<float>* r = new vector<float>();
r->resize(this->f);
ptr->get_item(item, &r->front());
v->fill_from_vector(r);
};
bool onDiskBuild(const char* filename) {
return ptr->on_disk_build(filename);
Expand Down
144 changes: 97 additions & 47 deletions src/annoygomodule.i
View file
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
%module annoyindex
%module annoy

namespace Annoy {}

Expand All @@ -9,6 +9,7 @@ namespace Annoy {}

// const float *
%typemap(gotype) (const float *) "[]float32"
%typemap(gotype) (int32_t) "int32"

%typemap(in) (const float *)
%{
Expand All @@ -21,76 +22,125 @@ namespace Annoy {}
$1 = &w[0];
%}

// vector<int32_t> *
%typemap(gotype) (vector<int32_t> *) "*[]int"

%typemap(in) (vector<int32_t> *)
%typemap(gotype) (const char *) "string"

%typemap(in) (const char *)
%{
$1 = new vector<int32_t>();
$1 = (char *)calloc((((_gostring_)$input).n + 1), sizeof(char));
strncpy($1, (((_gostring_)$input).p), ((_gostring_)$input).n);
%}

%typemap(freearg) (vector<int32_t> *)
%typemap(freearg) (const char *)
%{
delete $1;
free($1);
%}

%typemap(argout) (vector<int32_t> *)
%{
{
$input->len = $1->size();
$input->cap = $1->size();
$input->array = malloc($input->len * sizeof(intgo));
for (int i = 0; i < $1->size(); i++) {
((intgo *)$input->array)[i] = (intgo)(*$1)[i];

%ignore fill_from_vector;
%rename(X_RawAnnoyVectorInt) AnnoyVectorInt;
%rename(X_RawAnnoyVectorFloat) AnnoyVectorFloat;

%insert(go_wrapper) %{

type AnnoyVectorInt interface {
X_RawAnnoyVectorInt
ToSlice() []int32
Copy(in *[]int32)
InnerArray() []int32
Free()
}

func NewAnnoyVectorInt() AnnoyVectorInt {
vec := NewX_RawAnnoyVectorInt()
return vec.(SwigcptrX_RawAnnoyVectorInt)
}

func (p SwigcptrX_RawAnnoyVectorInt) ToSlice() []int32 {
var out []int32
p.Copy(&out)
return out
}

func (p SwigcptrX_RawAnnoyVectorInt) Copy(in *[]int32) {
out := *in
inner := p.InnerArray()
if cap(out) >= len(inner) {
if len(out) != len(inner) {
out = out[:len(inner)]
}
} else {
out = make([]int32, len(inner))
}
}
%}

copy(out, inner)
*in = out
}

// vector<float> *
%typemap(gotype) (vector<float> *) "*[]float32"
func (p SwigcptrX_RawAnnoyVectorInt) Free() {
DeleteX_RawAnnoyVectorInt(p)
}

%typemap(in) (vector<float> *)
%{
$1 = new vector<float>();
%}
func (p SwigcptrX_RawAnnoyVectorInt) InnerArray() []int32 {
length := p.Len()
ptr := unsafe.Pointer(p.ArrayPtr())
return ((*[1 << 30]int32)(ptr))[:length:length]
}

%typemap(freearg) (vector<float> *)
%{
delete $1;
%}

%typemap(argout) (vector<float> *)
%{
{
$input->len = $1->size();
$input->cap = $1->size();
$input->array = malloc($input->len * sizeof(float));
for (int i = 0; i < $1->size(); i++) {
((float *)$input->array)[i] = (float)(*$1)[i];
%insert(go_wrapper) %{

type AnnoyVectorFloat interface {
X_RawAnnoyVectorFloat
ToSlice() []float32
Copy(in *[]float32)
InnerArray() []float32
Free()
}

func NewAnnoyVectorFloat() AnnoyVectorFloat {
vec := NewX_RawAnnoyVectorFloat()
return vec.(SwigcptrX_RawAnnoyVectorFloat)
}

func (p SwigcptrX_RawAnnoyVectorFloat) ToSlice() []float32 {
var out []float32
p.Copy(&out)
return out
}

func (p SwigcptrX_RawAnnoyVectorFloat) Copy(in *[]float32) {
out := *in
inner := p.InnerArray()
if cap(out) >= len(inner) {
if len(out) != len(inner) {
out = out[:len(inner)]
}
} else {
out = make([]float32, len(inner))
}
}
%}

copy(out, inner)
*in = out
}

%typemap(gotype) (const char *) "string"
func (p SwigcptrX_RawAnnoyVectorFloat) Free() {
DeleteX_RawAnnoyVectorFloat(p)
}

%typemap(in) (const char *)
%{
$1 = (char *)calloc((((_gostring_)$input).n + 1), sizeof(char));
strncpy($1, (((_gostring_)$input).p), ((_gostring_)$input).n);
%}
func (p SwigcptrX_RawAnnoyVectorFloat) InnerArray() []float32 {
length := p.Len()
ptr := unsafe.Pointer(p.ArrayPtr())
return ((*[1 << 30]float32)(ptr))[:length:length]
}

%typemap(freearg) (const char *)
%{
free($1);
%}


/* Let's just grab the original header file here */
%include "annoygomodule.h"

%feature("notabstract") GoAnnoyIndexAngular;
%feature("notabstract") GoAnnoyIndexEuclidean;
%feature("notabstract") GoAnnoyIndexManhattan;
%feature("notabstract") GoAnnoyIndexDotProduct;
%feature("notabstract") GoAnnoyIndexDotProduct;
Loading

0 comments on commit 8a7e82c

Please sign in to comment.