Skip to content

Commit

Permalink
move memoize code to cnquery
Browse files Browse the repository at this point in the history
Signed-off-by: Ivan Milchev <[email protected]>
  • Loading branch information
imilchev committed Feb 22, 2024
1 parent 6a42bb1 commit 7b6a601
Show file tree
Hide file tree
Showing 5 changed files with 125 additions and 8 deletions.
48 changes: 48 additions & 0 deletions providers-sdk/v1/util/memoize/memoize.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,48 @@
package memoize

import (
"time"

"github.com/patrickmn/go-cache"

Check failure on line 6 in providers-sdk/v1/util/memoize/memoize.go

View workflow job for this annotation

GitHub Actions / go-test

no required module provides package github.com/patrickmn/go-cache; to add it:

Check failure on line 6 in providers-sdk/v1/util/memoize/memoize.go

View workflow job for this annotation

GitHub Actions / go-test

no required module provides package github.com/patrickmn/go-cache; to add it:

Check failure on line 6 in providers-sdk/v1/util/memoize/memoize.go

View workflow job for this annotation

GitHub Actions / go-test

no required module provides package github.com/patrickmn/go-cache; to add it:

Check failure on line 6 in providers-sdk/v1/util/memoize/memoize.go

View workflow job for this annotation

GitHub Actions / go-test-cli

no required module provides package github.com/patrickmn/go-cache; to add it:
"golang.org/x/sync/singleflight"
)

// Memoizer allows you to memoize function calls. Memoizer is safe for concurrent use by multiple goroutines.
type Memoizer struct {
// Storage exposes the underlying cache of memoized results to manipulate as desired - for example, to Flush().
Storage *cache.Cache

group singleflight.Group
}

// NewMemoizer creates a new Memoizer with the configured expiry and cleanup policies.
// If desired, use cache.NoExpiration to cache values forever.
func NewMemoizer(defaultExpiration, cleanupInterval time.Duration) *Memoizer {
return &Memoizer{
Storage: cache.New(defaultExpiration, cleanupInterval),
group: singleflight.Group{},
}
}

// Memoize executes and returns the results of the given function, unless there was a cached value of the same key.
// Only one execution is in-flight for a given key at a time.
// The boolean return value indicates whether v was previously stored.
func (m *Memoizer) Memoize(key string, fn func() (interface{}, error)) (interface{}, error, bool) {
// Check cache
value, found := m.Storage.Get(key)
if found {
return value, nil, true
}

// Combine memoized function with a cache store
value, err, _ := m.group.Do(key, func() (interface{}, error) {
data, innerErr := fn()

if innerErr == nil {
m.Storage.Set(key, data, cache.DefaultExpiration)
}

return data, innerErr
})
return value, err, false
}
76 changes: 76 additions & 0 deletions providers-sdk/v1/util/memoize/memoize_test.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,76 @@
package memoize

import (
"errors"
"testing"
"time"

"github.com/stretchr/testify/assert"
)

// TestBasic adopts the code from readme.md into a simple test case
func TestBasic(t *testing.T) {
expensiveCalls := 0

// Function tracks how many times its been called
expensive := func() (interface{}, error) {
expensiveCalls++
return expensiveCalls, nil
}

cache := NewMemoizer(90*time.Second, 10*time.Minute)

// First call SHOULD NOT be cached
result, err, cached := cache.Memoize("key1", expensive)
assert.NoError(t, err)
assert.Equal(t, result.(int), 1)
assert.False(t, cached)

// Second call on same key SHOULD be cached
result, err, cached = cache.Memoize("key1", expensive)
assert.NoError(t, err)
assert.Equal(t, result.(int), 1)
assert.True(t, cached)

// First call on a new key SHOULD NOT be cached
result, err, cached = cache.Memoize("key2", expensive)
assert.NoError(t, err)
assert.Equal(t, result.(int), 2)
assert.False(t, cached)
}

// TestFailure checks that failed function values are not cached
func TestFailure(t *testing.T) {
calls := 0

// This function will fail IFF it has not been called before.
twoForTheMoney := func() (interface{}, error) {
calls++

if calls == 1 {
return calls, errors.New("Try again")
} else {
return calls, nil
}
}

cache := NewMemoizer(90*time.Second, 10*time.Minute)

// First call should fail, and not be cached
result, err, cached := cache.Memoize("key1", twoForTheMoney)
assert.Error(t, err)
assert.Equal(t, result.(int), 1)
assert.False(t, cached)

// Second call should succeed, and not be cached
result, err, cached = cache.Memoize("key1", twoForTheMoney)
assert.NoError(t, err)
assert.Equal(t, result.(int), 2)
assert.False(t, cached)

// Third call should succeed, and be cached
result, err, cached = cache.Memoize("key1", twoForTheMoney)
assert.NoError(t, err)
assert.Equal(t, result.(int), 2)
assert.True(t, cached)
}
1 change: 0 additions & 1 deletion providers/github/go.mod
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,6 @@ toolchain go1.22.0
require (
github.com/cockroachdb/errors v1.11.1
github.com/google/go-github/v59 v59.0.0
github.com/kofalt/go-memoize v0.0.0-20220914132407-0b5d6a304579
github.com/rs/zerolog v1.32.0
github.com/stretchr/testify v1.8.4
go.mondoo.com/cnquery/v10 v10.4.0
Expand Down
6 changes: 0 additions & 6 deletions providers/github/go.sum
Original file line number Diff line number Diff line change
Expand Up @@ -691,8 +691,6 @@ github.com/klauspost/compress v1.17.2 h1:RlWWUY/Dr4fL8qk9YG7DTZ7PDgME2V4csBXA8L/
github.com/klauspost/compress v1.17.2/go.mod h1:ntbaceVETuRiXiv4DpjP66DpAtAGkEQskQzEyD//IeE=
github.com/knqyf263/go-rpmdb v0.0.0-20231206071317-a8af76a6220f h1:G7KFUNfKsJDCvOP2ZSiqDdHzBX3tPJ05pO7eKVYL1jo=
github.com/knqyf263/go-rpmdb v0.0.0-20231206071317-a8af76a6220f/go.mod h1:9LQcoMCMQ9vrF7HcDtXfvqGO4+ddxFQ8+YF/0CVGDww=
github.com/kofalt/go-memoize v0.0.0-20220914132407-0b5d6a304579 h1:RbY+urZu3ri7Medi8pY3ovt1+XQxxv7zSkgmEZ5E0CU=
github.com/kofalt/go-memoize v0.0.0-20220914132407-0b5d6a304579/go.mod h1:PifxINf6wYU0USPBk0z1Z8Pka1AqeyCJAp9ecCcNL5Q=
github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ=
github.com/konsorten/go-windows-terminal-sequences v1.0.3/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ=
github.com/kr/fs v0.1.0 h1:Jskdu9ieNAYnjxsi0LbQp1ulIKZV1LAFgK1tWhpZgl8=
Expand Down Expand Up @@ -960,12 +958,8 @@ github.com/sivchari/tenv v1.7.1 h1:PSpuD4bu6fSmtWMxSGWcvqUUgIn7k3yOJhOIzVWn8Ak=
github.com/sivchari/tenv v1.7.1/go.mod h1:64yStXKSOxDfX47NlhVwND4dHwfZDdbp2Lyl018Icvg=
github.com/smarty/assertions v1.15.1 h1:812oFiXI+G55vxsFf+8bIZ1ux30qtkdqzKbEFwyX3Tk=
github.com/smarty/assertions v1.15.1/go.mod h1:yABtdzeQs6l1brC900WlRNwj6ZR55d7B+E8C6HtKdec=
github.com/smartystreets/assertions v1.2.0 h1:42S6lae5dvLc7BrLu/0ugRtcFVjoJNMC/N3yZFZkDFs=
github.com/smartystreets/assertions v1.2.0/go.mod h1:tcbTF8ujkAEcZ8TElKY+i30BzYlVhC/LOxJk7iOWnoo=
github.com/smartystreets/goconvey v1.8.1 h1:qGjIddxOk4grTu9JPOU31tVfq3cNdBlNa5sSznIX1xY=
github.com/smartystreets/goconvey v1.8.1/go.mod h1:+/u4qLyY6x1jReYOp7GOM2FSt8aP9CzCZL03bI28W60=
github.com/smartystreets/gunit v1.4.2 h1:tyWYZffdPhQPfK5VsMQXfauwnJkqg7Tv5DLuQVYxq3Q=
github.com/smartystreets/gunit v1.4.2/go.mod h1:ZjM1ozSIMJlAz/ay4SG8PeKF00ckUp+zMHZXV9/bvak=
github.com/sonatard/noctx v0.0.2 h1:L7Dz4De2zDQhW8S0t+KUjY0MAQJd6SgVwhzNIc4ok00=
github.com/sonatard/noctx v0.0.2/go.mod h1:kzFz+CzWSjQ2OzIm46uJZoXuBpa2+0y3T36U18dWqIo=
github.com/sourcegraph/conc v0.3.0 h1:OQTbbt6P72L20UqAkXXuLOj79LfEanQ+YQFNpLA9ySo=
Expand Down
2 changes: 1 addition & 1 deletion providers/github/resources/github.go
Original file line number Diff line number Diff line change
Expand Up @@ -8,10 +8,10 @@ import (
"time"

"github.com/google/go-github/v59/github"
"github.com/kofalt/go-memoize"
"github.com/rs/zerolog/log"
"go.mondoo.com/cnquery/v10/llx"
"go.mondoo.com/cnquery/v10/providers-sdk/v1/plugin"
"go.mondoo.com/cnquery/v10/providers-sdk/v1/util/memoize"
"go.mondoo.com/cnquery/v10/providers/github/connection"
)

Expand Down

0 comments on commit 7b6a601

Please sign in to comment.