From ef035eb07a1ebd857c67575056def90bfd4031df Mon Sep 17 00:00:00 2001 From: Martin Redolatti Date: Sat, 11 Nov 2023 16:49:50 -0300 Subject: [PATCH] controller wip --- CHANGES.txt | 3 + go.mod | 3 +- go.sum | 5 +- splitio/commitversion.go | 2 +- splitio/producer/conf/sections.go | 1 + splitio/producer/initialization.go | 6 +- splitio/proxy/caching/caching.go | 12 --- splitio/proxy/caching/workers.go | 1 + splitio/proxy/conf/sections.go | 24 +++--- splitio/proxy/controllers/sdk.go | 7 ++ splitio/proxy/controllers/sdk_test.go | 11 +++ splitio/proxy/flagsets/flagsets.go | 35 +++++++++ splitio/proxy/flagsets/flagsets_test.go | 21 ++++++ splitio/proxy/initialization.go | 9 ++- splitio/proxy/proxy.go | 6 ++ splitio/proxy/storage/optimized/historic.go | 25 +++++-- .../proxy/storage/optimized/historic_test.go | 2 +- .../proxy/storage/optimized/mocks/mocks.go | 23 ++++++ splitio/proxy/storage/splits.go | 15 ++-- splitio/proxy/storage/splits_test.go | 74 ++++++++++--------- splitio/version.go | 2 +- 21 files changed, 205 insertions(+), 82 deletions(-) create mode 100644 splitio/proxy/flagsets/flagsets.go create mode 100644 splitio/proxy/flagsets/flagsets_test.go create mode 100644 splitio/proxy/storage/optimized/mocks/mocks.go diff --git a/CHANGES.txt b/CHANGES.txt index c6a00fa7..6a00f353 100644 --- a/CHANGES.txt +++ b/CHANGES.txt @@ -1,3 +1,6 @@ +5.5.0 (Month XX, 2023) +- FlagSet + 5.4.0 (July 18, 2023) - Improved streaming architecture implementation to apply feature flag updates from the notification received which is now enhanced, improving efficiency and reliability of the whole update system. - Fixed possible edge case issue where deleting a feature flag doesn’t propagate immediately. diff --git a/go.mod b/go.mod index 3485f219..39147527 100644 --- a/go.mod +++ b/go.mod @@ -8,7 +8,7 @@ require ( github.com/gin-gonic/gin v1.9.1 github.com/google/uuid v1.3.0 github.com/splitio/gincache v1.0.1 - github.com/splitio/go-split-commons/v5 v5.0.1-0.20230926022914-2101c4dc74c0 + github.com/splitio/go-split-commons/v5 v5.0.1-0.20231004184048-81902536fc1f github.com/splitio/go-toolkit/v5 v5.3.2-0.20230920032539-d08915cf020a github.com/stretchr/testify v1.8.4 go.etcd.io/bbolt v1.3.6 @@ -38,6 +38,7 @@ require ( github.com/pelletier/go-toml/v2 v2.0.8 // indirect github.com/pmezard/go-difflib v1.0.0 // indirect github.com/redis/go-redis/v9 v9.0.4 // indirect + github.com/stretchr/objx v0.5.0 // indirect github.com/twitchyliquid64/golang-asm v0.15.1 // indirect github.com/ugorji/go/codec v1.2.11 // indirect golang.org/x/arch v0.3.0 // indirect diff --git a/go.sum b/go.sum index ff9301c7..c2bebaeb 100644 --- a/go.sum +++ b/go.sum @@ -90,12 +90,13 @@ github.com/rogpeppe/go-internal v1.8.0 h1:FCbCCtXNOY3UtUuHUYaghJg4y7Fd14rXifAYUA github.com/rogpeppe/go-internal v1.8.0/go.mod h1:WmiCO8CzOY8rg0OYDC4/i/2WRWAB6poM+XZ2dLUbcbE= github.com/splitio/gincache v1.0.1 h1:dLYdANY/BqH4KcUMCe/LluLyV5WtuE/LEdQWRE06IXU= github.com/splitio/gincache v1.0.1/go.mod h1:CcgJDSM9Af75kyBH0724v55URVwMBuSj5x1eCWIOECY= -github.com/splitio/go-split-commons/v5 v5.0.1-0.20230926022914-2101c4dc74c0 h1:t7QuH0+4T2LeJOc2gdRP+PkFPkQEB017arfxBccsArg= -github.com/splitio/go-split-commons/v5 v5.0.1-0.20230926022914-2101c4dc74c0/go.mod h1:ksVZQYLs+3ZuzU81vEvf1aCjk24pdrVWjUXNq6Qcayo= +github.com/splitio/go-split-commons/v5 v5.0.1-0.20231004184048-81902536fc1f h1:g3rsXA0cdMx2uz3MrTEz2tiittf+HDXpHooyYnuYg6w= +github.com/splitio/go-split-commons/v5 v5.0.1-0.20231004184048-81902536fc1f/go.mod h1:ksVZQYLs+3ZuzU81vEvf1aCjk24pdrVWjUXNq6Qcayo= github.com/splitio/go-toolkit/v5 v5.3.2-0.20230920032539-d08915cf020a h1:2wjh5hSGlFRuh6Lbmodr0VRqtry2m9pEBNmwiLsY+ss= github.com/splitio/go-toolkit/v5 v5.3.2-0.20230920032539-d08915cf020a/go.mod h1:xYhUvV1gga9/1029Wbp5pjnR6Cy8nvBpjw99wAbsMko= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= +github.com/stretchr/objx v0.5.0 h1:1zr/of2m5FGMsad5YfcqgdqdWrIhu+EBEJRhR1U7z/c= github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= diff --git a/splitio/commitversion.go b/splitio/commitversion.go index fb43ed6d..0d9c8362 100644 --- a/splitio/commitversion.go +++ b/splitio/commitversion.go @@ -5,4 +5,4 @@ This file is created automatically, please do not edit */ // CommitVersion is the version of the last commit previous to release -const CommitVersion = "353237e" +const CommitVersion = "4867a2d" diff --git a/splitio/producer/conf/sections.go b/splitio/producer/conf/sections.go index 32808ab6..ba26d57e 100644 --- a/splitio/producer/conf/sections.go +++ b/splitio/producer/conf/sections.go @@ -9,6 +9,7 @@ import ( type Main struct { Apikey string `json:"apikey" s-cli:"apikey" s-def:"" s-desc:"Split server side SDK key"` IPAddressEnabled bool `json:"ipAddressEnabled" s-cli:"ip-address-enabled" s-def:"true" s-desc:"Bundle host's ip address when sending data to Split"` + FlagSetsFilter []string `json:"flagSetsFilter" s-cli:"flag-sets-filter" s-def:"" s-desc:"Flag Sets Filter provided"` Initialization Initialization `json:"initialization" s-nested:"true"` Storage Storage `json:"storage" s-nested:"true"` Sync Sync `json:"sync" s-nested:"true"` diff --git a/splitio/producer/initialization.go b/splitio/producer/initialization.go index cf8da384..bd9ba3f9 100644 --- a/splitio/producer/initialization.go +++ b/splitio/producer/initialization.go @@ -47,6 +47,7 @@ const ( func Start(logger logging.LoggerInterface, cfg *conf.Main) error { // Getting initial config data advanced := cfg.BuildAdvancedConfig() + advanced.FlagSetsFilter = cfg.FlagSetsFilter metadata := util.GetMetadata(false, cfg.IPAddressEnabled) clientKey, err := util.GetClientKey(cfg.Apikey) @@ -85,8 +86,11 @@ func Start(logger logging.LoggerInterface, cfg *conf.Main) error { syncTelemetryStorage, _ := inmemory.NewTelemetryStorage() sdkTelemetryStorage := storage.NewRedisTelemetryCosumerclient(redisClient, logger) + // FlagSetsFilter + flagSetsFilter := flagsets.NewFlagSetFilter(cfg.FlagSetsFilter) + // These storages are forwarded to the dashboard, the sdk-telemetry is irrelevant there - splitStorage, err := observability.NewObservableSplitStorage(redis.NewSplitStorage(redisClient, logger), logger) + splitStorage, err := observability.NewObservableSplitStorage(redis.NewSplitStorage(redisClient, logger, flagSetsFilter), logger) if err != nil { return fmt.Errorf("error instantiating observable feature flag storage: %w", err) } diff --git a/splitio/proxy/caching/caching.go b/splitio/proxy/caching/caching.go index 29b65456..89d153ec 100644 --- a/splitio/proxy/caching/caching.go +++ b/splitio/proxy/caching/caching.go @@ -34,18 +34,6 @@ func MakeSurrogateForSegmentChanges(segmentName string) string { // MakeSurrogateForMySegments creates a list surrogate keys for all the segments involved func MakeSurrogateForMySegments(mysegments []dtos.MySegmentDTO) []string { - if len(mysegments) == 0 { - return nil - } - - /* - surrogates := make([]string, 0, len(mysegments)) - for idx := range mysegments { - surrogates = append(surrogates, segmentPrefix+mysegments[idx].Name) - } - return surrogates - */ - // Since we are now evicting individually for every updated key, we don't need surrogates for mySegments return nil } diff --git a/splitio/proxy/caching/workers.go b/splitio/proxy/caching/workers.go index 3931286c..b2aea986 100644 --- a/splitio/proxy/caching/workers.go +++ b/splitio/proxy/caching/workers.go @@ -28,6 +28,7 @@ func NewCacheAwareSplitSync( runtimeTelemetry storage.TelemetryRuntimeProducer, cacheFlusher gincache.CacheFlusher, appMonitor application.MonitorProducerInterface, + flagSetsFilter flagsets.FlagSetFilter, ) *CacheAwareSplitSynchronizer { return &CacheAwareSplitSynchronizer{ wrapped: split.NewSplitUpdater(splitStorage, splitFetcher, logger, runtimeTelemetry, appMonitor, flagsets.NewFlagSetFilter(nil)), // TODO(mredolatti): fix this diff --git a/splitio/proxy/conf/sections.go b/splitio/proxy/conf/sections.go index 204720f5..3f7d796a 100644 --- a/splitio/proxy/conf/sections.go +++ b/splitio/proxy/conf/sections.go @@ -7,17 +7,19 @@ import ( // Main configuration options type Main struct { - Apikey string `json:"apikey" s-cli:"apikey" s-def:"" s-desc:"Split server side SDK key"` - IPAddressEnabled bool `json:"ipAddressEnabled" s-cli:"ip-address-enabled" s-def:"true" s-desc:"Bundle host's ip address when sending data to Split"` - Initialization Initialization `json:"initialization" s-nested:"true"` - Server Server `json:"server" s-nested:"true"` - Admin conf.Admin `json:"admin" s-nested:"true"` - Storage Storage `json:"storage" s-nested:"true"` - Sync Sync `json:"sync" s-nested:"true"` - Integrations conf.Integrations `json:"integrations" s-nested:"true"` - Logging conf.Logging `json:"logging" s-nested:"true"` - Healthcheck Healthcheck `json:"healthcheck" s-nested:"true"` - Observability Observability `json:"observability" s-nested:"true"` + Apikey string `json:"apikey" s-cli:"apikey" s-def:"" s-desc:"Split server side SDK key"` + IPAddressEnabled bool `json:"ipAddressEnabled" s-cli:"ip-address-enabled" s-def:"true" s-desc:"Bundle host's ip address when sending data to Split"` + FlagSetsFilter []string `json:"flagSetsFilter" s-cli:"flag-sets-filter" s-def:"" s-desc:"Flag Sets Filter provided"` + FlagSetStrictMatching bool `json:"flagSetStrictMatching" s-cli:"flag-sets-strict-matching" s-def:"false" s-desc:"filter sets not present in cache when building splitChanges responses"` + Initialization Initialization `json:"initialization" s-nested:"true"` + Server Server `json:"server" s-nested:"true"` + Admin conf.Admin `json:"admin" s-nested:"true"` + Storage Storage `json:"storage" s-nested:"true"` + Sync Sync `json:"sync" s-nested:"true"` + Integrations conf.Integrations `json:"integrations" s-nested:"true"` + Logging conf.Logging `json:"logging" s-nested:"true"` + Healthcheck Healthcheck `json:"healthcheck" s-nested:"true"` + Observability Observability `json:"observability" s-nested:"true"` } // BuildAdvancedConfig generates a commons-compatible advancedconfig with default + overriden parameters diff --git a/splitio/proxy/controllers/sdk.go b/splitio/proxy/controllers/sdk.go index 492266a1..def1e11a 100644 --- a/splitio/proxy/controllers/sdk.go +++ b/splitio/proxy/controllers/sdk.go @@ -14,6 +14,7 @@ import ( "github.com/splitio/go-toolkit/v5/logging" "github.com/splitio/split-synchronizer/v5/splitio/proxy/caching" + "github.com/splitio/split-synchronizer/v5/splitio/proxy/flagsets" "github.com/splitio/split-synchronizer/v5/splitio/proxy/storage" ) @@ -23,6 +24,7 @@ type SdkServerController struct { fetcher service.SplitFetcher proxySplitStorage storage.ProxySplitStorage proxySegmentStorage storage.ProxySegmentStorage + fsmatcher flagsets.FlagSetMatcher } // NewSdkServerController instantiates a new sdk server controller @@ -31,12 +33,15 @@ func NewSdkServerController( fetcher service.SplitFetcher, proxySplitStorage storage.ProxySplitStorage, proxySegmentStorage storage.ProxySegmentStorage, + fsmatcher flagsets.FlagSetMatcher, + ) *SdkServerController { return &SdkServerController{ logger: logger, fetcher: fetcher, proxySplitStorage: proxySplitStorage, proxySegmentStorage: proxySegmentStorage, + fsmatcher: fsmatcher, } } @@ -61,6 +66,8 @@ func (c *SdkServerController) SplitChanges(ctx *gin.Context) { slices.Sort(sets) } + sets = c.fsmatcher.Sanitize(sets) + c.logger.Debug(fmt.Sprintf("SDK Fetches Feature Flags Since: %d", since)) splits, err := c.fetchSplitChangesSince(since, sets) diff --git a/splitio/proxy/controllers/sdk_test.go b/splitio/proxy/controllers/sdk_test.go index 049503cc..450a1c57 100644 --- a/splitio/proxy/controllers/sdk_test.go +++ b/splitio/proxy/controllers/sdk_test.go @@ -14,6 +14,7 @@ import ( "github.com/splitio/go-split-commons/v5/service/mocks" "github.com/splitio/go-toolkit/v5/logging" + "github.com/splitio/split-synchronizer/v5/splitio/proxy/flagsets" "github.com/splitio/split-synchronizer/v5/splitio/proxy/storage" psmocks "github.com/splitio/split-synchronizer/v5/splitio/proxy/storage/mocks" ) @@ -54,6 +55,7 @@ func TestSplitChangesCachedRecipe(t *testing.T) { }, }, nil, + flagsets.NewMatcher(false, nil), ) controller.Register(group) @@ -116,6 +118,7 @@ func TestSplitChangesNonCachedRecipe(t *testing.T) { }, }, nil, + flagsets.NewMatcher(false, nil), ) controller.Register(group) @@ -170,6 +173,7 @@ func TestSplitChangesNonCachedRecipeAndFetchFails(t *testing.T) { }, }, nil, + flagsets.NewMatcher(false, nil), ) controller.Register(group) @@ -211,6 +215,7 @@ func TestSegmentChanges(t *testing.T) { }, nil }, }, + flagsets.NewMatcher(false, nil), ) controller.Register(group) @@ -253,6 +258,7 @@ func TestSegmentChangesNotFound(t *testing.T) { return nil, storage.ErrSegmentNotFound }, }, + flagsets.NewMatcher(false, nil), ) controller.Register(group) @@ -289,6 +295,7 @@ func TestMySegments(t *testing.T) { return []string{"segment1", "segment2"}, nil }, }, + flagsets.NewMatcher(false, nil), ) controller.Register(group) @@ -337,6 +344,7 @@ func TestMySegmentsError(t *testing.T) { return nil, errors.New("something") }, }, + flagsets.NewMatcher(false, nil), ) controller.Register(group) @@ -351,3 +359,6 @@ func TestMySegmentsError(t *testing.T) { t.Error("Status code should be 500 and is ", resp.Code) } } + +func TestSplitChangesWithFlagSetsNonStrict(t *testing.T) { +} diff --git a/splitio/proxy/flagsets/flagsets.go b/splitio/proxy/flagsets/flagsets.go new file mode 100644 index 00000000..0419a66c --- /dev/null +++ b/splitio/proxy/flagsets/flagsets.go @@ -0,0 +1,35 @@ +package flagsets + +type FlagSetMatcher struct { + strict bool + sets map[string]struct{} +} + +func NewMatcher(strict bool, fetched []string) FlagSetMatcher { + out := FlagSetMatcher{ + strict: strict, + sets: make(map[string]struct{}, len(fetched)), + } + + for idx := range fetched { + out.sets[fetched[idx]] = struct{}{} + } + + return out +} + +func (f *FlagSetMatcher) Sanitize(input []string) []string { + if !f.strict || len(input) == 0 { + return input + } + + for idx := range input { + if _, ok := f.sets[input[idx]]; !ok { + if idx+1 < len(input) { + input[idx] = input[len(input)-1] + } + input = input[:len(input)-1] + } + } + return input +} diff --git a/splitio/proxy/flagsets/flagsets_test.go b/splitio/proxy/flagsets/flagsets_test.go new file mode 100644 index 00000000..2059712f --- /dev/null +++ b/splitio/proxy/flagsets/flagsets_test.go @@ -0,0 +1,21 @@ +package flagsets + +import ( + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestFlagSetsMatcher(t *testing.T) { + + m := NewMatcher(false, []string{"s1", "s2", "s3"}) + assert.Equal(t, []string{"s1", "s2", "s3"}, m.Sanitize([]string{"s1", "s2", "s3"})) + assert.Equal(t, []string{"s1", "s2"}, m.Sanitize([]string{"s1", "s2"})) + assert.Equal(t, []string{"s4"}, m.Sanitize([]string{"s4"})) + + m = NewMatcher(true, []string{"s1", "s2", "s3"}) + assert.Equal(t, []string{"s1", "s2", "s3"}, m.Sanitize([]string{"s1", "s2", "s3"})) + assert.Equal(t, []string{"s1", "s2"}, m.Sanitize([]string{"s1", "s2"})) + assert.Equal(t, []string{"s1", "s2"}, m.Sanitize([]string{"s1", "s2", "s7"})) + assert.Equal(t, []string{}, m.Sanitize([]string{"s4"})) +} diff --git a/splitio/proxy/initialization.go b/splitio/proxy/initialization.go index 8f9a9e5a..fab1011c 100644 --- a/splitio/proxy/initialization.go +++ b/splitio/proxy/initialization.go @@ -71,14 +71,19 @@ func Start(logger logging.LoggerInterface, cfg *pconf.Main) error { // Getting initial config data advanced := cfg.BuildAdvancedConfig() + // advanced.FlagSetsFilter = cfg.FlagSetsFilter + advanced.FlagSetsFilter = make([]string, 0) metadata := util.GetMetadata(cfg.IPAddressEnabled, true) + // FlagSetsFilter + flagSetsFilter := flagsets.NewFlagSetFilter(cfg.FlagSetsFilter) + // Setup fetchers & recorders splitAPI := api.NewSplitAPI(cfg.Apikey, *advanced, logger, metadata) // Proxy storages already implement the observable interface, so no need to wrap them // TODO(mredolatti): add a config for flagsets and build it properly here - splitStorage := storage.NewProxySplitStorage(dbInstance, logger, flagsets.NewFlagSetFilter(nil), cfg.Initialization.Snapshot != "") + splitStorage := storage.NewProxySplitStorage(dbInstance, logger, flagsets.NewFlagSetFilter(cfg.FlagSetsFilter), cfg.Initialization.Snapshot != "") segmentStorage := storage.NewProxySegmentStorage(dbInstance, logger, cfg.Initialization.Snapshot != "") // Local telemetry @@ -114,7 +119,7 @@ func Start(logger logging.LoggerInterface, cfg *pconf.Main) error { // setup feature flags, segments & local telemetry API interactions workers := synchronizer.Workers{ - SplitUpdater: caching.NewCacheAwareSplitSync(splitStorage, splitAPI.SplitFetcher, logger, localTelemetryStorage, httpCache, appMonitor), + SplitUpdater: caching.NewCacheAwareSplitSync(splitStorage, splitAPI.SplitFetcher, logger, localTelemetryStorage, httpCache, appMonitor, flagSetsFilter), SegmentUpdater: caching.NewCacheAwareSegmentSync(splitStorage, segmentStorage, splitAPI.SegmentFetcher, logger, localTelemetryStorage, httpCache, appMonitor), TelemetryRecorder: telemetry.NewTelemetrySynchronizer(localTelemetryStorage, telemetryRecorder, splitStorage, segmentStorage, logger, diff --git a/splitio/proxy/proxy.go b/splitio/proxy/proxy.go index f35dac9e..274f6a76 100644 --- a/splitio/proxy/proxy.go +++ b/splitio/proxy/proxy.go @@ -11,6 +11,7 @@ import ( "github.com/splitio/split-synchronizer/v5/splitio/common/impressionlistener" "github.com/splitio/split-synchronizer/v5/splitio/proxy/controllers" "github.com/splitio/split-synchronizer/v5/splitio/proxy/controllers/middleware" + "github.com/splitio/split-synchronizer/v5/splitio/proxy/flagsets" "github.com/splitio/split-synchronizer/v5/splitio/proxy/storage" "github.com/splitio/split-synchronizer/v5/splitio/proxy/tasks" @@ -78,6 +79,10 @@ type Options struct { // Proxy TLS configuration TLSConfig *tls.Config + + FlagSets []string + + FlagSetsStrictMatchibg bool } // API bundles all components required to answer API calls from Split sdks @@ -154,6 +159,7 @@ func setupSdkController(options *Options) *controllers.SdkServerController { options.SplitFetcher, options.ProxySplitStorage, options.ProxySegmentStorage, + flagsets.NewMatcher(options.FlagSetsStrictMatchibg, options.FlagSets), ) } diff --git a/splitio/proxy/storage/optimized/historic.go b/splitio/proxy/storage/optimized/historic.go index c7b59b74..c8581066 100644 --- a/splitio/proxy/storage/optimized/historic.go +++ b/splitio/proxy/storage/optimized/historic.go @@ -9,12 +9,23 @@ import ( "github.com/splitio/go-split-commons/v5/dtos" ) -type HistoricChanges struct { +type HistoricChanges interface { + GetUpdatedSince(since int64, flagSets []string) []FeatureView + Update(toAdd []dtos.SplitDTO, toRemove []dtos.SplitDTO, newCN int64) +} + +type HistoricChangesImpl struct { data []FeatureView mutex sync.RWMutex } -func (h *HistoricChanges) GetUpdatedSince(since int64, flagSets []string) []FeatureView { +func NewHistoricSplitChanges(capacity int) *HistoricChangesImpl { + return &HistoricChangesImpl{ + data: make([]FeatureView, 0, capacity), + } +} + +func (h *HistoricChangesImpl) GetUpdatedSince(since int64, flagSets []string) []FeatureView { slices.Sort(flagSets) h.mutex.RLock() views := h.findNewerThan(since) @@ -23,7 +34,7 @@ func (h *HistoricChanges) GetUpdatedSince(since int64, flagSets []string) []Feat return toRet } -func (h *HistoricChanges) Update(toAdd []dtos.SplitDTO, toRemove []dtos.SplitDTO, newCN int64) { +func (h *HistoricChangesImpl) Update(toAdd []dtos.SplitDTO, toRemove []dtos.SplitDTO, newCN int64) { h.mutex.Lock() h.updateFrom(toAdd) h.updateFrom(toRemove) @@ -33,7 +44,7 @@ func (h *HistoricChanges) Update(toAdd []dtos.SplitDTO, toRemove []dtos.SplitDTO // public interface ends here -func (h *HistoricChanges) updateFrom(source []dtos.SplitDTO) { +func (h *HistoricChangesImpl) updateFrom(source []dtos.SplitDTO) { for idx := range source { if current := h.findByName(source[idx].Name); current != nil { current.updateFrom(&source[idx]) @@ -46,7 +57,7 @@ func (h *HistoricChanges) updateFrom(source []dtos.SplitDTO) { } -func (h *HistoricChanges) findByName(name string) *FeatureView { +func (h *HistoricChangesImpl) findByName(name string) *FeatureView { for idx := range h.data { if h.data[idx].Name == name { // TODO(mredolatti): optimize! return &h.data[idx] @@ -55,7 +66,7 @@ func (h *HistoricChanges) findByName(name string) *FeatureView { return nil } -func (h *HistoricChanges) findNewerThan(since int64) []FeatureView { +func (h *HistoricChangesImpl) findNewerThan(since int64) []FeatureView { // precondition: h.data is sorted by CN start := sort.Search(len(h.data), func(i int) bool { return h.data[i].LastUpdated > since }) if start == len(h.data) { @@ -211,3 +222,5 @@ func incrUpTo(toIncr *int, limit int) { } *toIncr++ } + +var _ HistoricChanges = (*HistoricChangesImpl)(nil) diff --git a/splitio/proxy/storage/optimized/historic_test.go b/splitio/proxy/storage/optimized/historic_test.go index 005e9994..6c826602 100644 --- a/splitio/proxy/storage/optimized/historic_test.go +++ b/splitio/proxy/storage/optimized/historic_test.go @@ -12,7 +12,7 @@ import ( func TestHistoricSplitStorage(t *testing.T) { - var historic HistoricChanges + var historic HistoricChangesImpl historic.Update([]dtos.SplitDTO{ {Name: "f1", Sets: []string{"s1", "s2"}, Status: "ACTIVE", ChangeNumber: 1, TrafficTypeName: "tt1"}, }, []dtos.SplitDTO{}, 1) diff --git a/splitio/proxy/storage/optimized/mocks/mocks.go b/splitio/proxy/storage/optimized/mocks/mocks.go new file mode 100644 index 00000000..24160c0f --- /dev/null +++ b/splitio/proxy/storage/optimized/mocks/mocks.go @@ -0,0 +1,23 @@ +package mocks + +import ( + "github.com/splitio/go-split-commons/v5/dtos" + "github.com/splitio/split-synchronizer/v5/splitio/proxy/storage/optimized" + "github.com/stretchr/testify/mock" +) + +type HistoricStorageMock struct { + mock.Mock +} + +// GetUpdatedSince implements optimized.HistoricChanges +func (h *HistoricStorageMock) GetUpdatedSince(since int64, flagSets []string) []optimized.FeatureView { + return h.Called(since, flagSets).Get(0).([]optimized.FeatureView) +} + +// Update implements optimized.HistoricChanges +func (h *HistoricStorageMock) Update(toAdd []dtos.SplitDTO, toRemove []dtos.SplitDTO, newCN int64) { + h.Called(toAdd, toRemove, newCN) +} + +var _ optimized.HistoricChanges = (*HistoricStorageMock)(nil) diff --git a/splitio/proxy/storage/splits.go b/splitio/proxy/storage/splits.go index a862bc6a..5bf2d8d7 100644 --- a/splitio/proxy/storage/splits.go +++ b/splitio/proxy/storage/splits.go @@ -34,7 +34,6 @@ type ProxySplitStorage interface { // ProxySplitStorageImpl implements the ProxySplitStorage interface and the SplitProducer interface type ProxySplitStorageImpl struct { snapshot mutexmap.MMSplitStorage - recipes *optimized.SplitChangesSummaries db *persistent.SplitChangesCollection flagSets flagsets.FlagSetFilter historic optimized.HistoricChanges @@ -46,16 +45,16 @@ type ProxySplitStorageImpl struct { // for snapshot purposes func NewProxySplitStorage(db persistent.DBWrapper, logger logging.LoggerInterface, flagSets flagsets.FlagSetFilter, restoreBackup bool) *ProxySplitStorageImpl { disk := persistent.NewSplitChangesCollection(db, logger) - snapshot := mutexmap.NewMMSplitStorage(flagSets) // TODO(mredolatti): fix this - recipes := optimized.NewSplitChangesSummaries(maxRecipes) + snapshot := mutexmap.NewMMSplitStorage(flagSets) + historic := optimized.NewHistoricSplitChanges(1000) if restoreBackup { - snapshotFromDisk(snapshot, recipes, disk, logger) + snapshotFromDisk(snapshot, historic, disk, logger) } return &ProxySplitStorageImpl{ snapshot: *snapshot, - recipes: recipes, db: disk, flagSets: flagSets, + historic: historic, } } @@ -110,7 +109,6 @@ func (p *ProxySplitStorageImpl) Update(toAdd []dtos.SplitDTO, toRemove []dtos.Sp p.mtx.Lock() p.snapshot.Update(toAdd, toRemove, changeNumber) p.historic.Update(toAdd, toRemove, changeNumber) - p.recipes.AddChanges(toAdd, toRemove, changeNumber) p.db.Update(toAdd, toRemove, changeNumber) p.mtx.Unlock() } @@ -127,7 +125,6 @@ func (p *ProxySplitStorageImpl) RegisterOlderCn(payload *dtos.SplitChangesDTO) { toDel = append(toDel, split) } } - p.recipes.AddOlderChange(toAdd, toDel, payload.Till) } // ChangeNumber returns the current change number @@ -172,7 +169,7 @@ func (p *ProxySplitStorageImpl) Count() int { return len(p.SplitNames()) } -func snapshotFromDisk(dst *mutexmap.MMSplitStorage, summary *optimized.SplitChangesSummaries, src *persistent.SplitChangesCollection, logger logging.LoggerInterface) { +func snapshotFromDisk(dst *mutexmap.MMSplitStorage, historic optimized.HistoricChanges, src *persistent.SplitChangesCollection, logger logging.LoggerInterface) { all, err := src.FetchAll() if err != nil { logger.Error("error parsing feature flags from snapshot. No data will be available!: ", err) @@ -193,7 +190,7 @@ func snapshotFromDisk(dst *mutexmap.MMSplitStorage, summary *optimized.SplitChan } dst.Update(filtered, nil, cn) - summary.AddChanges(filtered, nil, cn) + historic.Update(filtered, nil, cn) } func archivedDTOForView(view *optimized.FeatureView) dtos.SplitDTO { diff --git a/splitio/proxy/storage/splits_test.go b/splitio/proxy/storage/splits_test.go index 798eed57..12abcdf0 100644 --- a/splitio/proxy/storage/splits_test.go +++ b/splitio/proxy/storage/splits_test.go @@ -3,6 +3,8 @@ package storage import ( "testing" + "github.com/splitio/split-synchronizer/v5/splitio/proxy/storage/optimized" + "github.com/splitio/split-synchronizer/v5/splitio/proxy/storage/optimized/mocks" "github.com/splitio/split-synchronizer/v5/splitio/proxy/storage/persistent" "github.com/splitio/go-split-commons/v5/dtos" @@ -14,53 +16,56 @@ import ( func TestSplitStorage(t *testing.T) { dbw, err := persistent.NewBoltWrapper(persistent.BoltInMemoryMode, nil) - if err != nil { - t.Error("error creating bolt wrapper: ", err) - } + assert.Nil(t, err) logger := logging.NewLogger(nil) - splitC := persistent.NewSplitChangesCollection(dbw, logger) - splitC.Update([]dtos.SplitDTO{ + toAdd := []dtos.SplitDTO{ {Name: "f1", ChangeNumber: 1, Status: "ACTIVE"}, {Name: "f2", ChangeNumber: 2, Status: "ACTIVE"}, - }, nil, 1) - - pss := NewProxySplitStorage(dbw, logger, flagsets.NewFlagSetFilter(nil), true) - - sinceMinus1, currentCN, err := pss.recipes.FetchSince(-1) - if err != nil { - t.Error("unexpected error: ", err) } + toAdd2 := []dtos.SplitDTO{{Name: "f3", ChangeNumber: 3, Status: "ACTIVE", TrafficTypeName: "ttt"}} - if currentCN != 2 { - t.Error("current cn should be 2. Is: ", currentCN) - } + splitC := persistent.NewSplitChangesCollection(dbw, logger) + splitC.Update(toAdd, nil, 2) - if _, ok := sinceMinus1.Updated["f1"]; !ok { - t.Error("s1 should be added") - } + var historicMock mocks.HistoricStorageMock + historicMock.On("Update", toAdd2, []dtos.SplitDTO(nil), int64(3)).Once() + historicMock.On("GetUpdatedSince", int64(2), []string(nil)). + Once(). + Return([]optimized.FeatureView{{Name: "f3", LastUpdated: 3, Active: true, TrafficTypeName: "ttt"}}) - if _, ok := sinceMinus1.Updated["f2"]; !ok { - t.Error("s2 should be added") - } + pss := NewProxySplitStorage(dbw, logger, flagsets.NewFlagSetFilter(nil), true) - since2, currentCN, err := pss.recipes.FetchSince(2) - if err != nil { - t.Error("unexpected error: ", err) - } + // validate initial state of the historic cache & replace it with a mock for the next validations + assert.ElementsMatch(t, + []optimized.FeatureView{ + {Name: "f1", Active: true, LastUpdated: 1, FlagSets: []optimized.FlagSetView{}}, + {Name: "f2", Active: true, LastUpdated: 2, FlagSets: []optimized.FlagSetView{}}, + }, pss.historic.GetUpdatedSince(-1, nil)) + pss.historic = &historicMock + // ---- - if currentCN != 2 { - t.Error("current cn should be 2. Is: ", currentCN) - } + changes, err := pss.ChangesSince(-1, nil) + assert.Nil(t, err) + assert.Equal(t, int64(-1), changes.Since) + assert.Equal(t, int64(2), changes.Till) + assert.ElementsMatch(t, changes.Splits, toAdd) - if len(since2.Updated) != 0 { - t.Error("nothing should have been added") - } + pss.Update(toAdd2, nil, 3) + changes, err = pss.ChangesSince(-1, nil) + assert.Nil(t, err) + assert.Equal(t, int64(-1), changes.Since) + assert.Equal(t, int64(3), changes.Till) + assert.ElementsMatch(t, changes.Splits, append(append([]dtos.SplitDTO(nil), toAdd...), toAdd2...)) - if len(since2.Removed) != 0 { - t.Error("nothing should have been removed") - } + changes, err = pss.ChangesSince(2, nil) + assert.Nil(t, err) + assert.Equal(t, int64(2), changes.Since) + assert.Equal(t, int64(3), changes.Till) + assert.ElementsMatch(t, changes.Splits, toAdd2) + + historicMock.AssertExpectations(t) } func TestSplitStorageWithFlagsets(t *testing.T) { @@ -139,5 +144,4 @@ func TestSplitStorageWithFlagsets(t *testing.T) { assert.ElementsMatch(t, []dtos.SplitDTO{ {Name: "f2", ChangeNumber: 2, Status: "ACTIVE", Sets: []string{"s2", "s3"}}, }, res.Splits) - } diff --git a/splitio/version.go b/splitio/version.go index 84a70c52..089f9cc9 100644 --- a/splitio/version.go +++ b/splitio/version.go @@ -2,4 +2,4 @@ package splitio // Version is the version of this Agent -const Version = "5.4.0" +const Version = "5.5.0-rc1"