controller wip

CHANGES.txt

@@ -1,3 +1,6 @@
+5.5.0 (Month XX, 2023)
+- FlagSet
+
 5.4.0 (July 18, 2023)
 - Improved streaming architecture implementation to apply feature flag updates from the notification received which is now enhanced, improving efficiency and reliability of the whole update system.
 - Fixed possible edge case issue where deleting a feature flag doesn’t propagate immediately.

go.mod

@@ -8,7 +8,7 @@ require (
 	github.com/gin-gonic/gin v1.9.1
 	github.com/google/uuid v1.3.0
 	github.com/splitio/gincache v1.0.1
-	github.com/splitio/go-split-commons/v5 v5.0.1-0.20230926022914-2101c4dc74c0
+	github.com/splitio/go-split-commons/v5 v5.0.1-0.20231004184048-81902536fc1f
 	github.com/splitio/go-toolkit/v5 v5.3.2-0.20230920032539-d08915cf020a
 	github.com/stretchr/testify v1.8.4
 	go.etcd.io/bbolt v1.3.6
@@ -38,6 +38,7 @@ require (
 	github.com/pelletier/go-toml/v2 v2.0.8 // indirect
 	github.com/pmezard/go-difflib v1.0.0 // indirect
 	github.com/redis/go-redis/v9 v9.0.4 // indirect
+	github.com/stretchr/objx v0.5.0 // indirect
 	github.com/twitchyliquid64/golang-asm v0.15.1 // indirect
 	github.com/ugorji/go/codec v1.2.11 // indirect
 	golang.org/x/arch v0.3.0 // indirect

go.sum

@@ -90,12 +90,13 @@ github.com/rogpeppe/go-internal v1.8.0 h1:FCbCCtXNOY3UtUuHUYaghJg4y7Fd14rXifAYUA
 github.com/rogpeppe/go-internal v1.8.0/go.mod h1:WmiCO8CzOY8rg0OYDC4/i/2WRWAB6poM+XZ2dLUbcbE=
 github.com/splitio/gincache v1.0.1 h1:dLYdANY/BqH4KcUMCe/LluLyV5WtuE/LEdQWRE06IXU=
 github.com/splitio/gincache v1.0.1/go.mod h1:CcgJDSM9Af75kyBH0724v55URVwMBuSj5x1eCWIOECY=
-github.com/splitio/go-split-commons/v5 v5.0.1-0.20230926022914-2101c4dc74c0 h1:t7QuH0+4T2LeJOc2gdRP+PkFPkQEB017arfxBccsArg=
-github.com/splitio/go-split-commons/v5 v5.0.1-0.20230926022914-2101c4dc74c0/go.mod h1:ksVZQYLs+3ZuzU81vEvf1aCjk24pdrVWjUXNq6Qcayo=
+github.com/splitio/go-split-commons/v5 v5.0.1-0.20231004184048-81902536fc1f h1:g3rsXA0cdMx2uz3MrTEz2tiittf+HDXpHooyYnuYg6w=
+github.com/splitio/go-split-commons/v5 v5.0.1-0.20231004184048-81902536fc1f/go.mod h1:ksVZQYLs+3ZuzU81vEvf1aCjk24pdrVWjUXNq6Qcayo=
 github.com/splitio/go-toolkit/v5 v5.3.2-0.20230920032539-d08915cf020a h1:2wjh5hSGlFRuh6Lbmodr0VRqtry2m9pEBNmwiLsY+ss=
 github.com/splitio/go-toolkit/v5 v5.3.2-0.20230920032539-d08915cf020a/go.mod h1:xYhUvV1gga9/1029Wbp5pjnR6Cy8nvBpjw99wAbsMko=
 github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
 github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw=
+github.com/stretchr/objx v0.5.0 h1:1zr/of2m5FGMsad5YfcqgdqdWrIhu+EBEJRhR1U7z/c=
 github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo=
 github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
 github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=

splitio/commitversion.go

@@ -5,4 +5,4 @@ This file is created automatically, please do not edit
 */
 
 // CommitVersion is the version of the last commit previous to release
-const CommitVersion = "353237e"
+const CommitVersion = "4867a2d"

splitio/producer/conf/sections.go

@@ -9,6 +9,7 @@ import (
 type Main struct {
 	Apikey           string            `json:"apikey" s-cli:"apikey" s-def:"" s-desc:"Split server side SDK key"`
 	IPAddressEnabled bool              `json:"ipAddressEnabled" s-cli:"ip-address-enabled" s-def:"true" s-desc:"Bundle host's ip address when sending data to Split"`
+	FlagSetsFilter   []string          `json:"flagSetsFilter" s-cli:"flag-sets-filter" s-def:"" s-desc:"Flag Sets Filter provided"`
 	Initialization   Initialization    `json:"initialization" s-nested:"true"`
 	Storage          Storage           `json:"storage" s-nested:"true"`
 	Sync             Sync              `json:"sync" s-nested:"true"`

splitio/producer/initialization.go

@@ -47,6 +47,7 @@ const (
 func Start(logger logging.LoggerInterface, cfg *conf.Main) error {
 	// Getting initial config data
 	advanced := cfg.BuildAdvancedConfig()
+	advanced.FlagSetsFilter = cfg.FlagSetsFilter
 	metadata := util.GetMetadata(false, cfg.IPAddressEnabled)
 
 	clientKey, err := util.GetClientKey(cfg.Apikey)
@@ -85,8 +86,11 @@ func Start(logger logging.LoggerInterface, cfg *conf.Main) error {
 	syncTelemetryStorage, _ := inmemory.NewTelemetryStorage()
 	sdkTelemetryStorage := storage.NewRedisTelemetryCosumerclient(redisClient, logger)
 
+	// FlagSetsFilter
+	flagSetsFilter := flagsets.NewFlagSetFilter(cfg.FlagSetsFilter)
+
 	// These storages are forwarded to the dashboard, the sdk-telemetry is irrelevant there
-	splitStorage, err := observability.NewObservableSplitStorage(redis.NewSplitStorage(redisClient, logger), logger)
+	splitStorage, err := observability.NewObservableSplitStorage(redis.NewSplitStorage(redisClient, logger, flagSetsFilter), logger)
 	if err != nil {
 		return fmt.Errorf("error instantiating observable feature flag storage: %w", err)
 	}

splitio/proxy/caching/caching.go

@@ -34,18 +34,6 @@ func MakeSurrogateForSegmentChanges(segmentName string) string {
 
 // MakeSurrogateForMySegments creates a list surrogate keys for all the segments involved
 func MakeSurrogateForMySegments(mysegments []dtos.MySegmentDTO) []string {
-	if len(mysegments) == 0 {
-		return nil
-	}
-
-	/*
-		surrogates := make([]string, 0, len(mysegments))
-		for idx := range mysegments {
-			surrogates = append(surrogates, segmentPrefix+mysegments[idx].Name)
-		}
-		return surrogates
-	*/
-
 	// Since we are now evicting individually for every updated key, we don't need surrogates for mySegments
 	return nil
 }

splitio/proxy/caching/workers.go

@@ -28,6 +28,7 @@ func NewCacheAwareSplitSync(
 	runtimeTelemetry storage.TelemetryRuntimeProducer,
 	cacheFlusher gincache.CacheFlusher,
 	appMonitor application.MonitorProducerInterface,
+	flagSetsFilter flagsets.FlagSetFilter,
 ) *CacheAwareSplitSynchronizer {
 	return &CacheAwareSplitSynchronizer{
 		wrapped:      split.NewSplitUpdater(splitStorage, splitFetcher, logger, runtimeTelemetry, appMonitor, flagsets.NewFlagSetFilter(nil)), // TODO(mredolatti): fix this

splitio/proxy/conf/sections.go

@@ -7,17 +7,19 @@ import (
 
 // Main configuration options
 type Main struct {
-	Apikey           string            `json:"apikey" s-cli:"apikey" s-def:"" s-desc:"Split server side SDK key"`
-	IPAddressEnabled bool              `json:"ipAddressEnabled" s-cli:"ip-address-enabled" s-def:"true" s-desc:"Bundle host's ip address when sending data to Split"`
-	Initialization   Initialization    `json:"initialization" s-nested:"true"`
-	Server           Server            `json:"server" s-nested:"true"`
-	Admin            conf.Admin        `json:"admin" s-nested:"true"`
-	Storage          Storage           `json:"storage" s-nested:"true"`
-	Sync             Sync              `json:"sync" s-nested:"true"`
-	Integrations     conf.Integrations `json:"integrations" s-nested:"true"`
-	Logging          conf.Logging      `json:"logging" s-nested:"true"`
-	Healthcheck      Healthcheck       `json:"healthcheck" s-nested:"true"`
-	Observability    Observability     `json:"observability" s-nested:"true"`
+	Apikey                string            `json:"apikey" s-cli:"apikey" s-def:"" s-desc:"Split server side SDK key"`
+	IPAddressEnabled      bool              `json:"ipAddressEnabled" s-cli:"ip-address-enabled" s-def:"true" s-desc:"Bundle host's ip address when sending data to Split"`
+	FlagSetsFilter        []string          `json:"flagSetsFilter" s-cli:"flag-sets-filter" s-def:"" s-desc:"Flag Sets Filter provided"`
+	FlagSetStrictMatching bool              `json:"flagSetStrictMatching" s-cli:"flag-sets-strict-matching" s-def:"false" s-desc:"filter sets not present in cache when building splitChanges responses"`
+	Initialization        Initialization    `json:"initialization" s-nested:"true"`
+	Server                Server            `json:"server" s-nested:"true"`
+	Admin                 conf.Admin        `json:"admin" s-nested:"true"`
+	Storage               Storage           `json:"storage" s-nested:"true"`
+	Sync                  Sync              `json:"sync" s-nested:"true"`
+	Integrations          conf.Integrations `json:"integrations" s-nested:"true"`
+	Logging               conf.Logging      `json:"logging" s-nested:"true"`
+	Healthcheck           Healthcheck       `json:"healthcheck" s-nested:"true"`
+	Observability         Observability     `json:"observability" s-nested:"true"`
 }
 
 // BuildAdvancedConfig generates a commons-compatible advancedconfig with default + overriden parameters

splitio/proxy/controllers/sdk.go

@@ -14,6 +14,7 @@ import (
 	"github.com/splitio/go-toolkit/v5/logging"
 
 	"github.com/splitio/split-synchronizer/v5/splitio/proxy/caching"
+	"github.com/splitio/split-synchronizer/v5/splitio/proxy/flagsets"
 	"github.com/splitio/split-synchronizer/v5/splitio/proxy/storage"
 )
 
@@ -23,6 +24,7 @@ type SdkServerController struct {
 	fetcher             service.SplitFetcher
 	proxySplitStorage   storage.ProxySplitStorage
 	proxySegmentStorage storage.ProxySegmentStorage
+	fsmatcher           flagsets.FlagSetMatcher
 }
 
 // NewSdkServerController instantiates a new sdk server controller
@@ -31,12 +33,15 @@ func NewSdkServerController(
 	fetcher service.SplitFetcher,
 	proxySplitStorage storage.ProxySplitStorage,
 	proxySegmentStorage storage.ProxySegmentStorage,
+	fsmatcher flagsets.FlagSetMatcher,
+
 ) *SdkServerController {
 	return &SdkServerController{
 		logger:              logger,
 		fetcher:             fetcher,
 		proxySplitStorage:   proxySplitStorage,
 		proxySegmentStorage: proxySegmentStorage,
+		fsmatcher:           fsmatcher,
 	}
 }
 
@@ -61,6 +66,8 @@ func (c *SdkServerController) SplitChanges(ctx *gin.Context) {
 		slices.Sort(sets)
 	}
 
+	sets = c.fsmatcher.Sanitize(sets)
+
 	c.logger.Debug(fmt.Sprintf("SDK Fetches Feature Flags Since: %d", since))
 
 	splits, err := c.fetchSplitChangesSince(since, sets)

splitio/proxy/controllers/sdk_test.go

@@ -14,6 +14,7 @@ import (
 	"github.com/splitio/go-split-commons/v5/service/mocks"
 	"github.com/splitio/go-toolkit/v5/logging"
 
+	"github.com/splitio/split-synchronizer/v5/splitio/proxy/flagsets"
 	"github.com/splitio/split-synchronizer/v5/splitio/proxy/storage"
 	psmocks "github.com/splitio/split-synchronizer/v5/splitio/proxy/storage/mocks"
 )
@@ -54,6 +55,7 @@ func TestSplitChangesCachedRecipe(t *testing.T) {
 			},
 		},
 		nil,
+		flagsets.NewMatcher(false, nil),
 	)
 	controller.Register(group)
 
@@ -116,6 +118,7 @@ func TestSplitChangesNonCachedRecipe(t *testing.T) {
 			},
 		},
 		nil,
+		flagsets.NewMatcher(false, nil),
 	)
 	controller.Register(group)
 
@@ -170,6 +173,7 @@ func TestSplitChangesNonCachedRecipeAndFetchFails(t *testing.T) {
 			},
 		},
 		nil,
+		flagsets.NewMatcher(false, nil),
 	)
 	controller.Register(group)
 
@@ -211,6 +215,7 @@ func TestSegmentChanges(t *testing.T) {
 				}, nil
 			},
 		},
+		flagsets.NewMatcher(false, nil),
 	)
 	controller.Register(group)
 
@@ -253,6 +258,7 @@ func TestSegmentChangesNotFound(t *testing.T) {
 				return nil, storage.ErrSegmentNotFound
 			},
 		},
+		flagsets.NewMatcher(false, nil),
 	)
 	controller.Register(group)
 
@@ -289,6 +295,7 @@ func TestMySegments(t *testing.T) {
 				return []string{"segment1", "segment2"}, nil
 			},
 		},
+		flagsets.NewMatcher(false, nil),
 	)
 	controller.Register(group)
 
@@ -337,6 +344,7 @@ func TestMySegmentsError(t *testing.T) {
 				return nil, errors.New("something")
 			},
 		},
+		flagsets.NewMatcher(false, nil),
 	)
 	controller.Register(group)
 
@@ -351,3 +359,6 @@ func TestMySegmentsError(t *testing.T) {
 		t.Error("Status code should be 500 and is ", resp.Code)
 	}
 }
+
+func TestSplitChangesWithFlagSetsNonStrict(t *testing.T) {
+}

splitio/proxy/flagsets/flagsets.go

@@ -0,0 +1,35 @@
+package flagsets
+
+type FlagSetMatcher struct {
+	strict bool
+	sets   map[string]struct{}
+}
+
+func NewMatcher(strict bool, fetched []string) FlagSetMatcher {
+	out := FlagSetMatcher{
+		strict: strict,
+		sets:   make(map[string]struct{}, len(fetched)),
+	}
+
+	for idx := range fetched {
+		out.sets[fetched[idx]] = struct{}{}
+	}
+
+	return out
+}
+
+func (f *FlagSetMatcher) Sanitize(input []string) []string {
+	if !f.strict || len(input) == 0 {
+		return input
+	}
+
+	for idx := range input {
+		if _, ok := f.sets[input[idx]]; !ok {
+			if idx+1 < len(input) {
+				input[idx] = input[len(input)-1]
+			}
+			input = input[:len(input)-1]
+		}
+	}
+	return input
+}

splitio/proxy/flagsets/flagsets_test.go

@@ -0,0 +1,21 @@
+package flagsets
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+)
+
+func TestFlagSetsMatcher(t *testing.T) {
+
+	m := NewMatcher(false, []string{"s1", "s2", "s3"})
+	assert.Equal(t, []string{"s1", "s2", "s3"}, m.Sanitize([]string{"s1", "s2", "s3"}))
+	assert.Equal(t, []string{"s1", "s2"}, m.Sanitize([]string{"s1", "s2"}))
+	assert.Equal(t, []string{"s4"}, m.Sanitize([]string{"s4"}))
+
+	m = NewMatcher(true, []string{"s1", "s2", "s3"})
+	assert.Equal(t, []string{"s1", "s2", "s3"}, m.Sanitize([]string{"s1", "s2", "s3"}))
+	assert.Equal(t, []string{"s1", "s2"}, m.Sanitize([]string{"s1", "s2"}))
+	assert.Equal(t, []string{"s1", "s2"}, m.Sanitize([]string{"s1", "s2", "s7"}))
+	assert.Equal(t, []string{}, m.Sanitize([]string{"s4"}))
+}

splitio/proxy/initialization.go

@@ -71,14 +71,19 @@ func Start(logger logging.LoggerInterface, cfg *pconf.Main) error {
 
 	// Getting initial config data
 	advanced := cfg.BuildAdvancedConfig()
+	// advanced.FlagSetsFilter = cfg.FlagSetsFilter
+	advanced.FlagSetsFilter = make([]string, 0)
 	metadata := util.GetMetadata(cfg.IPAddressEnabled, true)
 
+	// FlagSetsFilter
+	flagSetsFilter := flagsets.NewFlagSetFilter(cfg.FlagSetsFilter)
+
 	// Setup fetchers & recorders
 	splitAPI := api.NewSplitAPI(cfg.Apikey, *advanced, logger, metadata)
 
 	// Proxy storages already implement the observable interface, so no need to wrap them
 	// TODO(mredolatti): add a config for flagsets and build it properly here
-	splitStorage := storage.NewProxySplitStorage(dbInstance, logger, flagsets.NewFlagSetFilter(nil), cfg.Initialization.Snapshot != "")
+	splitStorage := storage.NewProxySplitStorage(dbInstance, logger, flagsets.NewFlagSetFilter(cfg.FlagSetsFilter), cfg.Initialization.Snapshot != "")
 	segmentStorage := storage.NewProxySegmentStorage(dbInstance, logger, cfg.Initialization.Snapshot != "")
 
 	// Local telemetry
@@ -114,7 +119,7 @@ func Start(logger logging.LoggerInterface, cfg *pconf.Main) error {
 
 	// setup feature flags, segments & local telemetry API interactions
 	workers := synchronizer.Workers{
-		SplitUpdater: caching.NewCacheAwareSplitSync(splitStorage, splitAPI.SplitFetcher, logger, localTelemetryStorage, httpCache, appMonitor),
+		SplitUpdater: caching.NewCacheAwareSplitSync(splitStorage, splitAPI.SplitFetcher, logger, localTelemetryStorage, httpCache, appMonitor, flagSetsFilter),
 		SegmentUpdater: caching.NewCacheAwareSegmentSync(splitStorage, segmentStorage, splitAPI.SegmentFetcher, logger, localTelemetryStorage, httpCache,
 			appMonitor),
 		TelemetryRecorder: telemetry.NewTelemetrySynchronizer(localTelemetryStorage, telemetryRecorder, splitStorage, segmentStorage, logger,

splitio/proxy/proxy.go

@@ -11,6 +11,7 @@ import (
 	"github.com/splitio/split-synchronizer/v5/splitio/common/impressionlistener"
 	"github.com/splitio/split-synchronizer/v5/splitio/proxy/controllers"
 	"github.com/splitio/split-synchronizer/v5/splitio/proxy/controllers/middleware"
+	"github.com/splitio/split-synchronizer/v5/splitio/proxy/flagsets"
 	"github.com/splitio/split-synchronizer/v5/splitio/proxy/storage"
 	"github.com/splitio/split-synchronizer/v5/splitio/proxy/tasks"
 
@@ -78,6 +79,10 @@ type Options struct {
 
 	// Proxy TLS configuration
 	TLSConfig *tls.Config
+
+	FlagSets []string
+
+	FlagSetsStrictMatchibg bool
 }
 
 // API bundles all components required to answer API calls from Split sdks
@@ -154,6 +159,7 @@ func setupSdkController(options *Options) *controllers.SdkServerController {
 		options.SplitFetcher,
 		options.ProxySplitStorage,
 		options.ProxySegmentStorage,
+		flagsets.NewMatcher(options.FlagSetsStrictMatchibg, options.FlagSets),
 	)
 }