From fb13067c94e0e2de7da9c91cc20f6e85bdc74f9c Mon Sep 17 00:00:00 2001 From: Mauro Antonio Sanz Date: Mon, 27 Jun 2022 18:47:06 +0200 Subject: [PATCH 01/25] updated factory to starting to use the new strategies for impressions --- go.mod | 4 + go.sum | 10 +- splitio/client/client_test.go | 396 +++++++++++++++++++++++-- splitio/client/factory.go | 225 +++++++++++--- splitio/client/input_validator_test.go | 11 +- splitio/conf/sdkconf.go | 2 + 6 files changed, 573 insertions(+), 75 deletions(-) diff --git a/go.mod b/go.mod index 325f4fec..9f371d58 100644 --- a/go.mod +++ b/go.mod @@ -2,6 +2,10 @@ module github.com/splitio/go-client/v6 go 1.13 +replace github.com/splitio/go-split-commons/v4 => /Users/mauroantoniosanz/go/src/github.com/splitio/go-split-commons + +replace github.com/splitio/go-toolkit/v5 => /Users/mauroantoniosanz/go/src/github.com/splitio/go-toolkit + require ( github.com/splitio/go-split-commons/v4 v4.1.0 github.com/splitio/go-toolkit/v5 v5.2.0 diff --git a/go.sum b/go.sum index 8af83dc2..d7a83fd6 100644 --- a/go.sum +++ b/go.sum @@ -1,3 +1,7 @@ +github.com/bits-and-blooms/bitset v1.2.0 h1:Kn4yilvwNtMACtf1eYDlG8H77R07mZSPbMjLyS07ChA= +github.com/bits-and-blooms/bitset v1.2.0/go.mod h1:gIdJ4wp64HaoK2YrL1Q5/N7Y16edYb8uY+O0FJTyyDA= +github.com/bits-and-blooms/bloom/v3 v3.2.0 h1:N+g3GTQ0TVbghahYyzwkQbMZR+IwIwFFC8dpIChtN0U= +github.com/bits-and-blooms/bloom/v3 v3.2.0/go.mod h1:MC8muvBzzPOFsrcdND/A7kU7kMhkqb9KI70JlZCP+C8= github.com/cespare/xxhash/v2 v2.1.2 h1:YRXhKfTDauu4ajMg1TPgFO5jnlC2HCbmLXMcTG5cbYE= github.com/cespare/xxhash/v2 v2.1.2/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI= @@ -47,10 +51,8 @@ github.com/onsi/gomega v1.17.0/go.mod h1:HnhC7FXeEQY45zxNK3PPoIUhzk/80Xly9PcubAl github.com/onsi/gomega v1.18.1 h1:M1GfJqGRrBrrGGsbxzV5dqM2U2ApXefZCQpkukxYRLE= github.com/onsi/gomega v1.18.1/go.mod h1:0q+aL8jAiMXy9hbwj2mr5GziHiwhAIQpFmmtT5hitRs= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= -github.com/splitio/go-split-commons/v4 v4.1.0 h1:bERGPdxHbjPFRy0SCiwqZzPRo51hleP3yHpzuPimlDQ= -github.com/splitio/go-split-commons/v4 v4.1.0/go.mod h1:ycM23TPERljz7Anp6W4nwxPNoaIqY2d4n/UWYV0V0PE= -github.com/splitio/go-toolkit/v5 v5.2.0 h1:9bej4sMHVdcc+aB8IXQYEK9OjJ44H/jYFYRkbRwKsu4= -github.com/splitio/go-toolkit/v5 v5.2.0/go.mod h1:SYi/svhhtEgdMSb5tNcDcMjOSUH/7XVkvjp5dPL+nBE= +github.com/spaolacci/murmur3 v1.1.0 h1:7c1g84S4BPRrfL5Xrdp6fOJ206sU9y293DDHaoy0bLI= +github.com/spaolacci/murmur3 v1.1.0/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA= github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= diff --git a/splitio/client/client_test.go b/splitio/client/client_test.go index 12bcdc29..e8e21a65 100644 --- a/splitio/client/client_test.go +++ b/splitio/client/client_test.go @@ -9,6 +9,7 @@ import ( "net/http" "net/http/httptest" "os" + "strings" "sync/atomic" "testing" "time" @@ -23,6 +24,7 @@ import ( "github.com/splitio/go-split-commons/v4/dtos" "github.com/splitio/go-split-commons/v4/healthcheck/application" "github.com/splitio/go-split-commons/v4/provisional" + "github.com/splitio/go-split-commons/v4/provisional/strategy" authMocks "github.com/splitio/go-split-commons/v4/service/mocks" "github.com/splitio/go-split-commons/v4/storage" "github.com/splitio/go-split-commons/v4/storage/inmemory" @@ -127,10 +129,11 @@ func getFactory() SplitFactory { cfg := conf.Default() cfg.LabelsEnabled = true logger := logging.NewLogger(nil) - impressionManager, _ := provisional.NewImpressionManager(commonsCfg.ManagerConfig{ - ImpressionsMode: cfg.ImpressionsMode, - OperationMode: cfg.OperationMode, - }, provisional.NewImpressionsCounter(), telemetryStorage) + + impressionObserver, _ := strategy.NewImpressionObserver(500) + impressionsCounter := strategy.NewImpressionsCounter() + impressionsStrategy := strategy.NewOptimizedImpl(impressionObserver, impressionsCounter, telemetryStorage, false) + impressionManager := provisional.NewImpressionManager(impressionsStrategy) return SplitFactory{ cfg: cfg, @@ -289,10 +292,11 @@ func TestClientPanicking(t *testing.T) { cfg := conf.Default() cfg.LabelsEnabled = true logger := logging.NewLogger(nil) - impressionManager, _ := provisional.NewImpressionManager(commonsCfg.ManagerConfig{ - ImpressionsMode: cfg.ImpressionsMode, - OperationMode: cfg.OperationMode, - }, provisional.NewImpressionsCounter(), telemetryMockedStorage) + + impressionObserver, _ := strategy.NewImpressionObserver(500) + impressionsCounter := strategy.NewImpressionsCounter() + impressionsStrategy := strategy.NewOptimizedImpl(impressionObserver, impressionsCounter, telemetryMockedStorage, false) + impressionManager := provisional.NewImpressionManager(impressionsStrategy) factory := SplitFactory{ cfg: cfg, @@ -448,6 +452,7 @@ func compareListener(ilTest map[string]interface{}, f string, k string, l string func getClientForListener() SplitClient { cfg := conf.Default() cfg.LabelsEnabled = true + logger := logging.NewLogger(nil) impTest := &ImpressionListenerTest{} @@ -461,11 +466,12 @@ func getClientForListener() SplitClient { RecordLatencyCall: func(method string, latency time.Duration) {}, } impressionStorage := mutexqueue.NewMQImpressionsStorage(cfg.Advanced.ImpressionsQueueSize, make(chan string, 1), logger, telemetryMockedStorage) - impressionManager, _ := provisional.NewImpressionManager(commonsCfg.ManagerConfig{ - ImpressionsMode: cfg.ImpressionsMode, - OperationMode: cfg.OperationMode, - ListenerEnabled: true, - }, provisional.NewImpressionsCounter(), telemetryMockedStorage) + + impressionObserver, _ := strategy.NewImpressionObserver(500) + impressionsCounter := strategy.NewImpressionsCounter() + impressionsStrategy := strategy.NewOptimizedImpl(impressionObserver, impressionsCounter, telemetryMockedStorage, true) + impressionManager := provisional.NewImpressionManager(impressionsStrategy) + factory := &SplitFactory{ cfg: cfg, storages: sdkStorages{ @@ -695,15 +701,14 @@ func TestBlockUntilReadyRedis(t *testing.T) { sdkConf.OperationMode = conf.RedisConsumer factory, _ := NewSplitFactory("something", sdkConf) - if !factory.IsReady() { t.Error("Factory should be ready immediately") } - client := factory.Client() if !client.factory.IsReady() { t.Error("Client should be ready immediately") } + err := client.BlockUntilReady(1) if err != nil { t.Error("Error was not expected") @@ -1135,7 +1140,7 @@ func isInvalidImpression(client SplitClient, key string, feature string, treatme func TestClient(t *testing.T) { cfg := conf.Default() - + cfg.ImpressionsMode = commonsCfg.ImpressionsModeDebug cfg.LabelsEnabled = true logger := logging.NewLogger(nil) @@ -1194,10 +1199,10 @@ func TestClient(t *testing.T) { RecordLatencyCall: func(method string, latency time.Duration) {}, } - impressionManager, _ := provisional.NewImpressionManager(commonsCfg.ManagerConfig{ - ImpressionsMode: commonsCfg.ImpressionsModeDebug, - OperationMode: cfg.OperationMode, - }, provisional.NewImpressionsCounter(), mockedTelemetryStorage) + impressionObserver, _ := strategy.NewImpressionObserver(500) + impressionsStrategy := strategy.NewDebugImpl(impressionObserver, true) + impressionManager := provisional.NewImpressionManager(impressionsStrategy) + factory := &SplitFactory{cfg: cfg, impressionManager: impressionManager} client := SplitClient{ evaluator: evaluator, @@ -1683,6 +1688,115 @@ func TestClientOptimized(t *testing.T) { } } +func TestClientNone(t *testing.T) { + var isDestroyCalled int64 + var splitsMock, _ = ioutil.ReadFile("../../testdata/splits_mock_2.json") + + postChannel := make(chan string, 1) + + ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + switch r.URL.Path { + case "/splitChanges": + fmt.Fprintln(w, string(splitsMock)) + return + case "/testImpressions/bulk": + t.Error("Should not post impressions") + case "/testImpressions/count": + fmt.Fprintln(w, "ok") + if atomic.LoadInt64(&isDestroyCalled) == 1 { + rBody, _ := ioutil.ReadAll(r.Body) + + var dataInPost map[string][]map[string]interface{} + err := json.Unmarshal(rBody, &dataInPost) + if err != nil { + t.Error(err) + return + } + + for _, v := range dataInPost["pf"] { + if int64(v["m"].(float64)) != util.TruncateTimeFrame(time.Now().UTC().UnixNano()) { + t.Error("Wrong timeFrame") + } + switch v["f"] { + case "DEMO_MURMUR2": + if v["rc"].(float64) != 4 { + t.Error("Wrong rc") + } + if int64(v["m"].(float64)) != util.TruncateTimeFrame(time.Now().UTC().UnixNano()) { + t.Error("Wrong timeFrame") + } + case "DEMO_MURMUR": + if v["rc"].(float64) != 1 { + t.Error("Wrong rc") + } + } + } + } + case "/keys/ss": + rBody, _ := ioutil.ReadAll(r.Body) + + var uniques dtos.Uniques + err := json.Unmarshal(rBody, &uniques) + if err != nil { + t.Error(err) + return + } + + if len(uniques.Keys) != 2 { + t.Error("Length should be 2") + } + for _, key := range uniques.Keys { + if key.Feature == "DEMO_MURMUR2" && len(key.Keys) != 2 { + t.Error("Length should be 2") + } + if key.Feature == "DEMO_MURMUR" && len(key.Keys) != 1 { + t.Error("Length should be 1") + } + } + + postChannel <- "finished" + case "/events/bulk": + fmt.Fprintln(w, "ok") + case "/segmentChanges": + fallthrough + default: + fmt.Fprintln(w, "ok") + } + })) + defer ts.Close() + + impTest := &ImpressionListenerTest{} + cfg := conf.Default() + cfg.LabelsEnabled = true + cfg.Advanced.EventsURL = ts.URL + cfg.Advanced.SdkURL = ts.URL + cfg.Advanced.TelemetryServiceURL = ts.URL + cfg.Advanced.ImpressionListener = impTest + cfg.ImpressionsMode = commonsCfg.ImpressionsModeNone + + factory, _ := NewSplitFactory("test", cfg) + client := factory.Client() + client.BlockUntilReady(2) + + // Calls treatments to generate one valid impression + time.Sleep(300 * time.Millisecond) // Let's wait until first call of recorders have finished + client.Treatment("user1", "DEMO_MURMUR2", nil) + client.Treatment("user1", "DEMO_MURMUR2", nil) + client.Treatments("user1", []string{"DEMO_MURMUR2", "DEMO_MURMUR"}, nil) + client.Treatment("user2", "DEMO_MURMUR2", nil) + + atomic.AddInt64(&isDestroyCalled, 1) + client.Destroy() + + select { + case <-postChannel: + return + case <-time.After(4 * time.Second): + t.Error("The test couldn't send impressions to check headers") + return + } +} + func TestClientDebug(t *testing.T) { var isDestroyCalled = false var splitsMock, _ = ioutil.ReadFile("../../testdata/splits_mock_2.json") @@ -1987,3 +2101,245 @@ func TestTelemetryRedis(t *testing.T) { deleteDataGenerated(prefixedClient) factory.Destroy() } + +func TestClientNoneRedis(t *testing.T) { + redisConfig := &commonsCfg.RedisConfig{ + Host: "localhost", + Port: 6379, + Password: "", + Prefix: "test-prefix-m", + } + + prefixedClient, _ := redis.NewRedisClient(redisConfig, logging.NewLogger(&logging.LoggerOptions{})) + raw, _ := json.Marshal(*valid) + prefixedClient.Set("SPLITIO.split.valid", raw, 0) + raw, _ = json.Marshal(*noConfig) + prefixedClient.Set("SPLITIO.split.noConfig", raw, 0) + + impTest := &ImpressionListenerTest{} + cfg := conf.Default() + cfg.LabelsEnabled = true + cfg.Advanced.ImpressionListener = impTest + cfg.ImpressionsMode = commonsCfg.ImpressionsModeNone + cfg.OperationMode = conf.RedisConsumer + cfg.Redis = *redisConfig + + factory, _ := NewSplitFactory("test", cfg) + client := factory.Client() + client.BlockUntilReady(2) + + // Calls treatments to generate one valid impression + time.Sleep(300 * time.Millisecond) // Let's wait until first call of recorders have finished + client.Treatment("user1", "valid", nil) + client.Treatment("user2", "valid", nil) + client.Treatment("user3", "valid", nil) + client.Treatment("user1", "valid", nil) + client.Treatment("user2", "valid", nil) + client.Treatment("user3", "valid", nil) + client.Treatment("user3", "noConfig", nil) + client.Treatment("user3", "noConfig", nil) + client.Destroy() + + // Validate unique keys + uniques, _ := prefixedClient.LRange("SPLITIO.uniquekeys", 0, -1) + var uniquesDto dtos.Uniques + _ = json.Unmarshal([]byte(uniques[0]), &uniquesDto) + + if len(uniquesDto.Keys) != 2 { + t.Error("Lenght should be 2") + } + + for _, unique := range uniquesDto.Keys { + if unique.Feature == "valid" && len(unique.Keys) != 3 { + t.Error("Keys should be 3") + } + if unique.Feature == "noConfig" && len(unique.Keys) != 1 { + t.Error("Keys should be 1") + } + } + + // Validate impression counts + impressionscount, _ := prefixedClient.HGetAll("SPLITIO.impressions.count") + + for key, count := range impressionscount { + if strings.HasPrefix(key, "valid::") && count != "6" { + t.Error("Expected: 6. actual: " + count) + } + if strings.HasPrefix(key, "vanoConfiglid::") && count != "2" { + t.Error("Expected: 2. actual: " + count) + } + } + + // Validate that impressions doesn't exist + exist, _ := prefixedClient.Exists("SPLITIO.impressions") + if exist != 0 { + t.Error("SPLITIO.impressions should not exist") + } + + // Clean redis + keys, _ := prefixedClient.Keys("SPLITIO*") + for _, k := range keys { + prefixedClient.Del(k) + } +} + +func TestClientOptimizedRedis(t *testing.T) { + redisConfig := &commonsCfg.RedisConfig{ + Host: "localhost", + Port: 6379, + Password: "", + Prefix: "test-prefix-m", + } + + prefixedClient, _ := redis.NewRedisClient(redisConfig, logging.NewLogger(&logging.LoggerOptions{})) + raw, _ := json.Marshal(*valid) + prefixedClient.Set("SPLITIO.split.valid", raw, 0) + raw, _ = json.Marshal(*noConfig) + prefixedClient.Set("SPLITIO.split.noConfig", raw, 0) + + impTest := &ImpressionListenerTest{} + cfg := conf.Default() + cfg.LabelsEnabled = true + cfg.Advanced.ImpressionListener = impTest + cfg.ImpressionsMode = commonsCfg.ImpressionsModeOptimized + cfg.OperationMode = conf.RedisConsumer + cfg.Redis = *redisConfig + + factory, _ := NewSplitFactory("test", cfg) + client := factory.Client() + client.BlockUntilReady(2) + + // Calls treatments to generate one valid impression + time.Sleep(300 * time.Millisecond) // Let's wait until first call of recorders have finished + client.Treatment("user1", "valid", nil) + client.Treatment("user2", "valid", nil) + client.Treatment("user3", "valid", nil) + client.Treatment("user1", "valid", nil) + client.Treatment("user2", "valid", nil) + client.Treatment("user3", "valid", nil) + client.Treatment("user3", "noConfig", nil) + client.Treatment("user3", "noConfig", nil) + client.Destroy() + + // Validate impressions + impressions, _ := prefixedClient.LRange("SPLITIO.impressions", 0, -1) + + if len(impressions) != 4 { + t.Error("Impression length shold be 4") + } + + for _, imp := range impressions { + var imprObject dtos.ImpressionQueueObject + _ = json.Unmarshal([]byte(imp), &imprObject) + + if imprObject.Impression.KeyName == "user1" && imprObject.Impression.FeatureName == "valid" && imprObject.Impression.Pt != 0 { + t.Error("Pt should be 0.") + } + if imprObject.Impression.KeyName == "user2" && imprObject.Impression.FeatureName == "valid" && imprObject.Impression.Pt != 0 { + t.Error("Pt should be 0.") + } + if imprObject.Impression.KeyName == "user3" && imprObject.Impression.FeatureName == "valid" && imprObject.Impression.Pt != 0 { + t.Error("Pt should be 0.") + } + if imprObject.Impression.KeyName == "user3" && imprObject.Impression.FeatureName == "noConfig" && imprObject.Impression.Pt != 0 { + t.Error("Pt should be 0.") + } + } + + // Validate impression counts + impressionscount, _ := prefixedClient.HGetAll("SPLITIO.impressions.count") + + for key, count := range impressionscount { + if strings.HasPrefix(key, "valid::") && count != "6" { + t.Error("Expected: 6. actual: " + count) + } + if strings.HasPrefix(key, "vanoConfiglid::") && count != "2" { + t.Error("Expected: 2. actual: " + count) + } + } + + // Validate that uniquekeys doesn't exist + exist, _ := prefixedClient.Exists("SPLITIO.uniquekeys") + if exist != 0 { + t.Error("SPLITIO.uniquekeys should not exist") + } + + // Clean redis + keys, _ := prefixedClient.Keys("SPLITIO*") + for _, k := range keys { + prefixedClient.Del(k) + } +} + +func TestClientDebugRedis(t *testing.T) { + redisConfig := &commonsCfg.RedisConfig{ + Host: "localhost", + Port: 6379, + Password: "", + Prefix: "test-prefix-m", + } + + prefixedClient, _ := redis.NewRedisClient(redisConfig, logging.NewLogger(&logging.LoggerOptions{})) + raw, _ := json.Marshal(*valid) + prefixedClient.Set("SPLITIO.split.valid", raw, 0) + raw, _ = json.Marshal(*noConfig) + prefixedClient.Set("SPLITIO.split.noConfig", raw, 0) + + impTest := &ImpressionListenerTest{} + cfg := conf.Default() + cfg.LabelsEnabled = true + cfg.Advanced.ImpressionListener = impTest + cfg.ImpressionsMode = commonsCfg.ImpressionsModeDebug + cfg.OperationMode = conf.RedisConsumer + cfg.Redis = *redisConfig + + factory, _ := NewSplitFactory("test", cfg) + client := factory.Client() + client.BlockUntilReady(2) + + // Calls treatments to generate one valid impression + time.Sleep(300 * time.Millisecond) // Let's wait until first call of recorders have finished + client.Treatment("user1", "valid", nil) + client.Treatment("user2", "valid", nil) + client.Treatment("user3", "valid", nil) + client.Treatment("user1", "valid", nil) + client.Treatment("user2", "valid", nil) + client.Treatment("user3", "valid", nil) + client.Treatment("user3", "noConfig", nil) + client.Treatment("user3", "noConfig", nil) + client.Destroy() + + // Validate impressions + impressions, _ := prefixedClient.LRange("SPLITIO.impressions", 0, -1) + if len(impressions) != 8 { + t.Error("Impression length shold be 4") + } + + // Validate impression counts + impressionscount, _ := prefixedClient.HGetAll("SPLITIO.impressions.count") + + for key, count := range impressionscount { + if strings.HasPrefix(key, "valid::") && count != "6" { + t.Error("Expected: 6. actual: " + count) + } + if strings.HasPrefix(key, "vanoConfiglid::") && count != "2" { + t.Error("Expected: 2. actual: " + count) + } + } + + // Validate that uniquekeys doesn't exist + exist, _ := prefixedClient.Exists("SPLITIO.uniquekeys") + if exist != 0 { + t.Error("SPLITIO.uniquekeys should not exist") + } + exist, _ = prefixedClient.Exists("SPLITIO.impressions.count") + if exist != 0 { + t.Error("SPLITIO.impressions.count should not exist") + } + + // Clean redis + keys, _ := prefixedClient.Keys("SPLITIO*") + for _, k := range keys { + prefixedClient.Del(k) + } +} diff --git a/splitio/client/factory.go b/splitio/client/factory.go index 350c7b93..f85e9781 100644 --- a/splitio/client/factory.go +++ b/splitio/client/factory.go @@ -19,9 +19,11 @@ import ( "github.com/splitio/go-split-commons/v4/dtos" "github.com/splitio/go-split-commons/v4/healthcheck/application" "github.com/splitio/go-split-commons/v4/provisional" + "github.com/splitio/go-split-commons/v4/provisional/strategy" "github.com/splitio/go-split-commons/v4/service/api" "github.com/splitio/go-split-commons/v4/service/local" "github.com/splitio/go-split-commons/v4/storage" + "github.com/splitio/go-split-commons/v4/storage/filter" "github.com/splitio/go-split-commons/v4/storage/inmemory" "github.com/splitio/go-split-commons/v4/storage/inmemory/mutexmap" "github.com/splitio/go-split-commons/v4/storage/inmemory/mutexqueue" @@ -46,6 +48,15 @@ const ( sdkInitializationFailed = -1 ) +const ( + bfExpectedElemenets = 10000000 + bfFalsePositiveProbability = 0.01 + uniqueKeysTaskPeriodInMemory = 900 + uniqueKeysTaskPeriodredis = 300 + impressionsCountPeriodTaskInMemory = 1800 // 30 min + impressionsCountPeriodTaskRedis = 300 +) + type sdkStorages struct { splits storage.SplitStorageConsumer segments storage.SegmentStorageConsumer @@ -54,6 +65,7 @@ type sdkStorages struct { initTelemetry storage.TelemetryConfigProducer runtimeTelemetry storage.TelemetryRuntimeProducer evaluationTelemetry storage.TelemetryEvaluationProducer + impressionsCount storage.ImpressionsCountProducer } // SplitFactory struct is responsible for instantiating and storing instances of client and manager. @@ -72,6 +84,7 @@ type SplitFactory struct { syncManager synchronizer.Manager telemetrySync telemetry.TelemetrySynchronizer // To execute SynchronizeInit impressionManager provisional.ImpressionManager + synchronizer synchronizer.Synchronizer } // Client returns the split client instantiated by the factory @@ -136,6 +149,10 @@ func (f *SplitFactory) initializationInMemory(readyChannel chan int) { } } +func (f *SplitFactory) initializationRedis(syncImp synchronizer.Synchronizer) { + syncImp.StartPeriodicDataRecording() +} + // recordInitTelemetry In charge of recording init stats from redis and memory func (f *SplitFactory) recordInitTelemetry(tags []string, currentFactories map[string]int64) { f.logger.Debug("Sending init telemetry") @@ -156,12 +173,9 @@ func (f *SplitFactory) recordInitTelemetry(tags []string, currentFactories map[s AuthServiceURL: f.cfg.Advanced.AuthServiceURL, StreamingServiceURL: f.cfg.Advanced.StreamingServiceURL, }, - TaskPeriods: config.TaskPeriods(f.cfg.TaskPeriods), - ManagerConfig: config.ManagerConfig{ - OperationMode: f.cfg.OperationMode, - ImpressionsMode: f.cfg.ImpressionsMode, - ListenerEnabled: f.cfg.Advanced.ImpressionListener != nil, - }, + TaskPeriods: config.TaskPeriods(f.cfg.TaskPeriods), + ImpressionsMode: f.cfg.ImpressionsMode, + ListenerEnabled: f.cfg.Advanced.ImpressionListener != nil, }, time.Now().UTC().Sub(f.startTime).Milliseconds(), currentFactories, @@ -254,6 +268,7 @@ func (f *SplitFactory) Destroy() { } if f.cfg.OperationMode == conf.RedisConsumer { + f.synchronizer.StopPeriodicDataRecording() return } @@ -293,36 +308,60 @@ func setupInMemoryFactory( return nil, err } - managerConfig := config.ManagerConfig{ - ImpressionsMode: cfg.ImpressionsMode, - OperationMode: cfg.OperationMode, - ListenerEnabled: cfg.Advanced.ImpressionListener != nil, - } - var dummyHC = &application.Dummy{} splitAPI := api.NewSplitAPI(apikey, advanced, logger, metadata) workers := synchronizer.Workers{ - SplitFetcher: split.NewSplitFetcher(splitsStorage, splitAPI.SplitFetcher, logger, telemetryStorage, dummyHC), - SegmentFetcher: segment.NewSegmentFetcher(splitsStorage, segmentsStorage, splitAPI.SegmentFetcher, logger, telemetryStorage, dummyHC), - EventRecorder: event.NewEventRecorderSingle(eventsStorage, splitAPI.EventRecorder, logger, metadata, telemetryStorage), - ImpressionRecorder: impression.NewRecorderSingle(impressionsStorage, splitAPI.ImpressionRecorder, logger, metadata, managerConfig, telemetryStorage), - TelemetryRecorder: telemetry.NewTelemetrySynchronizer(telemetryStorage, splitAPI.TelemetryRecorder, splitsStorage, segmentsStorage, logger, metadata, telemetryStorage), + SplitFetcher: split.NewSplitFetcher(splitsStorage, splitAPI.SplitFetcher, logger, telemetryStorage, dummyHC), + SegmentFetcher: segment.NewSegmentFetcher(splitsStorage, segmentsStorage, splitAPI.SegmentFetcher, logger, telemetryStorage, dummyHC), + EventRecorder: event.NewEventRecorderSingle(eventsStorage, splitAPI.EventRecorder, logger, metadata, telemetryStorage), + TelemetryRecorder: telemetry.NewTelemetrySynchronizer(telemetryStorage, splitAPI.TelemetryRecorder, splitsStorage, segmentsStorage, logger, metadata, telemetryStorage), } splitTasks := synchronizer.SplitTasks{ - SplitSyncTask: tasks.NewFetchSplitsTask(workers.SplitFetcher, cfg.TaskPeriods.SplitSync, logger), - SegmentSyncTask: tasks.NewFetchSegmentsTask(workers.SegmentFetcher, cfg.TaskPeriods.SegmentSync, advanced.SegmentWorkers, advanced.SegmentQueueSize, logger), - EventSyncTask: tasks.NewRecordEventsTask(workers.EventRecorder, advanced.EventsBulkSize, cfg.TaskPeriods.EventsSync, logger), - ImpressionSyncTask: tasks.NewRecordImpressionsTask(workers.ImpressionRecorder, cfg.TaskPeriods.ImpressionSync, logger, advanced.ImpressionsBulkSize), - TelemetrySyncTask: tasks.NewRecordTelemetryTask(workers.TelemetryRecorder, cfg.TaskPeriods.TelemetrySync, logger), - } - var impressionsCounter *provisional.ImpressionsCounter - if cfg.ImpressionsMode == config.ImpressionsModeOptimized { - impressionsCounter = provisional.NewImpressionsCounter() + SplitSyncTask: tasks.NewFetchSplitsTask(workers.SplitFetcher, cfg.TaskPeriods.SplitSync, logger), + SegmentSyncTask: tasks.NewFetchSegmentsTask(workers.SegmentFetcher, cfg.TaskPeriods.SegmentSync, advanced.SegmentWorkers, advanced.SegmentQueueSize, logger), + EventSyncTask: tasks.NewRecordEventsTask(workers.EventRecorder, advanced.EventsBulkSize, cfg.TaskPeriods.EventsSync, logger), + TelemetrySyncTask: tasks.NewRecordTelemetryTask(workers.TelemetryRecorder, cfg.TaskPeriods.TelemetrySync, logger), + } + + listenerEnabled := cfg.Advanced.ImpressionListener != nil + + var impressionManager provisional.ImpressionManager + switch cfg.ImpressionsMode { + case config.ImpressionsModeNone: + impressionsCounter := strategy.NewImpressionsCounter() + filter := filter.NewBloomFilter(bfExpectedElemenets, bfFalsePositiveProbability) + uniqueKeysTracker := strategy.NewUniqueKeysTracker(filter) + impressionsStrategy := strategy.NewNoneImpl(impressionsCounter, uniqueKeysTracker, listenerEnabled) + impressionManager = provisional.NewImpressionManager(impressionsStrategy) + workers.ImpressionsCountRecorder = impressionscount.NewRecorderSingle(impressionsCounter, splitAPI.ImpressionRecorder, metadata, logger, telemetryStorage) - splitTasks.ImpressionsCountSyncTask = tasks.NewRecordImpressionsCountTask(workers.ImpressionsCountRecorder, logger) + splitTasks.ImpressionsCountSyncTask = tasks.NewRecordImpressionsCountTask(workers.ImpressionsCountRecorder, logger, impressionsCountPeriodTaskInMemory) + splitTasks.UniqueKeysTask = tasks.NewRecordUniqueKeysTask(workers.TelemetryRecorder, uniqueKeysTracker, uniqueKeysTaskPeriodInMemory, logger) + case config.ImpressionsModeDebug: + impressionObserver, err := strategy.NewImpressionObserver(500) + if err != nil { + return nil, err + } + impressionsStrategy := strategy.NewDebugImpl(impressionObserver, listenerEnabled) + impressionManager = provisional.NewImpressionManager(impressionsStrategy) + + workers.ImpressionRecorder = impression.NewRecorderSingle(impressionsStorage, splitAPI.ImpressionRecorder, logger, metadata, cfg.ImpressionsMode, telemetryStorage) + splitTasks.ImpressionSyncTask = tasks.NewRecordImpressionsTask(workers.ImpressionRecorder, cfg.TaskPeriods.ImpressionSync, logger, advanced.ImpressionsBulkSize) + default: + impressionObserver, err := strategy.NewImpressionObserver(500) + if err != nil { + return nil, err + } + impressionsCounter := strategy.NewImpressionsCounter() + impressionsStrategy := strategy.NewOptimizedImpl(impressionObserver, impressionsCounter, telemetryStorage, listenerEnabled) + workers.ImpressionsCountRecorder = impressionscount.NewRecorderSingle(impressionsCounter, splitAPI.ImpressionRecorder, metadata, logger, telemetryStorage) + splitTasks.ImpressionsCountSyncTask = tasks.NewRecordImpressionsCountTask(workers.ImpressionsCountRecorder, logger, impressionsCountPeriodTaskInMemory) + impressionManager = provisional.NewImpressionManager(impressionsStrategy) + + workers.ImpressionRecorder = impression.NewRecorderSingle(impressionsStorage, splitAPI.ImpressionRecorder, logger, metadata, cfg.ImpressionsMode, telemetryStorage) + splitTasks.ImpressionSyncTask = tasks.NewRecordImpressionsTask(workers.ImpressionRecorder, cfg.TaskPeriods.ImpressionSync, logger, advanced.ImpressionsBulkSize) } - impressionManager, _ := provisional.NewImpressionManager(managerConfig, impressionsCounter, telemetryStorage) syncImpl := synchronizer.NewSynchronizer( advanced, @@ -370,9 +409,9 @@ func setupInMemoryFactory( readinessSubscriptors: make(map[int]chan int), syncManager: syncManager, telemetrySync: workers.TelemetryRecorder, + impressionManager: impressionManager, } splitFactory.status.Store(sdkStatusInitializing) - splitFactory.impressionManager = impressionManager setFactory(splitFactory.apikey, splitFactory.logger) go splitFactory.initializationInMemory(readyChannel) @@ -395,8 +434,57 @@ func setupRedisFactory(apikey string, cfg *conf.SplitSdkConfig, logger logging.L events: redis.NewEventsStorage(redisClient, metadata, logger), initTelemetry: telemetryStorage, evaluationTelemetry: telemetryStorage, + impressionsCount: redis.NewImpressionsCountStorage(redisClient, logger), + } + + splitTasks := synchronizer.SplitTasks{} + + listenerEnabled := cfg.Advanced.ImpressionListener != nil + var impressionManager provisional.ImpressionManager + switch cfg.ImpressionsMode { + case config.ImpressionsModeNone: + impressionsCounter := strategy.NewImpressionsCounter() + filter := filter.NewBloomFilter(bfExpectedElemenets, bfFalsePositiveProbability) + uniqueKeysTracker := strategy.NewUniqueKeysTracker(filter) + impressionsStrategy := strategy.NewNoneImpl(impressionsCounter, uniqueKeysTracker, listenerEnabled) + impressionManager = provisional.NewImpressionManager(impressionsStrategy) + + impressionsCountRecorder := impressionscount.NewRecorderRedis(impressionsCounter, storages.impressionsCount, logger) + splitTasks.ImpressionsCountSyncTask = tasks.NewRecordImpressionsCountTask(impressionsCountRecorder, logger, impressionsCountPeriodTaskRedis) + telemetryRecorder := telemetry.NewSynchronizerRedis(telemetryStorage, logger) + splitTasks.UniqueKeysTask = tasks.NewRecordUniqueKeysTask(telemetryRecorder, uniqueKeysTracker, uniqueKeysTaskPeriodredis, logger) + case config.ImpressionsModeDebug: + impressionObserver, err := strategy.NewImpressionObserver(500) + if err != nil { + return nil, err + } + impressionsStrategy := strategy.NewDebugImpl(impressionObserver, listenerEnabled) + impressionManager = provisional.NewImpressionManager(impressionsStrategy) + default: + impressionObserver, err := strategy.NewImpressionObserver(500) + if err != nil { + return nil, err + } + impressionsCounter := strategy.NewImpressionsCounter() + impressionsStrategy := strategy.NewOptimizedImpl(impressionObserver, impressionsCounter, mocks.MockTelemetryStorage{ + RecordSyncLatencyCall: func(resource int, latency time.Duration) {}, + RecordImpressionsStatsCall: func(dataType int, count int64) {}, + }, listenerEnabled) + impressionManager = provisional.NewImpressionManager(impressionsStrategy) + + impressionsCountRecorder := impressionscount.NewRecorderRedis(impressionsCounter, storages.impressionsCount, logger) + splitTasks.ImpressionsCountSyncTask = tasks.NewRecordImpressionsCountTask(impressionsCountRecorder, logger, impressionsCountPeriodTaskRedis) } + syncImpl := synchronizer.NewSynchronizer( + config.AdvancedConfig{}, + splitTasks, + synchronizer.Workers{}, + logger, + nil, + nil, + ) + factory := &SplitFactory{ startTime: time.Now().UTC(), apikey: apikey, @@ -407,22 +495,16 @@ func setupRedisFactory(apikey string, cfg *conf.SplitSdkConfig, logger logging.L storages: storages, readinessSubscriptors: make(map[int]chan int), telemetrySync: telemetry.NewSynchronizerRedis(telemetryStorage, logger), + impressionManager: impressionManager, + synchronizer: syncImpl, } factory.status.Store(sdkStatusInitializing) - impressionManager, _ := provisional.NewImpressionManager(config.ManagerConfig{ - OperationMode: cfg.OperationMode, - ImpressionsMode: cfg.ImpressionsMode, - ListenerEnabled: cfg.Advanced.ImpressionListener != nil, - }, nil, mocks.MockTelemetryStorage{ - RecordSyncLatencyCall: func(resource int, latency time.Duration) {}, - RecordImpressionsStatsCall: func(dataType int, count int64) {}, - }) - if err != nil { - return nil, err - } - factory.impressionManager = impressionManager setFactory(factory.apikey, factory.logger) + + go factory.initializationRedis(syncImpl) + factory.broadcastReadiness(sdkStatusReady, make([]string, 0)) + return factory, nil } @@ -481,15 +563,12 @@ func setupLocalhostFactory( } splitFactory.status.Store(sdkStatusInitializing) - impressionManager, _ := provisional.NewImpressionManager(config.ManagerConfig{ - OperationMode: cfg.OperationMode, - ImpressionsMode: cfg.ImpressionsMode, - ListenerEnabled: cfg.Advanced.ImpressionListener != nil, - }, nil, telemetryStorage) + impressionObserver, err := strategy.NewImpressionObserver(500) if err != nil { return nil, err } - splitFactory.impressionManager = impressionManager + impressionsStrategy := strategy.NewDebugImpl(impressionObserver, cfg.Advanced.ImpressionListener != nil) + splitFactory.impressionManager = provisional.NewImpressionManager(impressionsStrategy) setFactory(splitFactory.apikey, splitFactory.logger) // Call fetching tasks as goroutine @@ -534,3 +613,55 @@ func newFactory(apikey string, cfg conf.SplitSdkConfig, logger logging.LoggerInt return splitFactory, nil } + +func buildImpressionManager( + cfg *conf.SplitSdkConfig, + splitTasks *synchronizer.SplitTasks, + impressionsCountStorage storage.ImpressionsCountProducer, + logger logging.LoggerInterface, + telemetryStorage storage.TelemetryRedisProducer, + uniqueKeysTaskPeriod int, + impressionsCountTaskPeriod int, +) (provisional.ImpressionManager, error) { + listenerEnabled := cfg.Advanced.ImpressionListener != nil + + var impressionManager provisional.ImpressionManager + switch cfg.ImpressionsMode { + case config.ImpressionsModeNone: + impressionsCounter := strategy.NewImpressionsCounter() + filter := filter.NewBloomFilter(bfExpectedElemenets, bfFalsePositiveProbability) + uniqueKeysTracker := strategy.NewUniqueKeysTracker(filter) + impressionsStrategy := strategy.NewNoneImpl(impressionsCounter, uniqueKeysTracker, listenerEnabled) + + impressionsCountRecorder := impressionscount.NewRecorderRedis(impressionsCounter, impressionsCountStorage, logger) + splitTasks.ImpressionsCountSyncTask = tasks.NewRecordImpressionsCountTask(impressionsCountRecorder, logger, impressionsCountTaskPeriod) + telemetryRecorder := telemetry.NewSynchronizerRedis(telemetryStorage, logger) + splitTasks.UniqueKeysTask = tasks.NewRecordUniqueKeysTask(telemetryRecorder, uniqueKeysTracker, uniqueKeysTaskPeriod, logger) + + impressionManager = provisional.NewImpressionManager(impressionsStrategy) + case config.ImpressionsModeDebug: + impressionObserver, err := strategy.NewImpressionObserver(500) + if err != nil { + return nil, err + } + impressionsStrategy := strategy.NewDebugImpl(impressionObserver, listenerEnabled) + + impressionManager = provisional.NewImpressionManager(impressionsStrategy) + default: + impressionObserver, err := strategy.NewImpressionObserver(500) + if err != nil { + return nil, err + } + impressionsCounter := strategy.NewImpressionsCounter() + impressionsStrategy := strategy.NewOptimizedImpl(impressionObserver, impressionsCounter, mocks.MockTelemetryStorage{ + RecordSyncLatencyCall: func(resource int, latency time.Duration) {}, + RecordImpressionsStatsCall: func(dataType int, count int64) {}, + }, listenerEnabled) + impressionsCountRecorder := impressionscount.NewRecorderRedis(impressionsCounter, impressionsCountStorage, logger) + splitTasks.ImpressionsCountSyncTask = tasks.NewRecordImpressionsCountTask(impressionsCountRecorder, logger, impressionsCountTaskPeriod) + + impressionManager = provisional.NewImpressionManager(impressionsStrategy) + } + + return impressionManager, nil +} diff --git a/splitio/client/input_validator_test.go b/splitio/client/input_validator_test.go index 3b5f21f9..b3b1eabd 100644 --- a/splitio/client/input_validator_test.go +++ b/splitio/client/input_validator_test.go @@ -14,6 +14,7 @@ import ( "github.com/splitio/go-split-commons/v4/dtos" "github.com/splitio/go-split-commons/v4/healthcheck/application" "github.com/splitio/go-split-commons/v4/provisional" + "github.com/splitio/go-split-commons/v4/provisional/strategy" "github.com/splitio/go-split-commons/v4/service/api" authMocks "github.com/splitio/go-split-commons/v4/service/mocks" "github.com/splitio/go-split-commons/v4/storage/inmemory/mutexmap" @@ -80,10 +81,12 @@ func getClient() SplitClient { RecordImpressionsStatsCall: func(dataType int, count int64) {}, RecordLatencyCall: func(method string, latency time.Duration) {}, } - impressionManager, _ := provisional.NewImpressionManager(spConf.ManagerConfig{ - ImpressionsMode: spConf.ImpressionsModeDebug, - OperationMode: cfg.OperationMode, - }, provisional.NewImpressionsCounter(), telemetryMockedStorage) + + impressionObserver, _ := strategy.NewImpressionObserver(500) + impressionsCounter := strategy.NewImpressionsCounter() + impressionsStrategy := strategy.NewOptimizedImpl(impressionObserver, impressionsCounter, telemetryMockedStorage, true) + impressionManager := provisional.NewImpressionManager(impressionsStrategy) + factory := &SplitFactory{cfg: cfg, impressionManager: impressionManager, storages: sdkStorages{ runtimeTelemetry: telemetryMockedStorage, diff --git a/splitio/conf/sdkconf.go b/splitio/conf/sdkconf.go index 094d825f..4dde631a 100644 --- a/splitio/conf/sdkconf.go +++ b/splitio/conf/sdkconf.go @@ -195,6 +195,8 @@ func validConfigRates(cfg *SplitSdkConfig) error { return fmt.Errorf("ImpressionSync must be >= %d. Actual is: %d", minImpressionSync, cfg.TaskPeriods.ImpressionSync) } } + case conf.ImpressionsModeNone: + return nil default: fmt.Println(`You passed an invalid impressionsMode, impressionsMode should be one of the following values: 'debug' or 'optimized'. Defaulting to 'optimized' mode.`) cfg.ImpressionsMode = conf.ImpressionsModeOptimized From 8d860acb3d671918d64061a38fb12060dbcc1a79 Mon Sep 17 00:00:00 2001 From: Mauro Antonio Sanz Date: Thu, 30 Jun 2022 15:46:42 +0200 Subject: [PATCH 02/25] removed unnecesary code --- splitio/client/factory.go | 52 --------------------------------------- 1 file changed, 52 deletions(-) diff --git a/splitio/client/factory.go b/splitio/client/factory.go index f85e9781..28e45362 100644 --- a/splitio/client/factory.go +++ b/splitio/client/factory.go @@ -613,55 +613,3 @@ func newFactory(apikey string, cfg conf.SplitSdkConfig, logger logging.LoggerInt return splitFactory, nil } - -func buildImpressionManager( - cfg *conf.SplitSdkConfig, - splitTasks *synchronizer.SplitTasks, - impressionsCountStorage storage.ImpressionsCountProducer, - logger logging.LoggerInterface, - telemetryStorage storage.TelemetryRedisProducer, - uniqueKeysTaskPeriod int, - impressionsCountTaskPeriod int, -) (provisional.ImpressionManager, error) { - listenerEnabled := cfg.Advanced.ImpressionListener != nil - - var impressionManager provisional.ImpressionManager - switch cfg.ImpressionsMode { - case config.ImpressionsModeNone: - impressionsCounter := strategy.NewImpressionsCounter() - filter := filter.NewBloomFilter(bfExpectedElemenets, bfFalsePositiveProbability) - uniqueKeysTracker := strategy.NewUniqueKeysTracker(filter) - impressionsStrategy := strategy.NewNoneImpl(impressionsCounter, uniqueKeysTracker, listenerEnabled) - - impressionsCountRecorder := impressionscount.NewRecorderRedis(impressionsCounter, impressionsCountStorage, logger) - splitTasks.ImpressionsCountSyncTask = tasks.NewRecordImpressionsCountTask(impressionsCountRecorder, logger, impressionsCountTaskPeriod) - telemetryRecorder := telemetry.NewSynchronizerRedis(telemetryStorage, logger) - splitTasks.UniqueKeysTask = tasks.NewRecordUniqueKeysTask(telemetryRecorder, uniqueKeysTracker, uniqueKeysTaskPeriod, logger) - - impressionManager = provisional.NewImpressionManager(impressionsStrategy) - case config.ImpressionsModeDebug: - impressionObserver, err := strategy.NewImpressionObserver(500) - if err != nil { - return nil, err - } - impressionsStrategy := strategy.NewDebugImpl(impressionObserver, listenerEnabled) - - impressionManager = provisional.NewImpressionManager(impressionsStrategy) - default: - impressionObserver, err := strategy.NewImpressionObserver(500) - if err != nil { - return nil, err - } - impressionsCounter := strategy.NewImpressionsCounter() - impressionsStrategy := strategy.NewOptimizedImpl(impressionObserver, impressionsCounter, mocks.MockTelemetryStorage{ - RecordSyncLatencyCall: func(resource int, latency time.Duration) {}, - RecordImpressionsStatsCall: func(dataType int, count int64) {}, - }, listenerEnabled) - impressionsCountRecorder := impressionscount.NewRecorderRedis(impressionsCounter, impressionsCountStorage, logger) - splitTasks.ImpressionsCountSyncTask = tasks.NewRecordImpressionsCountTask(impressionsCountRecorder, logger, impressionsCountTaskPeriod) - - impressionManager = provisional.NewImpressionManager(impressionsStrategy) - } - - return impressionManager, nil -} From 2843c9266c2c36df883ef16428f49cdacf81e638 Mon Sep 17 00:00:00 2001 From: Mauro Antonio Sanz Date: Mon, 4 Jul 2022 15:04:04 +0200 Subject: [PATCH 03/25] removed replaces in go.mod --- go.mod | 8 ++------ go.sum | 4 ++++ splitio/client/factory.go | 12 +++++++----- 3 files changed, 13 insertions(+), 11 deletions(-) diff --git a/go.mod b/go.mod index 9f371d58..cf586dce 100644 --- a/go.mod +++ b/go.mod @@ -2,11 +2,7 @@ module github.com/splitio/go-client/v6 go 1.13 -replace github.com/splitio/go-split-commons/v4 => /Users/mauroantoniosanz/go/src/github.com/splitio/go-split-commons - -replace github.com/splitio/go-toolkit/v5 => /Users/mauroantoniosanz/go/src/github.com/splitio/go-toolkit - require ( - github.com/splitio/go-split-commons/v4 v4.1.0 - github.com/splitio/go-toolkit/v5 v5.2.0 + github.com/splitio/go-split-commons/v4 v4.1.1-0.20220630192502-e5ac0357f8f6 + github.com/splitio/go-toolkit/v5 v5.2.1-0.20220624162919-a134050c80a9 ) diff --git a/go.sum b/go.sum index d7a83fd6..7efff77c 100644 --- a/go.sum +++ b/go.sum @@ -53,6 +53,10 @@ github.com/onsi/gomega v1.18.1/go.mod h1:0q+aL8jAiMXy9hbwj2mr5GziHiwhAIQpFmmtT5h github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/spaolacci/murmur3 v1.1.0 h1:7c1g84S4BPRrfL5Xrdp6fOJ206sU9y293DDHaoy0bLI= github.com/spaolacci/murmur3 v1.1.0/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA= +github.com/splitio/go-split-commons/v4 v4.1.1-0.20220630192502-e5ac0357f8f6 h1:Wa+SqkqvD1dqwhrZraqHwvzfRCvmM9rdbV/LD8ExvtU= +github.com/splitio/go-split-commons/v4 v4.1.1-0.20220630192502-e5ac0357f8f6/go.mod h1:bV5ipOguPyV0u9mfMTOG/pPlCnU9688QFZqZx48uyKQ= +github.com/splitio/go-toolkit/v5 v5.2.1-0.20220624162919-a134050c80a9 h1:hprZ/6LWQtYdWSsWo4GgZJIpvfFNzlB1WNbbpqx9Oyc= +github.com/splitio/go-toolkit/v5 v5.2.1-0.20220624162919-a134050c80a9/go.mod h1:SYi/svhhtEgdMSb5tNcDcMjOSUH/7XVkvjp5dPL+nBE= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA= github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= diff --git a/splitio/client/factory.go b/splitio/client/factory.go index 28e45362..eea7957f 100644 --- a/splitio/client/factory.go +++ b/splitio/client/factory.go @@ -51,10 +51,10 @@ const ( const ( bfExpectedElemenets = 10000000 bfFalsePositiveProbability = 0.01 - uniqueKeysTaskPeriodInMemory = 900 - uniqueKeysTaskPeriodredis = 300 + uniqueKeysPeriodTaskInMemory = 900 // 15 min + uniqueKeysPeriodTaskRedis = 300 // 5 min impressionsCountPeriodTaskInMemory = 1800 // 30 min - impressionsCountPeriodTaskRedis = 300 + impressionsCountPeriodTaskRedis = 300 // 5 min ) type sdkStorages struct { @@ -337,7 +337,8 @@ func setupInMemoryFactory( workers.ImpressionsCountRecorder = impressionscount.NewRecorderSingle(impressionsCounter, splitAPI.ImpressionRecorder, metadata, logger, telemetryStorage) splitTasks.ImpressionsCountSyncTask = tasks.NewRecordImpressionsCountTask(workers.ImpressionsCountRecorder, logger, impressionsCountPeriodTaskInMemory) - splitTasks.UniqueKeysTask = tasks.NewRecordUniqueKeysTask(workers.TelemetryRecorder, uniqueKeysTracker, uniqueKeysTaskPeriodInMemory, logger) + splitTasks.UniqueKeysTask = tasks.NewRecordUniqueKeysTask(workers.TelemetryRecorder, uniqueKeysTracker, uniqueKeysPeriodTaskInMemory, logger) + splitTasks.CleanFilterTask = tasks.NewCleanFilterTask(filter, logger) case config.ImpressionsModeDebug: impressionObserver, err := strategy.NewImpressionObserver(500) if err != nil { @@ -452,7 +453,8 @@ func setupRedisFactory(apikey string, cfg *conf.SplitSdkConfig, logger logging.L impressionsCountRecorder := impressionscount.NewRecorderRedis(impressionsCounter, storages.impressionsCount, logger) splitTasks.ImpressionsCountSyncTask = tasks.NewRecordImpressionsCountTask(impressionsCountRecorder, logger, impressionsCountPeriodTaskRedis) telemetryRecorder := telemetry.NewSynchronizerRedis(telemetryStorage, logger) - splitTasks.UniqueKeysTask = tasks.NewRecordUniqueKeysTask(telemetryRecorder, uniqueKeysTracker, uniqueKeysTaskPeriodredis, logger) + splitTasks.UniqueKeysTask = tasks.NewRecordUniqueKeysTask(telemetryRecorder, uniqueKeysTracker, uniqueKeysPeriodTaskRedis, logger) + splitTasks.CleanFilterTask = tasks.NewCleanFilterTask(filter, logger) case config.ImpressionsModeDebug: impressionObserver, err := strategy.NewImpressionObserver(500) if err != nil { From 24aff58d362226d086dc2fa96fa5238f60a2f0b1 Mon Sep 17 00:00:00 2001 From: Mauro Antonio Sanz Date: Mon, 4 Jul 2022 19:09:20 +0200 Subject: [PATCH 04/25] pr feedback --- go.mod | 2 +- go.sum | 4 +- splitio/client/factory.go | 211 ++++++++++++++++++++------------------ 3 files changed, 112 insertions(+), 105 deletions(-) diff --git a/go.mod b/go.mod index cf586dce..3cbaef97 100644 --- a/go.mod +++ b/go.mod @@ -3,6 +3,6 @@ module github.com/splitio/go-client/v6 go 1.13 require ( - github.com/splitio/go-split-commons/v4 v4.1.1-0.20220630192502-e5ac0357f8f6 + github.com/splitio/go-split-commons/v4 v4.1.1-0.20220704164729-e48c7c6ccf9a github.com/splitio/go-toolkit/v5 v5.2.1-0.20220624162919-a134050c80a9 ) diff --git a/go.sum b/go.sum index 7efff77c..86d841f9 100644 --- a/go.sum +++ b/go.sum @@ -53,8 +53,8 @@ github.com/onsi/gomega v1.18.1/go.mod h1:0q+aL8jAiMXy9hbwj2mr5GziHiwhAIQpFmmtT5h github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/spaolacci/murmur3 v1.1.0 h1:7c1g84S4BPRrfL5Xrdp6fOJ206sU9y293DDHaoy0bLI= github.com/spaolacci/murmur3 v1.1.0/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA= -github.com/splitio/go-split-commons/v4 v4.1.1-0.20220630192502-e5ac0357f8f6 h1:Wa+SqkqvD1dqwhrZraqHwvzfRCvmM9rdbV/LD8ExvtU= -github.com/splitio/go-split-commons/v4 v4.1.1-0.20220630192502-e5ac0357f8f6/go.mod h1:bV5ipOguPyV0u9mfMTOG/pPlCnU9688QFZqZx48uyKQ= +github.com/splitio/go-split-commons/v4 v4.1.1-0.20220704164729-e48c7c6ccf9a h1:/eQ9KWzz/W4bIRo32AaoVGIOTycVHCdXU7kBuP3v/hE= +github.com/splitio/go-split-commons/v4 v4.1.1-0.20220704164729-e48c7c6ccf9a/go.mod h1:bV5ipOguPyV0u9mfMTOG/pPlCnU9688QFZqZx48uyKQ= github.com/splitio/go-toolkit/v5 v5.2.1-0.20220624162919-a134050c80a9 h1:hprZ/6LWQtYdWSsWo4GgZJIpvfFNzlB1WNbbpqx9Oyc= github.com/splitio/go-toolkit/v5 v5.2.1-0.20220624162919-a134050c80a9/go.mod h1:SYi/svhhtEgdMSb5tNcDcMjOSUH/7XVkvjp5dPL+nBE= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= diff --git a/splitio/client/factory.go b/splitio/client/factory.go index eea7957f..8ee9f581 100644 --- a/splitio/client/factory.go +++ b/splitio/client/factory.go @@ -60,6 +60,7 @@ const ( type sdkStorages struct { splits storage.SplitStorageConsumer segments storage.SegmentStorageConsumer + impressionsConsumer storage.ImpressionStorageConsumer impressions storage.ImpressionStorageProducer events storage.EventStorageProducer initTelemetry storage.TelemetryConfigProducer @@ -84,7 +85,6 @@ type SplitFactory struct { syncManager synchronizer.Manager telemetrySync telemetry.TelemetrySynchronizer // To execute SynchronizeInit impressionManager provisional.ImpressionManager - synchronizer synchronizer.Synchronizer } // Client returns the split client instantiated by the factory @@ -149,8 +149,9 @@ func (f *SplitFactory) initializationInMemory(readyChannel chan int) { } } -func (f *SplitFactory) initializationRedis(syncImp synchronizer.Synchronizer) { - syncImp.StartPeriodicDataRecording() +func (f *SplitFactory) initializationRedis() { + go f.syncManager.Start() + f.broadcastReadiness(sdkStatusReady, make([]string, 0)) } // recordInitTelemetry In charge of recording init stats from redis and memory @@ -267,11 +268,6 @@ func (f *SplitFactory) Destroy() { f.storages.runtimeTelemetry.RecordSessionLength(int64(time.Since(f.startTime) * time.Millisecond)) } - if f.cfg.OperationMode == conf.RedisConsumer { - f.synchronizer.StopPeriodicDataRecording() - return - } - f.syncManager.Stop() } @@ -324,44 +320,20 @@ func setupInMemoryFactory( TelemetrySyncTask: tasks.NewRecordTelemetryTask(workers.TelemetryRecorder, cfg.TaskPeriods.TelemetrySync, logger), } - listenerEnabled := cfg.Advanced.ImpressionListener != nil - - var impressionManager provisional.ImpressionManager - switch cfg.ImpressionsMode { - case config.ImpressionsModeNone: - impressionsCounter := strategy.NewImpressionsCounter() - filter := filter.NewBloomFilter(bfExpectedElemenets, bfFalsePositiveProbability) - uniqueKeysTracker := strategy.NewUniqueKeysTracker(filter) - impressionsStrategy := strategy.NewNoneImpl(impressionsCounter, uniqueKeysTracker, listenerEnabled) - impressionManager = provisional.NewImpressionManager(impressionsStrategy) - - workers.ImpressionsCountRecorder = impressionscount.NewRecorderSingle(impressionsCounter, splitAPI.ImpressionRecorder, metadata, logger, telemetryStorage) - splitTasks.ImpressionsCountSyncTask = tasks.NewRecordImpressionsCountTask(workers.ImpressionsCountRecorder, logger, impressionsCountPeriodTaskInMemory) - splitTasks.UniqueKeysTask = tasks.NewRecordUniqueKeysTask(workers.TelemetryRecorder, uniqueKeysTracker, uniqueKeysPeriodTaskInMemory, logger) - splitTasks.CleanFilterTask = tasks.NewCleanFilterTask(filter, logger) - case config.ImpressionsModeDebug: - impressionObserver, err := strategy.NewImpressionObserver(500) - if err != nil { - return nil, err - } - impressionsStrategy := strategy.NewDebugImpl(impressionObserver, listenerEnabled) - impressionManager = provisional.NewImpressionManager(impressionsStrategy) - - workers.ImpressionRecorder = impression.NewRecorderSingle(impressionsStorage, splitAPI.ImpressionRecorder, logger, metadata, cfg.ImpressionsMode, telemetryStorage) - splitTasks.ImpressionSyncTask = tasks.NewRecordImpressionsTask(workers.ImpressionRecorder, cfg.TaskPeriods.ImpressionSync, logger, advanced.ImpressionsBulkSize) - default: - impressionObserver, err := strategy.NewImpressionObserver(500) - if err != nil { - return nil, err - } - impressionsCounter := strategy.NewImpressionsCounter() - impressionsStrategy := strategy.NewOptimizedImpl(impressionObserver, impressionsCounter, telemetryStorage, listenerEnabled) - workers.ImpressionsCountRecorder = impressionscount.NewRecorderSingle(impressionsCounter, splitAPI.ImpressionRecorder, metadata, logger, telemetryStorage) - splitTasks.ImpressionsCountSyncTask = tasks.NewRecordImpressionsCountTask(workers.ImpressionsCountRecorder, logger, impressionsCountPeriodTaskInMemory) - impressionManager = provisional.NewImpressionManager(impressionsStrategy) + storages := sdkStorages{ + splits: splitsStorage, + events: eventsStorage, + impressionsConsumer: impressionsStorage, + impressions: impressionsStorage, + segments: segmentsStorage, + initTelemetry: telemetryStorage, + evaluationTelemetry: telemetryStorage, + runtimeTelemetry: telemetryStorage, + } - workers.ImpressionRecorder = impression.NewRecorderSingle(impressionsStorage, splitAPI.ImpressionRecorder, logger, metadata, cfg.ImpressionsMode, telemetryStorage) - splitTasks.ImpressionSyncTask = tasks.NewRecordImpressionsTask(workers.ImpressionRecorder, cfg.TaskPeriods.ImpressionSync, logger, advanced.ImpressionsBulkSize) + impressionManager, err := buildImpressionManager(cfg, advanced, logger, true, &splitTasks, &workers, storages, metadata, splitAPI) + if err != nil { + return nil, err } syncImpl := synchronizer.NewSynchronizer( @@ -392,21 +364,13 @@ func setupInMemoryFactory( } splitFactory := SplitFactory{ - startTime: time.Now().UTC(), - apikey: apikey, - cfg: cfg, - metadata: metadata, - logger: logger, - operationMode: conf.InMemoryStandAlone, - storages: sdkStorages{ - splits: splitsStorage, - events: eventsStorage, - impressions: impressionsStorage, - segments: segmentsStorage, - initTelemetry: telemetryStorage, - evaluationTelemetry: telemetryStorage, - runtimeTelemetry: telemetryStorage, - }, + startTime: time.Now().UTC(), + apikey: apikey, + cfg: cfg, + metadata: metadata, + logger: logger, + operationMode: conf.InMemoryStandAlone, + storages: storages, readinessSubscriptors: make(map[int]chan int), syncManager: syncManager, telemetrySync: workers.TelemetryRecorder, @@ -436,57 +400,33 @@ func setupRedisFactory(apikey string, cfg *conf.SplitSdkConfig, logger logging.L initTelemetry: telemetryStorage, evaluationTelemetry: telemetryStorage, impressionsCount: redis.NewImpressionsCountStorage(redisClient, logger), + runtimeTelemetry: mocks.MockTelemetryStorage{ + RecordSyncLatencyCall: func(resource int, latency time.Duration) {}, + RecordImpressionsStatsCall: func(dataType int, count int64) {}, + RecordSessionLengthCall: func(session int64) {}, + }, } splitTasks := synchronizer.SplitTasks{} + workers := synchronizer.Workers{} + advanced := config.AdvancedConfig{} - listenerEnabled := cfg.Advanced.ImpressionListener != nil - var impressionManager provisional.ImpressionManager - switch cfg.ImpressionsMode { - case config.ImpressionsModeNone: - impressionsCounter := strategy.NewImpressionsCounter() - filter := filter.NewBloomFilter(bfExpectedElemenets, bfFalsePositiveProbability) - uniqueKeysTracker := strategy.NewUniqueKeysTracker(filter) - impressionsStrategy := strategy.NewNoneImpl(impressionsCounter, uniqueKeysTracker, listenerEnabled) - impressionManager = provisional.NewImpressionManager(impressionsStrategy) - - impressionsCountRecorder := impressionscount.NewRecorderRedis(impressionsCounter, storages.impressionsCount, logger) - splitTasks.ImpressionsCountSyncTask = tasks.NewRecordImpressionsCountTask(impressionsCountRecorder, logger, impressionsCountPeriodTaskRedis) - telemetryRecorder := telemetry.NewSynchronizerRedis(telemetryStorage, logger) - splitTasks.UniqueKeysTask = tasks.NewRecordUniqueKeysTask(telemetryRecorder, uniqueKeysTracker, uniqueKeysPeriodTaskRedis, logger) - splitTasks.CleanFilterTask = tasks.NewCleanFilterTask(filter, logger) - case config.ImpressionsModeDebug: - impressionObserver, err := strategy.NewImpressionObserver(500) - if err != nil { - return nil, err - } - impressionsStrategy := strategy.NewDebugImpl(impressionObserver, listenerEnabled) - impressionManager = provisional.NewImpressionManager(impressionsStrategy) - default: - impressionObserver, err := strategy.NewImpressionObserver(500) - if err != nil { - return nil, err - } - impressionsCounter := strategy.NewImpressionsCounter() - impressionsStrategy := strategy.NewOptimizedImpl(impressionObserver, impressionsCounter, mocks.MockTelemetryStorage{ - RecordSyncLatencyCall: func(resource int, latency time.Duration) {}, - RecordImpressionsStatsCall: func(dataType int, count int64) {}, - }, listenerEnabled) - impressionManager = provisional.NewImpressionManager(impressionsStrategy) - - impressionsCountRecorder := impressionscount.NewRecorderRedis(impressionsCounter, storages.impressionsCount, logger) - splitTasks.ImpressionsCountSyncTask = tasks.NewRecordImpressionsCountTask(impressionsCountRecorder, logger, impressionsCountPeriodTaskRedis) + impressionManager, err := buildImpressionManager(cfg, advanced, logger, false, &splitTasks, &workers, storages, metadata, nil) + if err != nil { + return nil, err } syncImpl := synchronizer.NewSynchronizer( - config.AdvancedConfig{}, + advanced, splitTasks, - synchronizer.Workers{}, + workers, logger, nil, nil, ) + syncManager := synchronizer.NewSynchronizerManagerRedis(syncImpl) + factory := &SplitFactory{ startTime: time.Now().UTC(), apikey: apikey, @@ -498,14 +438,12 @@ func setupRedisFactory(apikey string, cfg *conf.SplitSdkConfig, logger logging.L readinessSubscriptors: make(map[int]chan int), telemetrySync: telemetry.NewSynchronizerRedis(telemetryStorage, logger), impressionManager: impressionManager, - synchronizer: syncImpl, + syncManager: syncManager, } factory.status.Store(sdkStatusInitializing) setFactory(factory.apikey, factory.logger) - go factory.initializationRedis(syncImpl) - - factory.broadcastReadiness(sdkStatusReady, make([]string, 0)) + factory.initializationRedis() return factory, nil } @@ -615,3 +553,72 @@ func newFactory(apikey string, cfg conf.SplitSdkConfig, logger logging.LoggerInt return splitFactory, nil } + +func buildImpressionManager( + cfg *conf.SplitSdkConfig, + advanced config.AdvancedConfig, + logger logging.LoggerInterface, + inMemory bool, + splitTasks *synchronizer.SplitTasks, + workers *synchronizer.Workers, + storages sdkStorages, + metadata dtos.Metadata, + splitAPI *api.SplitAPI, +) (provisional.ImpressionManager, error) { + listenerEnabled := cfg.Advanced.ImpressionListener != nil + switch cfg.ImpressionsMode { + case config.ImpressionsModeNone: + impressionsCounter := strategy.NewImpressionsCounter() + filter := filter.NewBloomFilter(bfExpectedElemenets, bfFalsePositiveProbability) + uniqueKeysTracker := strategy.NewUniqueKeysTracker(filter) + + if inMemory { + workers.ImpressionsCountRecorder = impressionscount.NewRecorderSingle(impressionsCounter, splitAPI.ImpressionRecorder, metadata, logger, storages.runtimeTelemetry) + splitTasks.ImpressionsCountSyncTask = tasks.NewRecordImpressionsCountTask(workers.ImpressionsCountRecorder, logger, impressionsCountPeriodTaskInMemory) + splitTasks.UniqueKeysTask = tasks.NewRecordUniqueKeysTask(workers.TelemetryRecorder, uniqueKeysTracker, uniqueKeysPeriodTaskInMemory, logger) + } else { + telemetryRecorder := telemetry.NewSynchronizerRedis(storages.initTelemetry, logger) + impressionsCountRecorder := impressionscount.NewRecorderRedis(impressionsCounter, storages.impressionsCount, logger) + splitTasks.ImpressionsCountSyncTask = tasks.NewRecordImpressionsCountTask(impressionsCountRecorder, logger, impressionsCountPeriodTaskRedis) + splitTasks.UniqueKeysTask = tasks.NewRecordUniqueKeysTask(telemetryRecorder, uniqueKeysTracker, uniqueKeysPeriodTaskRedis, logger) + } + + splitTasks.CleanFilterTask = tasks.NewCleanFilterTask(filter, logger) + impressionsStrategy := strategy.NewNoneImpl(impressionsCounter, uniqueKeysTracker, listenerEnabled) + + return provisional.NewImpressionManager(impressionsStrategy), nil + case config.ImpressionsModeDebug: + if inMemory { + workers.ImpressionRecorder = impression.NewRecorderSingle(storages.impressionsConsumer, splitAPI.ImpressionRecorder, logger, metadata, cfg.ImpressionsMode, storages.runtimeTelemetry) + splitTasks.ImpressionSyncTask = tasks.NewRecordImpressionsTask(workers.ImpressionRecorder, cfg.TaskPeriods.ImpressionSync, logger, advanced.ImpressionsBulkSize) + } + + impressionObserver, err := strategy.NewImpressionObserver(500) + if err != nil { + return nil, err + } + impressionsStrategy := strategy.NewDebugImpl(impressionObserver, listenerEnabled) + + return provisional.NewImpressionManager(impressionsStrategy), nil + default: + impressionsCounter := strategy.NewImpressionsCounter() + + if inMemory { + workers.ImpressionsCountRecorder = impressionscount.NewRecorderSingle(impressionsCounter, splitAPI.ImpressionRecorder, metadata, logger, storages.runtimeTelemetry) + workers.ImpressionRecorder = impression.NewRecorderSingle(storages.impressionsConsumer, splitAPI.ImpressionRecorder, logger, metadata, cfg.ImpressionsMode, storages.runtimeTelemetry) + splitTasks.ImpressionsCountSyncTask = tasks.NewRecordImpressionsCountTask(workers.ImpressionsCountRecorder, logger, impressionsCountPeriodTaskInMemory) + splitTasks.ImpressionSyncTask = tasks.NewRecordImpressionsTask(workers.ImpressionRecorder, cfg.TaskPeriods.ImpressionSync, logger, advanced.ImpressionsBulkSize) + } else { + impressionsCountRecorder := impressionscount.NewRecorderRedis(impressionsCounter, storages.impressionsCount, logger) + splitTasks.ImpressionsCountSyncTask = tasks.NewRecordImpressionsCountTask(impressionsCountRecorder, logger, impressionsCountPeriodTaskRedis) + } + + impressionObserver, err := strategy.NewImpressionObserver(500) + if err != nil { + return nil, err + } + impressionsStrategy := strategy.NewOptimizedImpl(impressionObserver, impressionsCounter, storages.runtimeTelemetry, listenerEnabled) + + return provisional.NewImpressionManager(impressionsStrategy), nil + } +} From c96f7558ad03ff014f681baf668d26af784d04d4 Mon Sep 17 00:00:00 2001 From: Mauro Antonio Sanz Date: Thu, 7 Jul 2022 13:06:27 +0200 Subject: [PATCH 05/25] updated commons dependency --- go.mod | 4 ++-- go.sum | 8 ++++---- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/go.mod b/go.mod index 3cbaef97..f69d3444 100644 --- a/go.mod +++ b/go.mod @@ -3,6 +3,6 @@ module github.com/splitio/go-client/v6 go 1.13 require ( - github.com/splitio/go-split-commons/v4 v4.1.1-0.20220704164729-e48c7c6ccf9a - github.com/splitio/go-toolkit/v5 v5.2.1-0.20220624162919-a134050c80a9 + github.com/splitio/go-split-commons/v4 v4.1.1-0.20220707110232-6f5ac5b26dd4 + github.com/splitio/go-toolkit/v5 v5.2.1-0.20220707101452-b7d19caecc1a ) diff --git a/go.sum b/go.sum index 86d841f9..e4a77d4c 100644 --- a/go.sum +++ b/go.sum @@ -53,10 +53,10 @@ github.com/onsi/gomega v1.18.1/go.mod h1:0q+aL8jAiMXy9hbwj2mr5GziHiwhAIQpFmmtT5h github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/spaolacci/murmur3 v1.1.0 h1:7c1g84S4BPRrfL5Xrdp6fOJ206sU9y293DDHaoy0bLI= github.com/spaolacci/murmur3 v1.1.0/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA= -github.com/splitio/go-split-commons/v4 v4.1.1-0.20220704164729-e48c7c6ccf9a h1:/eQ9KWzz/W4bIRo32AaoVGIOTycVHCdXU7kBuP3v/hE= -github.com/splitio/go-split-commons/v4 v4.1.1-0.20220704164729-e48c7c6ccf9a/go.mod h1:bV5ipOguPyV0u9mfMTOG/pPlCnU9688QFZqZx48uyKQ= -github.com/splitio/go-toolkit/v5 v5.2.1-0.20220624162919-a134050c80a9 h1:hprZ/6LWQtYdWSsWo4GgZJIpvfFNzlB1WNbbpqx9Oyc= -github.com/splitio/go-toolkit/v5 v5.2.1-0.20220624162919-a134050c80a9/go.mod h1:SYi/svhhtEgdMSb5tNcDcMjOSUH/7XVkvjp5dPL+nBE= +github.com/splitio/go-split-commons/v4 v4.1.1-0.20220707110232-6f5ac5b26dd4 h1:WYtaVji+P0YzoXQ3g021UnoxgtiNHo5ic0QYbE9tIcU= +github.com/splitio/go-split-commons/v4 v4.1.1-0.20220707110232-6f5ac5b26dd4/go.mod h1:jzoZQLQWxeK3FpfPJ7xbA7TEYisnQQlpjT9EB37JSQs= +github.com/splitio/go-toolkit/v5 v5.2.1-0.20220707101452-b7d19caecc1a h1:oHkI6Gh3LLTbCh9uQl4F6cw63o4Z0wG8R3GR84NEtvg= +github.com/splitio/go-toolkit/v5 v5.2.1-0.20220707101452-b7d19caecc1a/go.mod h1:SYi/svhhtEgdMSb5tNcDcMjOSUH/7XVkvjp5dPL+nBE= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA= github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= From 0ae2ca669af673386b95adbe639ba2be408024fa Mon Sep 17 00:00:00 2001 From: Mauro Antonio Sanz Date: Fri, 15 Jul 2022 18:07:04 +0200 Subject: [PATCH 06/25] fixed build --- go.mod | 2 +- go.sum | 4 ++-- splitio/client/factory.go | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/go.mod b/go.mod index f69d3444..5f42786b 100644 --- a/go.mod +++ b/go.mod @@ -3,6 +3,6 @@ module github.com/splitio/go-client/v6 go 1.13 require ( - github.com/splitio/go-split-commons/v4 v4.1.1-0.20220707110232-6f5ac5b26dd4 + github.com/splitio/go-split-commons/v4 v4.1.1-0.20220715132317-eb0f154ffdec github.com/splitio/go-toolkit/v5 v5.2.1-0.20220707101452-b7d19caecc1a ) diff --git a/go.sum b/go.sum index e4a77d4c..66d6ac7f 100644 --- a/go.sum +++ b/go.sum @@ -53,8 +53,8 @@ github.com/onsi/gomega v1.18.1/go.mod h1:0q+aL8jAiMXy9hbwj2mr5GziHiwhAIQpFmmtT5h github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/spaolacci/murmur3 v1.1.0 h1:7c1g84S4BPRrfL5Xrdp6fOJ206sU9y293DDHaoy0bLI= github.com/spaolacci/murmur3 v1.1.0/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA= -github.com/splitio/go-split-commons/v4 v4.1.1-0.20220707110232-6f5ac5b26dd4 h1:WYtaVji+P0YzoXQ3g021UnoxgtiNHo5ic0QYbE9tIcU= -github.com/splitio/go-split-commons/v4 v4.1.1-0.20220707110232-6f5ac5b26dd4/go.mod h1:jzoZQLQWxeK3FpfPJ7xbA7TEYisnQQlpjT9EB37JSQs= +github.com/splitio/go-split-commons/v4 v4.1.1-0.20220715132317-eb0f154ffdec h1:6Gr95BbLqSkz80W2Vy5O+isiUWK665/ht+S6Ek/DEU8= +github.com/splitio/go-split-commons/v4 v4.1.1-0.20220715132317-eb0f154ffdec/go.mod h1:jzoZQLQWxeK3FpfPJ7xbA7TEYisnQQlpjT9EB37JSQs= github.com/splitio/go-toolkit/v5 v5.2.1-0.20220707101452-b7d19caecc1a h1:oHkI6Gh3LLTbCh9uQl4F6cw63o4Z0wG8R3GR84NEtvg= github.com/splitio/go-toolkit/v5 v5.2.1-0.20220707101452-b7d19caecc1a/go.mod h1:SYi/svhhtEgdMSb5tNcDcMjOSUH/7XVkvjp5dPL+nBE= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= diff --git a/splitio/client/factory.go b/splitio/client/factory.go index 8ee9f581..9c2c5f1b 100644 --- a/splitio/client/factory.go +++ b/splitio/client/factory.go @@ -425,7 +425,7 @@ func setupRedisFactory(apikey string, cfg *conf.SplitSdkConfig, logger logging.L nil, ) - syncManager := synchronizer.NewSynchronizerManagerRedis(syncImpl) + syncManager := synchronizer.NewSynchronizerManagerRedis(syncImpl, logger) factory := &SplitFactory{ startTime: time.Now().UTC(), From 0e3a6f0d89dfbf63d9f90230f9f319d468752d24 Mon Sep 17 00:00:00 2001 From: Mauro Antonio Sanz Date: Fri, 15 Jul 2022 20:12:11 +0200 Subject: [PATCH 07/25] polishing --- go.mod | 2 +- go.sum | 2 ++ splitio/client/factory.go | 12 +++++++----- 3 files changed, 10 insertions(+), 6 deletions(-) diff --git a/go.mod b/go.mod index 5f42786b..23582733 100644 --- a/go.mod +++ b/go.mod @@ -3,6 +3,6 @@ module github.com/splitio/go-client/v6 go 1.13 require ( - github.com/splitio/go-split-commons/v4 v4.1.1-0.20220715132317-eb0f154ffdec + github.com/splitio/go-split-commons/v4 v4.1.1-0.20220715163621-15225bf79464 github.com/splitio/go-toolkit/v5 v5.2.1-0.20220707101452-b7d19caecc1a ) diff --git a/go.sum b/go.sum index 66d6ac7f..346da82e 100644 --- a/go.sum +++ b/go.sum @@ -55,6 +55,8 @@ github.com/spaolacci/murmur3 v1.1.0 h1:7c1g84S4BPRrfL5Xrdp6fOJ206sU9y293DDHaoy0b github.com/spaolacci/murmur3 v1.1.0/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA= github.com/splitio/go-split-commons/v4 v4.1.1-0.20220715132317-eb0f154ffdec h1:6Gr95BbLqSkz80W2Vy5O+isiUWK665/ht+S6Ek/DEU8= github.com/splitio/go-split-commons/v4 v4.1.1-0.20220715132317-eb0f154ffdec/go.mod h1:jzoZQLQWxeK3FpfPJ7xbA7TEYisnQQlpjT9EB37JSQs= +github.com/splitio/go-split-commons/v4 v4.1.1-0.20220715163621-15225bf79464 h1:QptMhUDCXi0a+OfYU7zj/yCvnJcVgVyPPt6fT0bwNGs= +github.com/splitio/go-split-commons/v4 v4.1.1-0.20220715163621-15225bf79464/go.mod h1:jzoZQLQWxeK3FpfPJ7xbA7TEYisnQQlpjT9EB37JSQs= github.com/splitio/go-toolkit/v5 v5.2.1-0.20220707101452-b7d19caecc1a h1:oHkI6Gh3LLTbCh9uQl4F6cw63o4Z0wG8R3GR84NEtvg= github.com/splitio/go-toolkit/v5 v5.2.1-0.20220707101452-b7d19caecc1a/go.mod h1:SYi/svhhtEgdMSb5tNcDcMjOSUH/7XVkvjp5dPL+nBE= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= diff --git a/splitio/client/factory.go b/splitio/client/factory.go index 9c2c5f1b..bcd1ef51 100644 --- a/splitio/client/factory.go +++ b/splitio/client/factory.go @@ -51,10 +51,12 @@ const ( const ( bfExpectedElemenets = 10000000 bfFalsePositiveProbability = 0.01 - uniqueKeysPeriodTaskInMemory = 900 // 15 min - uniqueKeysPeriodTaskRedis = 300 // 5 min - impressionsCountPeriodTaskInMemory = 1800 // 30 min - impressionsCountPeriodTaskRedis = 300 // 5 min + bfCleaningPeriod = 86400 // 24 hours + uniqueKeysPeriodTaskInMemory = 900 // 15 min + uniqueKeysPeriodTaskRedis = 300 // 5 min + impressionsCountPeriodTaskInMemory = 1800 // 30 min + impressionsCountPeriodTaskRedis = 300 // 5 min + ) type sdkStorages struct { @@ -583,7 +585,7 @@ func buildImpressionManager( splitTasks.UniqueKeysTask = tasks.NewRecordUniqueKeysTask(telemetryRecorder, uniqueKeysTracker, uniqueKeysPeriodTaskRedis, logger) } - splitTasks.CleanFilterTask = tasks.NewCleanFilterTask(filter, logger) + splitTasks.CleanFilterTask = tasks.NewCleanFilterTask(filter, logger, bfCleaningPeriod) impressionsStrategy := strategy.NewNoneImpl(impressionsCounter, uniqueKeysTracker, listenerEnabled) return provisional.NewImpressionManager(impressionsStrategy), nil From c2ad67931c53a8934521d36208abf481d0be5eb7 Mon Sep 17 00:00:00 2001 From: Mauro Antonio Sanz Date: Mon, 8 Aug 2022 17:48:18 +0200 Subject: [PATCH 08/25] updated commons commit version --- go.mod | 2 +- go.sum | 6 ++---- splitio/client/client_test.go | 8 ++++---- 3 files changed, 7 insertions(+), 9 deletions(-) diff --git a/go.mod b/go.mod index 23582733..bf84c380 100644 --- a/go.mod +++ b/go.mod @@ -3,6 +3,6 @@ module github.com/splitio/go-client/v6 go 1.13 require ( - github.com/splitio/go-split-commons/v4 v4.1.1-0.20220715163621-15225bf79464 + github.com/splitio/go-split-commons/v4 v4.1.1-0.20220804175011-c2496bdacf4a github.com/splitio/go-toolkit/v5 v5.2.1-0.20220707101452-b7d19caecc1a ) diff --git a/go.sum b/go.sum index 346da82e..5e6502eb 100644 --- a/go.sum +++ b/go.sum @@ -53,10 +53,8 @@ github.com/onsi/gomega v1.18.1/go.mod h1:0q+aL8jAiMXy9hbwj2mr5GziHiwhAIQpFmmtT5h github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/spaolacci/murmur3 v1.1.0 h1:7c1g84S4BPRrfL5Xrdp6fOJ206sU9y293DDHaoy0bLI= github.com/spaolacci/murmur3 v1.1.0/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA= -github.com/splitio/go-split-commons/v4 v4.1.1-0.20220715132317-eb0f154ffdec h1:6Gr95BbLqSkz80W2Vy5O+isiUWK665/ht+S6Ek/DEU8= -github.com/splitio/go-split-commons/v4 v4.1.1-0.20220715132317-eb0f154ffdec/go.mod h1:jzoZQLQWxeK3FpfPJ7xbA7TEYisnQQlpjT9EB37JSQs= -github.com/splitio/go-split-commons/v4 v4.1.1-0.20220715163621-15225bf79464 h1:QptMhUDCXi0a+OfYU7zj/yCvnJcVgVyPPt6fT0bwNGs= -github.com/splitio/go-split-commons/v4 v4.1.1-0.20220715163621-15225bf79464/go.mod h1:jzoZQLQWxeK3FpfPJ7xbA7TEYisnQQlpjT9EB37JSQs= +github.com/splitio/go-split-commons/v4 v4.1.1-0.20220804175011-c2496bdacf4a h1:19lJ8nGjMyCOPYiBf6Z2+DXVFgX7imN31fb7WB+ql3g= +github.com/splitio/go-split-commons/v4 v4.1.1-0.20220804175011-c2496bdacf4a/go.mod h1:jzoZQLQWxeK3FpfPJ7xbA7TEYisnQQlpjT9EB37JSQs= github.com/splitio/go-toolkit/v5 v5.2.1-0.20220707101452-b7d19caecc1a h1:oHkI6Gh3LLTbCh9uQl4F6cw63o4Z0wG8R3GR84NEtvg= github.com/splitio/go-toolkit/v5 v5.2.1-0.20220707101452-b7d19caecc1a/go.mod h1:SYi/svhhtEgdMSb5tNcDcMjOSUH/7XVkvjp5dPL+nBE= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= diff --git a/splitio/client/client_test.go b/splitio/client/client_test.go index e8e21a65..aab92f55 100644 --- a/splitio/client/client_test.go +++ b/splitio/client/client_test.go @@ -2142,14 +2142,14 @@ func TestClientNoneRedis(t *testing.T) { // Validate unique keys uniques, _ := prefixedClient.LRange("SPLITIO.uniquekeys", 0, -1) - var uniquesDto dtos.Uniques + var uniquesDto []dtos.Key _ = json.Unmarshal([]byte(uniques[0]), &uniquesDto) - if len(uniquesDto.Keys) != 2 { - t.Error("Lenght should be 2") + if len(uniquesDto) != 2 { + t.Errorf("Lenght should be 2, Actual %d", len(uniquesDto)) } - for _, unique := range uniquesDto.Keys { + for _, unique := range uniquesDto { if unique.Feature == "valid" && len(unique.Keys) != 3 { t.Error("Keys should be 3") } From 1de9ffdcf3dec5db19d8afeee9a4ca978c11d627 Mon Sep 17 00:00:00 2001 From: Mauro Antonio Sanz Date: Mon, 8 Aug 2022 17:53:28 +0200 Subject: [PATCH 09/25] 6.2.0-rc1 --- splitio/version.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/splitio/version.go b/splitio/version.go index 93b77340..0f1e1325 100644 --- a/splitio/version.go +++ b/splitio/version.go @@ -1,4 +1,4 @@ package splitio // Version contains a string with the split sdk version -const Version = "6.1.6" +const Version = "6.2.0-rc1" From e49987b9bb32f1c8bc4f17859fe5bea3381343a4 Mon Sep 17 00:00:00 2001 From: Mauro Antonio Sanz Date: Wed, 10 Aug 2022 17:38:33 +0200 Subject: [PATCH 10/25] updated factory --- go.mod | 2 +- go.sum | 2 ++ splitio/client/client_test.go | 56 +++++++++++++++++++---------------- splitio/client/factory.go | 43 +++++++++++++++++++-------- 4 files changed, 64 insertions(+), 39 deletions(-) diff --git a/go.mod b/go.mod index bf84c380..e1f3a6bc 100644 --- a/go.mod +++ b/go.mod @@ -3,6 +3,6 @@ module github.com/splitio/go-client/v6 go 1.13 require ( - github.com/splitio/go-split-commons/v4 v4.1.1-0.20220804175011-c2496bdacf4a + github.com/splitio/go-split-commons/v4 v4.1.1-0.20220809175331-6ebadff5afe5 github.com/splitio/go-toolkit/v5 v5.2.1-0.20220707101452-b7d19caecc1a ) diff --git a/go.sum b/go.sum index 5e6502eb..09a25b07 100644 --- a/go.sum +++ b/go.sum @@ -55,6 +55,8 @@ github.com/spaolacci/murmur3 v1.1.0 h1:7c1g84S4BPRrfL5Xrdp6fOJ206sU9y293DDHaoy0b github.com/spaolacci/murmur3 v1.1.0/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA= github.com/splitio/go-split-commons/v4 v4.1.1-0.20220804175011-c2496bdacf4a h1:19lJ8nGjMyCOPYiBf6Z2+DXVFgX7imN31fb7WB+ql3g= github.com/splitio/go-split-commons/v4 v4.1.1-0.20220804175011-c2496bdacf4a/go.mod h1:jzoZQLQWxeK3FpfPJ7xbA7TEYisnQQlpjT9EB37JSQs= +github.com/splitio/go-split-commons/v4 v4.1.1-0.20220809175331-6ebadff5afe5 h1:KnRLqnE61qG7DfQ0Idd/L4s1VQxT9Uued8azTuBBOi4= +github.com/splitio/go-split-commons/v4 v4.1.1-0.20220809175331-6ebadff5afe5/go.mod h1:jzoZQLQWxeK3FpfPJ7xbA7TEYisnQQlpjT9EB37JSQs= github.com/splitio/go-toolkit/v5 v5.2.1-0.20220707101452-b7d19caecc1a h1:oHkI6Gh3LLTbCh9uQl4F6cw63o4Z0wG8R3GR84NEtvg= github.com/splitio/go-toolkit/v5 v5.2.1-0.20220707101452-b7d19caecc1a/go.mod h1:SYi/svhhtEgdMSb5tNcDcMjOSUH/7XVkvjp5dPL+nBE= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= diff --git a/splitio/client/client_test.go b/splitio/client/client_test.go index aab92f55..2b573362 100644 --- a/splitio/client/client_test.go +++ b/splitio/client/client_test.go @@ -1310,9 +1310,9 @@ func TestLocalhostModeYAML(t *testing.T) { expectedTreatmentAndConfig(resultTreatmentsWithConfig["other_feature"], "control", "", t) } -func getRedisConfWithIP(IPAddressesEnabled bool) *predis.PrefixedRedisClient { +func getRedisConfWithIP(IPAddressesEnabled bool) (*predis.PrefixedRedisClient, *SplitClient) { // Create prefixed client for adding Split - prefixedClient, _ := redis.NewRedisClient(&commonsCfg.RedisConfig{ + prefixedClient, err := redis.NewRedisClient(&commonsCfg.RedisConfig{ Host: "localhost", Port: 6379, Database: 1, @@ -1322,7 +1322,7 @@ func getRedisConfWithIP(IPAddressesEnabled bool) *predis.PrefixedRedisClient { raw, err := json.Marshal(*valid) if err != nil { - return nil + return nil, nil } prefixedClient.Set("SPLITIO.split.valid", raw, 0) prefixedClient.Set("SPLITIO.splits.till", 1494593336752, 0) @@ -1351,7 +1351,7 @@ func getRedisConfWithIP(IPAddressesEnabled bool) *predis.PrefixedRedisClient { client.Treatment("user1", "valid", nil) client.Track("user1", "my-traffic", "my-event", nil, nil) - return prefixedClient + return prefixedClient, client } func deleteDataGenerated(prefixedClient *predis.PrefixedRedisClient) { @@ -1361,12 +1361,25 @@ func deleteDataGenerated(prefixedClient *predis.PrefixedRedisClient) { } func TestRedisClientWithIPDisabled(t *testing.T) { - prefixedClient := getRedisConfWithIP(false) + prefixedClient, splitClient := getRedisConfWithIP(false) + + // Grabs created event + resEvent, _ := prefixedClient.LRange("SPLITIO.events", 0, 1) + event := make(map[string]map[string]interface{}) + json.Unmarshal([]byte(resEvent[0]), &event) + metadata := event["m"] + // Checks if metadata was created with "NA" values + if metadata["i"] != "NA" || metadata["n"] != "NA" { + t.Error("Instance Name and Machine IP should have 'NA' values") + } + + splitClient.Destroy() + // Grabs created impression resImpression, _ := prefixedClient.LRange("SPLITIO.impressions", 0, 1) impression := make(map[string]map[string]interface{}) json.Unmarshal([]byte(resImpression[0]), &impression) - metadata := impression["m"] + metadata = impression["m"] // Checks if metadata was created with "NA" values if metadata["i"] != "NA" || metadata["n"] != "NA" { t.Error("Instance Name and Machine IP should have 'NA' values") @@ -1376,26 +1389,29 @@ func TestRedisClientWithIPDisabled(t *testing.T) { t.Error("InstanceName should be 'NA") } + deleteDataGenerated(prefixedClient) +} + +func TestRedisClientWithIPEnabled(t *testing.T) { + prefixedClient, splitClient := getRedisConfWithIP(true) + // Grabs created event resEvent, _ := prefixedClient.LRange("SPLITIO.events", 0, 1) event := make(map[string]map[string]interface{}) json.Unmarshal([]byte(resEvent[0]), &event) - metadata = event["m"] + metadata := event["m"] // Checks if metadata was created with "NA" values - if metadata["i"] != "NA" || metadata["n"] != "NA" { - t.Error("Instance Name and Machine IP should have 'NA' values") + if metadata["i"] == "NA" || metadata["n"] == "NA" { + t.Error("Instance Name and Machine IP should not have 'NA' values") } - deleteDataGenerated(prefixedClient) -} + splitClient.Destroy() -func TestRedisClientWithIPEnabled(t *testing.T) { - prefixedClient := getRedisConfWithIP(true) // Grabs created impression resImpression, _ := prefixedClient.LRange("SPLITIO.impressions", 0, 1) impression := make(map[string]map[string]interface{}) json.Unmarshal([]byte(resImpression[0]), &impression) - metadata := impression["m"] + metadata = impression["m"] // Checks if metadata was created with "NA" values if metadata["i"] == "NA" || metadata["n"] == "NA" { t.Error("Instance Name and Machine IP should not have 'NA' values") @@ -1405,16 +1421,6 @@ func TestRedisClientWithIPEnabled(t *testing.T) { t.Error("InstanceName should not be 'NA") } - // Grabs created event - resEvent, _ := prefixedClient.LRange("SPLITIO.events", 0, 1) - event := make(map[string]map[string]interface{}) - json.Unmarshal([]byte(resEvent[0]), &event) - metadata = event["m"] - // Checks if metadata was created with "NA" values - if metadata["i"] == "NA" || metadata["n"] == "NA" { - t.Error("Instance Name and Machine IP should not have 'NA' values") - } - deleteDataGenerated(prefixedClient) } @@ -2312,7 +2318,7 @@ func TestClientDebugRedis(t *testing.T) { // Validate impressions impressions, _ := prefixedClient.LRange("SPLITIO.impressions", 0, -1) if len(impressions) != 8 { - t.Error("Impression length shold be 4") + t.Errorf("Impression length should be 8. Actual %d", len(impressions)) } // Validate impression counts diff --git a/splitio/client/factory.go b/splitio/client/factory.go index bcd1ef51..6114994a 100644 --- a/splitio/client/factory.go +++ b/splitio/client/factory.go @@ -56,7 +56,8 @@ const ( uniqueKeysPeriodTaskRedis = 300 // 5 min impressionsCountPeriodTaskInMemory = 1800 // 30 min impressionsCountPeriodTaskRedis = 300 // 5 min - + impressionsPeriodTaskRedis = 300 // 5 min + impressionsBulkSizeRedis = 100 ) type sdkStorages struct { @@ -269,8 +270,9 @@ func (f *SplitFactory) Destroy() { if f.storages.runtimeTelemetry != nil { f.storages.runtimeTelemetry.RecordSessionLength(int64(time.Since(f.startTime) * time.Millisecond)) } - + fmt.Println("#### STOP") f.syncManager.Stop() + fmt.Println("#### AFET STOP") } // setupLogger sets up the logger according to the parameters submitted by the sdk user @@ -333,7 +335,7 @@ func setupInMemoryFactory( runtimeTelemetry: telemetryStorage, } - impressionManager, err := buildImpressionManager(cfg, advanced, logger, true, &splitTasks, &workers, storages, metadata, splitAPI) + impressionManager, err := buildImpressionManager(cfg, advanced, logger, true, &splitTasks, &workers, storages, metadata, splitAPI, nil) if err != nil { return nil, err } @@ -386,7 +388,12 @@ func setupInMemoryFactory( return &splitFactory, nil } -func setupRedisFactory(apikey string, cfg *conf.SplitSdkConfig, logger logging.LoggerInterface, metadata dtos.Metadata) (*SplitFactory, error) { +func setupRedisFactory( + apikey string, + cfg *conf.SplitSdkConfig, + logger logging.LoggerInterface, + metadata dtos.Metadata, +) (*SplitFactory, error) { redisClient, err := redis.NewRedisClient(&cfg.Redis, logger) if err != nil { logger.Error("Failed to instantiate redis client.") @@ -394,26 +401,30 @@ func setupRedisFactory(apikey string, cfg *conf.SplitSdkConfig, logger logging.L } telemetryStorage := redis.NewTelemetryStorage(redisClient, logger, metadata) + runtimeTelemetry := mocks.MockTelemetryStorage{ + RecordSyncLatencyCall: func(resource int, latency time.Duration) {}, + RecordImpressionsStatsCall: func(dataType int, count int64) {}, + RecordSessionLengthCall: func(session int64) {}, + } + inMememoryFullQueue := make(chan string, 2) // Size 2: So that it's able to accept one event from each resource simultaneously. + impressionStorage := mutexqueue.NewMQImpressionsStorage(100, inMememoryFullQueue, logger, runtimeTelemetry) // TODO: update params storages := sdkStorages{ splits: redis.NewSplitStorage(redisClient, logger), segments: redis.NewSegmentStorage(redisClient, logger), - impressions: redis.NewImpressionStorage(redisClient, metadata, logger), + impressionsConsumer: impressionStorage, + impressions: impressionStorage, events: redis.NewEventsStorage(redisClient, metadata, logger), initTelemetry: telemetryStorage, evaluationTelemetry: telemetryStorage, impressionsCount: redis.NewImpressionsCountStorage(redisClient, logger), - runtimeTelemetry: mocks.MockTelemetryStorage{ - RecordSyncLatencyCall: func(resource int, latency time.Duration) {}, - RecordImpressionsStatsCall: func(dataType int, count int64) {}, - RecordSessionLengthCall: func(session int64) {}, - }, + runtimeTelemetry: runtimeTelemetry, } splitTasks := synchronizer.SplitTasks{} workers := synchronizer.Workers{} advanced := config.AdvancedConfig{} - impressionManager, err := buildImpressionManager(cfg, advanced, logger, false, &splitTasks, &workers, storages, metadata, nil) + impressionManager, err := buildImpressionManager(cfg, advanced, logger, false, &splitTasks, &workers, storages, metadata, nil, redis.NewImpressionStorage(redisClient, metadata, logger)) if err != nil { return nil, err } @@ -566,6 +577,7 @@ func buildImpressionManager( storages sdkStorages, metadata dtos.Metadata, splitAPI *api.SplitAPI, + impressionRedisStorage storage.ImpressionStorageProducer, ) (provisional.ImpressionManager, error) { listenerEnabled := cfg.Advanced.ImpressionListener != nil switch cfg.ImpressionsMode { @@ -593,6 +605,9 @@ func buildImpressionManager( if inMemory { workers.ImpressionRecorder = impression.NewRecorderSingle(storages.impressionsConsumer, splitAPI.ImpressionRecorder, logger, metadata, cfg.ImpressionsMode, storages.runtimeTelemetry) splitTasks.ImpressionSyncTask = tasks.NewRecordImpressionsTask(workers.ImpressionRecorder, cfg.TaskPeriods.ImpressionSync, logger, advanced.ImpressionsBulkSize) + } else { + workers.ImpressionRecorder = impression.NewRecorderRedis(storages.impressionsConsumer, impressionRedisStorage, logger) + splitTasks.ImpressionSyncTask = tasks.NewRecordImpressionsTask(workers.ImpressionRecorder, impressionsPeriodTaskRedis, logger, impressionsBulkSizeRedis) } impressionObserver, err := strategy.NewImpressionObserver(500) @@ -611,8 +626,10 @@ func buildImpressionManager( splitTasks.ImpressionsCountSyncTask = tasks.NewRecordImpressionsCountTask(workers.ImpressionsCountRecorder, logger, impressionsCountPeriodTaskInMemory) splitTasks.ImpressionSyncTask = tasks.NewRecordImpressionsTask(workers.ImpressionRecorder, cfg.TaskPeriods.ImpressionSync, logger, advanced.ImpressionsBulkSize) } else { - impressionsCountRecorder := impressionscount.NewRecorderRedis(impressionsCounter, storages.impressionsCount, logger) - splitTasks.ImpressionsCountSyncTask = tasks.NewRecordImpressionsCountTask(impressionsCountRecorder, logger, impressionsCountPeriodTaskRedis) + workers.ImpressionsCountRecorder = impressionscount.NewRecorderRedis(impressionsCounter, storages.impressionsCount, logger) + workers.ImpressionRecorder = impression.NewRecorderRedis(storages.impressionsConsumer, impressionRedisStorage, logger) + splitTasks.ImpressionsCountSyncTask = tasks.NewRecordImpressionsCountTask(workers.ImpressionsCountRecorder, logger, impressionsCountPeriodTaskRedis) + splitTasks.ImpressionSyncTask = tasks.NewRecordImpressionsTask(workers.ImpressionRecorder, impressionsPeriodTaskRedis, logger, impressionsBulkSizeRedis) } impressionObserver, err := strategy.NewImpressionObserver(500) From 1e96d010bdd11dd84037033a06ac47bdab9a8d91 Mon Sep 17 00:00:00 2001 From: Mauro Antonio Sanz Date: Wed, 10 Aug 2022 17:59:29 +0200 Subject: [PATCH 11/25] removing logs --- splitio/client/factory.go | 2 -- 1 file changed, 2 deletions(-) diff --git a/splitio/client/factory.go b/splitio/client/factory.go index 6114994a..6857c126 100644 --- a/splitio/client/factory.go +++ b/splitio/client/factory.go @@ -270,9 +270,7 @@ func (f *SplitFactory) Destroy() { if f.storages.runtimeTelemetry != nil { f.storages.runtimeTelemetry.RecordSessionLength(int64(time.Since(f.startTime) * time.Millisecond)) } - fmt.Println("#### STOP") f.syncManager.Stop() - fmt.Println("#### AFET STOP") } // setupLogger sets up the logger according to the parameters submitted by the sdk user From 1a46f0d7fce5aa2a730ec8550dc5054d78ccc18f Mon Sep 17 00:00:00 2001 From: Mauro Antonio Sanz Date: Wed, 10 Aug 2022 18:33:20 +0200 Subject: [PATCH 12/25] pr feedback --- splitio/client/factory.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/splitio/client/factory.go b/splitio/client/factory.go index 6857c126..35042692 100644 --- a/splitio/client/factory.go +++ b/splitio/client/factory.go @@ -404,8 +404,8 @@ func setupRedisFactory( RecordImpressionsStatsCall: func(dataType int, count int64) {}, RecordSessionLengthCall: func(session int64) {}, } - inMememoryFullQueue := make(chan string, 2) // Size 2: So that it's able to accept one event from each resource simultaneously. - impressionStorage := mutexqueue.NewMQImpressionsStorage(100, inMememoryFullQueue, logger, runtimeTelemetry) // TODO: update params + inMememoryFullQueue := make(chan string, 2) // Size 2: So that it's able to accept one event from each resource simultaneously. + impressionStorage := mutexqueue.NewMQImpressionsStorage(cfg.Advanced.ImpressionsQueueSize, inMememoryFullQueue, logger, runtimeTelemetry) storages := sdkStorages{ splits: redis.NewSplitStorage(redisClient, logger), segments: redis.NewSegmentStorage(redisClient, logger), @@ -432,7 +432,7 @@ func setupRedisFactory( splitTasks, workers, logger, - nil, + inMememoryFullQueue, nil, ) From 871d8167929ddbb5a8f6c6757f584639848b4402 Mon Sep 17 00:00:00 2001 From: Mauro Antonio Sanz Date: Thu, 11 Aug 2022 16:16:41 +0200 Subject: [PATCH 13/25] pr feedback --- splitio/client/factory.go | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/splitio/client/factory.go b/splitio/client/factory.go index 35042692..e3a7de12 100644 --- a/splitio/client/factory.go +++ b/splitio/client/factory.go @@ -386,12 +386,7 @@ func setupInMemoryFactory( return &splitFactory, nil } -func setupRedisFactory( - apikey string, - cfg *conf.SplitSdkConfig, - logger logging.LoggerInterface, - metadata dtos.Metadata, -) (*SplitFactory, error) { +func setupRedisFactory(apikey string, cfg *conf.SplitSdkConfig, logger logging.LoggerInterface, metadata dtos.Metadata) (*SplitFactory, error) { redisClient, err := redis.NewRedisClient(&cfg.Redis, logger) if err != nil { logger.Error("Failed to instantiate redis client.") From c841e57ef4b2157010711e9c04552f27880617ec Mon Sep 17 00:00:00 2001 From: Matias Melograno Date: Wed, 5 Oct 2022 18:08:22 -0300 Subject: [PATCH 14/25] preparing release --- CHANGES.txt | 8 +++++++- go.mod | 2 +- go.sum | 4 ++-- splitio/version.go | 2 +- 4 files changed, 11 insertions(+), 5 deletions(-) diff --git a/CHANGES.txt b/CHANGES.txt index aee83d31..2e62341a 100644 --- a/CHANGES.txt +++ b/CHANGES.txt @@ -1,3 +1,6 @@ +6.2.0 (Oct 6, 2022) +- Added a new impressions mode for the SDK called NONE , to be used in factory when there is no desire to capture impressions on an SDK factory to feed Split's analytics engine. Running NONE mode, the SDK will only capture unique keys evaluated for a particular feature flag instead of full blown impressions. + 6.1.8 (Sep 9, 2022) - Updated BooleanMatcher logging: demoted the log message to "warning". @@ -37,7 +40,10 @@ - BREAKING CHANGE: Migrated to go modules (dep & bare-bones go-dep no longer supported). 5.3.0 (Oct 6, 2020) -- Added local impressions deduping (enabled by default). +- Added impressions dedupe logic to avoid sending duplicated impressions: + - Added `OPTIMIZED` and `DEBUG` modes in order to enabling/disabling how impressions are going to be sent into Split servers, + - `OPTIMIZED`: will send unique impressions in a timeframe in order to reduce how many times impressions are posted to Split. + - `DEBUG`: will send every impression generated to Split. 5.2.2 (Sep 10, 2020) - Fixed possible issue with SSE client using 100% cpu. diff --git a/go.mod b/go.mod index 51c87439..d98b89d0 100644 --- a/go.mod +++ b/go.mod @@ -3,6 +3,6 @@ module github.com/splitio/go-client/v6 go 1.13 require ( - github.com/splitio/go-split-commons/v4 v4.1.4-0.20221003143756-56a8c75521fb + github.com/splitio/go-split-commons/v4 v4.2.0 github.com/splitio/go-toolkit/v5 v5.2.2 ) diff --git a/go.sum b/go.sum index 14118ef0..f84eea70 100644 --- a/go.sum +++ b/go.sum @@ -51,8 +51,8 @@ github.com/onsi/gomega v1.17.0/go.mod h1:HnhC7FXeEQY45zxNK3PPoIUhzk/80Xly9PcubAl github.com/onsi/gomega v1.18.1 h1:M1GfJqGRrBrrGGsbxzV5dqM2U2ApXefZCQpkukxYRLE= github.com/onsi/gomega v1.18.1/go.mod h1:0q+aL8jAiMXy9hbwj2mr5GziHiwhAIQpFmmtT5hitRs= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= -github.com/splitio/go-split-commons/v4 v4.1.4-0.20221003143756-56a8c75521fb h1:GCxm5ORPV1yPtwd+F9rHmNTZ7TxrBlyKPwZkR06m7HI= -github.com/splitio/go-split-commons/v4 v4.1.4-0.20221003143756-56a8c75521fb/go.mod h1:mzanM00PV8t1FL6IHc2UXepIH2z79d49ArZ2LoJHGrY= +github.com/splitio/go-split-commons/v4 v4.2.0 h1:p49DFvyvsiHnVFkuNNouypzemY9TkvTNG0WBfihDNpo= +github.com/splitio/go-split-commons/v4 v4.2.0/go.mod h1:mzanM00PV8t1FL6IHc2UXepIH2z79d49ArZ2LoJHGrY= github.com/splitio/go-toolkit/v5 v5.2.2 h1:VHSJoIH9tsRt2cCzGKN4WG3BoGCr0tCPZIl8APtJ4bw= github.com/splitio/go-toolkit/v5 v5.2.2/go.mod h1:SYi/svhhtEgdMSb5tNcDcMjOSUH/7XVkvjp5dPL+nBE= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= diff --git a/splitio/version.go b/splitio/version.go index 358b176f..208b9b78 100644 --- a/splitio/version.go +++ b/splitio/version.go @@ -1,4 +1,4 @@ package splitio // Version contains a string with the split sdk version -const Version = "6.2.0-rc2" +const Version = "6.2.0-rc3" From 9ced60a6f14d900945b26940b7bc4bc09889a23f Mon Sep 17 00:00:00 2001 From: Matias Melograno Date: Thu, 6 Oct 2022 16:22:51 -0300 Subject: [PATCH 15/25] updated tests --- splitio/client/client_test.go | 22 ++++++++++++++-------- splitio/version.go | 2 +- 2 files changed, 15 insertions(+), 9 deletions(-) diff --git a/splitio/client/client_test.go b/splitio/client/client_test.go index 2b573362..97ba5e61 100644 --- a/splitio/client/client_test.go +++ b/splitio/client/client_test.go @@ -1312,7 +1312,7 @@ func TestLocalhostModeYAML(t *testing.T) { func getRedisConfWithIP(IPAddressesEnabled bool) (*predis.PrefixedRedisClient, *SplitClient) { // Create prefixed client for adding Split - prefixedClient, err := redis.NewRedisClient(&commonsCfg.RedisConfig{ + prefixedClient, _ := redis.NewRedisClient(&commonsCfg.RedisConfig{ Host: "localhost", Port: 6379, Database: 1, @@ -1431,6 +1431,8 @@ func getInMemoryClientWithIP(IPAddressesEnabled bool, ts *httptest.Server) Split cfg.IPAddressesEnabled = IPAddressesEnabled cfg.Advanced.EventsURL = ts.URL cfg.Advanced.SdkURL = ts.URL + cfg.Advanced.TelemetryServiceURL = ts.URL + cfg.Advanced.AuthServiceURL = ts.URL cfg.Advanced.ImpressionListener = &ImpressionListenerTest{} cfg.TaskPeriods.ImpressionSync = 1 cfg.TaskPeriods.EventsSync = 1 @@ -1625,7 +1627,7 @@ func TestClientOptimized(t *testing.T) { } switch v["f"] { case "DEMO_MURMUR2": - if v["rc"].(float64) != 3 { + if v["rc"].(float64) != 2 { t.Error("Wrong rc") } if int64(v["m"].(float64)) != util.TruncateTimeFrame(time.Now().UTC().UnixNano()) { @@ -1654,6 +1656,8 @@ func TestClientOptimized(t *testing.T) { cfg.LabelsEnabled = true cfg.Advanced.EventsURL = ts.URL cfg.Advanced.SdkURL = ts.URL + cfg.Advanced.TelemetryServiceURL = ts.URL + cfg.Advanced.AuthServiceURL = ts.URL cfg.Advanced.ImpressionListener = impTest factory, _ := NewSplitFactory("test", cfg) @@ -1854,6 +1858,8 @@ func TestClientDebug(t *testing.T) { cfg.LabelsEnabled = true cfg.Advanced.EventsURL = ts.URL cfg.Advanced.SdkURL = ts.URL + cfg.Advanced.TelemetryServiceURL = ts.URL + cfg.Advanced.AuthServiceURL = ts.URL cfg.Advanced.ImpressionListener = impTest cfg.ImpressionsMode = "Debug" @@ -2171,7 +2177,7 @@ func TestClientNoneRedis(t *testing.T) { if strings.HasPrefix(key, "valid::") && count != "6" { t.Error("Expected: 6. actual: " + count) } - if strings.HasPrefix(key, "vanoConfiglid::") && count != "2" { + if strings.HasPrefix(key, "noConfig::") && count != "2" { t.Error("Expected: 2. actual: " + count) } } @@ -2256,11 +2262,11 @@ func TestClientOptimizedRedis(t *testing.T) { impressionscount, _ := prefixedClient.HGetAll("SPLITIO.impressions.count") for key, count := range impressionscount { - if strings.HasPrefix(key, "valid::") && count != "6" { - t.Error("Expected: 6. actual: " + count) + if strings.HasPrefix(key, "valid::") && count != "3" { + t.Error("Expected: 3. actual: " + count) } - if strings.HasPrefix(key, "vanoConfiglid::") && count != "2" { - t.Error("Expected: 2. actual: " + count) + if strings.HasPrefix(key, "noConfig::") && count != "1" { + t.Error("Expected: 1. actual: " + count) } } @@ -2328,7 +2334,7 @@ func TestClientDebugRedis(t *testing.T) { if strings.HasPrefix(key, "valid::") && count != "6" { t.Error("Expected: 6. actual: " + count) } - if strings.HasPrefix(key, "vanoConfiglid::") && count != "2" { + if strings.HasPrefix(key, "noConfig::") && count != "2" { t.Error("Expected: 2. actual: " + count) } } diff --git a/splitio/version.go b/splitio/version.go index 208b9b78..14001f49 100644 --- a/splitio/version.go +++ b/splitio/version.go @@ -1,4 +1,4 @@ package splitio // Version contains a string with the split sdk version -const Version = "6.2.0-rc3" +const Version = "6.2.0-rc4" From eb3efad7376513d7897e42ded73f5b061a7277bf Mon Sep 17 00:00:00 2001 From: Mauro Antonio Sanz Date: Tue, 11 Oct 2022 17:25:13 -0300 Subject: [PATCH 16/25] configurable period --- splitio/client/factory.go | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/splitio/client/factory.go b/splitio/client/factory.go index e3a7de12..94e6d8ae 100644 --- a/splitio/client/factory.go +++ b/splitio/client/factory.go @@ -56,7 +56,6 @@ const ( uniqueKeysPeriodTaskRedis = 300 // 5 min impressionsCountPeriodTaskInMemory = 1800 // 30 min impressionsCountPeriodTaskRedis = 300 // 5 min - impressionsPeriodTaskRedis = 300 // 5 min impressionsBulkSizeRedis = 100 ) @@ -333,6 +332,10 @@ func setupInMemoryFactory( runtimeTelemetry: telemetryStorage, } + if cfg.ImpressionsMode == "" { + cfg.ImpressionsMode = config.ImpressionsModeOptimized + } + impressionManager, err := buildImpressionManager(cfg, advanced, logger, true, &splitTasks, &workers, storages, metadata, splitAPI, nil) if err != nil { return nil, err @@ -417,6 +420,10 @@ func setupRedisFactory(apikey string, cfg *conf.SplitSdkConfig, logger logging.L workers := synchronizer.Workers{} advanced := config.AdvancedConfig{} + if cfg.ImpressionsMode == "" { + cfg.ImpressionsMode = config.ImpressionsModeDebug + } + impressionManager, err := buildImpressionManager(cfg, advanced, logger, false, &splitTasks, &workers, storages, metadata, nil, redis.NewImpressionStorage(redisClient, metadata, logger)) if err != nil { return nil, err @@ -600,7 +607,7 @@ func buildImpressionManager( splitTasks.ImpressionSyncTask = tasks.NewRecordImpressionsTask(workers.ImpressionRecorder, cfg.TaskPeriods.ImpressionSync, logger, advanced.ImpressionsBulkSize) } else { workers.ImpressionRecorder = impression.NewRecorderRedis(storages.impressionsConsumer, impressionRedisStorage, logger) - splitTasks.ImpressionSyncTask = tasks.NewRecordImpressionsTask(workers.ImpressionRecorder, impressionsPeriodTaskRedis, logger, impressionsBulkSizeRedis) + splitTasks.ImpressionSyncTask = tasks.NewRecordImpressionsTask(workers.ImpressionRecorder, cfg.TaskPeriods.ImpressionSync, logger, impressionsBulkSizeRedis) } impressionObserver, err := strategy.NewImpressionObserver(500) @@ -622,7 +629,7 @@ func buildImpressionManager( workers.ImpressionsCountRecorder = impressionscount.NewRecorderRedis(impressionsCounter, storages.impressionsCount, logger) workers.ImpressionRecorder = impression.NewRecorderRedis(storages.impressionsConsumer, impressionRedisStorage, logger) splitTasks.ImpressionsCountSyncTask = tasks.NewRecordImpressionsCountTask(workers.ImpressionsCountRecorder, logger, impressionsCountPeriodTaskRedis) - splitTasks.ImpressionSyncTask = tasks.NewRecordImpressionsTask(workers.ImpressionRecorder, impressionsPeriodTaskRedis, logger, impressionsBulkSizeRedis) + splitTasks.ImpressionSyncTask = tasks.NewRecordImpressionsTask(workers.ImpressionRecorder, cfg.TaskPeriods.ImpressionSync, logger, impressionsBulkSizeRedis) } impressionObserver, err := strategy.NewImpressionObserver(500) From f4122498d7fab4a65894f5c1222ed3264baa065b Mon Sep 17 00:00:00 2001 From: Matias Melograno Date: Wed, 12 Oct 2022 15:27:19 -0300 Subject: [PATCH 17/25] prep 6.2.0 with none --- CHANGES.txt | 4 ++-- splitio/version.go | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/CHANGES.txt b/CHANGES.txt index 2e62341a..07f7b07d 100644 --- a/CHANGES.txt +++ b/CHANGES.txt @@ -1,5 +1,5 @@ -6.2.0 (Oct 6, 2022) -- Added a new impressions mode for the SDK called NONE , to be used in factory when there is no desire to capture impressions on an SDK factory to feed Split's analytics engine. Running NONE mode, the SDK will only capture unique keys evaluated for a particular feature flag instead of full blown impressions. +6.2.0 (Oct 12, 2022) +- Added a new impressions mode for the SDK called NONE, to be used in factory when there is no desire to capture impressions on an SDK factory to feed Split's analytics engine. Running NONE mode, the SDK will only capture unique keys evaluated for a particular feature flag instead of full blown impressions. 6.1.8 (Sep 9, 2022) - Updated BooleanMatcher logging: demoted the log message to "warning". diff --git a/splitio/version.go b/splitio/version.go index 14001f49..850fff19 100644 --- a/splitio/version.go +++ b/splitio/version.go @@ -1,4 +1,4 @@ package splitio // Version contains a string with the split sdk version -const Version = "6.2.0-rc4" +const Version = "6.2.0" From 5622bdd7baebd0b7051770290fbfb8bed327d194 Mon Sep 17 00:00:00 2001 From: Matias Melograno Date: Wed, 12 Oct 2022 15:32:25 -0300 Subject: [PATCH 18/25] added none mode to msg err --- splitio/client/input_validator_test.go | 2 +- splitio/conf/sdkconf.go | 6 +++--- splitio/conf/sdkconf_test.go | 2 +- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/splitio/client/input_validator_test.go b/splitio/client/input_validator_test.go index b3b1eabd..e8952c22 100644 --- a/splitio/client/input_validator_test.go +++ b/splitio/client/input_validator_test.go @@ -133,7 +133,7 @@ func TestFactoryWithNilApiKey(t *testing.T) { t.Error("Should be error") } - expected := "Factory instantiation: you passed an empty apikey, apikey must be a non-empty string" + expected := "factory instantiation: you passed an empty apikey, apikey must be a non-empty string" if !mW.Matches(expected) { t.Error("Error is distinct from the expected one") } diff --git a/splitio/conf/sdkconf.go b/splitio/conf/sdkconf.go index d24f24c3..e3a311b5 100644 --- a/splitio/conf/sdkconf.go +++ b/splitio/conf/sdkconf.go @@ -198,7 +198,7 @@ func validConfigRates(cfg *SplitSdkConfig) error { case conf.ImpressionsModeNone: return nil default: - fmt.Println(`You passed an invalid impressionsMode, impressionsMode should be one of the following values: 'debug' or 'optimized'. Defaulting to 'optimized' mode.`) + fmt.Println(`You passed an invalid impressionsMode, impressionsMode should be one of the following values: 'debug', 'optimized' or 'none'. Defaulting to 'optimized' mode.`) cfg.ImpressionsMode = conf.ImpressionsModeOptimized err := checkImpressionSync(cfg) if err != nil { @@ -213,7 +213,7 @@ func validConfigRates(cfg *SplitSdkConfig) error { return fmt.Errorf("TelemetrySync must be >= %d. Actual is: %d", minTelemetrySync, cfg.TaskPeriods.TelemetrySync) } if cfg.Advanced.SegmentWorkers <= 0 { - return errors.New("Number of workers for fetching segments MUST be greater than zero") + return errors.New("number of workers for fetching segments MUST be greater than zero") } return nil } @@ -223,7 +223,7 @@ func validConfigRates(cfg *SplitSdkConfig) error { func Normalize(apikey string, cfg *SplitSdkConfig) error { // Fail if no apikey is provided if apikey == "" && cfg.OperationMode != Localhost { - return errors.New("Factory instantiation: you passed an empty apikey, apikey must be a non-empty string") + return errors.New("factory instantiation: you passed an empty apikey, apikey must be a non-empty string") } // To keep the interface consistent with other sdks we accept "localhost" as an apikey, diff --git a/splitio/conf/sdkconf_test.go b/splitio/conf/sdkconf_test.go index ef6653d5..80509ac8 100644 --- a/splitio/conf/sdkconf_test.go +++ b/splitio/conf/sdkconf_test.go @@ -104,7 +104,7 @@ func TestValidRates(t *testing.T) { cfg = Default() cfg.Advanced.SegmentWorkers = 0 err = Normalize("asd", cfg) - if err == nil || err.Error() != "Number of workers for fetching segments MUST be greater than zero" { + if err == nil || err.Error() != "number of workers for fetching segments MUST be greater than zero" { t.Error("It should return err") } From 8b2fef1a7fa09f6a4bed2aa243388eb1930c8e96 Mon Sep 17 00:00:00 2001 From: Nadia Mayor Date: Mon, 24 Oct 2022 18:48:15 -0300 Subject: [PATCH 19/25] [SDKS-6232] Replace error to warning --- splitio/engine/grammar/matchers/startswith.go | 3 ++- splitio/version.go | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/splitio/engine/grammar/matchers/startswith.go b/splitio/engine/grammar/matchers/startswith.go index ba1cae1d..a1971783 100644 --- a/splitio/engine/grammar/matchers/startswith.go +++ b/splitio/engine/grammar/matchers/startswith.go @@ -1,6 +1,7 @@ package matchers import ( + "fmt" "strings" ) @@ -14,7 +15,7 @@ type StartsWithMatcher struct { func (m *StartsWithMatcher) Match(key string, attributes map[string]interface{}, bucketingKey *string) bool { matchingKey, err := m.matchingKey(key, attributes) if err != nil { - m.logger.Error("StartsWithMatcher: ", err) + m.logger.Warning(fmt.Sprintf("StartsWithMatcher: %s", err.Error())) return false } diff --git a/splitio/version.go b/splitio/version.go index 850fff19..539b4454 100644 --- a/splitio/version.go +++ b/splitio/version.go @@ -1,4 +1,4 @@ package splitio // Version contains a string with the split sdk version -const Version = "6.2.0" +const Version = "6.2.1-rc" From ec1d88543e5a46dd0b0cc9b394ea0ac8db65a7f0 Mon Sep 17 00:00:00 2001 From: Nadia Mayor Date: Tue, 25 Oct 2022 17:11:37 -0300 Subject: [PATCH 20/25] [SDKS-6232] Change error to warning --- splitio/engine/grammar/matchers/allofset.go | 5 +++-- splitio/engine/grammar/matchers/anyofset.go | 4 +++- splitio/engine/grammar/matchers/between.go | 5 +++-- splitio/engine/grammar/matchers/contains.go | 3 ++- splitio/engine/grammar/matchers/endswith.go | 3 ++- splitio/engine/grammar/matchers/equalto.go | 5 +++-- splitio/engine/grammar/matchers/equaltoset.go | 4 +++- splitio/engine/grammar/matchers/gtoet.go | 4 +++- splitio/engine/grammar/matchers/ltoet.go | 4 +++- splitio/engine/grammar/matchers/partofset.go | 4 +++- splitio/engine/grammar/matchers/regex.go | 3 ++- splitio/engine/grammar/matchers/whitelist.go | 4 +++- 12 files changed, 33 insertions(+), 15 deletions(-) diff --git a/splitio/engine/grammar/matchers/allofset.go b/splitio/engine/grammar/matchers/allofset.go index aa456a82..8792309d 100644 --- a/splitio/engine/grammar/matchers/allofset.go +++ b/splitio/engine/grammar/matchers/allofset.go @@ -2,8 +2,9 @@ package matchers import ( "fmt" - "github.com/splitio/go-toolkit/v5/datastructures/set" "reflect" + + "github.com/splitio/go-toolkit/v5/datastructures/set" ) // ContainsAllOfSetMatcher matches if the set supplied to the getTreatment is a superset of the one in the split @@ -16,7 +17,7 @@ type ContainsAllOfSetMatcher struct { func (m *ContainsAllOfSetMatcher) Match(key string, attributes map[string]interface{}, bucketingKey *string) bool { matchingKey, err := m.matchingKey(key, attributes) if err != nil { - m.logger.Error("AllOfSetMatcher: ", err) + m.logger.Warning(fmt.Sprintf("AllOfSetMatcher: %s", err.Error())) return false } diff --git a/splitio/engine/grammar/matchers/anyofset.go b/splitio/engine/grammar/matchers/anyofset.go index 6cde2df5..d77cdfc3 100644 --- a/splitio/engine/grammar/matchers/anyofset.go +++ b/splitio/engine/grammar/matchers/anyofset.go @@ -1,6 +1,8 @@ package matchers import ( + "fmt" + "github.com/splitio/go-toolkit/v5/datastructures/set" ) @@ -14,7 +16,7 @@ type ContainsAnyOfSetMatcher struct { func (m *ContainsAnyOfSetMatcher) Match(key string, attributes map[string]interface{}, bucketingKey *string) bool { matchingKey, err := m.matchingKey(key, attributes) if err != nil { - m.logger.Error("AnyOfSetMatcher: ", err) + m.logger.Warning(fmt.Sprintf("AnyOfSetMatcher %s", err.Error())) return false } diff --git a/splitio/engine/grammar/matchers/between.go b/splitio/engine/grammar/matchers/between.go index 3b8f1f78..202c9f2c 100644 --- a/splitio/engine/grammar/matchers/between.go +++ b/splitio/engine/grammar/matchers/between.go @@ -2,8 +2,9 @@ package matchers import ( "fmt" - "github.com/splitio/go-client/v6/splitio/engine/grammar/matchers/datatypes" "reflect" + + "github.com/splitio/go-client/v6/splitio/engine/grammar/matchers/datatypes" ) // BetweenMatcher will match if two numbers or two datetimes are equal @@ -18,7 +19,7 @@ type BetweenMatcher struct { func (m *BetweenMatcher) Match(key string, attributes map[string]interface{}, bucketingKey *string) bool { matchingRaw, err := m.matchingKey(key, attributes) if err != nil { - m.logger.Error("BetweenMatcher: Could not retrieve matching key. ", err) + m.logger.Warning(fmt.Sprintf("BetweenMatcher: %s", err.Error())) return false } diff --git a/splitio/engine/grammar/matchers/contains.go b/splitio/engine/grammar/matchers/contains.go index 191662b8..35a5708e 100644 --- a/splitio/engine/grammar/matchers/contains.go +++ b/splitio/engine/grammar/matchers/contains.go @@ -1,6 +1,7 @@ package matchers import ( + "fmt" "strings" ) @@ -14,7 +15,7 @@ type ContainsStringMatcher struct { func (m *ContainsStringMatcher) Match(key string, attributes map[string]interface{}, bucketingKey *string) bool { matchingKey, err := m.matchingKey(key, attributes) if err != nil { - m.logger.Error("ContainsAllOfSetMatcher: Error retrieving matching key") + m.logger.Warning(fmt.Sprintf("ContainsAllOfSetMatcher: %s", err.Error())) return false } diff --git a/splitio/engine/grammar/matchers/endswith.go b/splitio/engine/grammar/matchers/endswith.go index bfc6f315..662e05ed 100644 --- a/splitio/engine/grammar/matchers/endswith.go +++ b/splitio/engine/grammar/matchers/endswith.go @@ -1,6 +1,7 @@ package matchers import ( + "fmt" "strings" ) @@ -14,7 +15,7 @@ type EndsWithMatcher struct { func (m *EndsWithMatcher) Match(key string, attributes map[string]interface{}, bucketingKey *string) bool { matchingKey, err := m.matchingKey(key, attributes) if err != nil { - m.logger.Error("EndsWithMatcher: ", err) + m.logger.Warning(fmt.Sprintf("EndsWithMatcher: %s", err.Error())) return false } diff --git a/splitio/engine/grammar/matchers/equalto.go b/splitio/engine/grammar/matchers/equalto.go index 5706634f..b4f13419 100644 --- a/splitio/engine/grammar/matchers/equalto.go +++ b/splitio/engine/grammar/matchers/equalto.go @@ -2,8 +2,9 @@ package matchers import ( "fmt" - "github.com/splitio/go-client/v6/splitio/engine/grammar/matchers/datatypes" "reflect" + + "github.com/splitio/go-client/v6/splitio/engine/grammar/matchers/datatypes" ) // EqualToMatcher will match if two numbers or two datetimes are equal @@ -18,7 +19,7 @@ func (m *EqualToMatcher) Match(key string, attributes map[string]interface{}, bu matchingRaw, err := m.matchingKey(key, attributes) if err != nil { - m.logger.Error("EqualToMatcher: ", err) + m.logger.Warning(fmt.Sprintf("EqualToMatcher: %s", err.Error())) return false } diff --git a/splitio/engine/grammar/matchers/equaltoset.go b/splitio/engine/grammar/matchers/equaltoset.go index 35959dd9..0b9ba519 100644 --- a/splitio/engine/grammar/matchers/equaltoset.go +++ b/splitio/engine/grammar/matchers/equaltoset.go @@ -1,6 +1,8 @@ package matchers import ( + "fmt" + "github.com/splitio/go-toolkit/v5/datastructures/set" ) @@ -14,7 +16,7 @@ type EqualToSetMatcher struct { func (m *EqualToSetMatcher) Match(key string, attributes map[string]interface{}, bucketingKey *string) bool { matchingKey, err := m.matchingKey(key, attributes) if err != nil { - m.logger.Error("EqualToSetMatcher: ", err) + m.logger.Warning(fmt.Sprintf("EqualToSetMatcher: %s", err.Error())) return false } diff --git a/splitio/engine/grammar/matchers/gtoet.go b/splitio/engine/grammar/matchers/gtoet.go index bc786c18..7470194d 100644 --- a/splitio/engine/grammar/matchers/gtoet.go +++ b/splitio/engine/grammar/matchers/gtoet.go @@ -1,6 +1,8 @@ package matchers import ( + "fmt" + "github.com/splitio/go-client/v6/splitio/engine/grammar/matchers/datatypes" ) @@ -15,7 +17,7 @@ type GreaterThanOrEqualToMatcher struct { func (m *GreaterThanOrEqualToMatcher) Match(key string, attributes map[string]interface{}, bucketingKey *string) bool { matchingRaw, err := m.matchingKey(key, attributes) if err != nil { - m.logger.Error("GreaterThanOrEqualToMatcher: ", err) + m.logger.Warning(fmt.Sprintf("GreaterThanOrEqualToMatcher: %s", err.Error())) return false } diff --git a/splitio/engine/grammar/matchers/ltoet.go b/splitio/engine/grammar/matchers/ltoet.go index e52782f0..47fa4354 100644 --- a/splitio/engine/grammar/matchers/ltoet.go +++ b/splitio/engine/grammar/matchers/ltoet.go @@ -1,6 +1,8 @@ package matchers import ( + "fmt" + "github.com/splitio/go-client/v6/splitio/engine/grammar/matchers/datatypes" ) @@ -16,7 +18,7 @@ func (m *LessThanOrEqualToMatcher) Match(key string, attributes map[string]inter matchingRaw, err := m.matchingKey(key, attributes) if err != nil { - m.logger.Error("LessThanOrEqualToMatcher: ", err) + m.logger.Warning(fmt.Sprintf("LessThanOrEqualToMatcher: %s", err.Error())) return false } diff --git a/splitio/engine/grammar/matchers/partofset.go b/splitio/engine/grammar/matchers/partofset.go index 40263726..b5c0f2bf 100644 --- a/splitio/engine/grammar/matchers/partofset.go +++ b/splitio/engine/grammar/matchers/partofset.go @@ -1,6 +1,8 @@ package matchers import ( + "fmt" + "github.com/splitio/go-toolkit/v5/datastructures/set" ) @@ -14,7 +16,7 @@ type PartOfSetMatcher struct { func (m *PartOfSetMatcher) Match(key string, attributes map[string]interface{}, bucketingKey *string) bool { matchingKey, err := m.matchingKey(key, attributes) if err != nil { - m.logger.Error("PartOfSetMatcher: ", err) + m.logger.Warning(fmt.Sprintf("PartOfSetMatcher: %s", err.Error())) return false } diff --git a/splitio/engine/grammar/matchers/regex.go b/splitio/engine/grammar/matchers/regex.go index 3a3a68c8..dddd1a7c 100644 --- a/splitio/engine/grammar/matchers/regex.go +++ b/splitio/engine/grammar/matchers/regex.go @@ -1,6 +1,7 @@ package matchers import ( + "fmt" "reflect" "regexp" ) @@ -15,7 +16,7 @@ type RegexMatcher struct { func (m *RegexMatcher) Match(key string, attributes map[string]interface{}, bucketingKey *string) bool { matchingKey, err := m.matchingKey(key, attributes) if err != nil { - m.logger.Error("RegexMatcher: ", err) + m.logger.Warning(fmt.Sprintf("RegexMatcher: %s", err.Error())) return false } diff --git a/splitio/engine/grammar/matchers/whitelist.go b/splitio/engine/grammar/matchers/whitelist.go index 195e6429..05961bf3 100644 --- a/splitio/engine/grammar/matchers/whitelist.go +++ b/splitio/engine/grammar/matchers/whitelist.go @@ -1,6 +1,8 @@ package matchers import ( + "fmt" + "github.com/splitio/go-toolkit/v5/datastructures/set" ) @@ -14,7 +16,7 @@ type WhitelistMatcher struct { func (m *WhitelistMatcher) Match(key string, attributes map[string]interface{}, bucketingKey *string) bool { matchingKey, err := m.matchingKey(key, attributes) if err != nil { - m.logger.Error("WhitelistMatcher: ", err) + m.logger.Warning(fmt.Sprintf("WhitelistMatcher: %s", err.Error())) return false } From a6519ddf4849cadc5f074e111bfea5027634d570 Mon Sep 17 00:00:00 2001 From: Matias Melograno Date: Tue, 25 Oct 2022 17:55:08 -0300 Subject: [PATCH 21/25] updated readme --- README.md | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 4a694d9a..8fb7afc8 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,8 @@ # Split GO SDK -[![Build Status](https://api.travis-ci.com/splitio/go-client.svg?branch=master)](https://api.travis-ci.com/splitio/go-client) +[![build workflow](https://https://github.com/splitio/go-client/actions/workflows/build.yml/badge.svg)](https://github.com/splitio/go-client/actions) +[![PkgGoDev](https://pkg.go.dev/github.com/splitio/go-client/v6)](https://pkg.go.dev/github.com/splitio/go-client/v6/splitio?tab=doc) +[![Documentation](https://help.split.io/hc/en-us/articles/360020093652-Go-SDK)](https://help.split.io/hc/en-us/articles/360020093652-Go-SDK) ## Overview @@ -16,7 +18,7 @@ This SDK is compatible with Go 1.8. Below is a simple example that describes the instantiation and most basic usage of our SDK: -Run `go get github.com/splitio/go-client/` or `dep ensure -add github.com/splitio/go-client` +Run `go get github.com/splitio/go-client/` ```go package main From b0e618f02afe6616516b1c4b97d0041b00ed2035 Mon Sep 17 00:00:00 2001 From: Matias Melograno Date: Tue, 25 Oct 2022 17:59:46 -0300 Subject: [PATCH 22/25] updated --- README.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index 8fb7afc8..05f6c23e 100644 --- a/README.md +++ b/README.md @@ -1,8 +1,8 @@ # Split GO SDK -[![build workflow](https://https://github.com/splitio/go-client/actions/workflows/build.yml/badge.svg)](https://github.com/splitio/go-client/actions) -[![PkgGoDev](https://pkg.go.dev/github.com/splitio/go-client/v6)](https://pkg.go.dev/github.com/splitio/go-client/v6/splitio?tab=doc) -[![Documentation](https://help.split.io/hc/en-us/articles/360020093652-Go-SDK)](https://help.split.io/hc/en-us/articles/360020093652-Go-SDK) +[![build workflow](https://github.com/splitio/go-client/actions/workflows/build.yml/badge.svg)](https://github.com/splitio/go-client/actions) +[![PkgGoDev](https://pkg.go.dev/badge/github.com/splitio/go-client/v6)](https://pkg.go.dev/github.com/splitio/go-client/v6/splitio?tab=doc) +[![Documentation](https://img.shields.io/badge/go_client-documentation-informational)](https://help.split.io/hc/en-us/articles/360020093652-Go-SDK) ## Overview From 3b8bec0691ae50c1c6c3251821257a842be4b3aa Mon Sep 17 00:00:00 2001 From: Matias Melograno Date: Tue, 25 Oct 2022 18:04:12 -0300 Subject: [PATCH 23/25] fix build --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 05f6c23e..2eb53c23 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@ # Split GO SDK -[![build workflow](https://github.com/splitio/go-client/actions/workflows/build.yml/badge.svg)](https://github.com/splitio/go-client/actions) +[![build workflow](https://github.com/splitio/go-client/actions/workflows/ci.yml/badge.svg)](https://github.com/splitio/go-client/actions) [![PkgGoDev](https://pkg.go.dev/badge/github.com/splitio/go-client/v6)](https://pkg.go.dev/github.com/splitio/go-client/v6/splitio?tab=doc) [![Documentation](https://img.shields.io/badge/go_client-documentation-informational)](https://help.split.io/hc/en-us/articles/360020093652-Go-SDK) From 592d04d3c24bcba4c89f7f19c2e53c4661d5c64a Mon Sep 17 00:00:00 2001 From: Matias Melograno Date: Tue, 25 Oct 2022 18:18:45 -0300 Subject: [PATCH 24/25] added streaming enabled false --- splitio/client/client_test.go | 1 + 1 file changed, 1 insertion(+) diff --git a/splitio/client/client_test.go b/splitio/client/client_test.go index 97ba5e61..2b2400b6 100644 --- a/splitio/client/client_test.go +++ b/splitio/client/client_test.go @@ -1778,6 +1778,7 @@ func TestClientNone(t *testing.T) { impTest := &ImpressionListenerTest{} cfg := conf.Default() cfg.LabelsEnabled = true + cfg.Advanced.StreamingEnabled = false cfg.Advanced.EventsURL = ts.URL cfg.Advanced.SdkURL = ts.URL cfg.Advanced.TelemetryServiceURL = ts.URL From a3c287821c2449c1ca8851483e75864b35ec56e8 Mon Sep 17 00:00:00 2001 From: Nadia Mayor Date: Fri, 28 Oct 2022 15:12:42 -0300 Subject: [PATCH 25/25] [SDKS-6232] Update version and changelog --- CHANGES.txt | 3 +++ splitio/version.go | 2 +- 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/CHANGES.txt b/CHANGES.txt index 07f7b07d..23b8dda9 100644 --- a/CHANGES.txt +++ b/CHANGES.txt @@ -1,3 +1,6 @@ +6.2.1 (Oct 28, 2022) +- Updated Matchers logging: demoted the log message to "warning". + 6.2.0 (Oct 12, 2022) - Added a new impressions mode for the SDK called NONE, to be used in factory when there is no desire to capture impressions on an SDK factory to feed Split's analytics engine. Running NONE mode, the SDK will only capture unique keys evaluated for a particular feature flag instead of full blown impressions. diff --git a/splitio/version.go b/splitio/version.go index 539b4454..5125a4ad 100644 --- a/splitio/version.go +++ b/splitio/version.go @@ -1,4 +1,4 @@ package splitio // Version contains a string with the split sdk version -const Version = "6.2.1-rc" +const Version = "6.2.1"