diff --git a/go.mod b/go.mod index 6e2636bd..1ddeff5b 100644 --- a/go.mod +++ b/go.mod @@ -8,8 +8,8 @@ require ( github.com/gin-gonic/gin v1.9.1 github.com/google/uuid v1.3.0 github.com/splitio/gincache v1.0.1 - github.com/splitio/go-split-commons/v5 v5.1.2-0.20240108145819-63cfece95155 - github.com/splitio/go-toolkit/v5 v5.3.3-0.20240108144147-a36a17c46788 + github.com/splitio/go-split-commons/v5 v5.2.0 + github.com/splitio/go-toolkit/v5 v5.4.0 github.com/stretchr/testify v1.8.4 go.etcd.io/bbolt v1.3.6 golang.org/x/exp v0.0.0-20231006140011-7918f672742d diff --git a/go.sum b/go.sum index d6b4e045..1a1bff0b 100644 --- a/go.sum +++ b/go.sum @@ -90,10 +90,10 @@ github.com/rogpeppe/go-internal v1.8.0 h1:FCbCCtXNOY3UtUuHUYaghJg4y7Fd14rXifAYUA github.com/rogpeppe/go-internal v1.8.0/go.mod h1:WmiCO8CzOY8rg0OYDC4/i/2WRWAB6poM+XZ2dLUbcbE= github.com/splitio/gincache v1.0.1 h1:dLYdANY/BqH4KcUMCe/LluLyV5WtuE/LEdQWRE06IXU= github.com/splitio/gincache v1.0.1/go.mod h1:CcgJDSM9Af75kyBH0724v55URVwMBuSj5x1eCWIOECY= -github.com/splitio/go-split-commons/v5 v5.1.2-0.20240108145819-63cfece95155 h1:SecApJYs4oumtZMIBwnE2DFyImlwiuswXIdZDaW42uE= -github.com/splitio/go-split-commons/v5 v5.1.2-0.20240108145819-63cfece95155/go.mod h1:sdXeIX4UdBf+EPdVAdBC+f2RuFUh/44bqHlTLy3rH/I= -github.com/splitio/go-toolkit/v5 v5.3.3-0.20240108144147-a36a17c46788 h1:URrg0BcgUzFE4pAacFlrWiTrmEdH4SY03XXLLWVtqLc= -github.com/splitio/go-toolkit/v5 v5.3.3-0.20240108144147-a36a17c46788/go.mod h1:xYhUvV1gga9/1029Wbp5pjnR6Cy8nvBpjw99wAbsMko= +github.com/splitio/go-split-commons/v5 v5.2.0 h1:1P66JdUV1Fj1DUeWU1rwkeObqinl9AecRxDsktBsx0g= +github.com/splitio/go-split-commons/v5 v5.2.0/go.mod h1:m1Od/jxiSUJXpdbRvRxTaKeSAdQVem5AZr7AjI4xXn8= +github.com/splitio/go-toolkit/v5 v5.4.0 h1:g5WFpRhQomnXCmvfsNOWV4s5AuUrWIZ+amM68G8NBKM= +github.com/splitio/go-toolkit/v5 v5.4.0/go.mod h1:xYhUvV1gga9/1029Wbp5pjnR6Cy8nvBpjw99wAbsMko= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= github.com/stretchr/objx v0.5.0 h1:1zr/of2m5FGMsad5YfcqgdqdWrIhu+EBEJRhR1U7z/c= diff --git a/main b/main new file mode 100755 index 00000000..98420ac9 Binary files /dev/null and b/main differ diff --git a/splitio/admin/controllers/dashboard.go b/splitio/admin/controllers/dashboard.go index a6718764..355f209b 100644 --- a/splitio/admin/controllers/dashboard.go +++ b/splitio/admin/controllers/dashboard.go @@ -157,5 +157,6 @@ func (c *DashboardController) gatherStats() *dashboard.GlobalStats { LoggedErrors: errorCount, LoggedMessages: errorMessages, Uptime: int64(c.runtime.Uptime().Seconds()), + FlagSets: getFlagSetsInfo(c.storages.SplitStorage), } } diff --git a/splitio/admin/controllers/helpers.go b/splitio/admin/controllers/helpers.go index 3a519d4f..1016c6d7 100644 --- a/splitio/admin/controllers/helpers.go +++ b/splitio/admin/controllers/helpers.go @@ -1,6 +1,8 @@ package controllers import ( + "sort" + "strings" "time" "github.com/splitio/go-split-commons/v5/storage" @@ -209,6 +211,26 @@ func getEventsSize(eventStorage storage.EventMultiSdkConsumer) int64 { return eventStorage.Count() } +func getFlagSetsInfo(splitsStorage storage.SplitStorage) []dashboard.FlagSetsSummary { + flagSetNames := splitsStorage.GetAllFlagSetNames() + + summaries := make([]dashboard.FlagSetsSummary, 0, len(flagSetNames)) + featureFlagsBySets := splitsStorage.GetNamesByFlagSets(flagSetNames) + + for key, featureFlags := range featureFlagsBySets { + summaries = append(summaries, dashboard.FlagSetsSummary{ + Name: key, + FeatureFlagsAssociated: int64(len(featureFlags)), + FeatureFlags: strings.Join(featureFlags, ", "), + }) + } + sort.Slice(summaries, func(i, j int) bool { + return summaries[j].Name > summaries[i].Name + }) + + return summaries +} + func getImpressionSize(impressionStorage storage.ImpressionMultiSdkConsumer) int64 { if impressionStorage == nil { return 0 diff --git a/splitio/admin/controllers/observability.go b/splitio/admin/controllers/observability.go index 1170f55c..d4ae8a75 100644 --- a/splitio/admin/controllers/observability.go +++ b/splitio/admin/controllers/observability.go @@ -12,6 +12,12 @@ import ( "github.com/gin-gonic/gin" ) +type ObservabilityDto struct { + ActiveSplits []string `json:"activeSplits"` + ActiveSegments map[string]int `json:"activeSegments"` + ActiveFlagSets []string `json:"activeFlagSets"` +} + // ObservabilityController interface is used to have a single constructor that returns the apropriate controller type ObservabilityController interface { Register(gin.IRouter) @@ -30,9 +36,10 @@ func (c *SyncObservabilityController) Register(router gin.IRouter) { } func (c *SyncObservabilityController) observability(ctx *gin.Context) { - ctx.JSON(200, gin.H{ - "activeSplits": c.splits.SplitNames(), - "activeSegments": c.segments.NamesAndCount(), + ctx.JSON(200, ObservabilityDto{ + ActiveSplits: c.splits.SplitNames(), + ActiveSegments: c.segments.NamesAndCount(), + ActiveFlagSets: c.splits.GetAllFlagSetNames(), }) } diff --git a/splitio/admin/controllers/observability_test.go b/splitio/admin/controllers/observability_test.go new file mode 100644 index 00000000..0a0d8512 --- /dev/null +++ b/splitio/admin/controllers/observability_test.go @@ -0,0 +1,131 @@ +package controllers + +import ( + "encoding/json" + "io" + "net/http" + "net/http/httptest" + "testing" + + "github.com/gin-gonic/gin" + "github.com/splitio/go-split-commons/v5/dtos" + "github.com/splitio/go-split-commons/v5/storage/mocks" + "github.com/splitio/go-toolkit/v5/datastructures/set" + "github.com/splitio/go-toolkit/v5/logging" + adminCommon "github.com/splitio/split-synchronizer/v5/splitio/admin/common" + "github.com/splitio/split-synchronizer/v5/splitio/provisional/observability" +) + +func TestSyncObservabilityEndpoint(t *testing.T) { + logger := logging.NewLogger(nil) + + extSplitStorage := &extMockSplitStorage{ + &mocks.MockSplitStorage{ + SplitNamesCall: func() []string { + return []string{"split1", "split2", "split3"} + }, + SegmentNamesCall: func() *set.ThreadUnsafeSet { + return set.NewSet("segment1") + }, + GetAllFlagSetNamesCall: func() []string { + return []string{"fSet1", "fSet2"} + }, + }, + nil, + } + + extSegmentStorage := &extMockSegmentStorage{ + MockSegmentStorage: &mocks.MockSegmentStorage{}, + SizeCall: func(name string) (int, error) { + switch name { + case "segment1": + return 10, nil + case "segment2": + return 20, nil + } + return 0, nil + }, + } + + oSplitStorage, err := observability.NewObservableSplitStorage(extSplitStorage, logger) + if err != nil { + t.Error(err) + return + } + + oSegmentStorage, err := observability.NewObservableSegmentStorage(logger, extSplitStorage, extSegmentStorage) + if err != nil { + t.Error(err) + return + } + + storages := adminCommon.Storages{ + SplitStorage: oSplitStorage, + SegmentStorage: oSegmentStorage, + } + + ctrl, err := NewObservabilityController(false, logger, storages) + + if err != nil { + t.Error(err) + return + } + + resp := httptest.NewRecorder() + ctx, router := gin.CreateTestContext(resp) + ctrl.Register(router) + + ctx.Request, _ = http.NewRequest(http.MethodGet, "/observability", nil) + router.ServeHTTP(resp, ctx.Request) + + if resp.Code != 200 { + t.Error("hay crap.") + } + + body, err := io.ReadAll(resp.Body) + if err != nil { + t.Error(err) + return + } + + var result ObservabilityDto + if err := json.Unmarshal(body, &result); err != nil { + t.Error("there should be no error ", err) + } + + if len(result.ActiveFlagSets) != 2 { + t.Errorf("Active flag sets should be 2. Actual %d", len(result.ActiveFlagSets)) + } + + if len(result.ActiveSplits) != 3 { + t.Errorf("Active splits should be 3. Actual %d", len(result.ActiveSplits)) + } + + if len(result.ActiveSegments) != 1 { + t.Errorf("Active segments should be 1. Actual %d", len(result.ActiveSegments)) + } +} + +// TODO: should unify this classes +type extMockSplitStorage struct { + *mocks.MockSplitStorage + UpdateWithErrorsCall func([]dtos.SplitDTO, []dtos.SplitDTO, int64) error +} + +func (e *extMockSplitStorage) UpdateWithErrors(toAdd []dtos.SplitDTO, toRemove []dtos.SplitDTO, cn int64) error { + return e.UpdateWithErrorsCall(toAdd, toRemove, cn) +} + +type extMockSegmentStorage struct { + *mocks.MockSegmentStorage + UpdateWithSummaryCall func(string, *set.ThreadUnsafeSet, *set.ThreadUnsafeSet, int64) (int, int, error) + SizeCall func(string) (int, error) +} + +func (e *extMockSegmentStorage) UpdateWithSummary(name string, toAdd *set.ThreadUnsafeSet, toRemove *set.ThreadUnsafeSet, till int64) (added int, removed int, err error) { + return e.UpdateWithSummaryCall(name, toAdd, toRemove, till) +} + +func (e *extMockSegmentStorage) Size(name string) (int, error) { + return e.SizeCall(name) +} diff --git a/splitio/admin/views/dashboard/datainspector.go b/splitio/admin/views/dashboard/datainspector.go index 7886e3b9..6081b894 100644 --- a/splitio/admin/views/dashboard/datainspector.go +++ b/splitio/admin/views/dashboard/datainspector.go @@ -9,122 +9,140 @@ const dataInspector = `
  • -  Segments - + + + + +  Segments + +
    • +
  • + + +  Flag Sets +
    • - + -
    +
    - -
    -
    -
    -
    - -
    -
    -
    - - - - - -
    + +
    +
    +
    +
    + +
    +
    +
    + + + + +
    -
    -
    - - - - - - - - - - - - - -
    Feature FlagStatusKilledTreatmentsFlag SetsLast Modified
    -
    +
    +
    +
    + + + + + + + + + + + + + +
    Feature FlagStatusKilledTreatmentsFlag SetsLast Modified
    +
    + + +
    +
    +
    +
    + + + + + {{if .ProxyMode}} + + + + + + + {{else}} + + + + + {{end}} + + + + +
     SegmentTotal KeysRemoved KeysActive KeysLast Modified SegmentActive KeysLast Modified
    +
    +
    +
    +
    - -
    -
    -
    -
    - - - - - {{if .ProxyMode}} - - - - - - - {{else}} - - - - - {{end}} - - - - -
     SegmentTotal KeysRemoved KeysActive KeysLast Modified SegmentActive KeysLast Modified
    -
    + +
    +
    +
    +
    + + + + {{if .ProxyMode}} + + {{else}} + + + + {{end}} + + + + +
     Flag SetTotal Feature Flags AssociatedFeature Flags
    +
    {{end}} ` diff --git a/splitio/admin/views/dashboard/js.go b/splitio/admin/views/dashboard/js.go index 8deb3f5c..eb584b99 100644 --- a/splitio/admin/views/dashboard/js.go +++ b/splitio/admin/views/dashboard/js.go @@ -255,6 +255,21 @@ const mainScript = ` } }; + function formatFlagSet(flagSet) { + return ( + '' + + ' ' + flagSet.name + '' + + ' ' + flagSet.featureFlagsAssociated + '' + + ' ' + flagSet.featureFlags + '' + + '\n'); + }; + + function updateFlagSets(flagSets) { + const formatted = flagSets.map(formatFlagSet).join('\n'); + $('#flag_sets_rows tbody').empty(); + $('#flag_sets_rows tbody').append(formatted); + } + function formatSegment(segment) { return '' + '' + @@ -318,6 +333,7 @@ const mainScript = ` $('#backend_total_requests').html(stats.backendTotalRequests); $('#feature_flags_number').html(stats.featureFlags.length); $('#segments_number').html(stats.segments.length); + $('#flag_sets_number').html(stats.flagSets.length); $('#impressions_queue_value').html(stats.impressionsQueueSize); $('#events_queue_value').html(stats.eventsQueueSize); $('#feature_flags_number').html(); @@ -409,6 +425,7 @@ const mainScript = ` updateFeatureFlags(stats.featureFlags); updateSegments(stats.segments); updateLogEntries(stats.loggedMessages); + updateFlagSets(stats.flagSets) renderBackendStatsChart(stats.backendLatencies); {{if .ProxyMode}} diff --git a/splitio/admin/views/dashboard/main.go b/splitio/admin/views/dashboard/main.go index d84279d9..b790e22a 100644 --- a/splitio/admin/views/dashboard/main.go +++ b/splitio/admin/views/dashboard/main.go @@ -87,23 +87,24 @@ type RootObject struct { // GlobalStats runtime stats used to render the dashboard type GlobalStats struct { - BackendTotalRequests int64 `json:"backendTotalRequests"` - RequestsOk int64 `json:"requestsOk"` - RequestsErrored int64 `json:"requestsErrored"` - BackendRequestsOk int64 `json:"backendRequestsOk"` - BackendRequestsErrored int64 `json:"backendRequestsErrored"` - SdksTotalRequests int64 `json:"sdksTotalRequests"` - LoggedErrors int64 `json:"loggedErrors"` - LoggedMessages []string `json:"loggedMessages"` - FeatureFlags []SplitSummary `json:"featureFlags"` - Segments []SegmentSummary `json:"segments"` - Latencies []ChartJSData `json:"latencies"` - BackendLatencies []ChartJSData `json:"backendLatencies"` - ImpressionsQueueSize int64 `json:"impressionsQueueSize"` - ImpressionsLambda float64 `json:"impressionsLambda"` - EventsQueueSize int64 `json:"eventsQueueSize"` - EventsLambda float64 `json:"eventsLambda"` - Uptime int64 `json:"uptime"` + BackendTotalRequests int64 `json:"backendTotalRequests"` + RequestsOk int64 `json:"requestsOk"` + RequestsErrored int64 `json:"requestsErrored"` + BackendRequestsOk int64 `json:"backendRequestsOk"` + BackendRequestsErrored int64 `json:"backendRequestsErrored"` + SdksTotalRequests int64 `json:"sdksTotalRequests"` + LoggedErrors int64 `json:"loggedErrors"` + LoggedMessages []string `json:"loggedMessages"` + FeatureFlags []SplitSummary `json:"featureFlags"` + Segments []SegmentSummary `json:"segments"` + Latencies []ChartJSData `json:"latencies"` + BackendLatencies []ChartJSData `json:"backendLatencies"` + ImpressionsQueueSize int64 `json:"impressionsQueueSize"` + ImpressionsLambda float64 `json:"impressionsLambda"` + EventsQueueSize int64 `json:"eventsQueueSize"` + EventsLambda float64 `json:"eventsLambda"` + Uptime int64 `json:"uptime"` + FlagSets []FlagSetsSummary `json:"flagSets"` } // SplitSummary encapsulates a minimalistic view of feature flag properties to be presented in the dashboard @@ -113,7 +114,7 @@ type SplitSummary struct { Killed bool `json:"killed"` DefaultTreatment string `json:"defaultTreatment"` Treatments []string `json:"treatments"` - FlagSets []string `json:"flagSets"` + FlagSets []string `json:"flagSets"` LastModified string `json:"cn"` ChangeNumber int64 `json:"changeNumber"` } @@ -135,6 +136,12 @@ type SegmentKeySummary struct { ChangeNumber int64 `json:"cn"` } +type FlagSetsSummary struct { + Name string `json:"name"` + FeatureFlagsAssociated int64 `json:"featureFlagsAssociated"` + FeatureFlags string `json:"featureFlags"` +} + // RGBA bundles input to CSS's rgba function type RGBA struct { Red int32 diff --git a/splitio/admin/views/dashboard/stats.go b/splitio/admin/views/dashboard/stats.go index a46a52d8..914b525d 100644 --- a/splitio/admin/views/dashboard/stats.go +++ b/splitio/admin/views/dashboard/stats.go @@ -68,18 +68,24 @@ const cards = `

    -
    +

    Cached Feature Flags

    -
    +

    Cached Segments

    +
    +
    +

    Cached Flag Sets

    +

    +
    +
    {{end}}
    diff --git a/splitio/producer/initialization.go b/splitio/producer/initialization.go index f1a865cb..884b4a6d 100644 --- a/splitio/producer/initialization.go +++ b/splitio/producer/initialization.go @@ -90,7 +90,7 @@ func Start(logger logging.LoggerInterface, cfg *conf.Main) error { flagSetsFilter := flagsets.NewFlagSetFilter(cfg.FlagSetsFilter) // These storages are forwarded to the dashboard, the sdk-telemetry is irrelevant there - splitStorage, err := observability.NewObservableSplitStorage(redis.NewSplitStorage(redisClient, logger, flagSetsFilter), logger) + splitStorage, err := observability.NewObservableSplitStorage(redis.NewSplitStorage(redisClient, logger, flagSetsFilter, int64(redisOptions.ScanCount)), logger) if err != nil { return fmt.Errorf("error instantiating observable feature flag storage: %w", err) } diff --git a/splitio/producer/util.go b/splitio/producer/util.go index f7cda7f2..bd151e01 100644 --- a/splitio/producer/util.go +++ b/splitio/producer/util.go @@ -30,6 +30,7 @@ import ( const ( impressionsCountPeriodTaskInMemory = 1800 // 30 min impressionObserverSize = 500 + ScanCount = 10 ) func parseTLSConfig(opt *conf.Redis) (*tls.Config, error) { @@ -99,6 +100,7 @@ func parseRedisOptions(cfg *conf.Redis) (*config.RedisConfig, error) { WriteTimeout: cfg.WriteTimeout, PoolSize: cfg.PoolSize, TLSConfig: tlsCfg, + ScanCount: ScanCount, } if cfg.SentinelReplication { diff --git a/splitio/proxy/storage/splits.go b/splitio/proxy/storage/splits.go index bb6090b8..4214dd52 100644 --- a/splitio/proxy/storage/splits.go +++ b/splitio/proxy/storage/splits.go @@ -182,8 +182,8 @@ func (*ProxySplitStorageImpl) GetNamesByFlagSets(sets []string) map[string][]str // GetAllFlagSetNames implements storage.SplitStorage func (*ProxySplitStorageImpl) GetAllFlagSetNames() []string { // NOTE: This method is NOT used by the proxy. - // we need to revisit our interfaces so that we're not obliged to do this smeely empty impls. - panic("unimplemented") + // we need to revisit our interfaces so that we're not obliged to do this smeely empty impls. + return nil } func (p *ProxySplitStorageImpl) setStartingPoint(cn int64) {