diff --git a/CHANGES.txt b/CHANGES.txt index ab778841..97b535b1 100644 --- a/CHANGES.txt +++ b/CHANGES.txt @@ -4,6 +4,13 @@ - Updated Syncrhonizer to properly handle flagsets in redis. - Added configuration options to specify which flagsets to use as a filter when downloading flag definitions. Please refer to our docs to learn more +5.4.2 (Nov 7, 2023) +- Updated docker images for vulnerability fixes. +- Updated dependencies for vulnerability fixes. + +5.4.1 (Oct 31, 2023) +- Fix issue in split proxy where removed segment keys would be returned as active at startup + 5.4.0 (July 18, 2023) - Improved streaming architecture implementation to apply feature flag updates from the notification received which is now enhanced, improving efficiency and reliability of the whole update system. - Fixed possible edge case issue where deleting a feature flag doesn’t propagate immediately. diff --git a/Makefile b/Makefile index 80641e2f..fefba63d 100644 --- a/Makefile +++ b/Makefile @@ -59,6 +59,10 @@ test: $(sources) go.sum test_coverage: $(sources) go.sum $(GO) test -v -cover -coverprofile=coverage.out $(ARGS) ./... +## display unit test coverage derived from last test run (use `make test display-coverage` for up-to-date results) +display-coverage: coverage.out + go tool cover -html=coverage.out + ## Generate binaires for all architectures, ready to upload for distribution (with and without version) release_assets: \ $(BUILD)/synchronizer \ @@ -96,10 +100,6 @@ images_release: # entrypoints @echo "$(DOCKER) push splitsoftware/split-proxy:$(version)" @echo "$(DOCKER) push splitsoftware/split-proxy:latest" -## display unit test coverage derived from last test run (use `make test display-coverage` for up-to-date results) -display-coverage: coverage.out - go tool cover -html=coverage.out - # -------------------------------------------------------------------------- # # Internal targets: @@ -172,6 +172,10 @@ table_header: @echo "| **Command line option** | **JSON option** | **Environment variable** (container-only) | **Description** |" @echo "| --- | --- | --- | --- |" +coverage.out: test_coverage + + + # Help target borrowed from: https://docs.cloudposse.com/reference/best-practices/make-best-practices/ ## This help screen help: diff --git a/go.mod b/go.mod index f06defe6..fd37ae1d 100644 --- a/go.mod +++ b/go.mod @@ -42,11 +42,11 @@ require ( github.com/twitchyliquid64/golang-asm v0.15.1 // indirect github.com/ugorji/go/codec v1.2.11 // indirect golang.org/x/arch v0.3.0 // indirect - golang.org/x/crypto v0.9.0 // indirect - golang.org/x/net v0.10.0 // indirect + golang.org/x/crypto v0.14.0 // indirect + golang.org/x/net v0.17.0 // indirect golang.org/x/sync v0.3.0 // indirect golang.org/x/sys v0.13.0 // indirect - golang.org/x/text v0.9.0 // indirect + golang.org/x/text v0.13.0 // indirect google.golang.org/protobuf v1.30.0 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect ) diff --git a/go.sum b/go.sum index 935bc428..5bf34df0 100644 --- a/go.sum +++ b/go.sum @@ -122,13 +122,13 @@ golang.org/x/arch v0.0.0-20210923205945-b76863e36670/go.mod h1:5om86z9Hs0C8fWVUu golang.org/x/arch v0.3.0 h1:02VY4/ZcO/gBOH6PUaoiptASxtXU10jazRCP865E97k= golang.org/x/arch v0.3.0/go.mod h1:5om86z9Hs0C8fWVUuoMHwpExlXzs5Tkyp9hOrfG7pp8= golang.org/x/crypto v0.0.0-20210711020723-a769d52b0f97/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= -golang.org/x/crypto v0.9.0 h1:LF6fAI+IutBocDJ2OT0Q1g8plpYljMZ4+lty+dsqw3g= -golang.org/x/crypto v0.9.0/go.mod h1:yrmDGqONDYtNj3tH8X9dzUun2m2lzPa9ngI6/RUPGR0= +golang.org/x/crypto v0.14.0 h1:wBqGXzWJW6m1XrIKlAH0Hs1JJ7+9KBwnIO8v66Q9cHc= +golang.org/x/crypto v0.14.0/go.mod h1:MVFd36DqK4CsrnJYDkBA3VC4m2GkXAM0PvzMCn4JQf4= golang.org/x/exp v0.0.0-20231006140011-7918f672742d h1:jtJma62tbqLibJ5sFQz8bKtEM8rJBtfilJ2qTU199MI= golang.org/x/exp v0.0.0-20231006140011-7918f672742d/go.mod h1:ldy0pHrwJyGW56pPQzzkH36rKxoZW1tw7ZJpeKx+hdo= golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= -golang.org/x/net v0.10.0 h1:X2//UzNDwYmtCLn7To6G58Wr6f5ahEAQgKNzv9Y951M= -golang.org/x/net v0.10.0/go.mod h1:0qNGK6F8kojg2nk9dLZ2mShWaEBan6FAoqfSigmmuDg= +golang.org/x/net v0.17.0 h1:pVaXccu2ozPjCXewfr1S7xza/zcXTity9cCdXQYSjIM= +golang.org/x/net v0.17.0/go.mod h1:NxSsAGuq816PNPmqtQdLE42eU2Fs7NoRIZrHJAlaCOE= golang.org/x/sync v0.3.0 h1:ftCYgMx6zT/asHUrPw8BLLscYtGznsLAnjq5RH9P66E= golang.org/x/sync v0.3.0/go.mod h1:FU7BRWz2tNW+3quACPkgCx/L+uEAv1htQ0V83Z9Rj+Y= golang.org/x/sys v0.0.0-20200923182605-d9f96fdee20d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -143,8 +143,8 @@ golang.org/x/sys v0.13.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= -golang.org/x/text v0.9.0 h1:2sjJmO8cDvYveuX97RDLsxlyUxLl+GHoLxBiRdHllBE= -golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= +golang.org/x/text v0.13.0 h1:ablQoSUd0tRdKxZewP80B+BaqeKJuVhuRxj/dkrun3k= +golang.org/x/text v0.13.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= diff --git a/splitio/commitversion.go b/splitio/commitversion.go index 75a51623..bee01bac 100644 --- a/splitio/commitversion.go +++ b/splitio/commitversion.go @@ -5,4 +5,4 @@ This file is created automatically, please do not edit */ // CommitVersion is the version of the last commit previous to release -const CommitVersion = "fccb322" +const CommitVersion = "cf3da63" diff --git a/splitio/proxy/storage/persistent/mocks/segment.go b/splitio/proxy/storage/persistent/mocks/segment.go new file mode 100644 index 00000000..e69d86ef --- /dev/null +++ b/splitio/proxy/storage/persistent/mocks/segment.go @@ -0,0 +1,28 @@ +package mocks + +import ( + "github.com/splitio/go-toolkit/v5/datastructures/set" + "github.com/splitio/split-synchronizer/v5/splitio/proxy/storage/persistent" + "github.com/stretchr/testify/mock" +) + +type SegmentChangesCollectionMock struct { + mock.Mock +} + +func (s *SegmentChangesCollectionMock) Update(name string, toAdd *set.ThreadUnsafeSet, toRemove *set.ThreadUnsafeSet, cn int64) error { + return s.Called(name, toAdd, toRemove, cn).Error(0) +} + +func (s *SegmentChangesCollectionMock) Fetch(name string) (*persistent.SegmentChangesItem, error) { + args := s.Called(name) + return args.Get(0).(*persistent.SegmentChangesItem), args.Error(1) +} + +func (s *SegmentChangesCollectionMock) ChangeNumber(segment string) int64 { + return s.Called(segment).Get(0).(int64) +} + +func (s *SegmentChangesCollectionMock) SetChangeNumber(segment string, cn int64) { + s.Called(segment, cn) +} diff --git a/splitio/proxy/storage/persistent/segments.go b/splitio/proxy/storage/persistent/segments.go index f7a8a295..0d95ad49 100644 --- a/splitio/proxy/storage/persistent/segments.go +++ b/splitio/proxy/storage/persistent/segments.go @@ -25,8 +25,15 @@ type SegmentChangesItem struct { Keys map[string]SegmentKey } -// SegmentChangesCollection represents a collection of SplitChangesItem -type SegmentChangesCollection struct { +type SegmentChangesCollection interface { + Update(name string, toAdd *set.ThreadUnsafeSet, toRemove *set.ThreadUnsafeSet, cn int64) error + Fetch(name string) (*SegmentChangesItem, error) + ChangeNumber(segment string) int64 + SetChangeNumber(segment string, cn int64) +} + +// SegmentChangesCollectionImpl represents a collection of SplitChangesItem +type SegmentChangesCollectionImpl struct { collection CollectionWrapper segmentsTill map[string]int64 logger logging.LoggerInterface @@ -34,8 +41,8 @@ type SegmentChangesCollection struct { } // NewSegmentChangesCollection returns an instance of SegmentChangesCollection -func NewSegmentChangesCollection(db DBWrapper, logger logging.LoggerInterface) *SegmentChangesCollection { - return &SegmentChangesCollection{ +func NewSegmentChangesCollection(db DBWrapper, logger logging.LoggerInterface) *SegmentChangesCollectionImpl { + return &SegmentChangesCollectionImpl{ collection: &BoltDBCollectionWrapper{db: db, name: segmentChangesCollectionName, logger: logger}, segmentsTill: make(map[string]int64, 0), logger: logger, @@ -43,7 +50,7 @@ func NewSegmentChangesCollection(db DBWrapper, logger logging.LoggerInterface) * } // Update persists a segmentChanges update -func (c *SegmentChangesCollection) Update(name string, toAdd *set.ThreadUnsafeSet, toRemove *set.ThreadUnsafeSet, cn int64) error { +func (c *SegmentChangesCollectionImpl) Update(name string, toAdd *set.ThreadUnsafeSet, toRemove *set.ThreadUnsafeSet, cn int64) error { c.mutex.Lock() defer c.mutex.Unlock() @@ -63,17 +70,10 @@ func (c *SegmentChangesCollection) Update(name string, toAdd *set.ThreadUnsafeSe continue } c.logger.Debug("Removing", strKey, "from", name) - if _, exists := segmentItem.Keys[strKey]; exists { - itemAux := segmentItem.Keys[strKey] - itemAux.Removed = true - itemAux.ChangeNumber = cn - segmentItem.Keys[strKey] = itemAux - } else { - segmentItem.Keys[strKey] = SegmentKey{ - Name: strKey, - Removed: true, - ChangeNumber: cn, - } + segmentItem.Keys[strKey] = SegmentKey{ + Name: strKey, + Removed: true, + ChangeNumber: cn, } } @@ -85,17 +85,10 @@ func (c *SegmentChangesCollection) Update(name string, toAdd *set.ThreadUnsafeSe continue } c.logger.Debug("Adding", strKey, "in", name) - if _, exists := segmentItem.Keys[strKey]; exists { - itemAux := segmentItem.Keys[strKey] - itemAux.Removed = false - itemAux.ChangeNumber = cn - segmentItem.Keys[strKey] = itemAux - } else { - segmentItem.Keys[strKey] = SegmentKey{ - Name: strKey, - Removed: false, - ChangeNumber: cn, - } + segmentItem.Keys[strKey] = SegmentKey{ + Name: strKey, + Removed: false, + ChangeNumber: cn, } } @@ -108,13 +101,13 @@ func (c *SegmentChangesCollection) Update(name string, toAdd *set.ThreadUnsafeSe } // Fetch return a SegmentChangesItem -func (c *SegmentChangesCollection) Fetch(name string) (*SegmentChangesItem, error) { +func (c *SegmentChangesCollectionImpl) Fetch(name string) (*SegmentChangesItem, error) { c.mutex.RLock() defer c.mutex.RUnlock() return c.fetch(name) } -func (c *SegmentChangesCollection) fetch(name string) (*SegmentChangesItem, error) { +func (c *SegmentChangesCollectionImpl) fetch(name string) (*SegmentChangesItem, error) { item, err := c.collection.FetchBy([]byte(name)) if err != nil { return nil, err @@ -133,7 +126,7 @@ func (c *SegmentChangesCollection) fetch(name string) (*SegmentChangesItem, erro } // FetchAll return a list of SegmentChangesItem -func (c *SegmentChangesCollection) FetchAll() ([]SegmentChangesItem, error) { +func (c *SegmentChangesCollectionImpl) FetchAll() ([]SegmentChangesItem, error) { c.mutex.RLock() defer c.mutex.RUnlock() items, err := c.collection.FetchAll() @@ -163,7 +156,7 @@ func (c *SegmentChangesCollection) FetchAll() ([]SegmentChangesItem, error) { } // ChangeNumber returns changeNumber -func (c *SegmentChangesCollection) ChangeNumber(segment string) int64 { +func (c *SegmentChangesCollectionImpl) ChangeNumber(segment string) int64 { c.mutex.RLock() defer c.mutex.RUnlock() value, exists := c.segmentsTill[segment] @@ -174,8 +167,10 @@ func (c *SegmentChangesCollection) ChangeNumber(segment string) int64 { } // SetChangeNumber returns changeNumber -func (c *SegmentChangesCollection) SetChangeNumber(segment string, cn int64) { +func (c *SegmentChangesCollectionImpl) SetChangeNumber(segment string, cn int64) { c.mutex.Lock() defer c.mutex.Unlock() c.segmentsTill[segment] = cn } + +var _ SegmentChangesCollection = (*SegmentChangesCollectionImpl)(nil) diff --git a/splitio/proxy/storage/segments.go b/splitio/proxy/storage/segments.go index 65168b5e..48fe5bc3 100644 --- a/splitio/proxy/storage/segments.go +++ b/splitio/proxy/storage/segments.go @@ -29,7 +29,7 @@ type ProxySegmentStorage interface { type ProxySegmentStorageImpl struct { logger logging.LoggerInterface nameCountCache *observability.ActiveSegmentTracker - db *persistent.SegmentChangesCollection + db persistent.SegmentChangesCollection mysegments optimized.MySegmentsCache } @@ -68,21 +68,25 @@ func (s *ProxySegmentStorageImpl) ChangesSince(name string, since int64) (*dtos. // Horrible loop borrowed from sdk-api for _, skey := range item.Keys { + if skey.ChangeNumber <= since { // if the key was updated in a previous/current CN, we don't need to return it continue } + if skey.Removed && since < 0 { + // removed keys should not be returned on initialization payloads + continue + } + // Add the key to the corresponding list - if skey.Removed && since > 0 { + if skey.Removed { removed = append(removed, skey.Name) } else { added = append(added, skey.Name) } // Update the till to be returned if necessary - if since > 0 && skey.ChangeNumber > till { - till = skey.ChangeNumber - } else if !skey.Removed && skey.ChangeNumber > till { + if skey.ChangeNumber > till { till = skey.ChangeNumber } } @@ -177,7 +181,7 @@ func (s *ProxySegmentStorageImpl) NamesAndCount() map[string]int { func populateCachesFromDisk( dst optimized.MySegmentsCache, names *observability.ActiveSegmentTracker, - src *persistent.SegmentChangesCollection, + src *persistent.SegmentChangesCollectionImpl, logger logging.LoggerInterface, ) { all, err := src.FetchAll() diff --git a/splitio/proxy/storage/segments_test.go b/splitio/proxy/storage/segments_test.go new file mode 100644 index 00000000..eba084c1 --- /dev/null +++ b/splitio/proxy/storage/segments_test.go @@ -0,0 +1,75 @@ +package storage + +import ( + "testing" + + "github.com/splitio/go-toolkit/v5/logging" + "github.com/splitio/split-synchronizer/v5/splitio/proxy/storage/optimized" + "github.com/splitio/split-synchronizer/v5/splitio/proxy/storage/persistent" + "github.com/splitio/split-synchronizer/v5/splitio/proxy/storage/persistent/mocks" + "github.com/stretchr/testify/assert" +) + +func TestSegmentStorage(t *testing.T) { + + psm := &mocks.SegmentChangesCollectionMock{} + psm.On("Fetch", "some").Return(&persistent.SegmentChangesItem{ + Name: "some", + Keys: map[string]persistent.SegmentKey{ + "k1": {Name: "k1", ChangeNumber: 1, Removed: false}, + "k2": {Name: "k2", ChangeNumber: 1, Removed: true}, + "k3": {Name: "k3", ChangeNumber: 2, Removed: false}, + "k4": {Name: "k4", ChangeNumber: 2, Removed: true}, + "k5": {Name: "k5", ChangeNumber: 3, Removed: false}, + "k6": {Name: "k6", ChangeNumber: 3, Removed: true}, + "k7": {Name: "k7", ChangeNumber: 4, Removed: false}, + }, + }, nil) + + ss := ProxySegmentStorageImpl{ + logger: logging.NewLogger(nil), + db: psm, + mysegments: optimized.NewMySegmentsCache(), + } + + changes, err := ss.ChangesSince("some", -1) + assert.Nil(t, err) + assert.Equal(t, "some", changes.Name) + assert.ElementsMatch(t, []string{"k1", "k3", "k5", "k7"}, changes.Added) + assert.ElementsMatch(t, []string{}, changes.Removed) + assert.Equal(t, int64(-1), changes.Since) + assert.Equal(t, int64(4), changes.Till) + + changes, err = ss.ChangesSince("some", 1) + assert.Nil(t, err) + assert.Equal(t, "some", changes.Name) + assert.ElementsMatch(t, []string{"k3", "k5", "k7"}, changes.Added) + assert.ElementsMatch(t, []string{"k4", "k6"}, changes.Removed) + assert.Equal(t, int64(1), changes.Since) + assert.Equal(t, int64(4), changes.Till) + + changes, err = ss.ChangesSince("some", 2) + assert.Nil(t, err) + assert.Equal(t, "some", changes.Name) + assert.ElementsMatch(t, []string{"k5", "k7"}, changes.Added) + assert.ElementsMatch(t, []string{"k6"}, changes.Removed) + assert.Equal(t, int64(2), changes.Since) + assert.Equal(t, int64(4), changes.Till) + + changes, err = ss.ChangesSince("some", 3) + assert.Nil(t, err) + assert.Equal(t, "some", changes.Name) + assert.ElementsMatch(t, []string{"k7"}, changes.Added) + assert.ElementsMatch(t, []string{}, changes.Removed) + assert.Equal(t, int64(3), changes.Since) + assert.Equal(t, int64(4), changes.Till) + + changes, err = ss.ChangesSince("some", 4) + assert.Nil(t, err) + assert.Equal(t, "some", changes.Name) + assert.ElementsMatch(t, []string{}, changes.Added) + assert.ElementsMatch(t, []string{}, changes.Removed) + assert.Equal(t, int64(4), changes.Since) + assert.Equal(t, int64(4), changes.Till) + +}