Merge branch 'master' of github.com:splitio/split-synchronizer

splitio · Dec 16, 2024 · 599fa3f · 599fa3f
2 parents dc0722a + 17d8154
commit 599fa3f
Show file tree

Hide file tree

Showing 33 changed files with 996 additions and 279 deletions.
diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml
@@ -36,25 +36,31 @@ jobs:
       - name: Set up Docker Buildx
         uses: docker/setup-buildx-action@v3
 
+      - name: Login to Dockerhub
+        uses: docker/login-action@v3
+        with:
+          username: ${{ vars.DOCKERHUB_USERNAME }}
+          password: ${{ secrets.DOCKERHUB_RO_TOKEN }}
+
       - name: Login to Artifactory
         if: ${{ github.event_name == 'push' }}
         uses: docker/login-action@v3
         with:
-          registry: splitio-docker.jfrog.io
-          username: ${{ secrets.ARTIFACTORY_DOCKER_USER }}
+          registry: ${{ vars.ARTIFACTORY_DOCKER_REGISTRY }}
+          username: ${{ vars.ARTIFACTORY_DOCKER_USER }}
           password: ${{ secrets.ARTIFACTORY_DOCKER_PASS }}
 
       - name: Get version
         run: echo "VERSION=$(awk '/^const Version/{gsub(/"/, "", $4); print $4}' splitio/version.go)" >> $GITHUB_ENV
 
       - name: Docker Build and Push
-        uses: docker/build-push-action@v5
+        uses: docker/build-push-action@v6
         with:
           context: .
           file: docker/Dockerfile.${{ matrix.app }}
           push: ${{ github.event_name == 'push' }}
           platforms: linux/amd64,linux/arm64
-          tags: splitio-docker.jfrog.io/split-${{ matrix.app }}${{ matrix.fips_mode == 'enabled' && '-fips' || ''}}:${{ env.VERSION }},splitio-docker.jfrog.io/split-${{ matrix.app }}${{ matrix.fips_mode == 'enabled' && '-fips' || '' }}:latest
+          tags: ${{ vars.ARTIFACTORY_DOCKER_REGISTRY }}/split-${{ matrix.app }}${{ matrix.fips_mode == 'enabled' && '-fips' || ''}}:${{ env.VERSION }},${{ vars.ARTIFACTORY_DOCKER_REGISTRY }}/split-${{ matrix.app }}${{ matrix.fips_mode == 'enabled' && '-fips' || '' }}:latest
           build-args: |
             FIPS_MODE=${{ matrix.fips_mode }}
 
@@ -77,21 +83,28 @@ jobs:
       - name: Get version
         run: echo "VERSION=$(awk '/^const Version/{gsub(/"/, "", $4); print $4}' splitio/version.go)" >> $GITHUB_ENV
 
+      - name: Login to Dockerhub
+        uses: docker/login-action@v3
+        with:
+          username: ${{ vars.DOCKERHUB_USERNAME }}
+          password: ${{ secrets.DOCKERHUB_RO_TOKEN }}
+
       - name: Docker Build and Push
-        uses: docker/build-push-action@v5
+        uses: docker/build-push-action@v6
         with:
           context: .
           file: docker/Dockerfile.${{ matrix.app }}
           push: false
-          tags: splitio-docker.jfrog.io/split-${{ matrix.app }}${{ matrix.fips_mode == 'enabled' && '-fips' || ''}}:${{ env.VERSION }}
+          tags: ${{ vars.ARTIFACTORY_DOCKER_REGISTRY }}/split-${{ matrix.app }}${{ matrix.fips_mode == 'enabled' && '-fips' || ''}}:${{ env.VERSION }}
           build-args: |
             FIPS_MODE=${{ matrix.fips_mode }}
 
       - name: Scan container using Lacework
-        uses: lacework/[email protected].1
+        uses: lacework/[email protected].3
         with:
-          LW_ACCOUNT_NAME: ${{ secrets.LW_ACCOUNT_NAME }}
+          LW_ACCOUNT_NAME: ${{ vars.LW_ACCOUNT_NAME }}
           LW_ACCESS_TOKEN: ${{ secrets.LW_ACCESS_TOKEN }}
-          IMAGE_NAME: splitio-docker.jfrog.io/split-${{ matrix.app }}${{ matrix.fips_mode == 'enabled' && '-fips' || ''}}
+          IMAGE_NAME: ${{ vars.ARTIFACTORY_DOCKER_REGISTRY }}/split-${{ matrix.app }}${{ matrix.fips_mode == 'enabled' && '-fips' || ''}}
           IMAGE_TAG: ${{ env.VERSION }}
           SAVE_RESULTS_IN_LACEWORK: true
+          RESULTS_IN_GITHUB_SUMMARY: true
diff --git a/.github/workflows/s3.yml b/.github/workflows/s3.yml
@@ -0,0 +1,48 @@
+name: cd
+
+on:
+  push:
+    branches:
+      - master
+  pull_request:
+    branches:
+      - master
+
+permissions:
+  contents: read
+  id-token: write
+
+jobs:
+  build-publish:
+    name: Build and publish to S3
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+
+      - name: Setup Go
+        uses: actions/setup-go@v5
+        with:
+          go-version: '1.23.3'
+
+      - name: Create build folder
+        run: mkdir -p build
+
+      - name: Execute build
+        run: make release_assets
+
+      - name: Configure AWS credentials
+        if: ${{ github.event_name == 'push' }}
+        uses: aws-actions/configure-aws-credentials@v4
+        with:
+          role-to-assume: arn:aws:iam::825951051969:role/gha-downloads-role
+          aws-region: us-east-1
+
+      - name: Deploy to S3
+        if: ${{ github.event_name == 'push' }}
+        run: aws s3 sync $SOURCE_DIR s3://$BUCKET
+        env:
+          BUCKET: downloads.split.io
+          SOURCE_DIR: ./build
diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml
@@ -1,6 +1,9 @@
 name: test
 
 on:
+  push:
+    branches:
+      - master
   pull_request:
     branches-ignore:
       - none
@@ -12,6 +15,9 @@ jobs:
     services:
       redis:
         image: redis
+        credentials:
+          username: ${{ vars.DOCKERHUB_USERNAME }}
+          password: ${{ secrets.DOCKERHUB_RO_TOKEN }}
         ports:
           - 6379:6379
     steps:
@@ -23,22 +29,17 @@ jobs:
       - name: Setup Go
         uses: actions/setup-go@v5
         with:
-          go-version: '^1.23.3'
+          go-version: '1.23.3'
 
       - name: Get version
         run: echo "VERSION=$(awk '/^const Version/{gsub(/"/, "", $4); print $4}' splitio/version.go)" >> $GITHUB_ENV
 
       - name: Run test
         run: make test_coverage
 
-      - name: Create build folder
-        run: mkdir -p build
-
-      - name: Build assets
-        run: make release_assets
-
       - name: SonarQube Scan (Pull Request)
-        uses: SonarSource/sonarcloud-github-action@v2
+        if: ${{ github.event_name == 'pull_request' }}
+        uses: SonarSource/sonarcloud-github-action@v3
         env:
           SONAR_TOKEN: ${{ secrets.SONARQUBE_TOKEN }}
         with:
@@ -49,3 +50,14 @@ jobs:
             -Dsonar.pullrequest.key=${{ github.event.pull_request.number }}
             -Dsonar.pullrequest.branch=${{ github.event.pull_request.head.ref }}
             -Dsonar.pullrequest.base=${{ github.event.pull_request.base.ref }}
+
+      - name: SonarQube Scan (Push)
+        if: ${{ github.event_name == 'push' }}
+        uses: SonarSource/sonarcloud-github-action@v3
+        env:
+          SONAR_TOKEN: ${{ secrets.SONARQUBE_TOKEN }}
+        with:
+          projectBaseDir: .
+          args: >
+            -Dsonar.host.url=${{ vars.SONARQUBE_HOST }}
+            -Dsonar.projectVersion=${{ env.VERSION }}
diff --git a/.github/workflows/unstable.yml b/.github/workflows/unstable.yml
@@ -29,18 +29,24 @@ jobs:
       - name: Set up Docker Buildx
         uses: docker/setup-buildx-action@v3
 
+      - name: Login to Dockerhub
+        uses: docker/login-action@v3
+        with:
+          username: ${{ vars.DOCKERHUB_USERNAME }}
+          password: ${{ secrets.DOCKERHUB_RO_TOKEN }}
+
       - name: Login to Artifactory
         uses: docker/login-action@v3
         with:
           registry: splitio-docker-dev.jfrog.io
-          username: ${{ secrets.ARTIFACTORY_DOCKER_USER }}
+          username: ${{ vars.ARTIFACTORY_DOCKER_USER }}
           password: ${{ secrets.ARTIFACTORY_DOCKER_PASS }}
 
       - name: Get short hash
         run: echo "SHORT_SHA=$(git rev-parse --short HEAD)" >> $GITHUB_ENV
 
       - name: Docker Build and Push
-        uses: docker/build-push-action@v5
+        uses: docker/build-push-action@v6
         with:
           context: .
           file: docker/Dockerfile.${{ matrix.app }}

diff --git a/CHANGES.txt b/CHANGES.txt
@@ -1,3 +1,9 @@
+
+5.9.0 (Dec 2, 2024)
+- Split Proxy:
+   - Added support for targeting rules based on large segments.
+   - Added `/memberships` endpoint to support SDKs running with large segments.
+
 5.8.3 (Nov 22, 2024)
 - Fixed vulnerabilities:
    - Updated debian image to 12.8

diff --git a/go.mod b/go.mod
@@ -8,7 +8,7 @@ require (
 	github.com/gin-gonic/gin v1.10.0
 	github.com/google/uuid v1.3.0
 	github.com/splitio/gincache v1.0.1
-	github.com/splitio/go-split-commons/v6 v6.0.1
+	github.com/splitio/go-split-commons/v6 v6.0.2
 	github.com/splitio/go-toolkit/v5 v5.4.0
 	github.com/stretchr/testify v1.9.0
 	go.etcd.io/bbolt v1.3.6

diff --git a/go.sum b/go.sum
@@ -93,8 +93,8 @@ github.com/rogpeppe/go-internal v1.8.0 h1:FCbCCtXNOY3UtUuHUYaghJg4y7Fd14rXifAYUA
 github.com/rogpeppe/go-internal v1.8.0/go.mod h1:WmiCO8CzOY8rg0OYDC4/i/2WRWAB6poM+XZ2dLUbcbE=
 github.com/splitio/gincache v1.0.1 h1:dLYdANY/BqH4KcUMCe/LluLyV5WtuE/LEdQWRE06IXU=
 github.com/splitio/gincache v1.0.1/go.mod h1:CcgJDSM9Af75kyBH0724v55URVwMBuSj5x1eCWIOECY=
-github.com/splitio/go-split-commons/v6 v6.0.1 h1:WJcvTk8lwWw6kLQvxt8hOkY/tGlBN4w+2agkINPGugY=
-github.com/splitio/go-split-commons/v6 v6.0.1/go.mod h1:TsvIh3XP7yjc7ly4vpj06AkoBND36SodPs5qfhb8rHc=
+github.com/splitio/go-split-commons/v6 v6.0.2 h1:uvrNjyGCOHUjxVTB1pDUA+UB20WypoxpeLkgkgX5v+U=
+github.com/splitio/go-split-commons/v6 v6.0.2/go.mod h1:D/XIY/9Hmfk9ivWsRsJVp439kEdmHbzUi3PKzQQDOXY=
 github.com/splitio/go-toolkit/v5 v5.4.0 h1:g5WFpRhQomnXCmvfsNOWV4s5AuUrWIZ+amM68G8NBKM=
 github.com/splitio/go-toolkit/v5 v5.4.0/go.mod h1:xYhUvV1gga9/1029Wbp5pjnR6Cy8nvBpjw99wAbsMko=
 github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=

diff --git a/splitio/admin/admin.go b/splitio/admin/admin.go
@@ -23,23 +23,24 @@ const baseShutdownPath = "/shutdown"
 
 // Options encapsulates dependencies & config options for the Admin server
 type Options struct {
-	Host              string
-	Port              int
-	Name              string
-	Proxy             bool
-	Username          string
-	Password          string
-	Logger            logging.LoggerInterface
-	Storages          adminCommon.Storages
-	ImpressionsEvCalc evcalc.Monitor
-	EventsEvCalc      evcalc.Monitor
-	Runtime           common.Runtime
-	HcAppMonitor      application.MonitorIterface
-	HcServicesMonitor services.MonitorIterface
-	Snapshotter       cstorage.Snapshotter
-	TLS               *tls.Config
-	FullConfig        interface{}
-	FlagSpecVersion   string
+	Host                string
+	Port                int
+	Name                string
+	Proxy               bool
+	Username            string
+	Password            string
+	Logger              logging.LoggerInterface
+	Storages            adminCommon.Storages
+	ImpressionsEvCalc   evcalc.Monitor
+	EventsEvCalc        evcalc.Monitor
+	Runtime             common.Runtime
+	HcAppMonitor        application.MonitorIterface
+	HcServicesMonitor   services.MonitorIterface
+	Snapshotter         cstorage.Snapshotter
+	TLS                 *tls.Config
+	FullConfig          interface{}
+	FlagSpecVersion     string
+	LargeSegmentVersion string
 }
 
 type AdminServer struct {
@@ -68,6 +69,7 @@ func NewServer(options *Options) (*AdminServer, error) {
 		options.Runtime,
 		options.HcAppMonitor,
 		options.FlagSpecVersion,
+		options.LargeSegmentVersion,
 	)
 	if err != nil {
 		return nil, fmt.Errorf("error instantiating dashboard controller: %w", err)

diff --git a/splitio/admin/common/config.go b/splitio/admin/common/config.go
@@ -10,4 +10,5 @@ type Storages struct {
 	EventStorage          storage.EventMultiSdkConsumer
 	ImpressionStorage     storage.ImpressionMultiSdkConsumer
 	UniqueKeysStorage     storage.UniqueKeysMultiSdkConsumer
+	LargeSegmentStorage   storage.LargeSegmentsStorage
 }
diff --git a/splitio/admin/controllers/dashboard.go b/splitio/admin/controllers/dashboard.go
@@ -20,16 +20,17 @@ import (
 
 // DashboardController contains handlers for rendering the dashboard and its associated FE queries
 type DashboardController struct {
-	title             string
-	proxy             bool
-	logger            logging.LoggerInterface
-	storages          adminCommon.Storages
-	layout            *template.Template
-	impressionsEvCalc evcalc.Monitor
-	eventsEvCalc      evcalc.Monitor
-	runtime           common.Runtime
-	appMonitor        application.MonitorIterface
-	FlagSpecVersion   string
+	title               string
+	proxy               bool
+	logger              logging.LoggerInterface
+	storages            adminCommon.Storages
+	layout              *template.Template
+	impressionsEvCalc   evcalc.Monitor
+	eventsEvCalc        evcalc.Monitor
+	runtime             common.Runtime
+	appMonitor          application.MonitorIterface
+	FlagSpecVersion     string
+	LargeSegmentVersion string
 }
 
 // NewDashboardController instantiates a new dashboard controller
@@ -43,18 +44,20 @@ func NewDashboardController(
 	runtime common.Runtime,
 	appMonitor application.MonitorIterface,
 	flagSpecVersion string,
+	largeSegmentVersion string,
 ) (*DashboardController, error) {
 
 	toReturn := &DashboardController{
-		title:             name,
-		proxy:             proxy,
-		logger:            logger,
-		runtime:           runtime,
-		storages:          storages,
-		eventsEvCalc:      eventsEvCalc,
-		impressionsEvCalc: impressionEvCalc,
-		appMonitor:        appMonitor,
-		FlagSpecVersion:   flagSpecVersion,
+		title:               name,
+		proxy:               proxy,
+		logger:              logger,
+		runtime:             runtime,
+		storages:            storages,
+		eventsEvCalc:        eventsEvCalc,
+		impressionsEvCalc:   impressionEvCalc,
+		appMonitor:          appMonitor,
+		FlagSpecVersion:     flagSpecVersion,
+		LargeSegmentVersion: largeSegmentVersion,
 	}
 
 	var err error
@@ -146,6 +149,7 @@ func (c *DashboardController) gatherStats() *dashboard.GlobalStats {
 	return &dashboard.GlobalStats{
 		FeatureFlags:           bundleSplitInfo(c.storages.SplitStorage),
 		Segments:               bundleSegmentInfo(c.storages.SplitStorage, c.storages.SegmentStorage),
+		LargeSegments:          bundleLargeSegmentInfo(c.storages.SplitStorage, c.storages.LargeSegmentStorage),
 		Latencies:              bundleProxyLatencies(c.storages.LocalTelemetryStorage),
 		BackendLatencies:       bundleLocalSyncLatencies(c.storages.LocalTelemetryStorage),
 		ImpressionsQueueSize:   getImpressionSize(c.storages.ImpressionStorage),

diff --git a/splitio/admin/controllers/helpers.go b/splitio/admin/controllers/helpers.go
@@ -48,6 +48,30 @@ func bundleSplitInfo(splitStorage storage.SplitStorageConsumer) []dashboard.Spli
 	return summaries
 }
 
+func bundleLargeSegmentInfo(splitStorage storage.SplitStorage, lsStorage storage.LargeSegmentStorageConsumer) []dashboard.LargeSegmentSummary {
+	if lsStorage == nil {
+		return []dashboard.LargeSegmentSummary{}
+	}
+
+	lsNames := splitStorage.LargeSegmentNames()
+
+	toReturn := make([]dashboard.LargeSegmentSummary, 0, lsNames.Size())
+	for _, name := range lsNames.List() {
+		strName, ok := name.(string)
+		if !ok {
+			continue
+		}
+		cn := lsStorage.ChangeNumber(strName)
+		toReturn = append(toReturn, dashboard.LargeSegmentSummary{
+			Name:         strName,
+			TotalKeys:    lsStorage.TotalKeys(strName),
+			LastModified: time.Unix(0, cn*int64(time.Millisecond)).UTC().Format(time.UnixDate),
+		})
+	}
+
+	return toReturn
+}
+
 func bundleSegmentInfo(splitStorage storage.SplitStorage, segmentStorage storage.SegmentStorageConsumer) []dashboard.SegmentSummary {
 	names := splitStorage.SegmentNames()
 	summaries := make([]dashboard.SegmentSummary, 0, names.Size())