From ad5ba46dfce2b776cb4888831d1af0c49fbcb3bc Mon Sep 17 00:00:00 2001 From: Martin Redolatti Date: Wed, 5 Jul 2023 16:11:40 -0300 Subject: [PATCH 01/42] add release action --- .github/workflows/ci.yml | 16 ++++++++++++++-- Makefile | 16 ++++++++-------- 2 files changed, 22 insertions(+), 10 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 65fe13d..11d073a 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -2,10 +2,12 @@ name: ci on: pull_request: branches: - - dev + - main push: branches: - - dev + - main + tags: + - "v*.*.*" concurrency: group: ${{ github.workflow }}-${{ github.event_name == 'push' && github.run_number || github.event.pull_request.number }} @@ -62,6 +64,16 @@ jobs: - name: Run tests run: make test + - name: Release + uses: softprops/action-gh-release@v1 + if: ${{ github.event_name == 'push' && startsWith(github.ref, 'refs/tags/') }} + with: + files: + - splitd-darwin-amd64-1.0.0.bin + - splitd-darwin-arm-1.0.0.bin + - splitd-linux-amd64-1.0.0.bin + - splitd-linux-arm-1.0.0.bin + # - name: SonarQube Scan (Push) # if: matrix.version == '8.2' && github.event_name == 'push' # uses: SonarSource/sonarcloud-github-action@v1.9 diff --git a/Makefile b/Makefile index 06ac9ff..0dd3de8 100644 --- a/Makefile +++ b/Makefile @@ -19,10 +19,10 @@ go.sum: go.mod clean: rm -Rf splitcli \ splitd \ - splitd.linux.amd64.$(VERSION).bin \ - splitd.darwin.amd64.$(VERSION).bin \ - splitd.linux.arm.$(VERSION).bin \ - splitd.darwin.arm.$(VERSION).bin + splitd-linux-amd64-$(VERSION).bin \ + splitd-darwin-amd64-$(VERSION).bin \ + splitd-linux-arm-$(VERSION).bin \ + splitd-darwin-arm-$(VERSION).bin ## build binaries for this platform build: splitd splitcli @@ -57,16 +57,16 @@ images_release: # entrypoints binaries_release: splitd.linux.amd64.$(VERSION).bin splitd.darwin.amd64.$(VERSION).bin splitd.linux.arm.$(VERSION).bin splitd.darwin.arm.$(VERSION).bin -splitd.linux.amd64.$(VERSION).bin: $(GO_FILES) +splitd-linux-amd64-$(VERSION).bin: $(GO_FILES) GOARCH=amd64 GOOS=linux $(GO) build -o $@ cmd/splitd/main.go -splitd.darwin.amd64.$(VERSION).bin: $(GO_FILES) +splitd-darwin-amd64-$(VERSION).bin: $(GO_FILES) GOARCH=amd64 GOOS=darwin $(GO) build -o $@ cmd/splitd/main.go -splitd.linux.arm.$(VERSION).bin: $(GO_FILES) +splitd-linux-arm-$(VERSION).bin: $(GO_FILES) GOARCH=arm64 GOOS=linux $(GO) build -o $@ cmd/splitd/main.go -splitd.darwin.arm.$(VERSION).bin: $(GO_FILES) +splitd-darwin-arm-$(VERSION).bin: $(GO_FILES) GOARCH=arm64 GOOS=darwin $(GO) build -o $@ cmd/splitd/main.go binaries_release: splitd From c3f53f69e1232f73825b2de4d174a7b8a5691538 Mon Sep 17 00:00:00 2001 From: Martin Redolatti Date: Wed, 5 Jul 2023 16:16:50 -0300 Subject: [PATCH 02/42] parametrize version --- .github/workflows/ci.yml | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 11d073a..80ebb0b 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -43,7 +43,7 @@ jobs: with: github_token: ${{ secrets.GITHUB_TOKEN }} custom_tag: ${{ env.VERSION }} - tag_prefix: '' + tag_prefix: 'v' - name: Setup Go version uses: actions/setup-go@v4 @@ -69,10 +69,10 @@ jobs: if: ${{ github.event_name == 'push' && startsWith(github.ref, 'refs/tags/') }} with: files: - - splitd-darwin-amd64-1.0.0.bin - - splitd-darwin-arm-1.0.0.bin - - splitd-linux-amd64-1.0.0.bin - - splitd-linux-arm-1.0.0.bin + - splitd-darwin-amd64-${{ env.VERSION }}.bin + - splitd-darwin-arm-${{ env.VERSION }}.bin + - splitd-linux-amd64-${{ env.VERSION }}.bin + - splitd-linux-arm-${{ env.VERSION }}.bin # - name: SonarQube Scan (Push) # if: matrix.version == '8.2' && github.event_name == 'push' From 750d958a36d1d9fffac01cbcf56e9bd00d8d02d2 Mon Sep 17 00:00:00 2001 From: Martin Redolatti Date: Wed, 5 Jul 2023 16:25:43 -0300 Subject: [PATCH 03/42] reorder steps, use VERSION as name --- .github/workflows/ci.yml | 21 ++++++++++----------- 1 file changed, 10 insertions(+), 11 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 80ebb0b..f2c7778 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -6,8 +6,6 @@ on: push: branches: - main - tags: - - "v*.*.*" concurrency: group: ${{ github.workflow }}-${{ github.event_name == 'push' && github.run_number || github.event.pull_request.number }} @@ -37,14 +35,6 @@ jobs: with: script: core.setFailed('[ERROR] Tag already exists.') - - name: Git tag - if: ${{ github.event_name == 'push' }} - uses: mathieudutour/github-tag-action@v6.1 - with: - github_token: ${{ secrets.GITHUB_TOKEN }} - custom_tag: ${{ env.VERSION }} - tag_prefix: 'v' - - name: Setup Go version uses: actions/setup-go@v4 with: @@ -64,10 +54,19 @@ jobs: - name: Run tests run: make test + - name: Git tag + if: ${{ github.event_name == 'push' }} + uses: mathieudutour/github-tag-action@v6.1 + with: + github_token: ${{ secrets.GITHUB_TOKEN }} + custom_tag: ${{ env.VERSION }} + tag_prefix: 'v' + - name: Release uses: softprops/action-gh-release@v1 - if: ${{ github.event_name == 'push' && startsWith(github.ref, 'refs/tags/') }} + if: ${{ github.event_name == 'push' }} with: + name: splitd-${{ env.VERSION }} files: - splitd-darwin-amd64-${{ env.VERSION }}.bin - splitd-darwin-arm-${{ env.VERSION }}.bin From e513e31a2992fb780cfed40bbea5b96d171e7d62 Mon Sep 17 00:00:00 2001 From: Martin Redolatti Date: Wed, 5 Jul 2023 16:42:32 -0300 Subject: [PATCH 04/42] enable sonarqube --- .github/workflows/ci.yml | 36 ++++++++++-------------------------- sonar-project.properties | 8 ++++++++ 2 files changed, 18 insertions(+), 26 deletions(-) create mode 100644 sonar-project.properties diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index f2c7778..f2121cd 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -73,29 +73,13 @@ jobs: - splitd-linux-amd64-${{ env.VERSION }}.bin - splitd-linux-arm-${{ env.VERSION }}.bin - # - name: SonarQube Scan (Push) - # if: matrix.version == '8.2' && github.event_name == 'push' - # uses: SonarSource/sonarcloud-github-action@v1.9 - # env: - # SONAR_TOKEN: ${{ secrets.SONARQUBE_TOKEN }} - # GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - # with: - # projectBaseDir: . - # args: > - # -Dsonar.host.url=${{ secrets.SONARQUBE_HOST }} - # -Dsonar.projectVersion=${{ env.VERSION }} - # - # - name: SonarQube Scan (Pull Request) - # if: matrix.version == '8.2' && github.event_name == 'pull_request' - # uses: SonarSource/sonarcloud-github-action@v1.9 - # env: - # SONAR_TOKEN: ${{ secrets.SONARQUBE_TOKEN }} - # GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - # with: - # projectBaseDir: . - # args: > - # -Dsonar.host.url=${{ secrets.SONARQUBE_HOST }} - # -Dsonar.projectVersion=${{ env.VERSION }} - # -Dsonar.pullrequest.key=${{ github.event.pull_request.number }} - # -Dsonar.pullrequest.branch=${{ github.event.pull_request.head.ref }} - # -Dsonar.pullrequest.base=${{ github.event.pull_request.base.ref }} + - name: SonarQube Scan + uses: SonarSource/sonarcloud-github-action@v1.9.1 + env: + SONAR_TOKEN: ${{ secrets.SONARQUBE_TOKEN }} + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + with: + projectBaseDir: . + args: > + -Dsonar.host.url=${{ secrets.SONARQUBE_HOST }} + -Dsonar.projectVersion=${{ env.VERSION }} diff --git a/sonar-project.properties b/sonar-project.properties new file mode 100644 index 0000000..c56998c --- /dev/null +++ b/sonar-project.properties @@ -0,0 +1,8 @@ +sonar.projectKey=go-occupancy-tracker +sonar.sources=. +sonar.tests=. +sonar.test.inclusions=**/*_test.go +sonar.go.coverage.reportPaths=coverage.out +#sonar.coverage.exclusions=conf/conf.go,splitio/services/initialize.go,**/main.go +sonar.links.ci=https://github.com/splitio/splitd +sonar.links.scm=https://github.com/splitio/splitd/actions From 00a7373134cd07e0f9b0ad8b7ca4c7fd35bf15e3 Mon Sep 17 00:00:00 2001 From: Martin Redolatti Date: Wed, 5 Jul 2023 16:44:56 -0300 Subject: [PATCH 05/42] . --- .github/workflows/ci.yml | 22 +++++++++++----------- sonar-project.properties | 2 +- 2 files changed, 12 insertions(+), 12 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index f2121cd..3bdc0ff 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -54,6 +54,17 @@ jobs: - name: Run tests run: make test + - name: SonarQube Scan + uses: SonarSource/sonarcloud-github-action@v1.9.1 + env: + SONAR_TOKEN: ${{ secrets.SONARQUBE_TOKEN }} + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + with: + projectBaseDir: . + args: > + -Dsonar.host.url=${{ secrets.SONARQUBE_HOST }} + -Dsonar.projectVersion=${{ env.VERSION }} + - name: Git tag if: ${{ github.event_name == 'push' }} uses: mathieudutour/github-tag-action@v6.1 @@ -72,14 +83,3 @@ jobs: - splitd-darwin-arm-${{ env.VERSION }}.bin - splitd-linux-amd64-${{ env.VERSION }}.bin - splitd-linux-arm-${{ env.VERSION }}.bin - - - name: SonarQube Scan - uses: SonarSource/sonarcloud-github-action@v1.9.1 - env: - SONAR_TOKEN: ${{ secrets.SONARQUBE_TOKEN }} - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - with: - projectBaseDir: . - args: > - -Dsonar.host.url=${{ secrets.SONARQUBE_HOST }} - -Dsonar.projectVersion=${{ env.VERSION }} diff --git a/sonar-project.properties b/sonar-project.properties index c56998c..dee84fb 100644 --- a/sonar-project.properties +++ b/sonar-project.properties @@ -1,4 +1,4 @@ -sonar.projectKey=go-occupancy-tracker +sonar.projectKey=splitd sonar.sources=. sonar.tests=. sonar.test.inclusions=**/*_test.go From 4ffa091db75af1c0fcbd6ff663e67b99d22e692c Mon Sep 17 00:00:00 2001 From: Martin Redolatti Date: Wed, 5 Jul 2023 17:06:23 -0300 Subject: [PATCH 06/42] pass files in newline-separated str --- .github/workflows/ci.yml | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 3bdc0ff..ff48dc5 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -78,8 +78,8 @@ jobs: if: ${{ github.event_name == 'push' }} with: name: splitd-${{ env.VERSION }} - files: - - splitd-darwin-amd64-${{ env.VERSION }}.bin - - splitd-darwin-arm-${{ env.VERSION }}.bin - - splitd-linux-amd64-${{ env.VERSION }}.bin - - splitd-linux-arm-${{ env.VERSION }}.bin + files: | + splitd-darwin-amd64-${{ env.VERSION }}.bin + splitd-darwin-arm-${{ env.VERSION }}.bin + splitd-linux-amd64-${{ env.VERSION }}.bin + splitd-linux-arm-${{ env.VERSION }}.bin From 376a394e57b1a2ba0e5200049e0b3e5546ffe43a Mon Sep 17 00:00:00 2001 From: Martin Redolatti Date: Wed, 5 Jul 2023 17:20:57 -0300 Subject: [PATCH 07/42] fix deps --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index 0dd3de8..4bdb786 100644 --- a/Makefile +++ b/Makefile @@ -54,7 +54,7 @@ images_release: # entrypoints @echo "$(DOCKER) push splitsoftware/splitd-sidecar:$(VERSION)" ## build release for binaires -binaries_release: splitd.linux.amd64.$(VERSION).bin splitd.darwin.amd64.$(VERSION).bin splitd.linux.arm.$(VERSION).bin splitd.darwin.arm.$(VERSION).bin +binaries_release: splitd-linux-amd64-$(VERSION).bin splitd-darwin-amd64-$(VERSION).bin splitd-linux-arm-$(VERSION).bin splitd-darwin-arm-$(VERSION).bin splitd-linux-amd64-$(VERSION).bin: $(GO_FILES) From 71374a0582383eab867daa5509e87b1638531abc Mon Sep 17 00:00:00 2001 From: Martin Redolatti Date: Wed, 5 Jul 2023 18:38:59 -0300 Subject: [PATCH 08/42] fix flakey test --- splitio/sdk/tasks/impressions_test.go | 27 +++++++++++++++++++++++---- 1 file changed, 23 insertions(+), 4 deletions(-) diff --git a/splitio/sdk/tasks/impressions_test.go b/splitio/sdk/tasks/impressions_test.go index 94e242e..0b77c4a 100644 --- a/splitio/sdk/tasks/impressions_test.go +++ b/splitio/sdk/tasks/impressions_test.go @@ -1,6 +1,8 @@ package tasks import ( + "reflect" + "sort" "testing" "time" @@ -22,7 +24,7 @@ func TestImpressionsTask(t *testing.T) { task := NewImpressionSyncTask(rec, is, logger, ts, &conf.Impressions{SyncPeriod: 1 * time.Second}) - var emptyMap map[string]string + var emptyMap map[string]string rec.On("Record", []dtos.ImpressionsDTO{{ TestName: "f1", KeyImpressions: []dtos.ImpressionDTO{{KeyName: "k1", Treatment: "on", Time: 123456, ChangeNumber: 123, Label: "l1"}}, @@ -37,7 +39,24 @@ func TestImpressionsTask(t *testing.T) { Return(nil). Once() - rec.On("Record", []dtos.ImpressionsDTO{ + rec.On("Record", + mock.MatchedBy(func(imps []dtos.ImpressionsDTO) bool { + sort.Slice(imps, func(i, j int) bool { return imps[i].TestName < imps[j].TestName }) + return reflect.DeepEqual(imps, []dtos.ImpressionsDTO{ + { + TestName: "f3", + KeyImpressions: []dtos.ImpressionDTO{{KeyName: "k3", Treatment: "on", Time: 123458, ChangeNumber: 789, Label: "l3"}}, + }, + { + TestName: "f4", + KeyImpressions: []dtos.ImpressionDTO{{KeyName: "k3", Treatment: "on", Time: 123459, ChangeNumber: 890, Label: "l4"}}, + }, + }) + }), + dtos.Metadata{SDKVersion: "python-1.2.3", MachineIP: "", MachineName: ""}, + emptyMap).Return(nil).Once() + + /*[]dtos.ImpressionsDTO{ { TestName: "f3", KeyImpressions: []dtos.ImpressionDTO{{KeyName: "k3", Treatment: "on", Time: 123458, ChangeNumber: 789, Label: "l3"}}, @@ -49,7 +68,7 @@ func TestImpressionsTask(t *testing.T) { }, dtos.Metadata{SDKVersion: "python-1.2.3", MachineIP: "", MachineName: ""}, emptyMap). Return(nil). Once() - + */ is.Push(types.ClientMetadata{ID: "i1", SdkVersion: "php-1.2.3"}, dtos.Impression{KeyName: "k1", FeatureName: "f1", Treatment: "on", Label: "l1", ChangeNumber: 123, Time: 123456}) is.Push(types.ClientMetadata{ID: "i2", SdkVersion: "go-1.2.3"}, @@ -66,7 +85,7 @@ func TestImpressionsTask(t *testing.T) { time.Sleep(1500 * time.Millisecond) task.Stop(true) - rec.AssertExpectations(t) + rec.AssertExpectations(t) } type RecorderMock struct { From bfa0e44e4af91645e12c61597b5d451fc4be4cc7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Cristian=20Daniel=20Fad=C3=B3n?= Date: Wed, 5 Jul 2023 19:08:05 -0300 Subject: [PATCH 09/42] Fix workflows --- .github/workflows/ci.yml | 16 ++++++++-------- .github/workflows/docker.yml | 2 +- .pre-commit-config.yaml | 9 +++++++++ README.md | 2 +- common/common.go | 2 -- infra/entrypoint.sh | 0 infra/test/assert.sh | 0 infra/test/test_entrypoint.sh | 4 ++-- splitd.yaml.tpl | 3 --- splitio/link/transfer/acceptor_test.go | 2 +- splitio/link/transfer/framing/lengthprefix.go | 1 - .../link/transfer/framing/lengthprefix_test.go | 4 ++-- splitio/sdk/sdk.go | 2 +- splitio/sdk/sdk_test.go | 2 +- splitio/sdk/storage/queue_test.go | 2 +- splitio/sdk/storage/storages.go | 1 - splitio/util/shutdown.go | 2 -- 17 files changed, 27 insertions(+), 27 deletions(-) create mode 100644 .pre-commit-config.yaml mode change 100644 => 100755 infra/entrypoint.sh mode change 100644 => 100755 infra/test/assert.sh mode change 100644 => 100755 infra/test/test_entrypoint.sh diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index ff48dc5..5dd4698 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -1,4 +1,5 @@ name: ci + on: pull_request: branches: @@ -16,6 +17,10 @@ jobs: name: Run unit tests runs-on: ubuntu-latest steps: + - name: Checkout code + uses: actions/checkout@v3 + with: + fetch-depth: 0 - name: Set VERSION env run: echo "VERSION=$(cat splitio/version.go | grep 'Version =' | awk '{print $3}' | tr -d '"')" >> $GITHUB_ENV @@ -40,20 +45,15 @@ jobs: with: go-version: '^1.19.1' - - name: Checkout code - uses: actions/checkout@v3 - with: - fetch-depth: 0 - - name: Build binaries for host machine run: make splitd splitcli - - name: Cross build for GNU Linux & Darwin x amd64 & arm64 - run: make binaries_release - - name: Run tests run: make test + - name: Cross build for GNU Linux & Darwin x amd64 & arm64 + run: make binaries_release + - name: SonarQube Scan uses: SonarSource/sonarcloud-github-action@v1.9.1 env: diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index d4469d9..74d8028 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -12,12 +12,12 @@ on: concurrency: group: ${{ github.workflow }}-${{ github.event_name == 'push' && github.run_number || github.event.pull_request.number }} + cancel-in-progress: true jobs: build-docker-image: name: Build and push Docker image runs-on: ubuntu-latest - steps: - name: Login to Artifactory if: ${{ github.event_name == 'push' }} diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml new file mode 100644 index 0000000..43ea7b5 --- /dev/null +++ b/.pre-commit-config.yaml @@ -0,0 +1,9 @@ +repos: + - repo: https://github.com/pre-commit/pre-commit-hooks + rev: v4.4.0 + hooks: + - id: check-added-large-files + - id: check-shebang-scripts-are-executable + - id: check-yaml + - id: end-of-file-fixer + - id: trailing-whitespace diff --git a/README.md b/README.md index 5db38ad..137ad78 100644 --- a/README.md +++ b/README.md @@ -1 +1 @@ -# splitd \ No newline at end of file +# splitd diff --git a/common/common.go b/common/common.go index 3f4e042..9ac273a 100644 --- a/common/common.go +++ b/common/common.go @@ -4,5 +4,3 @@ type Message struct { Func string Args []interface{} } - - diff --git a/infra/entrypoint.sh b/infra/entrypoint.sh old mode 100644 new mode 100755 diff --git a/infra/test/assert.sh b/infra/test/assert.sh old mode 100644 new mode 100755 diff --git a/infra/test/test_entrypoint.sh b/infra/test/test_entrypoint.sh old mode 100644 new mode 100755 index d9f21ef..21be58e --- a/infra/test/test_entrypoint.sh +++ b/infra/test/test_entrypoint.sh @@ -36,10 +36,10 @@ function testAllVars { export SPLITD_LINK_WRITE_TIMEOUT_MS=3 export SPLITD_LINK_ACCEPT_TIMEOUT_MS=4 export SPLITD_LOG_LEVEL="WARNING" - + # Exec entrypoint [ -f "./testcfg" ] && rm ./testcfg - export SPLITD_CFG_OUTPUT="./testcfg" + export SPLITD_CFG_OUTPUT="./testcfg" export SPLITD_EXEC="${SCRIPT_DIR}/../../splitd" export TPL_FILE="${SCRIPT_DIR}/../../splitd.yaml.tpl" conf_json=$(bash "${SCRIPT_DIR}/../entrypoint.sh" -outputConfig | awk '/^Config:/ {print $2}') diff --git a/splitd.yaml.tpl b/splitd.yaml.tpl index 6a05118..7c067d4 100644 --- a/splitd.yaml.tpl +++ b/splitd.yaml.tpl @@ -14,6 +14,3 @@ link: type: "unix-seqpacket" address: "/var/run/splitd.sock" serialization: "msgpack" - - - diff --git a/splitio/link/transfer/acceptor_test.go b/splitio/link/transfer/acceptor_test.go index 07e87e3..c8fb8e6 100644 --- a/splitio/link/transfer/acceptor_test.go +++ b/splitio/link/transfer/acceptor_test.go @@ -24,7 +24,7 @@ func TestAcceptor(t *testing.T) { // Second client's server-end of the socket will be closed after the timeout // The write not error out (though nothing is written), but will notice that the socket hasbeen remotely closed and update it's state // The following read will report an EOF - + logger := logging.NewLogger(nil) dir, err := os.MkdirTemp(os.TempDir(), "acceptortest") diff --git a/splitio/link/transfer/framing/lengthprefix.go b/splitio/link/transfer/framing/lengthprefix.go index ce4dcdc..39a6600 100644 --- a/splitio/link/transfer/framing/lengthprefix.go +++ b/splitio/link/transfer/framing/lengthprefix.go @@ -82,4 +82,3 @@ func encodeSize(size int, target []byte) []byte { func decodeSize(size []byte) uint32 { return binary.LittleEndian.Uint32(size[:]) } - diff --git a/splitio/link/transfer/framing/lengthprefix_test.go b/splitio/link/transfer/framing/lengthprefix_test.go index 8e21ef2..6a08f1c 100644 --- a/splitio/link/transfer/framing/lengthprefix_test.go +++ b/splitio/link/transfer/framing/lengthprefix_test.go @@ -35,7 +35,7 @@ func TestLengthPrefixRead(t *testing.T) { Run(func(args mock.Arguments) { copy(args.Get(0).([]byte), []byte(" SEND")) }). Return(len(" SEND"), (error)(nil)).Once() - + var lp LengthPrefixImpl var buffer [2048]byte n, err := lp.ReadFrame(sock, buffer[:]) @@ -60,7 +60,7 @@ func TestLengthPrefixWrite(t *testing.T) { sock. On("Write", []byte("TO SEND")). Return(7, (error)(nil)).Once() - + var lp LengthPrefixImpl n, err := lp.WriteFrame(sock, []byte("SOMETHING TO SEND")) assert.Nil(t, err) diff --git a/splitio/sdk/sdk.go b/splitio/sdk/sdk.go index e40c335..ba79777 100644 --- a/splitio/sdk/sdk.go +++ b/splitio/sdk/sdk.go @@ -110,7 +110,7 @@ func (i *Impl) Treatments(cfg *types.ClientConfig, key string, bk *string, featu res := i.ev.EvaluateFeatures(key, bk, features, attributes) toRet := make(map[string]Result, len(res.Evaluations)) for _, feature := range features { - + curr, ok := res.Evaluations[feature] if !ok { toRet[feature] = Result{Treatment: "control"} diff --git a/splitio/sdk/sdk_test.go b/splitio/sdk/sdk_test.go index 8cf9422..cc57ce8 100644 --- a/splitio/sdk/sdk_test.go +++ b/splitio/sdk/sdk_test.go @@ -171,7 +171,7 @@ func TestTreatments(t *testing.T) { } res, err := client.Treatments( - &types.ClientConfig{Metadata: types.ClientMetadata{ID: "some", SdkVersion: "go-1.2.3"}}, + &types.ClientConfig{Metadata: types.ClientMetadata{ID: "some", SdkVersion: "go-1.2.3"}}, "key1", nil, []string{"f1", "f2", "f3"}, Attributes{"a": 1}) assert.Nil(t, err) assert.Nil(t, res["f1"].Config) diff --git a/splitio/sdk/storage/queue_test.go b/splitio/sdk/storage/queue_test.go index f0e2210..1732c54 100644 --- a/splitio/sdk/storage/queue_test.go +++ b/splitio/sdk/storage/queue_test.go @@ -40,7 +40,7 @@ func TestLockingQueueBasic(t *testing.T) { assert.Nil(t, err) assert.Equal(t, 3, n) assert.Equal(t, 3, st.Len()) - + n, err = st.Pop(3, &buf) assert.Nil(t, err) assert.Equal(t, 3, n ) diff --git a/splitio/sdk/storage/storages.go b/splitio/sdk/storage/storages.go index ce33041..58851d6 100644 --- a/splitio/sdk/storage/storages.go +++ b/splitio/sdk/storage/storages.go @@ -24,4 +24,3 @@ func NewEventsQueue(approxSize int) (st *EventsStorage, realSize int) { return NewLKQueue[dtos.EventDTO](bits) }), int(math.Pow(2, float64(bits))) } - diff --git a/splitio/util/shutdown.go b/splitio/util/shutdown.go index 1cfaa6c..1978e6c 100644 --- a/splitio/util/shutdown.go +++ b/splitio/util/shutdown.go @@ -61,5 +61,3 @@ func (s *ShutdownHandler) TriggerAndWait() { defer s.Wait() s.incoming <- syscall.SIGTERM } - - From e6a4e65fbd1c268e81793830a7b2e6546ca9c2a1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Cristian=20Daniel=20Fad=C3=B3n?= Date: Wed, 5 Jul 2023 19:13:59 -0300 Subject: [PATCH 10/42] Enable coverage --- .gitignore | 6 ++++++ Makefile | 2 +- 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/.gitignore b/.gitignore index 34aecac..9057784 100644 --- a/.gitignore +++ b/.gitignore @@ -22,3 +22,9 @@ shared splitd.darwin.* splitd.linux.* + +# Sonarqube files +.scannerwork + +# DS Store +.DS_Store diff --git a/Makefile b/Makefile index 4bdb786..4484540 100644 --- a/Makefile +++ b/Makefile @@ -32,7 +32,7 @@ test: unit-tests entrypoint-test ## run go unit tests unit-tests: - $(GO) test ./... -count=1 -race + $(GO) test ./... -count=1 -race -coverprofile=coverage.out ## run bash entrypoint tests entrypoint-test: From b537937861cb219ba5a1da2c16e26c795ab85acb Mon Sep 17 00:00:00 2001 From: Martin Redolatti Date: Tue, 25 Jul 2023 12:11:40 -0300 Subject: [PATCH 11/42] simplify internal configs flow --- Makefile | 15 +- cmd/splitcli/main.go | 79 +++++-- cmd/splitd/main.go | 7 +- common/common.go | 8 - splitio/conf/splitd.go | 106 +++++----- splitio/conf/splitd_test.go | 93 +++++++++ splitio/link/client/client.go | 31 ++- splitio/link/client/types/interfaces.go | 16 ++ splitio/link/client/v1/impl.go | 106 ++++++++-- splitio/link/client/v1/impl_test.go | 216 ++++++++++++++++++++ splitio/link/link.go | 163 +++------------ splitio/link/link_test.go | 42 ---- splitio/link/protocol/v1/mocks/mocks.go | 90 ++++++++ splitio/link/protocol/v1/rpcs.go | 74 +++++-- splitio/link/serializer/mocks/serializer.go | 24 +++ splitio/link/service/service.go | 21 +- splitio/link/service/v1/clientmgr.go | 3 - splitio/link/service/v1/clientmgr_test.go | 174 +++------------- splitio/link/transfer/acceptor.go | 23 ++- splitio/link/transfer/acceptor_test.go | 42 ++-- splitio/link/transfer/mocks/rawconn.go | 30 +++ splitio/link/transfer/setup.go | 27 +-- splitio/sdk/conf/conf.go | 41 ---- splitio/sdk/mocks/sdk.go | 38 ++++ splitio/sdk/sdk.go | 6 +- splitio/sdk/tasks/impressions_test.go | 15 +- splitio/util/conf/helpers.go | 27 +++ splitio/util/lfqueue/lfqueue.go | 63 ------ 28 files changed, 935 insertions(+), 645 deletions(-) delete mode 100644 common/common.go create mode 100644 splitio/conf/splitd_test.go create mode 100644 splitio/link/client/types/interfaces.go create mode 100644 splitio/link/client/v1/impl_test.go delete mode 100644 splitio/link/link_test.go create mode 100644 splitio/link/protocol/v1/mocks/mocks.go create mode 100644 splitio/link/serializer/mocks/serializer.go create mode 100644 splitio/link/transfer/mocks/rawconn.go create mode 100644 splitio/sdk/mocks/sdk.go create mode 100644 splitio/util/conf/helpers.go delete mode 100644 splitio/util/lfqueue/lfqueue.go diff --git a/Makefile b/Makefile index 4bdb786..59aa95b 100644 --- a/Makefile +++ b/Makefile @@ -4,9 +4,12 @@ GO ?=go DOCKER ?= docker SHELL = /usr/bin/env bash -o pipefail + +# setup platform specific docker image builds PLATFORM ?= -VERSION := $(shell cat splitio/version.go | grep 'const Version' | sed 's/const Version = //' | tr -d '"') +PLATFORM_STR := $(if $(PLATFORM),--platform=$(PLATFORM),) +VERSION := $(shell cat splitio/version.go | grep 'const Version' | sed 's/const Version = //' | tr -d '"') GO_FILES := $(shell find . -name "*.go") go.sum default: help @@ -35,7 +38,7 @@ unit-tests: $(GO) test ./... -count=1 -race ## run bash entrypoint tests -entrypoint-test: +entrypoint-test: splitd # requires splitd binary to generate a config and validate env var forwarding bash infra/test/test_entrypoint.sh ## build splitd for local machine @@ -48,7 +51,7 @@ splitcli: $(GO_FILES) ## build docker images for sidecar images_release: # entrypoints - $(DOCKER) build $(platform_str) -t splitsoftware/splitd-sidecar:latest -t splitsoftware/splitd-sidecar:$(VERSION) -f infra/sidecar.Dockerfile . + $(DOCKER) build $(PLATFORM_STR) -t splitsoftware/splitd-sidecar:latest -t splitsoftware/splitd-sidecar:$(VERSION) -f infra/sidecar.Dockerfile . @echo "Image created. Make sure everything works ok, and then run the following commands to push them." @echo "$(DOCKER) push splitsoftware/splitd-sidecar:latest" @echo "$(DOCKER) push splitsoftware/splitd-sidecar:$(VERSION)" @@ -56,7 +59,6 @@ images_release: # entrypoints ## build release for binaires binaries_release: splitd-linux-amd64-$(VERSION).bin splitd-darwin-amd64-$(VERSION).bin splitd-linux-arm-$(VERSION).bin splitd-darwin-arm-$(VERSION).bin - splitd-linux-amd64-$(VERSION).bin: $(GO_FILES) GOARCH=amd64 GOOS=linux $(GO) build -o $@ cmd/splitd/main.go @@ -69,11 +71,6 @@ splitd-linux-arm-$(VERSION).bin: $(GO_FILES) splitd-darwin-arm-$(VERSION).bin: $(GO_FILES) GOARCH=arm64 GOOS=darwin $(GO) build -o $@ cmd/splitd/main.go -binaries_release: splitd - -# helper macros -platform_str = $(if $(PLATFORM),--platform=$(PLATFORM),) - # Help target borrowed from: https://docs.cloudposse.com/reference/best-practices/make-best-practices/ ## This help screen help: diff --git a/cmd/splitcli/main.go b/cmd/splitcli/main.go index 82403e1..493da6f 100644 --- a/cmd/splitcli/main.go +++ b/cmd/splitcli/main.go @@ -11,8 +11,9 @@ import ( "github.com/splitio/go-toolkit/v5/logging" "github.com/splitio/splitd/splitio/link" - "github.com/splitio/splitd/splitio/link/client" + "github.com/splitio/splitd/splitio/link/client/types" "github.com/splitio/splitd/splitio/util" + cc "github.com/splitio/splitd/splitio/util/conf" ) func main() { @@ -23,9 +24,23 @@ func main() { os.Exit(1) } - logger := logging.NewLogger(nil) + linkOpts, err := args.linkOpts() + if err != nil { + fmt.Println("error building options from arguments: ", err.Error()) + os.Exit(1) + } + + logLevel := logging.Level(args.logLevel) + logger := logging.NewLogger(&logging.LoggerOptions{ + LogLevel: logLevel, + ErrorWriter: os.Stderr, + WarningWriter: os.Stderr, + InfoWriter: os.Stderr, + DebugWriter: os.Stderr, + VerboseWriter: os.Stderr, + }) - c, err := link.Consumer(logger, args.linkOpts()...) + c, err := link.Consumer(logger, linkOpts) if err != nil { logger.Error("error creating client wrapper: ", err) os.Exit(2) @@ -42,7 +57,7 @@ func main() { before := time.Now() result, err := executeCall(c, args) - fmt.Printf("took: %d\n", time.Since(before).Microseconds()) + logger.Debug(fmt.Sprintf("took: %d\n", time.Since(before).Microseconds())) if err != nil { logger.Error("error executing call: ", err.Error()) os.Exit(3) @@ -51,10 +66,11 @@ func main() { fmt.Println(result) } -func executeCall(c client.Interface, a *cliArgs) (string, error) { +func executeCall(c types.ClientInterface, a *cliArgs) (string, error) { switch a.method { case "treatment": - return c.Treatment(a.key, a.bucketingKey, a.feature, a.attributes) + res, err := c.Treatment(a.key, a.bucketingKey, a.feature, a.attributes) + return res.Treatment, err case "treatments", "treatmentWithConfig", "treatmentsWithConfig", "track": return "", fmt.Errorf("method '%s' is not yet implemented", a.method) default: @@ -63,9 +79,16 @@ func executeCall(c client.Interface, a *cliArgs) (string, error) { } type cliArgs struct { - connType string - connAddr string - bufSize int + logLevel string + protocol string + serialization string + connType string + connAddr string + bufSize int + readTimeoutMS int + writeTimeoutMS int + + // command method string key string bucketingKey string @@ -77,21 +100,40 @@ type cliArgs struct { attributes map[string]interface{} } -func (a *cliArgs) linkOpts() []link.Option { - var ret []link.Option - if a.connType != "" { - ret = append(ret, link.WithSockType(a.connType)) +func (a *cliArgs) linkOpts() (*link.ConsumerOptions, error) { + + opts := link.DefaultConsumerOptions() + + var err error + if a.protocol != "" { + if opts.Consumer.Protocol, err = cc.ParseProtocolVersion(a.protocol); err != nil { + return nil, fmt.Errorf("invalid protocol version %s", a.protocol) + } } - if a.connAddr != "" { - ret = append(ret, link.WithAddress(a.connAddr)) + + if a.connType != "" { + if opts.Transfer.ConnType, err = cc.ParseConnType(a.connType); err != nil { + return nil, fmt.Errorf("invalid connection type %s", a.connType) + } } - if a.bufSize != 0 { - ret = append(ret, link.WithBufSize(a.bufSize)) + + if a.serialization != "" { + if opts.Serialization, err = cc.ParseSerializer(a.serialization); err != nil { + return nil, fmt.Errorf("invalid serialization %s", a.serialization) + } } - return ret + + durationFromMS := func(i int) time.Duration { return time.Duration(i) * time.Millisecond } + cc.SetIfNotEmpty(&opts.Transfer.Address, &a.connAddr) + cc.SetIfNotEmpty(&opts.Transfer.BufferSize, &a.bufSize) + cc.MapIfNotEmpty(&opts.Transfer.ReadTimeout, &a.readTimeoutMS, durationFromMS) + cc.MapIfNotEmpty(&opts.Transfer.WriteTimeout, &a.writeTimeoutMS, durationFromMS) + + return &opts, nil } func parseArgs() (*cliArgs, error) { + ll := flag.String("log-level", "INFO", "log level [ERROR,WARNING,INFO,DEBUG]") ct := flag.String("conn-type", "", "unix-seqpacket|unix-stream") ca := flag.String("conn-address", "", "path/ipv4-address") bs := flag.Int("buffer-size", 0, "read buffer size in bytes") @@ -121,6 +163,7 @@ func parseArgs() (*cliArgs, error) { } return &cliArgs{ + logLevel: *ll, connType: *ct, connAddr: *ca, bufSize: *bs, diff --git a/cmd/splitd/main.go b/cmd/splitd/main.go index 8528a35..da34b7f 100644 --- a/cmd/splitd/main.go +++ b/cmd/splitd/main.go @@ -26,10 +26,13 @@ func main() { logger := logging.NewLogger(cfg.Logger.ToLoggerOptions()) - splitSDK, err := sdk.New(logger, cfg.SDK.Apikey, cfg.SDK.ToSDKConf()...) + splitSDK, err := sdk.New(logger, cfg.SDK.Apikey, cfg.SDK.ToSDKConf()) exitOnErr("sdk initialization", err) - errc, lShutdown, err := link.Listen(logger, splitSDK, cfg.Link.ToLinkOpts()...) + linkCFG, err := cfg.Link.ToListenerOpts() + exitOnErr("link config", err) + + errc, lShutdown, err := link.Listen(logger, splitSDK, linkCFG) exitOnErr("rpc listener setup", err) shutdown := util.NewShutdownHandler() diff --git a/common/common.go b/common/common.go deleted file mode 100644 index 3f4e042..0000000 --- a/common/common.go +++ /dev/null @@ -1,8 +0,0 @@ -package common - -type Message struct { - Func string - Args []interface{} -} - - diff --git a/splitio/conf/splitd.go b/splitio/conf/splitd.go index 0c1534e..615edc0 100644 --- a/splitio/conf/splitd.go +++ b/splitio/conf/splitd.go @@ -7,10 +7,12 @@ import ( "log" "os" "strings" + "time" "github.com/splitio/go-toolkit/v5/logging" "github.com/splitio/splitd/splitio/link" - "github.com/splitio/splitd/splitio/sdk/conf" + sdkConf "github.com/splitio/splitd/splitio/sdk/conf" + cc "github.com/splitio/splitd/splitio/util/conf" "gopkg.in/yaml.v3" ) @@ -23,12 +25,12 @@ type Config struct { } func (c Config) String() string { - if len(c.SDK.Apikey) > 4 { - c.SDK.Apikey = c.SDK.Apikey[:4] + "xxxxxxx" - } + if len(c.SDK.Apikey) > 4 { + c.SDK.Apikey = c.SDK.Apikey[:4] + "xxxxxxx" + } - output, _ := json.Marshal(c) - return string(output) + output, _ := json.Marshal(c) + return string(output) } func (c *Config) parse(fn string) error { @@ -49,38 +51,46 @@ func (c *Config) parse(fn string) error { type Link struct { Type *string `yaml:"type"` Address *string `yaml:"address"` - Serialization *string `yaml:"serialization"` MaxSimultaneousConns *int `yaml:"maxSimultaneousConns"` ReadTimeoutMS *int `yaml:"readTimeoutMS"` WriteTimeoutMS *int `yaml:"writeTimeoutMS"` AcceptTimeoutMS *int `yaml:"acceptTimeoutMS"` + Serialization *string `yaml:"serialization"` + BufferSize *int `yaml:"bufferSize"` + Protocol *string `yaml:"protocol"` } -func (l *Link) ToLinkOpts() []link.Option { - var opts []link.Option - if l.Type != nil { - opts = append(opts, link.WithSockType(*l.Type)) +func (l *Link) ToListenerOpts() (*link.ListenerOptions, error) { + opts := link.DefaultListenerOptions() + + var err error + if l.Protocol != nil { + if opts.Protocol, err = cc.ParseProtocolVersion(*l.Protocol); err != nil { + return nil, fmt.Errorf("invalid protocol version %s", *l.Protocol) + } } - if l.Address != nil { - opts = append(opts, link.WithAddress(*l.Address)) + + if l.Type != nil { + if opts.Transfer.ConnType, err = cc.ParseConnType(*l.Type); err != nil { + return nil, fmt.Errorf("invalid connection type %s", *l.Type) + } } + if l.Serialization != nil { - opts = append(opts, link.WithSerialization(*l.Serialization)) - } - if l.MaxSimultaneousConns != nil { - opts = append(opts, link.WithMaxSimultaneousConns(*l.MaxSimultaneousConns)) - } - if l.AcceptTimeoutMS != nil { - opts = append(opts, link.WithAcceptTimeoutMs(*l.AcceptTimeoutMS)) - } - if l.ReadTimeoutMS != nil { - opts = append(opts, link.WithReadTimeoutMs(*l.ReadTimeoutMS)) - } - if l.WriteTimeoutMS != nil { - opts = append(opts, link.WithWriteTimeoutMs(*l.WriteTimeoutMS)) + if opts.Serialization, err = cc.ParseSerializer(*l.Serialization); err != nil { + return nil, fmt.Errorf("invalid serialization %s", *l.Serialization) + } } - return opts + durationFromMS := func(i int) time.Duration { return time.Duration(i) * time.Millisecond } + cc.SetIfNotNil(&opts.Transfer.Address, l.Address) + cc.SetIfNotNil(&opts.Transfer.BufferSize, l.BufferSize) + cc.SetIfNotNil(&opts.Acceptor.MaxSimultaneousConnections, l.MaxSimultaneousConns) + cc.MapIfNotNil(&opts.Transfer.ReadTimeout, l.ReadTimeoutMS, durationFromMS) + cc.MapIfNotNil(&opts.Transfer.WriteTimeout, l.WriteTimeoutMS, durationFromMS) + cc.MapIfNotNil(&opts.Acceptor.AcceptTimeout, l.AcceptTimeoutMS, durationFromMS) + + return &opts, nil } type SDK struct { @@ -90,16 +100,13 @@ type SDK struct { URLs URLs `yaml:"urls"` } -func (s *SDK) ToSDKConf() []conf.Option { - var opts []conf.Option - if s.LabelsEnabled != nil { - opts = append(opts, conf.WithLabelsEnabled(*s.LabelsEnabled)) - } - if s.StreamingEnabled != nil { - opts = append(opts, conf.WithStreamingEnabled(*s.StreamingEnabled)) - } - opts = append(opts, s.URLs.ToSDKConf()...) - return opts +func (s *SDK) ToSDKConf() *sdkConf.Config { + + cfg := sdkConf.DefaultConfig() + cc.SetIfNotNil(&cfg.LabelsEnabled, s.LabelsEnabled) + cc.SetIfNotNil(&cfg.StreamingEnabled, s.StreamingEnabled) + s.URLs.updateSDKConfURLs(&cfg.URLs) + return cfg } @@ -111,25 +118,12 @@ type URLs struct { Telemetry *string `yaml:"telemetry"` } -func (u *URLs) ToSDKConf() []conf.Option { - var opts []conf.Option - if u.Auth != nil { - opts = append(opts, conf.WithAuthURL(*u.Auth)) - } - if u.SDK != nil { - opts = append(opts, conf.WithSDKURL(*u.SDK)) - } - if u.Events != nil { - opts = append(opts, conf.WithEventsURL(*u.Events)) - } - if u.Streaming != nil { - opts = append(opts, conf.WithStreamingURL(*u.Streaming)) - } - if u.Telemetry != nil { - opts = append(opts, conf.WithTelemetryURL(*u.Telemetry)) - } - return opts - +func (u *URLs) updateSDKConfURLs(dst *sdkConf.URLs) { + cc.SetIfNotNil(&dst.SDK, u.SDK) + cc.SetIfNotNil(&dst.Events, u.Events) + cc.SetIfNotNil(&dst.Auth, u.Auth) + cc.SetIfNotNil(&dst.Streaming, u.Streaming) + cc.SetIfNotNil(&dst.Telemetry, u.Telemetry) } type Logger struct { diff --git a/splitio/conf/splitd_test.go b/splitio/conf/splitd_test.go new file mode 100644 index 0000000..7283ff7 --- /dev/null +++ b/splitio/conf/splitd_test.go @@ -0,0 +1,93 @@ +package conf + +import ( + "testing" + "time" + + "github.com/splitio/splitd/splitio/link" + "github.com/splitio/splitd/splitio/link/protocol" + "github.com/splitio/splitd/splitio/link/serializer" + "github.com/splitio/splitd/splitio/link/transfer" + "github.com/splitio/splitd/splitio/sdk/conf" + "github.com/stretchr/testify/assert" +) + +func TestLink(t *testing.T) { + + linkCFG := &Link{ + Type: ref("unix-stream"), + Address: ref("some/file"), + MaxSimultaneousConns: ref(1), + ReadTimeoutMS: ref(2), + WriteTimeoutMS: ref(3), + AcceptTimeoutMS: ref(4), + Serialization: ref("msgpack"), + BufferSize: ref(5), + Protocol: ref("v1"), + } + + expected := link.DefaultListenerOptions() + expected.Acceptor.AcceptTimeout = 4 * time.Millisecond + expected.Acceptor.MaxSimultaneousConnections = 1 + expected.Protocol = protocol.V1 + expected.Serialization = serializer.MsgPack + expected.Transfer.Address = "some/file" + expected.Transfer.ConnType = transfer.ConnTypeUnixStream + expected.Transfer.ReadTimeout = 2 * time.Millisecond + expected.Transfer.WriteTimeout = 3 * time.Millisecond + expected.Transfer.BufferSize = 5 + lopts, err := linkCFG.ToListenerOpts() + assert.Nil(t, err) + assert.Equal(t, &expected, lopts) + + // invalid protocol + linkCFG.Protocol = ref("sarasa") + lopts, err = linkCFG.ToListenerOpts() + assert.NotNil(t, err) + assert.Nil(t, lopts) + + // invalid serialization + linkCFG.Protocol = ref("v1") // restore valid protocol + linkCFG.Serialization = ref("sarasa") + lopts, err = linkCFG.ToListenerOpts() + assert.NotNil(t, err) + assert.Nil(t, lopts) + + // invalid conn type + linkCFG.Serialization = ref("msgpack") // restore valid serialization mechanism + linkCFG.Type = ref("sarasa") + lopts, err = linkCFG.ToListenerOpts() + assert.NotNil(t, err) + assert.Nil(t, lopts) +} + +func TestSDK(t *testing.T) { + + sdkCFG := &SDK{ + Apikey: "some", + LabelsEnabled: ref(false), + StreamingEnabled: ref(false), + URLs: URLs{ + Auth: ref("authURL"), + SDK: ref("sdkURL"), + Events: ref("eventsURL"), + Streaming: ref("streamingURL"), + Telemetry: ref("telemetryURL"), + }, + } + + expected := conf.DefaultConfig() + expected.StreamingEnabled = false + expected.LabelsEnabled = false + expected.URLs.Auth = "authURL" + expected.URLs.SDK = "sdkURL" + expected.URLs.Events = "eventsURL" + expected.URLs.Streaming = "streamingURL" + expected.URLs.Telemetry = "telemetryURL" + + assert.Equal(t, expected, sdkCFG.ToSDKConf()) +} + +func ref[T any](v T) *T { + return &v +} diff --git a/splitio/link/client/client.go b/splitio/link/client/client.go index 97bd2ab..0f9d45f 100644 --- a/splitio/link/client/client.go +++ b/splitio/link/client/client.go @@ -4,30 +4,29 @@ import ( "fmt" "github.com/splitio/go-toolkit/v5/logging" + "github.com/splitio/splitd/splitio/link/client/types" clientv1 "github.com/splitio/splitd/splitio/link/client/v1" - "github.com/splitio/splitd/splitio/link/common" "github.com/splitio/splitd/splitio/link/protocol" "github.com/splitio/splitd/splitio/link/serializer" "github.com/splitio/splitd/splitio/link/transfer" ) -type Interface interface { - Treatment(key string, bk string, feature string, attributes map[string]interface{}) (string, error) - Shutdown() error +func New(logger logging.LoggerInterface, conn transfer.RawConn, serial serializer.Interface, opts Options) (types.ClientInterface, error) { + switch opts.Protocol { + case protocol.V1: + return clientv1.New(logger, conn, serial, opts.ImpressionsFeedback) + } + return nil, fmt.Errorf("unknown protocol version: '%d'", opts.Protocol) } -func New(logger logging.LoggerInterface, conn transfer.RawConn, os ...common.Option) (Interface, error) { - o := common.DefaultOpts() - o.Parse(os) - - s, err := serializer.Setup(o.Serial) - if err != nil { - return nil, fmt.Errorf("error building serializer") - } +type Options struct { + Protocol protocol.Version + ImpressionsFeedback bool +} - switch o.ProtoV { - case protocol.V1: - return clientv1.New(logger, conn, s) +func DefaultOptions() Options { + return Options{ + Protocol: protocol.V1, + ImpressionsFeedback: false, } - return nil, fmt.Errorf("unknown protocol version: '%d'", o.ProtoV) } diff --git a/splitio/link/client/types/interfaces.go b/splitio/link/client/types/interfaces.go new file mode 100644 index 0000000..638e9b5 --- /dev/null +++ b/splitio/link/client/types/interfaces.go @@ -0,0 +1,16 @@ +package types + +import "github.com/splitio/go-split-commons/v4/dtos" + +type ClientInterface interface { + Treatment(key string, bucketingKey string, feature string, attrs map[string]interface{}) (*Result, error) + Treatments(key string, bucketingKey string, features []string, attrs map[string]interface{}) (Results, error) + Shutdown() error +} + +type Result struct { + Treatment string + Impression *dtos.Impression +} + +type Results = map[string]Result diff --git a/splitio/link/client/v1/impl.go b/splitio/link/client/v1/impl.go index e5dcaf0..7c1bea8 100644 --- a/splitio/link/client/v1/impl.go +++ b/splitio/link/client/v1/impl.go @@ -5,32 +5,36 @@ import ( "os" "strconv" + "github.com/splitio/go-split-commons/v4/dtos" "github.com/splitio/go-toolkit/v5/logging" "github.com/splitio/splitd/splitio" + "github.com/splitio/splitd/splitio/link/client/types" "github.com/splitio/splitd/splitio/link/protocol" protov1 "github.com/splitio/splitd/splitio/link/protocol/v1" "github.com/splitio/splitd/splitio/link/serializer" "github.com/splitio/splitd/splitio/link/transfer" ) -type Interface interface { - Treatment(key string, bucketingKey string, feature string, attrs map[string]interface{}) (string, error) -} +const ( + Control = "control" +) type Impl struct { - logger logging.LoggerInterface - conn transfer.RawConn - serializer serializer.Interface + logger logging.LoggerInterface + conn transfer.RawConn + serializer serializer.Interface + listenerFeedback bool } -func New(logger logging.LoggerInterface, conn transfer.RawConn, serializer serializer.Interface) (*Impl, error) { +func New(logger logging.LoggerInterface, conn transfer.RawConn, serializer serializer.Interface, listenerFeedback bool) (*Impl, error) { i := &Impl{ - logger: logger, - conn: conn, - serializer: serializer, + logger: logger, + conn: conn, + serializer: serializer, + listenerFeedback: listenerFeedback, } - if err := i.register(); err != nil { + if err := i.register(listenerFeedback); err != nil { i.conn.Shutdown() return nil, fmt.Errorf("error during client registration: %w", err) } @@ -39,30 +43,92 @@ func New(logger logging.LoggerInterface, conn transfer.RawConn, serializer seria } // Treatment implements Interface -func (c *Impl) Treatment(key string, bucketingKey string, feature string, attrs map[string]interface{}) (string, error) { +func (c *Impl) Treatment(key string, bucketingKey string, feature string, attrs map[string]interface{}) (*types.Result, error) { + var bkp *string + if bucketingKey != "" { + bkp = &bucketingKey + } rpc := protov1.RPC{ RPCBase: protocol.RPCBase{Version: protocol.V1}, OpCode: protov1.OCTreatment, - Args: []interface{}{key, bucketingKey, feature, attrs}, + Args: protov1.TreatmentArgs{Key: key, BucketingKey: bkp, Feature: feature, Attributes: attrs}.Encode(), } resp, err := doRPC[protov1.ResponseWrapper[protov1.TreatmentPayload]](c, &rpc) if err != nil { - return "control", fmt.Errorf("error executing rpc: %w", err) + return &types.Result{Treatment: Control}, fmt.Errorf("error executing rpc: %w", err) + } + + if resp.Status != protov1.ResultOk { + return &types.Result{Treatment: Control}, fmt.Errorf("server responded with error %d", resp.Status) + } + + var imp *dtos.Impression + if c.listenerFeedback && resp.Payload.ListenerData != nil { + imp = &dtos.Impression{ + KeyName: key, + FeatureName: feature, + Treatment: resp.Payload.Treatment, + Time: resp.Payload.ListenerData.Timestamp, + ChangeNumber: resp.Payload.ListenerData.ChangeNumber, + Label: resp.Payload.ListenerData.Label, + BucketingKey: bucketingKey, + } + } + + return &types.Result{Treatment: resp.Payload.Treatment, Impression: imp}, nil +} + +// Treatment implements Interface +func (c *Impl) Treatments(key string, bucketingKey string, features []string, attrs map[string]interface{}) (types.Results, error) { + var bkp *string + if bucketingKey != "" { + bkp = &bucketingKey + } + rpc := protov1.RPC{ + RPCBase: protocol.RPCBase{Version: protocol.V1}, + OpCode: protov1.OCTreatments, + Args: protov1.TreatmentsArgs{Key: key, BucketingKey: bkp, Features: features, Attributes: attrs}.Encode(), + } + + resp, err := doRPC[protov1.ResponseWrapper[protov1.TreatmentsPayload]](c, &rpc) + if err != nil { + return nil, fmt.Errorf("error executing rpc: %w", err) } if resp.Status != protov1.ResultOk { - return "control", fmt.Errorf("server responded with error %d", resp.Status) + return nil, fmt.Errorf("server responded with error %d", resp.Status) } - return resp.Payload.Treatment, nil + results := make(types.Results) + for idx := range features { + var imp *dtos.Impression + if c.listenerFeedback && resp.Payload.Results[idx].ListenerData != nil { + imp = &dtos.Impression{ + KeyName: key, + FeatureName: features[idx], + Treatment: resp.Payload.Results[idx].Treatment, + Time: resp.Payload.Results[idx].ListenerData.Timestamp, + ChangeNumber: resp.Payload.Results[idx].ListenerData.ChangeNumber, + Label: resp.Payload.Results[idx].ListenerData.Label, + BucketingKey: bucketingKey, + } + } + results[features[idx]] = types.Result{Treatment: resp.Payload.Results[idx].Treatment, Impression: imp} + } + + return results, nil } -func (c *Impl) register() error { +func (c *Impl) register(impressionsFeedback bool) error { + var flags protov1.RegisterFlags + if impressionsFeedback { + flags |= 1 << protov1.RegisterFlagReturnImpressionData + } rpc := protov1.RPC{ RPCBase: protocol.RPCBase{Version: protocol.V1}, OpCode: protov1.OCRegister, - Args: []interface{}{strconv.Itoa(os.Getpid()), fmt.Sprintf("splitd-%s", splitio.Version)}, + Args: protov1.RegisterArgs{ID: strconv.Itoa(os.Getpid()), SDKVersion: fmt.Sprintf("splitd-%s", splitio.Version), Flags: flags}.Encode(), } resp, err := doRPC[protov1.ResponseWrapper[protov1.RegisterPayload]](c, &rpc) @@ -78,7 +144,7 @@ func (c *Impl) register() error { } func doRPC[T any](c *Impl, rpc *protov1.RPC) (*T, error) { - serialized, err := c.serializer.Serialize(&rpc) + serialized, err := c.serializer.Serialize(rpc) if err != nil { return nil, fmt.Errorf("error serializing rpc: %w", err) } @@ -106,4 +172,4 @@ func (c *Impl) Shutdown() error { return c.conn.Shutdown() } -var _ Interface = (*Impl)(nil) +var _ types.ClientInterface = (*Impl)(nil) diff --git a/splitio/link/client/v1/impl_test.go b/splitio/link/client/v1/impl_test.go new file mode 100644 index 0000000..e365ab7 --- /dev/null +++ b/splitio/link/client/v1/impl_test.go @@ -0,0 +1,216 @@ +package v1 + +import ( + "testing" + + "github.com/splitio/go-split-commons/v4/dtos" + "github.com/splitio/go-toolkit/v5/logging" + v1 "github.com/splitio/splitd/splitio/link/protocol/v1" + proto1Mocks "github.com/splitio/splitd/splitio/link/protocol/v1/mocks" + serializerMocks "github.com/splitio/splitd/splitio/link/serializer/mocks" + transferMocks "github.com/splitio/splitd/splitio/link/transfer/mocks" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/mock" +) + +func TestClientGetTreatmentNoImpression(t *testing.T) { + + logger := logging.NewLogger(nil) + + rawConnMock := &transferMocks.RawConnMock{} + rawConnMock.On("SendMessage", []byte("registrationMessage")).Return(nil).Once() + rawConnMock.On("ReceiveMessage").Return([]byte("registrationSuccess"), nil).Once() + rawConnMock.On("SendMessage", []byte("treatmentMessage")).Return(nil).Once() + rawConnMock.On("ReceiveMessage").Return([]byte("treatmentResult"), nil).Once() + + serializerMock := &serializerMocks.SerializerMock{} + serializerMock.On("Serialize", proto1Mocks.NewRegisterRPC(false)).Return([]byte("registrationMessage"), nil).Once() + serializerMock.On("Parse", []byte("registrationSuccess"), mock.Anything).Return(nil).Run(func(args mock.Arguments) { + *args.Get(1).(*v1.ResponseWrapper[v1.RegisterPayload]) = v1.ResponseWrapper[v1.RegisterPayload]{Status: v1.ResultOk} + }).Once() + + serializerMock.On("Serialize", proto1Mocks.NewTreatmentRPC("key1", "buck1", "feat1", map[string]interface{}{"a": 1})). + Return([]byte("treatmentMessage"), nil).Once() + serializerMock.On("Parse", []byte("treatmentResult"), mock.Anything).Return(nil).Run(func(args mock.Arguments) { + *args.Get(1).(*v1.ResponseWrapper[v1.TreatmentPayload]) = v1.ResponseWrapper[v1.TreatmentPayload]{ + Status: v1.ResultOk, + Payload: v1.TreatmentPayload{Treatment: "on"}, + } + }).Once() + client, err := New(logger, rawConnMock, serializerMock, false) + assert.NotNil(t, client) + assert.Nil(t, err) + + res, err := client.Treatment("key1", "buck1", "feat1", map[string]interface{}{"a": 1}) + assert.Nil(t, err) + assert.Equal(t, "on", res.Treatment) + assert.Nil(t, res.Impression) +} + +func TestClientGetTreatmentWithImpression(t *testing.T) { + + logger := logging.NewLogger(nil) + + rawConnMock := &transferMocks.RawConnMock{} + rawConnMock.On("SendMessage", []byte("registrationMessage")).Return(nil).Once() + rawConnMock.On("ReceiveMessage").Return([]byte("registrationSuccess"), nil).Once() + rawConnMock.On("SendMessage", []byte("treatmentMessage")).Return(nil).Once() + rawConnMock.On("ReceiveMessage").Return([]byte("treatmentResult"), nil).Once() + + serializerMock := &serializerMocks.SerializerMock{} + serializerMock.On("Serialize", proto1Mocks.NewRegisterRPC(true)).Return([]byte("registrationMessage"), nil).Once() + serializerMock.On("Parse", []byte("registrationSuccess"), mock.Anything).Return(nil).Run(func(args mock.Arguments) { + *args.Get(1).(*v1.ResponseWrapper[v1.RegisterPayload]) = v1.ResponseWrapper[v1.RegisterPayload]{Status: v1.ResultOk} + }).Once() + + serializerMock.On("Serialize", proto1Mocks.NewTreatmentRPC("key1", "buck1", "feat1", map[string]interface{}{"a": 1})). + Return([]byte("treatmentMessage"), nil).Once() + serializerMock.On("Parse", []byte("treatmentResult"), mock.Anything).Return(nil).Run(func(args mock.Arguments) { + *args.Get(1).(*v1.ResponseWrapper[v1.TreatmentPayload]) = v1.ResponseWrapper[v1.TreatmentPayload]{ + Status: v1.ResultOk, + Payload: v1.TreatmentPayload{ + Treatment: "on", + ListenerData: &v1.ListenerExtraData{Label: "l1", Timestamp: 123, ChangeNumber: 1234}, + }, + } + }).Once() + client, err := New(logger, rawConnMock, serializerMock, true) + assert.NotNil(t, client) + assert.Nil(t, err) + + res, err := client.Treatment("key1", "buck1", "feat1", map[string]interface{}{"a": 1}) + assert.Nil(t, err) + assert.Equal(t, "on", res.Treatment) + validateImpression(t, &dtos.Impression{ + KeyName: "key1", + BucketingKey: "buck1", + FeatureName: "feat1", + Treatment: "on", + Label: "l1", + ChangeNumber: 1234, + Time: 123, + }, res.Impression) + +} + +func TestClientGetTreatmentsNoImpression(t *testing.T) { + + logger := logging.NewLogger(nil) + + rawConnMock := &transferMocks.RawConnMock{} + rawConnMock.On("SendMessage", []byte("registrationMessage")).Return(nil).Once() + rawConnMock.On("ReceiveMessage").Return([]byte("registrationSuccess"), nil).Once() + rawConnMock.On("SendMessage", []byte("treatmentsMessage")).Return(nil).Once() + rawConnMock.On("ReceiveMessage").Return([]byte("treatmentsResult"), nil).Once() + + serializerMock := &serializerMocks.SerializerMock{} + serializerMock.On("Serialize", proto1Mocks.NewRegisterRPC(false)).Return([]byte("registrationMessage"), nil).Once() + serializerMock.On("Parse", []byte("registrationSuccess"), mock.Anything).Return(nil).Run(func(args mock.Arguments) { + *args.Get(1).(*v1.ResponseWrapper[v1.RegisterPayload]) = v1.ResponseWrapper[v1.RegisterPayload]{Status: v1.ResultOk} + }).Once() + + serializerMock.On("Serialize", proto1Mocks.NewTreatmentsRPC("key1", "buck1", []string{"a", "b", "c"}, map[string]interface{}{"a": 1})). + Return([]byte("treatmentsMessage"), nil).Once() + serializerMock.On("Parse", []byte("treatmentsResult"), mock.Anything).Return(nil).Run(func(args mock.Arguments) { + *args.Get(1).(*v1.ResponseWrapper[v1.TreatmentsPayload]) = v1.ResponseWrapper[v1.TreatmentsPayload]{ + Status: v1.ResultOk, + Payload: v1.TreatmentsPayload{Results: []v1.TreatmentPayload{{Treatment: "on"}, {Treatment: "off"}, {Treatment: "na"}}}} + }).Once() + client, err := New(logger, rawConnMock, serializerMock, false) + assert.NotNil(t, client) + assert.Nil(t, err) + + res, err := client.Treatments("key1", "buck1", []string{"a", "b", "c"}, map[string]interface{}{"a": 1}) + assert.Nil(t, err) + assert.Equal(t, "on", res["a"].Treatment) + assert.Nil(t, res["a"].Impression) + assert.Equal(t, "off", res["b"].Treatment) + assert.Nil(t, res["b"].Impression) + assert.Equal(t, "na", res["c"].Treatment) + assert.Nil(t, res["c"].Impression) + +} + +func TestClientGetTreatmentsWithImpression(t *testing.T) { + + logger := logging.NewLogger(nil) + + rawConnMock := &transferMocks.RawConnMock{} + rawConnMock.On("SendMessage", []byte("registrationMessage")).Return(nil).Once() + rawConnMock.On("ReceiveMessage").Return([]byte("registrationSuccess"), nil).Once() + rawConnMock.On("SendMessage", []byte("treatmentsMessage")).Return(nil).Once() + rawConnMock.On("ReceiveMessage").Return([]byte("treatmentsResult"), nil).Once() + + serializerMock := &serializerMocks.SerializerMock{} + serializerMock.On("Serialize", proto1Mocks.NewRegisterRPC(true)).Return([]byte("registrationMessage"), nil).Once() + serializerMock.On("Parse", []byte("registrationSuccess"), mock.Anything).Return(nil).Run(func(args mock.Arguments) { + *args.Get(1).(*v1.ResponseWrapper[v1.RegisterPayload]) = v1.ResponseWrapper[v1.RegisterPayload]{Status: v1.ResultOk} + }).Once() + + serializerMock.On("Serialize", proto1Mocks.NewTreatmentsRPC("key1", "buck1", []string{"a", "b", "c"}, map[string]interface{}{"a": 1})). + Return([]byte("treatmentsMessage"), nil).Once() + serializerMock.On("Parse", []byte("treatmentsResult"), mock.Anything).Return(nil).Run(func(args mock.Arguments) { + *args.Get(1).(*v1.ResponseWrapper[v1.TreatmentsPayload]) = v1.ResponseWrapper[v1.TreatmentsPayload]{ + Status: v1.ResultOk, + Payload: v1.TreatmentsPayload{Results: []v1.TreatmentPayload{ + {Treatment: "on", ListenerData: &v1.ListenerExtraData{Label: "l1", Timestamp: 1, ChangeNumber: 5}}, + {Treatment: "off", ListenerData: &v1.ListenerExtraData{Label: "l2", Timestamp: 2, ChangeNumber: 6}}, + {Treatment: "na", ListenerData: &v1.ListenerExtraData{Label: "l3", Timestamp: 3, ChangeNumber: 7}}, + }}} + }).Once() + client, err := New(logger, rawConnMock, serializerMock, true) + assert.NotNil(t, client) + assert.Nil(t, err) + + res, err := client.Treatments("key1", "buck1", []string{"a", "b", "c"}, map[string]interface{}{"a": 1}) + assert.Nil(t, err) + assert.Equal(t, "on", res["a"].Treatment) + assert.Equal(t, "off", res["b"].Treatment) + assert.Equal(t, "na", res["c"].Treatment) + + validateImpression(t, &dtos.Impression{ + KeyName: "key1", + BucketingKey: "buck1", + FeatureName: "a", + Treatment: "on", + Label: "l1", + ChangeNumber: 5, + Time: 1, + }, res["a"].Impression) + validateImpression(t, &dtos.Impression{ + KeyName: "key1", + BucketingKey: "buck1", + FeatureName: "b", + Treatment: "off", + Label: "l2", + ChangeNumber: 6, + Time: 2, + }, res["b"].Impression) + validateImpression(t, &dtos.Impression{ + KeyName: "key1", + BucketingKey: "buck1", + FeatureName: "c", + Treatment: "na", + Label: "l3", + ChangeNumber: 7, + Time: 3, + }, res["c"].Impression) + +} + +func ref[T any](t T) *T { + return &t +} + +func validateImpression(t *testing.T, expected *dtos.Impression, actual *dtos.Impression) { + t.Helper() + assert.Equal(t, expected.BucketingKey, actual.BucketingKey) + assert.Equal(t, expected.ChangeNumber, actual.ChangeNumber) + assert.Equal(t, expected.FeatureName, actual.FeatureName) + assert.Equal(t, expected.KeyName, actual.KeyName) + assert.Equal(t, expected.Label, actual.Label) + assert.Equal(t, expected.Time, actual.Time) + assert.Equal(t, expected.Treatment, actual.Treatment) + assert.Equal(t, expected.Label, actual.Label) + +} diff --git a/splitio/link/link.go b/splitio/link/link.go index db5aa3f..8e35efb 100644 --- a/splitio/link/link.go +++ b/splitio/link/link.go @@ -2,11 +2,10 @@ package link import ( "fmt" - "time" "github.com/splitio/go-toolkit/v5/logging" "github.com/splitio/splitd/splitio/link/client" - "github.com/splitio/splitd/splitio/link/common" + "github.com/splitio/splitd/splitio/link/client/types" "github.com/splitio/splitd/splitio/link/protocol" "github.com/splitio/splitd/splitio/link/serializer" "github.com/splitio/splitd/splitio/link/service" @@ -14,20 +13,19 @@ import ( "github.com/splitio/splitd/splitio/sdk" ) -func Listen(logger logging.LoggerInterface, sdkFacade sdk.Interface, os ...Option) (<-chan error, func() error, error) { +func Listen(logger logging.LoggerInterface, sdkFacade sdk.Interface, opts *ListenerOptions) (<-chan error, func() error, error) { - var opts Options - err := opts.populate(os) + acceptor, err := transfer.NewAcceptor(logger, &opts.Transfer, &opts.Acceptor) if err != nil { - return nil, nil, fmt.Errorf("error parsing config options: %w", err) + return nil, nil, fmt.Errorf("error setting up transfer module: %w", err) } - acceptor, err := transfer.NewAcceptor(opts.forTransfer()...) + s, err := serializer.Setup(opts.Serialization) if err != nil { - return nil, nil, fmt.Errorf("error setting up transfer module: %w", err) + return nil, nil, fmt.Errorf("error building serializer") } - svc, err := service.New(logger, sdkFacade, opts.forApp()...) + svc, err := service.New(logger, sdkFacade, s, opts.Protocol) if err != nil { return nil, nil, fmt.Errorf("error setting up service handler: %w", err) } @@ -40,140 +38,47 @@ func Listen(logger logging.LoggerInterface, sdkFacade sdk.Interface, os ...Optio return ec, acceptor.Shutdown, nil } -func Consumer(logger logging.LoggerInterface, os ...Option) (client.Interface, error) { +func Consumer(logger logging.LoggerInterface, opts *ConsumerOptions) (types.ClientInterface, error) { - var opts Options - err := opts.populate(os) + s, err := serializer.Setup(opts.Serialization) if err != nil { - return nil, fmt.Errorf("error parsing config options: %w", err) + return nil, fmt.Errorf("error building serializer") } - conn, err := transfer.NewClientConn(opts.forTransfer()...) + conn, err := transfer.NewClientConn(&opts.Transfer) if err != nil { return nil, fmt.Errorf("errpr creating connection: %w", err) } - return client.New(logger, conn, opts.forApp()...) -} - -type Option func(*Options) error - -func WithSockType(s string) Option { - return func(o *Options) error { - switch s { - case "unix-seqpacket": - o.sockType = transfer.ConnTypeUnixSeqPacket - return nil - case "unix-stream": - o.sockType = transfer.ConnTypeUnixStream - return nil - } - return fmt.Errorf("unknown listener type '%s'", s) - } -} - -func WithAddress(s string) Option { - return func(o *Options) error { o.address = s; return nil } -} - -func WithBufSize(b int) Option { - return func(o *Options) error { o.bufSize = b; return nil } -} - -func WithSerialization(s string) Option { - return func(o *Options) error { - switch s { - case "msgpack": - o.serialization = serializer.MsgPack - return nil - } - return fmt.Errorf("unknown serialization mechanism '%s'", s) - } -} - -func WithProtocol(p string) Option { - return func(o *Options) error { - switch p { - case "v1": - o.protocolV = protocol.V1 - return nil - } - return fmt.Errorf("unkown protocol version '%s'", p) - } - -} - -func WithMaxSimultaneousConns(m int) Option { - return func(o *Options) error { o.maxSimulateneousConns = m; return nil } -} - -func WithReadTimeoutMs(m int) Option { - return func(o *Options) error { o.readTimeoutMS = m; return nil } + return client.New(logger, conn, s, opts.Consumer) } -func WithWriteTimeoutMs(m int) Option { - return func(o *Options) error { o.writeTimeoutMS = m; return nil } +type ListenerOptions struct { + Transfer transfer.Options + Acceptor transfer.AcceptorConfig + Serialization serializer.Mechanism + Protocol protocol.Version } -func WithAcceptTimeoutMs(m int) Option { - return func(o *Options) error { o.acceptTimeoutMS = m; return nil } +func DefaultListenerOptions() ListenerOptions { + return ListenerOptions{ + Transfer: transfer.DefaultOpts(), + Acceptor: transfer.DefaultAcceptorConfig(), + Serialization: serializer.MsgPack, + Protocol: protocol.V1, + } } -type Options struct { - sockType transfer.ConnType - address string - serialization serializer.Mechanism - protocolV protocol.Version - bufSize int - maxSimulateneousConns int - readTimeoutMS int - writeTimeoutMS int - acceptTimeoutMS int +type ConsumerOptions struct { + Transfer transfer.Options + Consumer client.Options + Serialization serializer.Mechanism } -func (o *Options) populate(options []Option) error { - for _, configure := range options { - err := configure(o) - if err != nil { - return err - } - } - return nil -} - -func (o *Options) forTransfer() []transfer.Option { - var toRet []transfer.Option - if o.sockType != 0 { - toRet = append(toRet, transfer.WithType(o.sockType)) - } - if o.address != "" { - toRet = append(toRet, transfer.WithAddress(o.address)) - } - if o.bufSize != 0 { - toRet = append(toRet, transfer.WithBufSize(o.bufSize)) - } - if o.maxSimulateneousConns != 0 { - toRet = append(toRet, transfer.WithMaxConns(o.maxSimulateneousConns)) - } - if o.readTimeoutMS != 0 { - toRet = append(toRet, transfer.WithReadTimeout(time.Duration(o.readTimeoutMS)*time.Millisecond)) - } - if o.writeTimeoutMS != 0 { - toRet = append(toRet, transfer.WithWriteTimeout(time.Duration(o.writeTimeoutMS)*time.Millisecond)) - } - if o.acceptTimeoutMS != 0 { - toRet = append(toRet, transfer.WithAcceptTimeout(time.Duration(o.acceptTimeoutMS)*time.Millisecond)) - } - return toRet -} - -func (o *Options) forApp() []common.Option { - var toRet []common.Option - if o.protocolV != 0 { - toRet = append(toRet, common.WithProtocolV(o.protocolV)) - } - if o.serialization != 0 { - toRet = append(toRet, common.WithSerialization(o.serialization)) - } - return toRet +func DefaultConsumerOptions() ConsumerOptions { + return ConsumerOptions{ + Transfer: transfer.DefaultOpts(), + Consumer: client.DefaultOptions(), + Serialization: serializer.MsgPack, + } } diff --git a/splitio/link/link_test.go b/splitio/link/link_test.go deleted file mode 100644 index ca5c859..0000000 --- a/splitio/link/link_test.go +++ /dev/null @@ -1,42 +0,0 @@ -package link - -import ( - "testing" - "time" - - "github.com/splitio/splitd/splitio/link/common" - "github.com/splitio/splitd/splitio/link/protocol" - "github.com/splitio/splitd/splitio/link/serializer" - "github.com/splitio/splitd/splitio/link/transfer" - "github.com/stretchr/testify/assert" -) - -func TestOptions(t *testing.T) { - var linkOpts Options - err := linkOpts.populate([]Option{ - WithAcceptTimeoutMs(5), - WithAddress("some_address"), - WithBufSize(123), - WithMaxSimultaneousConns(234), - WithProtocol("v1"), - WithReadTimeoutMs(6), - WithSerialization("msgpack"), - WithSockType("unix-stream"), - WithWriteTimeoutMs(7), - }) - assert.Nil(t, err) - - var transferOpts transfer.Options - transferOpts.Parse(linkOpts.forTransfer()) - assert.Equal(t, 5*time.Millisecond, transferOpts.AcceptTimeout) - assert.Equal(t, 6*time.Millisecond, transferOpts.ReadTimeout) - assert.Equal(t, 7*time.Millisecond, transferOpts.WriteTimeout) - assert.Equal(t, 123, transferOpts.BufferSize) - assert.Equal(t, transfer.ConnTypeUnixStream, transferOpts.ConnType) - assert.Equal(t, 234, transferOpts.MaxSimultaneousConnections) - - var hlOpts common.Opts - hlOpts.Parse(linkOpts.forApp()) - assert.Equal(t, protocol.V1, hlOpts.ProtoV) - assert.Equal(t, serializer.MsgPack, hlOpts.Serial) -} diff --git a/splitio/link/protocol/v1/mocks/mocks.go b/splitio/link/protocol/v1/mocks/mocks.go new file mode 100644 index 0000000..0787802 --- /dev/null +++ b/splitio/link/protocol/v1/mocks/mocks.go @@ -0,0 +1,90 @@ +package mocks + +import ( + "fmt" + "os" + "strconv" + + "github.com/splitio/splitd/splitio" + "github.com/splitio/splitd/splitio/link/protocol" + v1 "github.com/splitio/splitd/splitio/link/protocol/v1" + "github.com/splitio/splitd/splitio/sdk" +) + +func NewRegisterRPC(listener bool) *v1.RPC { + var flags v1.RegisterFlags + if listener { + flags = 1 << v1.RegisterFlagReturnImpressionData + } + return &v1.RPC{ + RPCBase: protocol.RPCBase{Version: protocol.V1}, + OpCode: v1.OCRegister, + Args: []interface{}{strconv.Itoa(os.Getpid()), fmt.Sprintf("splitd-%s", splitio.Version), flags}, + } +} + +func NewTreatmentRPC(key string, bucketing string, feature string, attrs map[string]interface{}) *v1.RPC { + return &v1.RPC{ + RPCBase: protocol.RPCBase{Version: protocol.V1}, + OpCode: v1.OCTreatment, + Args: []interface{}{key, bucketing, feature, attrs}, + } +} + +func NewTreatmentsRPC(key string, bucketing string, features []string, attrs map[string]interface{}) *v1.RPC { + return &v1.RPC{ + RPCBase: protocol.RPCBase{Version: protocol.V1}, + OpCode: v1.OCTreatments, + Args: []interface{}{key, bucketing, features, attrs}, + } +} + +func NewRegisterResp(ok bool) *v1.ResponseWrapper[v1.RegisterPayload] { + res := v1.ResultOk + if !ok { + res = v1.ResultInternalError + } + return &v1.ResponseWrapper[v1.RegisterPayload]{ + Status: res, + Payload: v1.RegisterPayload{}, + } +} + +func NewTreatmentResp(ok bool, treatment string, ilData *v1.ListenerExtraData) *v1.ResponseWrapper[v1.TreatmentPayload] { + res := v1.ResultOk + if !ok { + res = v1.ResultInternalError + } + return &v1.ResponseWrapper[v1.TreatmentPayload]{ + Status: res, + Payload: v1.TreatmentPayload{ + Treatment: treatment, + ListenerData: ilData, + }, + } +} + +func NewTreatmentsResp(ok bool, data []sdk.Result) *v1.ResponseWrapper[v1.TreatmentsPayload] { + res := v1.ResultOk + if !ok { + res = v1.ResultInternalError + } + + payload := make([]v1.TreatmentPayload, 0, len(data)) + for _, r := range data { + p := v1.TreatmentPayload{Treatment: r.Treatment} + if r.Impression != nil { + p.ListenerData = &v1.ListenerExtraData{ + Label: r.Impression.Label, + Timestamp: r.Impression.Time, + ChangeNumber: r.Impression.ChangeNumber, + } + } + payload = append(payload, p) + } + + return &v1.ResponseWrapper[v1.TreatmentsPayload]{ + Status: res, + Payload: v1.TreatmentsPayload{Results: payload}, + } +} diff --git a/splitio/link/protocol/v1/rpcs.go b/splitio/link/protocol/v1/rpcs.go index 79d3210..d0bed80 100644 --- a/splitio/link/protocol/v1/rpcs.go +++ b/splitio/link/protocol/v1/rpcs.go @@ -29,6 +29,17 @@ type RPC struct { Args []interface{} `msgpack:"a"` } +type Arguments interface { + PopulateFromRPC(rpc *RPC) error + Encode() []interface{} +} + +type RegisterArgs struct { + ID string `msgpack:"i"` + SDKVersion string `msgpack:"s"` + Flags RegisterFlags `msgpack:"f"` +} + const ( RegisterArgIDIdx = 0 RegisterArgSDKVersionIdx = 1 @@ -41,10 +52,8 @@ const ( RegisterFlagReturnImpressionData RegisterFlags = (1 << 0) ) -type RegisterArgs struct { - ID string `msgpack:"i"` - SDKVersion string `msgpack:"s"` - Flags RegisterFlags `msgpack:"f"` +func (r RegisterArgs) Encode() []interface{} { + return []interface{}{r.ID, r.SDKVersion, r.Flags} } func (r *RegisterArgs) PopulateFromRPC(rpc *RPC) error { @@ -86,6 +95,14 @@ type TreatmentArgs struct { Attributes map[string]interface{} `msgpack:"a"` } +func (r TreatmentArgs) Encode() []interface{} { + var bk string + if r.BucketingKey != nil { + bk = *r.BucketingKey + } + return []interface{}{r.Key, bk, r.Feature, r.Attributes} +} + func (t *TreatmentArgs) PopulateFromRPC(rpc *RPC) error { if rpc.OpCode != OCTreatment && rpc.OpCode != OCTreatmentWithConfig { return RPCParseError{Code: PECOpCodeMismatch} @@ -136,6 +153,14 @@ type TreatmentsArgs struct { Attributes map[string]interface{} `msgpack:"a"` } +func (r TreatmentsArgs) Encode() []interface{} { + var bk string + if r.BucketingKey != nil { + bk = *r.BucketingKey + } + return []interface{}{r.Key, bk, r.Features, r.Attributes} +} + func (t *TreatmentsArgs) PopulateFromRPC(rpc *RPC) error { if rpc.OpCode != OCTreatments && rpc.OpCode != OCTreatmentsWithConfig { return RPCParseError{Code: PECOpCodeMismatch} @@ -156,15 +181,15 @@ func (t *TreatmentsArgs) PopulateFromRPC(rpc *RPC) error { } - rawFeatureList, ok := rpc.Args[TreatmentsArgFeaturesIdx].([]interface{}) - if !ok { + rawFeatureList, ok := rpc.Args[TreatmentsArgFeaturesIdx].([]interface{}) + if !ok { return RPCParseError{Code: PECInvalidArgType, Data: int64(TreatmentsArgFeaturesIdx)} } - t.Features, ok = sanitizeFeatureList(rawFeatureList) - if !ok { - return RPCParseError{Code: PECInvalidArgType, Data: int64(TreatmentsArgFeaturesIdx)} - } + t.Features, ok = sanitizeFeatureList(rawFeatureList) + if !ok { + return RPCParseError{Code: PECInvalidArgType, Data: int64(TreatmentsArgFeaturesIdx)} + } rawAttrs, err := getOptional[map[string]interface{}](rpc.Args[TreatmentsArgAttributesIdx]) if err != nil { @@ -193,6 +218,10 @@ type TrackArgs struct { Timestamp int64 `msgpack:"m"` } +func (r TrackArgs) Encode() []interface{} { + return []interface{}{r.Key, r.TrafficType, r.EventType, r.Value, r.Properties, r.Timestamp} +} + func (t *TrackArgs) PopulateFromRPC(rpc *RPC) error { if rpc.OpCode != OCTrack { return RPCParseError{Code: PECOpCodeMismatch} @@ -294,18 +323,18 @@ func sanitizeAttributes(attrs map[string]interface{}) map[string]interface{} { } func sanitizeFeatureList(raw []interface{}) ([]string, bool) { - features := make([]string, 0, len(raw)) - for _, f := range raw { - asStr, ok := f.(string) - if !ok { - return nil, false - } - features = append(features, asStr) - } - return features, true + features := make([]string, 0, len(raw)) + for _, f := range raw { + asStr, ok := f.(string) + if !ok { + return nil, false + } + features = append(features, asStr) + } + return features, true } -func tryInt2[T int8|int16|int32|int64|uint8|uint16|uint32|uint64](x interface{}) (T, bool) { +func tryInt2[T int8 | int16 | int32 | int64 | uint8 | uint16 | uint32 | uint64](x interface{}) (T, bool) { switch parsed := x.(type) { case uint8: return T(parsed), true @@ -345,3 +374,8 @@ func tryNumberAsFloat(x interface{}) (float64, bool) { return 0, false } + +var _ Arguments = (*RegisterArgs)(nil) +var _ Arguments = (*TreatmentArgs)(nil) +var _ Arguments = (*TreatmentsArgs)(nil) +var _ Arguments = (*TrackArgs)(nil) diff --git a/splitio/link/serializer/mocks/serializer.go b/splitio/link/serializer/mocks/serializer.go new file mode 100644 index 0000000..0f6059e --- /dev/null +++ b/splitio/link/serializer/mocks/serializer.go @@ -0,0 +1,24 @@ +package mocks + +import ( + "github.com/splitio/splitd/splitio/link/serializer" + "github.com/stretchr/testify/mock" +) + +type SerializerMock struct { + mock.Mock +} + +// Parse implements serializer.Interface +func (m *SerializerMock) Parse(data []byte, v interface{}) error { + args := m.Called(data, v) + return args.Error(0) +} + +// Serialize implements serializer.Interface +func (m *SerializerMock) Serialize(v interface{}) ([]byte, error) { + args := m.Called(v) + return args.Get(0).([]byte), args.Error(1) +} + +var _ serializer.Interface = (*SerializerMock)(nil) diff --git a/splitio/link/service/service.go b/splitio/link/service/service.go index a500988..a122189 100644 --- a/splitio/link/service/service.go +++ b/splitio/link/service/service.go @@ -5,7 +5,6 @@ import ( "fmt" "github.com/splitio/go-toolkit/v5/logging" - "github.com/splitio/splitd/splitio/link/common" "github.com/splitio/splitd/splitio/link/protocol" "github.com/splitio/splitd/splitio/link/serializer" "github.com/splitio/splitd/splitio/link/transfer" @@ -34,13 +33,11 @@ func (s *Impl) HandleNewClient(cc transfer.RawConn) { // TODO(mredolatti): Track active connections } -func New(logger logging.LoggerInterface, splitSDK sdk.Interface, opts ...common.Option) (*Impl, error) { - o := common.DefaultOpts() - o.Parse(opts) +func New(logger logging.LoggerInterface, splitSDK sdk.Interface, serial serializer.Interface, proto protocol.Version) (*Impl, error) { - switch o.ProtoV { + switch proto { case protocol.V1: - cmf, err := newCMFactoryForV1(logger, splitSDK, o.Serial) + cmf, err := newCMFactoryForV1(logger, splitSDK, serial) if err != nil { return nil, fmt.Errorf("error setting up client-manager factory: %w", err) } @@ -50,7 +47,9 @@ func New(logger logging.LoggerInterface, splitSDK sdk.Interface, opts ...common. newClientManager: cmf, }, nil } - return nil, fmt.Errorf("unknown protocol version: '%d'", o.ProtoV) + + return nil, fmt.Errorf("unknown protocol version: '%d'", proto) + } type ClientManager interface { @@ -59,13 +58,9 @@ type ClientManager interface { type ClientManagerFactory func(transfer.RawConn) ClientManager -func newCMFactoryForV1(logger logging.LoggerInterface, splitSDK sdk.Interface, serialization serializer.Mechanism) (ClientManagerFactory, error) { - ser, err := serializer.Setup(serialization) - if err != nil { - return nil, err - } +func newCMFactoryForV1(logger logging.LoggerInterface, splitSDK sdk.Interface, serial serializer.Interface) (ClientManagerFactory, error) { return func(conn transfer.RawConn) ClientManager { - return serviceV1.NewClientManager(conn, logger, splitSDK, ser) + return serviceV1.NewClientManager(conn, logger, splitSDK, serial) }, nil } diff --git a/splitio/link/service/v1/clientmgr.go b/splitio/link/service/v1/clientmgr.go index 16bb353..8655956 100644 --- a/splitio/link/service/v1/clientmgr.go +++ b/splitio/link/service/v1/clientmgr.go @@ -68,7 +68,6 @@ func (m *ClientManager) handleClientInteractions() error { response, err := m.handleRPC(rpc) if err != nil { - // TODO(mredolatti): see if this is recoverable return fmt.Errorf("error handling RPC: %w", err) } @@ -96,13 +95,11 @@ func (m *ClientManager) sendResponse(response interface{}) error { serialized, err := m.serializer.Serialize(response) if err != nil { - // TODO(mredolatti): see if this is recoverable return fmt.Errorf("error serializing response: %w", err) } err = m.cc.SendMessage(serialized) if err != nil { - // TODO(mredolatti): see if this is recoverable return fmt.Errorf("error sending response back to the client: %w", err) } diff --git a/splitio/link/service/v1/clientmgr_test.go b/splitio/link/service/v1/clientmgr_test.go index 922d027..769e3e4 100644 --- a/splitio/link/service/v1/clientmgr_test.go +++ b/splitio/link/service/v1/clientmgr_test.go @@ -9,16 +9,18 @@ import ( "github.com/splitio/go-toolkit/v5/logging" "github.com/splitio/splitd/splitio/link/protocol" v1 "github.com/splitio/splitd/splitio/link/protocol/v1" - "github.com/splitio/splitd/splitio/link/serializer" - "github.com/splitio/splitd/splitio/link/transfer" + serializerMocks "github.com/splitio/splitd/splitio/link/serializer/mocks" + transferMocks "github.com/splitio/splitd/splitio/link/transfer/mocks" + proto1Mocks "github.com/splitio/splitd/splitio/link/protocol/v1/mocks" "github.com/splitio/splitd/splitio/sdk" + sdkMocks "github.com/splitio/splitd/splitio/sdk/mocks" "github.com/splitio/splitd/splitio/sdk/types" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/mock" ) func TestRegisterAndTreatmentHappyPath(t *testing.T) { - rawConnMock := &RawConnMock{} + rawConnMock := &transferMocks.RawConnMock{} rawConnMock.On("ReceiveMessage").Return([]byte("registrationMessage"), nil).Once() rawConnMock.On("SendMessage", []byte("successRegistration")).Return(nil).Once() rawConnMock.On("ReceiveMessage").Return([]byte("treatmentMessage"), nil).Once() @@ -26,7 +28,7 @@ func TestRegisterAndTreatmentHappyPath(t *testing.T) { rawConnMock.On("ReceiveMessage").Return([]byte(nil), io.EOF).Once() rawConnMock.On("Shutdown").Return(nil).Once() - serializerMock := &SerializerMock{} + serializerMock := &serializerMocks.SerializerMock{} serializerMock.On("Parse", []byte("registrationMessage"), mock.Anything).Return(nil).Run(func(args mock.Arguments) { *args.Get(1).(*v1.RPC) = v1.RPC{ RPCBase: protocol.RPCBase{Version: protocol.V1}, @@ -34,7 +36,7 @@ func TestRegisterAndTreatmentHappyPath(t *testing.T) { Args: []interface{}{"someID", "some_sdk-1.2.3", uint64(0)}, } }).Once() - serializerMock.On("Serialize", newRegisterResp(true)).Return([]byte("successRegistration"), nil).Once() + serializerMock.On("Serialize", proto1Mocks.NewRegisterResp(true)).Return([]byte("successRegistration"), nil).Once() serializerMock.On("Parse", []byte("treatmentMessage"), mock.Anything).Return(nil).Run(func(args mock.Arguments) { *args.Get(1).(*v1.RPC) = v1.RPC{ RPCBase: protocol.RPCBase{Version: protocol.V1}, @@ -42,11 +44,11 @@ func TestRegisterAndTreatmentHappyPath(t *testing.T) { Args: []interface{}{"key", nil, "someFeature", map[string]interface{}(nil)}, } }).Once() - serializerMock.On("Serialize", newTreatmentResp(true, "on", nil)).Return([]byte("successPayload"), nil).Once() + serializerMock.On("Serialize", proto1Mocks.NewTreatmentResp(true, "on", nil)).Return([]byte("successPayload"), nil).Once() - sdkMock := &SDKMock{} + sdkMock := &sdkMocks.SDKMock{} sdkMock. - On("Treatment", &types.ClientConfig{Metadata: types.ClientMetadata{ID: "someID", SdkVersion: "some_sdk-1.2.3"}}, "key", (*string)(nil), "someFeature", map[string]interface{}(nil)). + On("Treatment", &types.ClientConfig{Metadata: types.ClientMetadata{ID: "someID", SdkVersion: "some_sdk-1.2.3"}}, "key", (*string)(nil), "someFeature", map[string]interface{}(nil)). Return(&sdk.Result{Treatment: "on"}, nil).Once() logger := logging.NewLogger(nil) @@ -57,7 +59,7 @@ func TestRegisterAndTreatmentHappyPath(t *testing.T) { } func TestRegisterAndTreatmentsHappyPath(t *testing.T) { - rawConnMock := &RawConnMock{} + rawConnMock := &transferMocks.RawConnMock{} rawConnMock.On("ReceiveMessage").Return([]byte("registrationMessage"), nil).Once() rawConnMock.On("SendMessage", []byte("successRegistration")).Return(nil).Once() rawConnMock.On("ReceiveMessage").Return([]byte("treatmentsMessage"), nil).Once() @@ -65,7 +67,7 @@ func TestRegisterAndTreatmentsHappyPath(t *testing.T) { rawConnMock.On("ReceiveMessage").Return([]byte(nil), io.EOF).Once() rawConnMock.On("Shutdown").Return(nil).Once() - serializerMock := &SerializerMock{} + serializerMock := &serializerMocks.SerializerMock{} serializerMock.On("Parse", []byte("registrationMessage"), mock.Anything).Return(nil).Run(func(args mock.Arguments) { *args.Get(1).(*v1.RPC) = v1.RPC{ RPCBase: protocol.RPCBase{Version: protocol.V1}, @@ -73,7 +75,7 @@ func TestRegisterAndTreatmentsHappyPath(t *testing.T) { Args: []interface{}{"someID", "some_sdk-1.2.3", uint64(0)}, } }).Once() - serializerMock.On("Serialize", newRegisterResp(true)).Return([]byte("successRegistration"), nil).Once() + serializerMock.On("Serialize", proto1Mocks.NewRegisterResp(true)).Return([]byte("successRegistration"), nil).Once() serializerMock.On("Parse", []byte("treatmentsMessage"), mock.Anything).Return(nil).Run(func(args mock.Arguments) { *args.Get(1).(*v1.RPC) = v1.RPC{ RPCBase: protocol.RPCBase{Version: protocol.V1}, @@ -81,15 +83,15 @@ func TestRegisterAndTreatmentsHappyPath(t *testing.T) { Args: []interface{}{"key", nil, []interface{}{"feat1", "feat2", "feat3"}, map[string]interface{}(nil)}, } }).Once() - serializerMock.On("Serialize", newTreatmentsResp(true, []sdk.Result{ + serializerMock.On("Serialize", proto1Mocks.NewTreatmentsResp(true, []sdk.Result{ {Treatment: "on"}, {Treatment: "off"}, {Treatment: "control"}, })).Return([]byte("successPayload"), nil).Once() - sdkMock := &SDKMock{} + sdkMock := &sdkMocks.SDKMock{} sdkMock. On( "Treatments", - &types.ClientConfig{Metadata: types.ClientMetadata{ID: "someID", SdkVersion: "some_sdk-1.2.3"}}, + &types.ClientConfig{Metadata: types.ClientMetadata{ID: "someID", SdkVersion: "some_sdk-1.2.3"}}, "key", (*string)(nil), []string{"feat1", "feat2", "feat3"}, @@ -108,7 +110,7 @@ func TestRegisterAndTreatmentsHappyPath(t *testing.T) { } func TestRegisterWithImpsAndTreatmentHappyPath(t *testing.T) { - rawConnMock := &RawConnMock{} + rawConnMock := &transferMocks.RawConnMock{} rawConnMock.On("ReceiveMessage").Return([]byte("registrationMessage"), nil).Once() rawConnMock.On("SendMessage", []byte("successRegistration")).Return(nil).Once() rawConnMock.On("ReceiveMessage").Return([]byte("treatmentMessage"), nil).Once() @@ -116,7 +118,7 @@ func TestRegisterWithImpsAndTreatmentHappyPath(t *testing.T) { rawConnMock.On("ReceiveMessage").Return([]byte(nil), io.EOF).Once() rawConnMock.On("Shutdown").Return(nil).Once() - serializerMock := &SerializerMock{} + serializerMock := &serializerMocks.SerializerMock{} serializerMock.On("Parse", []byte("registrationMessage"), mock.Anything).Return(nil).Run(func(args mock.Arguments) { *args.Get(1).(*v1.RPC) = v1.RPC{ RPCBase: protocol.RPCBase{Version: protocol.V1}, @@ -124,7 +126,7 @@ func TestRegisterWithImpsAndTreatmentHappyPath(t *testing.T) { Args: []interface{}{"someID", "some_sdk-1.2.3", uint64(v1.RegisterFlagReturnImpressionData)}, } }).Once() - serializerMock.On("Serialize", newRegisterResp(true)).Return([]byte("successRegistration"), nil).Once() + serializerMock.On("Serialize", proto1Mocks.NewRegisterResp(true)).Return([]byte("successRegistration"), nil).Once() serializerMock.On("Parse", []byte("treatmentMessage"), mock.Anything).Return(nil).Run(func(args mock.Arguments) { *args.Get(1).(*v1.RPC) = v1.RPC{ RPCBase: protocol.RPCBase{Version: protocol.V1}, @@ -132,13 +134,13 @@ func TestRegisterWithImpsAndTreatmentHappyPath(t *testing.T) { Args: []interface{}{"key", nil, "someFeature", map[string]interface{}(nil)}, } }).Once() - serializerMock.On("Serialize", newTreatmentResp(true, "on", &v1.ListenerExtraData{Label: "l1", Timestamp: 1234556})). + serializerMock.On("Serialize", proto1Mocks.NewTreatmentResp(true, "on", &v1.ListenerExtraData{Label: "l1", Timestamp: 1234556})). Return([]byte("successPayload"), nil).Once() - sdkMock := &SDKMock{} + sdkMock := &sdkMocks.SDKMock{} sdkMock. On("Treatment", - &types.ClientConfig{Metadata: types.ClientMetadata{ID: "someID", SdkVersion: "some_sdk-1.2.3"}, ReturnImpressionData: true}, + &types.ClientConfig{Metadata: types.ClientMetadata{ID: "someID", SdkVersion: "some_sdk-1.2.3"}, ReturnImpressionData: true}, "key", (*string)(nil), "someFeature", map[string]interface{}(nil)). Return(&sdk.Result{Treatment: "on", Impression: &dtos.Impression{Label: "l1", Time: 1234556}}, nil).Once() @@ -150,11 +152,11 @@ func TestRegisterWithImpsAndTreatmentHappyPath(t *testing.T) { } func TestTreatmentWithoutRegister(t *testing.T) { - rawConnMock := &RawConnMock{} + rawConnMock := &transferMocks.RawConnMock{} rawConnMock.On("ReceiveMessage").Return([]byte("treatmentMessage"), nil).Once() rawConnMock.On("Shutdown").Return(nil).Once() - serializerMock := &SerializerMock{} + serializerMock := &serializerMocks.SerializerMock{} serializerMock.On("Parse", []byte("treatmentMessage"), mock.Anything).Return(nil).Run(func(args mock.Arguments) { *args.Get(1).(*v1.RPC) = v1.RPC{ RPCBase: protocol.RPCBase{Version: protocol.V1}, @@ -163,7 +165,7 @@ func TestTreatmentWithoutRegister(t *testing.T) { } }).Once() - sdkMock := &SDKMock{} + sdkMock := &sdkMocks.SDKMock{} logger := logging.NewLogger(nil) cm := NewClientManager(rawConnMock, logger, sdkMock, serializerMock) err := cm.handleClientInteractions() @@ -172,12 +174,12 @@ func TestTreatmentWithoutRegister(t *testing.T) { } func TestConnectionFailureWhenReading(t *testing.T) { - rawConnMock := &RawConnMock{} + rawConnMock := &transferMocks.RawConnMock{} rawConnMock.On("ReceiveMessage").Return([]byte(nil), errors.New("something")).Once() rawConnMock.On("Shutdown").Return(nil).Once() - serializerMock := &SerializerMock{} - sdkMock := &SDKMock{} + serializerMock := &serializerMocks.SerializerMock{} + sdkMock := &sdkMocks.SDKMock{} logger := logging.NewLogger(nil) cm := NewClientManager(rawConnMock, logger, sdkMock, serializerMock) err := cm.handleClientInteractions() @@ -185,124 +187,4 @@ func TestConnectionFailureWhenReading(t *testing.T) { rawConnMock.AssertNumberOfCalls(t, "Shutdown", 1) } -func newRegisterResp(ok bool) *v1.ResponseWrapper[v1.RegisterPayload] { - res := v1.ResultOk - if !ok { - res = v1.ResultInternalError - } - return &v1.ResponseWrapper[v1.RegisterPayload]{ - Status: res, - Payload: v1.RegisterPayload{}, - } -} - -func newTreatmentResp(ok bool, treatment string, ilData *v1.ListenerExtraData) *v1.ResponseWrapper[v1.TreatmentPayload] { - res := v1.ResultOk - if !ok { - res = v1.ResultInternalError - } - return &v1.ResponseWrapper[v1.TreatmentPayload]{ - Status: res, - Payload: v1.TreatmentPayload{ - Treatment: treatment, - ListenerData: ilData, - }, - } -} - -func newTreatmentsResp(ok bool, data []sdk.Result) *v1.ResponseWrapper[v1.TreatmentsPayload] { - res := v1.ResultOk - if !ok { - res = v1.ResultInternalError - } - - payload := make([]v1.TreatmentPayload, 0, len(data)) - for _, r := range data { - p := v1.TreatmentPayload{Treatment: r.Treatment} - if r.Impression != nil { - p.ListenerData = &v1.ListenerExtraData{ - Label: r.Impression.Label, - Timestamp: r.Impression.Time, - ChangeNumber: r.Impression.ChangeNumber, - } - } - payload = append(payload, p) - } - - return &v1.ResponseWrapper[v1.TreatmentsPayload]{ - Status: res, - Payload: v1.TreatmentsPayload{Results: payload}, - } -} - -// mocks - -type RawConnMock struct { - mock.Mock -} - -// ReceiveMessage implements transfer.RawConn -func (m *RawConnMock) ReceiveMessage() ([]byte, error) { - args := m.Called() - return args.Get(0).([]byte), args.Error(1) -} - -// SendMessage implements transfer.RawConn -func (m *RawConnMock) SendMessage(data []byte) error { - args := m.Called(data) - return args.Error(0) -} - -// Shutdown implements transfer.RawConn -func (m *RawConnMock) Shutdown() error { - args := m.Called() - return args.Error(0) -} - -type SerializerMock struct { - mock.Mock -} - -// Parse implements serializer.Interface -func (m *SerializerMock) Parse(data []byte, v interface{}) error { - args := m.Called(data, v) - return args.Error(0) -} - -// Serialize implements serializer.Interface -func (m *SerializerMock) Serialize(v interface{}) ([]byte, error) { - args := m.Called(v) - return args.Get(0).([]byte), args.Error(1) -} - -type SDKMock struct { - mock.Mock -} - -// Treatment implements sdk.Interface -func (m *SDKMock) Treatment( - md *types.ClientConfig, - key string, - bucketingKey *string, - feature string, - attributes map[string]interface{}, -) (*sdk.Result, error) { - args := m.Called(md, key, bucketingKey, feature, attributes) - return args.Get(0).(*sdk.Result), nil -} - -// Treatments implements sdk.Interface -func (m *SDKMock) Treatments( - md *types.ClientConfig, - key string, - bucketingKey *string, - features []string, - attributes map[string]interface{}, -) (map[string]sdk.Result, error) { - args := m.Called(md, key, bucketingKey, features, attributes) - return args.Get(0).(map[string]sdk.Result), nil -} -var _ transfer.RawConn = (*RawConnMock)(nil) -var _ serializer.Interface = (*SerializerMock)(nil) -var _ sdk.Interface = (*SDKMock)(nil) diff --git a/splitio/link/transfer/acceptor.go b/splitio/link/transfer/acceptor.go index cd3d9ec..a1dcdf5 100644 --- a/splitio/link/transfer/acceptor.go +++ b/splitio/link/transfer/acceptor.go @@ -34,14 +34,27 @@ type Acceptor struct { var errNoSetDeadline = errors.New("listener doesn't support setting a deadline") -func newAcceptor(address net.Addr, rawConnFactory RawConnFactory, o *Options) *Acceptor { +type AcceptorConfig struct { + AcceptTimeout time.Duration + MaxSimultaneousConnections int +} + +func DefaultAcceptorConfig() AcceptorConfig { + return AcceptorConfig{ + MaxSimultaneousConnections: 32, + AcceptTimeout: 1 * time.Second, + } +} + + +func newAcceptor(address net.Addr, rawConnFactory RawConnFactory, logger logging.LoggerInterface, cfg *AcceptorConfig) *Acceptor { return &Acceptor{ rawConnFactory: rawConnFactory, - logger: o.Logger, + logger: logger, address: address, - maxConns: o.MaxSimultaneousConnections, - sem: semaphore.NewWeighted(int64(o.MaxSimultaneousConnections)), - maxWait: o.AcceptTimeout, + maxConns: cfg.MaxSimultaneousConnections, + sem: semaphore.NewWeighted(int64(cfg.MaxSimultaneousConnections)), + maxWait: cfg.AcceptTimeout, } } diff --git a/splitio/link/transfer/acceptor_test.go b/splitio/link/transfer/acceptor_test.go index 07e87e3..da50059 100644 --- a/splitio/link/transfer/acceptor_test.go +++ b/splitio/link/transfer/acceptor_test.go @@ -14,17 +14,16 @@ import ( ) func TestAcceptor(t *testing.T) { - // This test sets up an acceptor with the following params: - // - a queue size of 1 - // - a 100ms timeout for items in the waitqueue - // - a 300ms delay in the handler (so that by the time the 2nd client tries to connect, the first one is busy) - // - // 2 clients will try to connect and do some write & reads. - // First client will successfully connect and exchange some information - // Second client's server-end of the socket will be closed after the timeout - // The write not error out (though nothing is written), but will notice that the socket hasbeen remotely closed and update it's state - // The following read will report an EOF - + // This test sets up an acceptor with the following params: + // - a queue size of 1 + // - a 100ms timeout for items in the waitqueue + // - a 300ms delay in the handler (so that by the time the 2nd client tries to connect, the first one is busy) + // + // 2 clients will try to connect and do some write & reads. + // First client will successfully connect and exchange some information + // Second client's server-end of the socket will be closed after the timeout + // The write not error out (though nothing is written), but will notice that the socket hasbeen remotely closed and update it's state + // The following read will report an EOF logger := logging.NewLogger(nil) dir, err := os.MkdirTemp(os.TempDir(), "acceptortest") @@ -32,13 +31,14 @@ func TestAcceptor(t *testing.T) { serverSockFN := path.Join(dir, "acctest.sock") - opts := defaultOpts() - opts.AcceptTimeout = 100 * time.Millisecond - opts.Logger = logger - opts.MaxSimultaneousConnections = 1 + connOpts := DefaultOpts() + + acceptorConfig := DefaultAcceptorConfig() + acceptorConfig.AcceptTimeout = 100 * time.Millisecond + acceptorConfig.MaxSimultaneousConnections = 1 acc := newAcceptor(&net.UnixAddr{Net: "unix", Name: serverSockFN}, func(c net.Conn) RawConn { - return newConnWrapper(c, &framing.LengthPrefixImpl{}, &opts) - }, &opts) + return newConnWrapper(c, &framing.LengthPrefixImpl{}, &connOpts) + }, logger, &acceptorConfig) endc, err := acc.Start(func(c RawConn) { message, err := c.ReceiveMessage() @@ -55,7 +55,11 @@ func TestAcceptor(t *testing.T) { time.Sleep(1 * time.Second) // to ensure server is started - client1, err := NewClientConn(WithType(ConnTypeUnixStream), WithAddress(serverSockFN)) + clientOpts := DefaultOpts() + clientOpts.Address = serverSockFN + clientOpts.ConnType = ConnTypeUnixStream + + client1, err := NewClientConn(&clientOpts) assert.Nil(t, err) assert.NotNil(t, client1) err = client1.SendMessage([]byte("some")) @@ -64,7 +68,7 @@ func TestAcceptor(t *testing.T) { assert.Nil(t, err) assert.Equal(t, []byte("thing"), recv) - client2, err := NewClientConn(WithType(ConnTypeUnixStream), WithAddress(serverSockFN)) + client2, err := NewClientConn(&clientOpts) assert.Nil(t, err) err = client2.SendMessage([]byte("some")) assert.Nil(t, err) // write doesn't fail. instead causes the transition of the socket to EOF state diff --git a/splitio/link/transfer/mocks/rawconn.go b/splitio/link/transfer/mocks/rawconn.go new file mode 100644 index 0000000..e43e6ed --- /dev/null +++ b/splitio/link/transfer/mocks/rawconn.go @@ -0,0 +1,30 @@ +package mocks + +import ( + "github.com/splitio/splitd/splitio/link/transfer" + "github.com/stretchr/testify/mock" +) + +type RawConnMock struct { + mock.Mock +} + +// ReceiveMessage implements transfer.RawConn +func (m *RawConnMock) ReceiveMessage() ([]byte, error) { + args := m.Called() + return args.Get(0).([]byte), args.Error(1) +} + +// SendMessage implements transfer.RawConn +func (m *RawConnMock) SendMessage(data []byte) error { + args := m.Called(data) + return args.Error(0) +} + +// Shutdown implements transfer.RawConn +func (m *RawConnMock) Shutdown() error { + args := m.Called() + return args.Error(0) +} + +var _ transfer.RawConn = (*RawConnMock)(nil) diff --git a/splitio/link/transfer/setup.go b/splitio/link/transfer/setup.go index 761cf2e..d26c9ce 100644 --- a/splitio/link/transfer/setup.go +++ b/splitio/link/transfer/setup.go @@ -21,10 +21,7 @@ var ( ErrInvalidConnType = errors.New("invalid listener type") ) -func NewAcceptor(opts ...Option) (*Acceptor, error) { - - o := defaultOpts() - o.Parse(opts) +func NewAcceptor(logger logging.LoggerInterface, o *Options, listenerConfig *AcceptorConfig) (*Acceptor, error) { var address net.Addr var framer framing.Interface @@ -38,13 +35,11 @@ func NewAcceptor(opts ...Option) (*Acceptor, error) { return nil, ErrInvalidConnType } - connFactory := func(c net.Conn) RawConn { return newConnWrapper(c, framer, &o) } - return newAcceptor(address, connFactory, &o), nil + connFactory := func(c net.Conn) RawConn { return newConnWrapper(c, framer, o) } + return newAcceptor(address, connFactory, logger, listenerConfig), nil } -func NewClientConn(opts ...Option) (RawConn, error) { - o := defaultOpts() - o.Parse(opts) +func NewClientConn(o *Options) (RawConn, error) { var address net.Addr var framer framing.Interface @@ -63,11 +58,12 @@ func NewClientConn(opts ...Option) (RawConn, error) { return nil, fmt.Errorf("error creating connection: %w", err) } - return newConnWrapper(c, framer, &o), nil + return newConnWrapper(c, framer, o), nil } type Option func(*Options) +/* func WithAddress(address string) Option { return func(o *Options) { o.Address = address } } func WithType(t ConnType) Option { return func(o *Options) { o.ConnType = t } } func WithLogger(logger logging.LoggerInterface) Option { return func(o *Options) { o.Logger = logger } } @@ -76,33 +72,30 @@ func WithMaxConns(m int) Option { return func(o *Options) func WithReadTimeout(d time.Duration) Option { return func(o *Options) { o.ReadTimeout = d } } func WithWriteTimeout(d time.Duration) Option { return func(o *Options) { o.WriteTimeout = d } } func WithAcceptTimeout(d time.Duration) Option { return func(o *Options) { o.AcceptTimeout = d } } +*/ type Options struct { ConnType ConnType Address string Logger logging.LoggerInterface BufferSize int - MaxSimultaneousConnections int ReadTimeout time.Duration WriteTimeout time.Duration - AcceptTimeout time.Duration } - +/* func (o *Options) Parse(opts []Option) { for _, configure := range opts { configure(o) } } - -func defaultOpts() Options { +*/ +func DefaultOpts() Options { return Options{ ConnType: ConnTypeUnixSeqPacket, Address: "/var/run/splitd.sock", Logger: logging.NewLogger(nil), BufferSize: 1024, - MaxSimultaneousConnections: 32, ReadTimeout: 1 * time.Second, WriteTimeout: 1 * time.Second, - AcceptTimeout: 1 * time.Second, } } diff --git a/splitio/sdk/conf/conf.go b/splitio/sdk/conf/conf.go index 8ccb5af..2f1bbe8 100644 --- a/splitio/sdk/conf/conf.go +++ b/splitio/sdk/conf/conf.go @@ -15,16 +15,6 @@ type Config struct { URLs URLs } -func (c *Config) ParseOptions(options []Option) error { - for _, apply := range options { - err := apply(c) - if err != nil { - return err - } - } - return nil -} - type Splits struct { SyncPeriod time.Duration UpdateBufferSize int @@ -98,34 +88,3 @@ func DefaultConfig() *Config { }, } } - -type Option func(c *Config) error - -func WithLabelsEnabled(v bool) Option { - return func(c *Config) error { c.LabelsEnabled = v; return nil } -} - -func WithStreamingEnabled(v bool) Option { - return func(c *Config) error { c.StreamingEnabled = v; return nil } -} - -func WithAuthURL(v string) Option { - return func(c *Config) error { c.URLs.Auth = v; return nil } -} - -func WithSDKURL(v string) Option { - return func(c *Config) error { c.URLs.SDK = v; return nil } -} - -func WithEventsURL(v string) Option { - return func(c *Config) error { c.URLs.Events = v; return nil } -} - -func WithStreamingURL(v string) Option { - return func(c *Config) error { c.URLs.Streaming = v; return nil } -} - - -func WithTelemetryURL(v string) Option { - return func(c *Config) error { c.URLs.Telemetry = v; return nil } -} diff --git a/splitio/sdk/mocks/sdk.go b/splitio/sdk/mocks/sdk.go new file mode 100644 index 0000000..8d3059e --- /dev/null +++ b/splitio/sdk/mocks/sdk.go @@ -0,0 +1,38 @@ +package mocks + +import ( + "github.com/splitio/splitd/splitio/sdk" + "github.com/splitio/splitd/splitio/sdk/types" + "github.com/stretchr/testify/mock" +) + +type SDKMock struct { + mock.Mock +} + +// Treatment implements sdk.Interface +func (m *SDKMock) Treatment( + md *types.ClientConfig, + key string, + bucketingKey *string, + feature string, + attributes map[string]interface{}, +) (*sdk.Result, error) { + args := m.Called(md, key, bucketingKey, feature, attributes) + return args.Get(0).(*sdk.Result), nil +} + +// Treatments implements sdk.Interface +func (m *SDKMock) Treatments( + md *types.ClientConfig, + key string, + bucketingKey *string, + features []string, + attributes map[string]interface{}, +) (map[string]sdk.Result, error) { + args := m.Called(md, key, bucketingKey, features, attributes) + return args.Get(0).(map[string]sdk.Result), nil +} + + +var _ sdk.Interface = (*SDKMock)(nil) diff --git a/splitio/sdk/sdk.go b/splitio/sdk/sdk.go index e40c335..0845d6a 100644 --- a/splitio/sdk/sdk.go +++ b/splitio/sdk/sdk.go @@ -40,13 +40,9 @@ type Impl struct { status chan int } -func New(logger logging.LoggerInterface, apikey string, opts ...conf.Option) (*Impl, error) { +func New(logger logging.LoggerInterface, apikey string, c *conf.Config) (*Impl, error) { md := dtos.Metadata{SDKVersion: fmt.Sprintf("splitd-%s", splitio.Version)} - c := sdkConf.DefaultConfig() - if err := c.ParseOptions(opts); err != nil { - return nil, fmt.Errorf("error parsing SDK config: %w", err) - } advCfg := c.ToAdvancedConfig() stores := setupStorages(c) diff --git a/splitio/sdk/tasks/impressions_test.go b/splitio/sdk/tasks/impressions_test.go index 0b77c4a..73389fa 100644 --- a/splitio/sdk/tasks/impressions_test.go +++ b/splitio/sdk/tasks/impressions_test.go @@ -39,6 +39,8 @@ func TestImpressionsTask(t *testing.T) { Return(nil). Once() + // ImpressionsDTO are built from the contents of a map so the ordering is undefined. + // to solve this, we sort the input by feature name (& provided an already sorted expected value) rec.On("Record", mock.MatchedBy(func(imps []dtos.ImpressionsDTO) bool { sort.Slice(imps, func(i, j int) bool { return imps[i].TestName < imps[j].TestName }) @@ -56,19 +58,6 @@ func TestImpressionsTask(t *testing.T) { dtos.Metadata{SDKVersion: "python-1.2.3", MachineIP: "", MachineName: ""}, emptyMap).Return(nil).Once() - /*[]dtos.ImpressionsDTO{ - { - TestName: "f3", - KeyImpressions: []dtos.ImpressionDTO{{KeyName: "k3", Treatment: "on", Time: 123458, ChangeNumber: 789, Label: "l3"}}, - }, - { - TestName: "f4", - KeyImpressions: []dtos.ImpressionDTO{{KeyName: "k3", Treatment: "on", Time: 123459, ChangeNumber: 890, Label: "l4"}}, - }, - }, dtos.Metadata{SDKVersion: "python-1.2.3", MachineIP: "", MachineName: ""}, emptyMap). - Return(nil). - Once() - */ is.Push(types.ClientMetadata{ID: "i1", SdkVersion: "php-1.2.3"}, dtos.Impression{KeyName: "k1", FeatureName: "f1", Treatment: "on", Label: "l1", ChangeNumber: 123, Time: 123456}) is.Push(types.ClientMetadata{ID: "i2", SdkVersion: "go-1.2.3"}, diff --git a/splitio/util/conf/helpers.go b/splitio/util/conf/helpers.go new file mode 100644 index 0000000..c6a3fe4 --- /dev/null +++ b/splitio/util/conf/helpers.go @@ -0,0 +1,27 @@ +package conf + +func SetIfNotNil[T any](dst *T, src *T) { + if src != nil { + *dst = *src + } +} + +func SetIfNotEmpty[T comparable](dst *T, src *T) { + var t T + if src != nil && *src != t { + *dst = *src + } +} + +func MapIfNotNil[T any, U any](dst *T, src *U, fn func(U) T) { + if src != nil { + *dst = fn(*src) + } +} + +func MapIfNotEmpty[T any, U comparable](dst *T, src *U, fn func(U) T) { + var u U + if src != nil && *src != u { + *dst = fn(*src) + } +} diff --git a/splitio/util/lfqueue/lfqueue.go b/splitio/util/lfqueue/lfqueue.go deleted file mode 100644 index 1d0325b..0000000 --- a/splitio/util/lfqueue/lfqueue.go +++ /dev/null @@ -1,63 +0,0 @@ -package lfqueue - -import ( - "sync/atomic" - "unsafe" -) - -type Interface[T any] interface { - Push(T) - Pop (T, bool) -} - -type Impl[T any] struct { - head unsafe.Pointer - tail unsafe.Pointer - dummy lfqNode[T] -} - -// NewLockfreeQueue is the only way to get a new, ready-to-use LockfreeQueue. -func NewLockfreeQueue[T any]() *Impl[T] { - var lfq Impl[T] - lfq.head = unsafe.Pointer(&lfq.dummy) - lfq.tail = lfq.head - return &lfq -} - -func (lfq *Impl[T]) Pop() (T, bool) { - for { - h := atomic.LoadPointer(&lfq.head) - rh := (*lfqNode[T])(h) - n := (*lfqNode[T])(atomic.LoadPointer(&rh.next)) - if n != nil { - if atomic.CompareAndSwapPointer(&lfq.head, h, rh.next) { - return n.val, true - } else { - continue - } - } else { - var v T - return v, false - } - } -} - -func (lfq *Impl[T]) Push(val T) { - node := unsafe.Pointer(&lfqNode[T]{val: val}) - for { - rt := (*lfqNode[T])(atomic.LoadPointer(&lfq.tail)) - if atomic.CompareAndSwapPointer(&rt.next, nil, node) { - atomic.StorePointer(&lfq.tail, node) - // If dead loop occurs, use CompareAndSwapPointer instead of StorePointer - // atomic.CompareAndSwapPointer(&lfq.tail, t, node) - return - } else { - continue - } - } -} - -type lfqNode[T any] struct { - val T - next unsafe.Pointer -} From c72a1afe17d0c121a4e58868add8b0c6dbebc313 Mon Sep 17 00:00:00 2001 From: Martin Redolatti Date: Tue, 1 Aug 2023 16:19:29 -0300 Subject: [PATCH 12/42] exclude mocks from coverage --- sonar-project.properties | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sonar-project.properties b/sonar-project.properties index dee84fb..2ec9177 100644 --- a/sonar-project.properties +++ b/sonar-project.properties @@ -3,6 +3,6 @@ sonar.sources=. sonar.tests=. sonar.test.inclusions=**/*_test.go sonar.go.coverage.reportPaths=coverage.out -#sonar.coverage.exclusions=conf/conf.go,splitio/services/initialize.go,**/main.go +sonar.coverage.exclusions=**/mocks/** sonar.links.ci=https://github.com/splitio/splitd sonar.links.scm=https://github.com/splitio/splitd/actions From 8b7d04d13375b2cd8f7be99b0eb64358cf7a2496 Mon Sep 17 00:00:00 2001 From: Martin Redolatti Date: Wed, 2 Aug 2023 11:52:21 -0300 Subject: [PATCH 13/42] reorder cli util conf & add tests --- cmd/splitcli/main.go | 124 +++------------------------------- splitio/conf/splitcli.go | 122 +++++++++++++++++++++++++++++++++ splitio/conf/splitcli_test.go | 98 +++++++++++++++++++++++++++ 3 files changed, 229 insertions(+), 115 deletions(-) create mode 100644 splitio/conf/splitcli.go create mode 100644 splitio/conf/splitcli_test.go diff --git a/cmd/splitcli/main.go b/cmd/splitcli/main.go index 493da6f..a55d520 100644 --- a/cmd/splitcli/main.go +++ b/cmd/splitcli/main.go @@ -1,36 +1,32 @@ package main import ( - "encoding/json" - "flag" "fmt" "os" - "strconv" - "strings" "time" "github.com/splitio/go-toolkit/v5/logging" + "github.com/splitio/splitd/splitio/conf" "github.com/splitio/splitd/splitio/link" "github.com/splitio/splitd/splitio/link/client/types" "github.com/splitio/splitd/splitio/util" - cc "github.com/splitio/splitd/splitio/util/conf" ) func main() { - args, err := parseArgs() + args, err := conf.ParseCliArgs() if err != nil { fmt.Println("error parsing arguments: ", err.Error()) os.Exit(1) } - linkOpts, err := args.linkOpts() + linkOpts, err := args.LinkOpts() if err != nil { fmt.Println("error building options from arguments: ", err.Error()) os.Exit(1) } - logLevel := logging.Level(args.logLevel) + logLevel := logging.Level(args.LogLevel) logger := logging.NewLogger(&logging.LoggerOptions{ LogLevel: logLevel, ErrorWriter: os.Stderr, @@ -66,116 +62,14 @@ func main() { fmt.Println(result) } -func executeCall(c types.ClientInterface, a *cliArgs) (string, error) { - switch a.method { +func executeCall(c types.ClientInterface, a *conf.CliArgs) (string, error) { + switch a.Method { case "treatment": - res, err := c.Treatment(a.key, a.bucketingKey, a.feature, a.attributes) + res, err := c.Treatment(a.Key, a.BucketingKey, a.Feature, a.Attributes) return res.Treatment, err case "treatments", "treatmentWithConfig", "treatmentsWithConfig", "track": - return "", fmt.Errorf("method '%s' is not yet implemented", a.method) + return "", fmt.Errorf("method '%s' is not yet implemented", a.Method) default: - return "", fmt.Errorf("unknwon method '%s'", a.method) + return "", fmt.Errorf("unknwon method '%s'", a.Method) } } - -type cliArgs struct { - logLevel string - protocol string - serialization string - connType string - connAddr string - bufSize int - readTimeoutMS int - writeTimeoutMS int - - // command - method string - key string - bucketingKey string - feature string - features []string - trafficType string - eventType string - eventVal float64 - attributes map[string]interface{} -} - -func (a *cliArgs) linkOpts() (*link.ConsumerOptions, error) { - - opts := link.DefaultConsumerOptions() - - var err error - if a.protocol != "" { - if opts.Consumer.Protocol, err = cc.ParseProtocolVersion(a.protocol); err != nil { - return nil, fmt.Errorf("invalid protocol version %s", a.protocol) - } - } - - if a.connType != "" { - if opts.Transfer.ConnType, err = cc.ParseConnType(a.connType); err != nil { - return nil, fmt.Errorf("invalid connection type %s", a.connType) - } - } - - if a.serialization != "" { - if opts.Serialization, err = cc.ParseSerializer(a.serialization); err != nil { - return nil, fmt.Errorf("invalid serialization %s", a.serialization) - } - } - - durationFromMS := func(i int) time.Duration { return time.Duration(i) * time.Millisecond } - cc.SetIfNotEmpty(&opts.Transfer.Address, &a.connAddr) - cc.SetIfNotEmpty(&opts.Transfer.BufferSize, &a.bufSize) - cc.MapIfNotEmpty(&opts.Transfer.ReadTimeout, &a.readTimeoutMS, durationFromMS) - cc.MapIfNotEmpty(&opts.Transfer.WriteTimeout, &a.writeTimeoutMS, durationFromMS) - - return &opts, nil -} - -func parseArgs() (*cliArgs, error) { - ll := flag.String("log-level", "INFO", "log level [ERROR,WARNING,INFO,DEBUG]") - ct := flag.String("conn-type", "", "unix-seqpacket|unix-stream") - ca := flag.String("conn-address", "", "path/ipv4-address") - bs := flag.Int("buffer-size", 0, "read buffer size in bytes") - m := flag.String("method", "", "treatment|treatments|treatmentWithConfig|treatmentsWithConfig|track") - k := flag.String("key", "", "user key") - bk := flag.String("bucketing-key", "", "bucketing key") - f := flag.String("feature", "", "feature to evaluate") - fs := flag.String("features", "", "features to evaluate (comma-separated list with no spaces in between)") - tt := flag.String("traffic-type", "", "traffic type of event") - et := flag.String("event-type", "", "event type") - ev := flag.String("value", "", "event associated value") - at := flag.String("attributes", "", "json representation of attributes") - - flag.Parse() - - val, err := strconv.ParseFloat(*ev, 64) - if *ev != "" && err != nil { - return nil, fmt.Errorf("error parsing event value") - } - - if *at == "" { - *at = "null" - } - attrs := make(map[string]interface{}) - if err = json.Unmarshal([]byte(*at), &attrs); err != nil { - return nil, fmt.Errorf("error parsing attributes: %w", err) - } - - return &cliArgs{ - logLevel: *ll, - connType: *ct, - connAddr: *ca, - bufSize: *bs, - method: *m, - key: *k, - bucketingKey: *bk, - feature: *f, - features: strings.Split(*fs, ","), - trafficType: *tt, - eventType: *et, - eventVal: val, - attributes: attrs, - }, nil - -} diff --git a/splitio/conf/splitcli.go b/splitio/conf/splitcli.go new file mode 100644 index 0000000..9555cdd --- /dev/null +++ b/splitio/conf/splitcli.go @@ -0,0 +1,122 @@ +package conf + +import ( + "encoding/json" + "flag" + "fmt" + "os" + "strconv" + "strings" + "time" + + "github.com/splitio/splitd/splitio/link" + cc "github.com/splitio/splitd/splitio/util/conf" +) + +type CliArgs struct { + LogLevel string + Protocol string + Serialization string + ConnType string + ConnAddr string + BufSize int + ReadTimeoutMS int + WriteTimeoutMS int + + // command + Method string + Key string + BucketingKey string + Feature string + Features []string + TrafficType string + EventType string + EventVal float64 + Attributes map[string]interface{} +} + +func (a *CliArgs) LinkOpts() (*link.ConsumerOptions, error) { + + opts := link.DefaultConsumerOptions() + + var err error + if a.Protocol != "" { + if opts.Consumer.Protocol, err = cc.ParseProtocolVersion(a.Protocol); err != nil { + return nil, fmt.Errorf("invalid protocol version %s", a.Protocol) + } + } + + if a.ConnType != "" { + if opts.Transfer.ConnType, err = cc.ParseConnType(a.ConnType); err != nil { + return nil, fmt.Errorf("invalid connection type %s", a.ConnType) + } + } + + if a.Serialization != "" { + if opts.Serialization, err = cc.ParseSerializer(a.Serialization); err != nil { + return nil, fmt.Errorf("invalid serialization %s", a.Serialization) + } + } + + durationFromMS := func(i int) time.Duration { return time.Duration(i) * time.Millisecond } + cc.SetIfNotEmpty(&opts.Transfer.Address, &a.ConnAddr) + cc.SetIfNotEmpty(&opts.Transfer.BufferSize, &a.BufSize) + cc.MapIfNotEmpty(&opts.Transfer.ReadTimeout, &a.ReadTimeoutMS, durationFromMS) + cc.MapIfNotEmpty(&opts.Transfer.WriteTimeout, &a.WriteTimeoutMS, durationFromMS) + return &opts, nil +} + +func ParseCliArgs() (*CliArgs, error) { + + cliFlags := flag.NewFlagSet(os.Args[0], flag.ContinueOnError) + p := cliFlags.String("protocol", "", "Protocol version [v1]") + s := cliFlags.String("serialization", "", "Client-Daemon communication serialization mechanism [msgpack]") + ll := cliFlags.String("log-level", "INFO", "log level [ERROR,WARNING,INFO,DEBUG]") + ct := cliFlags.String("conn-type", "", "unix-seqpacket|unix-stream") + ca := cliFlags.String("conn-address", "", "path/ipv4-address") + bs := cliFlags.Int("buffer-size", 0, "read buffer size in bytes") + m := cliFlags.String("method", "", "treatment|treatments|treatmentWithConfig|treatmentsWithConfig|track") + k := cliFlags.String("key", "", "user key") + bk := cliFlags.String("bucketing-key", "", "bucketing key") + f := cliFlags.String("feature", "", "feature to evaluate") + fs := cliFlags.String("features", "", "features to evaluate (comma-separated list with no spaces in between)") + tt := cliFlags.String("traffic-type", "", "traffic type of event") + et := cliFlags.String("event-type", "", "event type") + ev := cliFlags.String("value", "", "event associated value") + at := cliFlags.String("attributes", "", "json representation of attributes") + err := cliFlags.Parse(os.Args[1:]) + if err != nil { + return nil, fmt.Errorf("error parsing arguments: %w", err) + } + + val, err := strconv.ParseFloat(*ev, 64) + if *ev != "" && err != nil { + return nil, fmt.Errorf("error parsing event value") + } + + if *at == "" { + *at = "null" + } + attrs := make(map[string]interface{}) + if err = json.Unmarshal([]byte(*at), &attrs); err != nil { + return nil, fmt.Errorf("error parsing attributes: %w", err) + } + + return &CliArgs{ + Serialization: *s, + Protocol: *p, + LogLevel: *ll, + ConnType: *ct, + ConnAddr: *ca, + BufSize: *bs, + Method: *m, + Key: *k, + BucketingKey: *bk, + Feature: *f, + Features: strings.Split(*fs, ","), + TrafficType: *tt, + EventType: *et, + EventVal: val, + Attributes: attrs, + }, nil +} diff --git a/splitio/conf/splitcli_test.go b/splitio/conf/splitcli_test.go new file mode 100644 index 0000000..e935a7e --- /dev/null +++ b/splitio/conf/splitcli_test.go @@ -0,0 +1,98 @@ +package conf + +import ( + "os" + "testing" + + "github.com/splitio/splitd/splitio/link" + "github.com/stretchr/testify/assert" +) + +func TestCliConfig(t *testing.T) { + os.Args = []string{ + os.Args[0], // keep program name + "-log-level=someLevel", + "-conn-type=someConnType", + "-conn-address=someAddr", + "-buffer-size=123", + "-method=someMethod", + "-key=someKey", + "-bucketing-key=someBucketing", + "-feature=someFeature", + "-features=someFeature1,someFeature2", + "-traffic-type=someTrafficType", + "-event-type=someEventType", + "-value=0.123", + `-attributes={"some": "attribute"}`, + } + + parsed, err := ParseCliArgs() + assert.Nil(t, err) + assert.Equal(t, "someLevel", parsed.LogLevel) + assert.Equal(t, "someConnType", parsed.ConnType) + assert.Equal(t, "someAddr", parsed.ConnAddr) + assert.Equal(t, 123, parsed.BufSize) + assert.Equal(t, "someMethod", parsed.Method) + assert.Equal(t, "someKey", parsed.Key) + assert.Equal(t, "someBucketing", parsed.BucketingKey) + assert.Equal(t, "someFeature", parsed.Feature) + assert.Equal(t, []string{"someFeature1", "someFeature2"}, parsed.Features) + assert.Equal(t, "someTrafficType", parsed.TrafficType) + assert.Equal(t, "someEventType", parsed.EventType) + assert.Equal(t, 0.123, parsed.EventVal) + assert.Equal(t, map[string]interface{}{"some": "attribute"}, parsed.Attributes) + + // test bad buffer size + os.Args = []string{os.Args[0], "-buffer-size=sarasa"} + _, err = ParseCliArgs() + assert.NotNil(t, err) + assert.ErrorContains(t, err, "buffer-size") + + // test bad event value + os.Args = []string{os.Args[0], "-value=sarasa"} + _, err = ParseCliArgs() + assert.NotNil(t, err) + assert.ErrorContains(t, err, "event value") + + // test bad attributes + os.Args = []string{os.Args[0], "-attributes=123"} + _, err = ParseCliArgs() + assert.NotNil(t, err) + assert.ErrorContains(t, err, "attributes") + +} + +func TestLinkOptions(t *testing.T) { + // test defaults + os.Args = os.Args[:1] + parsed, err := ParseCliArgs() + assert.Nil(t, err) + lo, err := parsed.LinkOpts() + assert.Nil(t, err) + assert.Equal(t, link.DefaultConsumerOptions(), *lo) + + // test bad protocol + os.Args = []string{os.Args[0], "-protocol=sarasa"} + parsed, err = ParseCliArgs() + assert.Nil(t, err) + lo, err = parsed.LinkOpts() + assert.NotNil(t, err) + assert.ErrorContains(t, err, "protocol") + + // test bad conn type + os.Args = []string{os.Args[0], "-conn-type=sarasa"} + parsed, err = ParseCliArgs() + assert.Nil(t, err) + lo, err = parsed.LinkOpts() + assert.NotNil(t, err) + assert.ErrorContains(t, err, "connection type") + + // test bad serialization + os.Args = []string{os.Args[0], "-serialization=pinpin"} + parsed, err = ParseCliArgs() + assert.Nil(t, err) + lo, err = parsed.LinkOpts() + assert.NotNil(t, err) + assert.ErrorContains(t, err, "serialization") + +} From 61c56ec654d512e68d51fc122e1f1dac652bdd9d Mon Sep 17 00:00:00 2001 From: Martin Redolatti Date: Fri, 4 Aug 2023 17:47:24 -0300 Subject: [PATCH 14/42] more tests --- go.mod | 4 +- splitio/sdk/helpers.go | 23 ++- splitio/sdk/sdk.go | 80 +++++----- splitio/sdk/sdk_test.go | 142 +++++++++++++++--- splitio/sdk/storage/storages.go | 16 +- splitio/sdk/storage/storages_test.go | 17 +++ splitio/sdk/tasks/impressions.go | 88 ++--------- splitio/sdk/workers/impressions.go | 96 ++++++++++++ .../{tasks => workers}/impressions_test.go | 16 +- 9 files changed, 322 insertions(+), 160 deletions(-) create mode 100644 splitio/sdk/storage/storages_test.go create mode 100644 splitio/sdk/workers/impressions.go rename splitio/sdk/{tasks => workers}/impressions_test.go (88%) diff --git a/go.mod b/go.mod index da93717..84fc0da 100644 --- a/go.mod +++ b/go.mod @@ -1,6 +1,6 @@ module github.com/splitio/splitd -go 1.19 +go 1.20 require ( github.com/splitio/go-client/v6 v6.2.1 @@ -8,6 +8,7 @@ require ( github.com/splitio/go-toolkit/v5 v5.2.2 github.com/stretchr/testify v1.8.1 github.com/vmihailenco/msgpack/v5 v5.3.5 + golang.org/x/sync v0.2.0 gopkg.in/yaml.v3 v3.0.1 ) @@ -16,5 +17,4 @@ require ( github.com/pmezard/go-difflib v1.0.0 // indirect github.com/stretchr/objx v0.5.0 // indirect github.com/vmihailenco/tagparser/v2 v2.0.0 // indirect - golang.org/x/sync v0.2.0 // indirect ) diff --git a/splitio/sdk/helpers.go b/splitio/sdk/helpers.go index 87b8b8d..2f4de7d 100644 --- a/splitio/sdk/helpers.go +++ b/splitio/sdk/helpers.go @@ -3,9 +3,9 @@ package sdk import ( "fmt" - sss "github.com/splitio/splitd/splitio/sdk/storage" - tss "github.com/splitio/splitd/splitio/sdk/tasks" sdkConf "github.com/splitio/splitd/splitio/sdk/conf" + sss "github.com/splitio/splitd/splitio/sdk/storage" + "github.com/splitio/splitd/splitio/sdk/workers" "github.com/splitio/go-split-commons/v4/conf" "github.com/splitio/go-split-commons/v4/dtos" @@ -26,12 +26,18 @@ import ( "github.com/splitio/go-toolkit/v5/logging" ) -func setupWorkers(logger logging.LoggerInterface, api *api.SplitAPI, str *storages, hc application.MonitorProducerInterface) *synchronizer.Workers { - splitChangeWorker := split.NewSplitFetcher(str.splits, api.SplitFetcher, logger, str.telemetry, hc) - segmentChangeWorker := segment.NewSegmentFetcher(str.splits, str.segments, api.SegmentFetcher, logger, str.telemetry, hc) +func setupWorkers( + logger logging.LoggerInterface, + api *api.SplitAPI, + str *storages, + hc application.MonitorProducerInterface, + cfg *sdkConf.Impressions, + +) *synchronizer.Workers { return &synchronizer.Workers{ - SplitFetcher: splitChangeWorker, - SegmentFetcher: segmentChangeWorker, + SplitFetcher: split.NewSplitFetcher(str.splits, api.SplitFetcher, logger, str.telemetry, hc), + SegmentFetcher: segment.NewSegmentFetcher(str.splits, str.segments, api.SegmentFetcher, logger, str.telemetry, hc), + ImpressionRecorder: workers.NewImpressionsWorker(logger, str.telemetry, api.ImpressionRecorder, str.impressions, cfg), } } @@ -48,7 +54,8 @@ func setupTasks( return &synchronizer.SplitTasks{ SplitSyncTask: tasks.NewFetchSplitsTask(workers.SplitFetcher, int(cfg.Splits.SyncPeriod.Seconds()), logger), SegmentSyncTask: tasks.NewFetchSegmentsTask(workers.SegmentFetcher, int(cfg.Segments.SyncPeriod.Seconds()), cfg.Segments.WorkerCount, cfg.Segments.QueueSize, logger), - ImpressionSyncTask: tss.NewImpressionSyncTask(api.ImpressionRecorder, str.impressions, logger, str.telemetry, &cfg.Impressions), + ImpressionSyncTask: tasks.NewRecordImpressionsTask(workers.ImpressionRecorder, int(impCfg.SyncPeriod.Seconds()), logger, 5000), + //ImpressionSyncTask: tss.NewImpressionSyncTask(workers.ImpressionRecorder, logger, cfg.Impressions), ImpressionsCountSyncTask: tasks.NewRecordImpressionsCountTask( impressionscount.NewRecorderSingle(impComponents.counter, api.ImpressionRecorder, md, logger, str.telemetry), logger, diff --git a/splitio/sdk/sdk.go b/splitio/sdk/sdk.go index cfdf24b..056b633 100644 --- a/splitio/sdk/sdk.go +++ b/splitio/sdk/sdk.go @@ -22,6 +22,10 @@ import ( "github.com/splitio/splitd/splitio" ) +const ( + impressionsFullNotif = "IMPRESSIONS_FULL" +) + type Attributes = map[string]interface{} type Interface interface { @@ -30,14 +34,15 @@ type Interface interface { } type Impl struct { - logger logging.LoggerInterface - ev evaluator.Interface - sm synchronizer.Manager - ss synchronizer.Synchronizer - is *storage.ImpressionsStorage - iq provisional.ImpressionManager - cfg sdkConf.Config - status chan int + logger logging.LoggerInterface + ev evaluator.Interface + sm synchronizer.Manager + ss synchronizer.Synchronizer + is *storage.ImpressionsStorage + iq provisional.ImpressionManager + cfg sdkConf.Config + status chan int + queueFullChan chan string } func New(logger logging.LoggerInterface, apikey string, c *conf.Config) (*Impl, error) { @@ -52,10 +57,12 @@ func New(logger logging.LoggerInterface, apikey string, c *conf.Config) (*Impl, } hc := &application.Dummy{} + + queueFullChan := make(chan string, 1) // Only one item so that we don't queue N flushes (which makes no sense) if we're getting hit too hard splitApi := api.NewSplitAPI(apikey, *advCfg, logger, md) - workers := setupWorkers(logger, splitApi, stores, hc) + workers := setupWorkers(logger, splitApi, stores, hc, &c.Impressions) tasks := setupTasks(c, stores, logger, workers, impc, md, splitApi) - sync := synchronizer.NewSynchronizer(*advCfg, *tasks, *workers, logger, nil, nil) + sync := synchronizer.NewSynchronizer(*advCfg, *tasks, *workers, logger, queueFullChan, nil) status := make(chan int, 10) manager, err := synchronizer.NewSynchronizerManager(sync, logger, *advCfg, splitApi.AuthClient, stores.splits, status, stores.telemetry, md, nil, hc) @@ -71,13 +78,14 @@ func New(logger logging.LoggerInterface, apikey string, c *conf.Config) (*Impl, } return &Impl{ - logger: logger, - sm: manager, - ss: sync, - ev: evaluator.NewEvaluator(stores.splits, stores.segments, engine.NewEngine(logger), logger), - is: stores.impressions, - iq: impc.manager, - cfg: *c, + logger: logger, + sm: manager, + ss: sync, + ev: evaluator.NewEvaluator(stores.splits, stores.segments, engine.NewEngine(logger), logger), + is: stores.impressions, + iq: impc.manager, + cfg: *c, + queueFullChan: queueFullChan, }, nil } @@ -88,11 +96,7 @@ func (i *Impl) Treatment(cfg *types.ClientConfig, key string, bk *string, featur return nil, fmt.Errorf("nil result") } - imp, err := i.handleImpression(key, bk, feature, res, cfg.Metadata) - if err != nil { - i.logger.Error("error handling impression: ", err) - } - + imp := i.handleImpression(key, bk, feature, res, cfg.Metadata) return &Result{ Treatment: res.Treatment, Impression: imp, @@ -107,27 +111,23 @@ func (i *Impl) Treatments(cfg *types.ClientConfig, key string, bk *string, featu toRet := make(map[string]Result, len(res.Evaluations)) for _, feature := range features { - curr, ok := res.Evaluations[feature] - if !ok { - toRet[feature] = Result{Treatment: "control"} - continue - } + curr, ok := res.Evaluations[feature] + if !ok { + toRet[feature] = Result{Treatment: "control"} + continue + } - var err error var eres Result eres.Treatment = curr.Treatment - eres.Impression, err = i.handleImpression(key, bk, feature, &curr, cfg.Metadata) + eres.Impression = i.handleImpression(key, bk, feature, &curr, cfg.Metadata) eres.Config = curr.Config - if err != nil { - i.logger.Error("error handling impression: ", err) - } toRet[feature] = eres } return toRet, nil } -func (i *Impl) handleImpression(key string, bk *string, f string, r *evaluator.Result, cm types.ClientMetadata) (*dtos.Impression, error) { +func (i *Impl) handleImpression(key string, bk *string, f string, r *evaluator.Result, cm types.ClientMetadata) *dtos.Impression { var label string if i.cfg.LabelsEnabled { label = r.Label @@ -146,10 +146,20 @@ func (i *Impl) handleImpression(key string, bk *string, f string, r *evaluator.R shouldStore := i.iq.ProcessSingle(imp) if shouldStore { _, err := i.is.Push(cm, *imp) - return imp, err + if err != nil { + if err == storage.ErrQueueFull { + select { + case i.queueFullChan <- impressionsFullNotif: + default: + i.logger.Warning("impressions queue has filled up and is currently performing a flush. Current impression will bedropped") + } + } else { + i.logger.Error("error handling impression: ", err) + } + } } - return imp, nil + return imp } func timeMillis() int64 { diff --git a/splitio/sdk/sdk_test.go b/splitio/sdk/sdk_test.go index cc57ce8..1001c77 100644 --- a/splitio/sdk/sdk_test.go +++ b/splitio/sdk/sdk_test.go @@ -1,16 +1,21 @@ package sdk import ( + "fmt" "testing" "time" "github.com/splitio/go-client/v6/splitio/engine/evaluator" "github.com/splitio/go-split-commons/v4/dtos" "github.com/splitio/go-split-commons/v4/provisional" + "github.com/splitio/go-split-commons/v4/service" + "github.com/splitio/go-split-commons/v4/storage/inmemory" + "github.com/splitio/go-split-commons/v4/synchronizer" "github.com/splitio/go-toolkit/v5/logging" "github.com/splitio/splitd/splitio/sdk/conf" "github.com/splitio/splitd/splitio/sdk/storage" "github.com/splitio/splitd/splitio/sdk/types" + "github.com/splitio/splitd/splitio/sdk/workers" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/mock" ) @@ -32,10 +37,8 @@ func TestTreatmentLabelsDisabled(t *testing.T) { } im := &ImpressionManagerMock{} im.On("ProcessSingle", mock.Anything). - Run(func(args mock.Arguments) { - // hay que hacer el assert aca en lugar del matcher por el timestamp - assertImpressionEquals(t, expectedImpression, args.Get(0).(*dtos.Impression)) - }). + // hay que hacer el assert aca en lugar del matcher por el timestamp + Run(func(a mock.Arguments) { assertImpEq(t, expectedImpression, a.Get(0).(*dtos.Impression)) }). Return(true). Once() @@ -50,7 +53,7 @@ func TestTreatmentLabelsDisabled(t *testing.T) { res, err := client.Treatment(&types.ClientConfig{Metadata: types.ClientMetadata{ID: "some", SdkVersion: "go-1.2.3"}}, "key1", nil, "f1", Attributes{"a": 1}) assert.Nil(t, err) assert.Nil(t, res.Config) - assertImpressionEquals(t, expectedImpression, res.Impression) + assertImpEq(t, expectedImpression, res.Impression) err = is.RangeAndClear(func(md types.ClientMetadata, st *storage.LockingQueue[dtos.Impression]) { assert.Equal(t, types.ClientMetadata{ID: "some", SdkVersion: "go-1.2.3"}, md) @@ -61,7 +64,7 @@ func TestTreatmentLabelsDisabled(t *testing.T) { assert.Nil(t, nil) assert.Equal(t, 1, n) assert.Equal(t, 1, len(imps)) - assertImpressionEquals(t, expectedImpression, &imps[0]) + assertImpEq(t, expectedImpression, &imps[0]) n, err = st.Pop(1, &imps) assert.ErrorIs(t, err, storage.ErrQueueEmpty) @@ -89,7 +92,7 @@ func TestTreatmentLabelsEnabled(t *testing.T) { im.On("ProcessSingle", mock.Anything). Run(func(args mock.Arguments) { // hay que hacer el assert aca en lugar del matcher por el timestamp - assertImpressionEquals(t, expectedImpression, args.Get(0).(*dtos.Impression)) + assertImpEq(t, expectedImpression, args.Get(0).(*dtos.Impression)) }). Return(true). Once() @@ -105,7 +108,7 @@ func TestTreatmentLabelsEnabled(t *testing.T) { res, err := client.Treatment(&types.ClientConfig{Metadata: types.ClientMetadata{ID: "some", SdkVersion: "go-1.2.3"}}, "key1", nil, "f1", Attributes{"a": 1}) assert.Nil(t, err) assert.Nil(t, res.Config) - assertImpressionEquals(t, expectedImpression, res.Impression) + assertImpEq(t, expectedImpression, res.Impression) err = is.RangeAndClear(func(md types.ClientMetadata, st *storage.LockingQueue[dtos.Impression]) { assert.Equal(t, types.ClientMetadata{ID: "some", SdkVersion: "go-1.2.3"}, md) @@ -116,9 +119,9 @@ func TestTreatmentLabelsEnabled(t *testing.T) { assert.Nil(t, nil) assert.Equal(t, 1, n) assert.Equal(t, 1, len(imps)) - assertImpressionEquals(t, expectedImpression, &imps[0]) + assertImpEq(t, expectedImpression, &imps[0]) n, err = st.Pop(1, &imps) - assert.Equal(t, 0, n) + assert.Equal(t, 0, n) assert.ErrorIs(t, err, storage.ErrQueueEmpty) }) @@ -145,19 +148,19 @@ func TestTreatments(t *testing.T) { im := &ImpressionManagerMock{} im.On("ProcessSingle", mock.Anything). Run(func(args mock.Arguments) { - assertImpressionEquals(t, &expectedImpressions[0], args.Get(0).(*dtos.Impression)) + assertImpEq(t, &expectedImpressions[0], args.Get(0).(*dtos.Impression)) }). Return(true). Once() im.On("ProcessSingle", mock.Anything). Run(func(args mock.Arguments) { - assertImpressionEquals(t, &expectedImpressions[1], args.Get(0).(*dtos.Impression)) + assertImpEq(t, &expectedImpressions[1], args.Get(0).(*dtos.Impression)) }). Return(true). Once() im.On("ProcessSingle", mock.Anything). Run(func(args mock.Arguments) { - assertImpressionEquals(t, &expectedImpressions[2], args.Get(0).(*dtos.Impression)) + assertImpEq(t, &expectedImpressions[2], args.Get(0).(*dtos.Impression)) }). Return(true). Once() @@ -171,15 +174,15 @@ func TestTreatments(t *testing.T) { } res, err := client.Treatments( - &types.ClientConfig{Metadata: types.ClientMetadata{ID: "some", SdkVersion: "go-1.2.3"}}, - "key1", nil, []string{"f1", "f2", "f3"}, Attributes{"a": 1}) + &types.ClientConfig{Metadata: types.ClientMetadata{ID: "some", SdkVersion: "go-1.2.3"}}, + "key1", nil, []string{"f1", "f2", "f3"}, Attributes{"a": 1}) assert.Nil(t, err) assert.Nil(t, res["f1"].Config) assert.Nil(t, res["f2"].Config) assert.Nil(t, res["f3"].Config) - assertImpressionEquals(t, &expectedImpressions[0], res["f1"].Impression) - assertImpressionEquals(t, &expectedImpressions[1], res["f2"].Impression) - assertImpressionEquals(t, &expectedImpressions[2], res["f3"].Impression) + assertImpEq(t, &expectedImpressions[0], res["f1"].Impression) + assertImpEq(t, &expectedImpressions[1], res["f2"].Impression) + assertImpEq(t, &expectedImpressions[2], res["f3"].Impression) err = is.RangeAndClear(func(md types.ClientMetadata, st *storage.LockingQueue[dtos.Impression]) { assert.Equal(t, types.ClientMetadata{ID: "some", SdkVersion: "go-1.2.3"}, md) @@ -190,11 +193,11 @@ func TestTreatments(t *testing.T) { assert.Nil(t, nil) assert.Equal(t, 3, n) assert.Equal(t, 3, len(imps)) - assertImpressionEquals(t, &expectedImpressions[0], &imps[0]) - assertImpressionEquals(t, &expectedImpressions[1], &imps[1]) - assertImpressionEquals(t, &expectedImpressions[2], &imps[2]) + assertImpEq(t, &expectedImpressions[0], &imps[0]) + assertImpEq(t, &expectedImpressions[1], &imps[1]) + assertImpEq(t, &expectedImpressions[2], &imps[2]) n, err = st.Pop(1, &imps) - assert.Equal(t, 0, n) + assert.Equal(t, 0, n) assert.ErrorIs(t, err, storage.ErrQueueEmpty) }) @@ -202,8 +205,82 @@ func TestTreatments(t *testing.T) { } -func assertImpressionEquals(t *testing.T, i1, i2 *dtos.Impression) { - t.Helper() +func TestImpressionsQueueFull(t *testing.T) { + + logger := logging.NewLogger(nil) + + impRecorder := &ImpressionRecorderMock{} + impRecorder.On("Record", mock.Anything, mock.Anything, mock.Anything).Return(nil).Times(2) + + // setup an impressions queue of 4, and a task with a large period for evicting, and a synchronizer + // with only enough hata to flush impressions when the queue-full signal arrives + // @{ + queueFullChan := make(chan string, 2) + is, _ := storage.NewImpressionsQueue(4) + ts, _ := inmemory.NewTelemetryStorage() + iw := workers.NewImpressionsWorker(logger, ts, impRecorder, is, &conf.Impressions{Mode: "optimized", SyncPeriod: 100 * time.Second}) + sworkers := synchronizer.Workers{ImpressionRecorder: iw} + sy := synchronizer.NewSynchronizer(*conf.DefaultConfig().ToAdvancedConfig(), synchronizer.SplitTasks{}, sworkers, logger, queueFullChan, nil) + sy.StartPeriodicDataRecording() + // @} + + ev := &EvaluatorMock{} + ev.On("EvaluateFeature", mock.Anything, mock.Anything, mock.Anything, mock.Anything). + Return(&evaluator.Result{Treatment: "on", Label: "label1", EvaluationTime: 1 * time.Millisecond, SplitChangeNumber: 123}). + Times(9) + + expectedImpression := &dtos.Impression{KeyName: "key1", BucketingKey: "", FeatureName: "f1", Treatment: "on", Label: "label1", ChangeNumber: 123} + im := &ImpressionManagerMock{} + im.On("ProcessSingle", mock.Anything). + // hay que hacer el assert aca en lugar del matcher por el timestamp + Run(func(args mock.Arguments) { assertImpEq(t, expectedImpression, args.Get(0).(*dtos.Impression)) }). + Return(true). + Times(9) + + client := &Impl{logger: logging.NewLogger(nil), ss: nil, is: is, ev: ev, iq: im, cfg: conf.Config{LabelsEnabled: true}, queueFullChan: queueFullChan} + clientConf := &types.ClientConfig{Metadata: types.ClientMetadata{ID: "some", SdkVersion: "go-1.2.3"}} + + // create 4 impressions to fill the queue (last one will be dropped and trigger a flush) + for idx := 0; idx < 4; idx++ { + feature := fmt.Sprintf("f%d", idx) + expectedImpression.FeatureName = feature + res, err := client.Treatment(clientConf, "key1", nil, feature, Attributes{"a": 1}) + assert.Nil(t, err) + assert.Nil(t, res.Config) + assertImpEq(t, expectedImpression, res.Impression) + } + + time.Sleep(time.Second) // wait 1 sec to allow for a context switch since the flush is async + + // same + for idx := 4; idx < 8; idx++ { + feature := fmt.Sprintf("f%d", idx) + expectedImpression.FeatureName = feature + res, err := client.Treatment(clientConf, "key1", nil, feature, Attributes{"a": 1}) + assert.Nil(t, err) + assert.Nil(t, res.Config) + assertImpEq(t, expectedImpression, res.Impression) + } + + time.Sleep(time.Second) // wait 1 sec to allow for a context switch since the flush is async + + feature := "f8" + expectedImpression.FeatureName = feature + res, err := client.Treatment(clientConf, "key1", nil, feature, Attributes{"a": 1}) + assert.Nil(t, err) + assert.Nil(t, res.Config) + assertImpEq(t, expectedImpression, res.Impression) + + ev.AssertExpectations(t) + im.AssertExpectations(t) + impRecorder.AssertExpectations(t) + var totalSize int + is.Range(func(md types.ClientMetadata, q *storage.LockingQueue[dtos.Impression]) { totalSize += q.Len() }) + assert.Equal(t, 1, totalSize) // assert no more impressions in queue +} + +func assertImpEq(t *testing.T, i1, i2 *dtos.Impression) { + t.Helper() assert.Equal(t, i1.KeyName, i2.KeyName) assert.Equal(t, i1.BucketingKey, i2.BucketingKey) assert.Equal(t, i1.FeatureName, i2.FeatureName) @@ -246,5 +323,22 @@ func (m *ImpressionManagerMock) ProcessSingle(impression *dtos.Impression) bool return args.Bool(0) } +type ImpressionRecorderMock struct { + mock.Mock +} + +// Record implements service.ImpressionsRecorder +func (m *ImpressionRecorderMock) Record(impressions []dtos.ImpressionsDTO, metadata dtos.Metadata, extraHeaders map[string]string) error { + args := m.Called(impressions, metadata, extraHeaders) + return args.Error(0) +} + +// RecordImpressionsCount implements service.ImpressionsRecorder +func (m *ImpressionRecorderMock) RecordImpressionsCount(pf dtos.ImpressionsCountDTO, metadata dtos.Metadata) error { + args := m.Called(pf, metadata) + return args.Error(0) +} + var _ evaluator.Interface = (*EvaluatorMock)(nil) var _ provisional.ImpressionManager = (*ImpressionManagerMock)(nil) +var _ service.ImpressionsRecorder = (*ImpressionRecorderMock)(nil) diff --git a/splitio/sdk/storage/storages.go b/splitio/sdk/storage/storages.go index 58851d6..082d599 100644 --- a/splitio/sdk/storage/storages.go +++ b/splitio/sdk/storage/storages.go @@ -12,15 +12,27 @@ type ImpressionsStorage = MultiMetaQueues[dtos.Impression, types.ClientMetadata, type EventsStorage = MultiMetaQueues[dtos.EventDTO, types.ClientMetadata, *LockingQueue[dtos.EventDTO]] func NewImpressionsQueue(approxSize int) (st *ImpressionsStorage, realSize int) { - bits := int(math.Log2(float64(approxSize))) + 1 + bits := getNearestSizePowerOf2(approxSize) return NewMultiMetaQueue[dtos.Impression, types.ClientMetadata](func() *LockingQueue[dtos.Impression] { return NewLKQueue[dtos.Impression](bits) }), int(math.Pow(2, float64(bits))) } func NewEventsQueue(approxSize int) (st *EventsStorage, realSize int) { - bits := int(math.Log2(float64(approxSize))) + 1 + bits := getNearestSizePowerOf2(approxSize) return NewMultiMetaQueue[dtos.EventDTO, types.ClientMetadata](func() *LockingQueue[dtos.EventDTO] { return NewLKQueue[dtos.EventDTO](bits) }), int(math.Pow(2, float64(bits))) } + +// to make the round-queue performant , we need to replace the modulo operation with an AND. +// For that approach to work, the size must be a power of 2. This function calculates the minimum power of 2 +// that guarantees final_size >= requested_size +func getNearestSizePowerOf2(approxSize int) int { + bits := int(math.Log2(float64(approxSize))) // floor(log2(approxSize)) + if math.Pow(2, float64(bits)) < float64(approxSize) { + // if resulting size is lower than requested (because of float -> int conversion), add 1 bit + bits++ + } + return bits +} diff --git a/splitio/sdk/storage/storages_test.go b/splitio/sdk/storage/storages_test.go new file mode 100644 index 0000000..e004c7e --- /dev/null +++ b/splitio/sdk/storage/storages_test.go @@ -0,0 +1,17 @@ +package storage + +import ( + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestGetNearestSizePowerOf2(t *testing.T) { + assert.Equal(t, 1, getNearestSizePowerOf2(2)) + assert.Equal(t, 2, getNearestSizePowerOf2(3)) + assert.Equal(t, 2, getNearestSizePowerOf2(4)) + assert.Equal(t, 3, getNearestSizePowerOf2(5)) + assert.Equal(t, 3, getNearestSizePowerOf2(8)) + assert.Equal(t, 10, getNearestSizePowerOf2(1024)) + assert.Equal(t, 11, getNearestSizePowerOf2(1025)) +} diff --git a/splitio/sdk/tasks/impressions.go b/splitio/sdk/tasks/impressions.go index 3f8f7e2..71bb17a 100644 --- a/splitio/sdk/tasks/impressions.go +++ b/splitio/sdk/tasks/impressions.go @@ -1,99 +1,29 @@ package tasks import ( - "errors" - - sdkconf "github.com/splitio/splitd/splitio/sdk/conf" - "github.com/splitio/splitd/splitio/sdk/types" - - "github.com/splitio/go-split-commons/v4/dtos" - "github.com/splitio/go-split-commons/v4/service" - "github.com/splitio/go-split-commons/v4/storage" "github.com/splitio/go-toolkit/v5/asynctask" "github.com/splitio/go-toolkit/v5/logging" - sss "github.com/splitio/splitd/splitio/sdk/storage" + sdkconf "github.com/splitio/splitd/splitio/sdk/conf" + "github.com/splitio/splitd/splitio/sdk/workers" ) -type impressionSyncTaskHelper struct { - logger logging.LoggerInterface - telemetry storage.TelemetryRuntimeProducer - llrec service.ImpressionsRecorder - iq *sss.ImpressionsStorage - cfg *sdkconf.Impressions -} +const ( + defaultBulkSize = 5000 +) func NewImpressionSyncTask( - llrec service.ImpressionsRecorder, - impStore *sss.ImpressionsStorage, + worker *workers.MultiMetaImpressionWorker, logger logging.LoggerInterface, - telemetry storage.TelemetryRuntimeProducer, cfg *sdkconf.Impressions, ) *asynctask.AsyncTask { - helper := &impressionSyncTaskHelper{ - logger: logger, - telemetry: telemetry, - llrec: llrec, - iq: impStore, - cfg: cfg, - } - + // TODO(mredolatti): pass a proper bulk size (currently ignored, everything is flushed) return asynctask.NewAsyncTask( "impressions-sender", - func(logging.LoggerInterface) error { helper.synchronize(cfg.PostConcurrency); return nil }, + func(logging.LoggerInterface) error { worker.SynchronizeImpressions(defaultBulkSize); return nil }, int(cfg.SyncPeriod.Seconds()), nil, - func(logging.LoggerInterface) { helper.synchronize(cfg.PostConcurrency) }, + func(logging.LoggerInterface) { worker.SynchronizeImpressions(defaultBulkSize) }, logger, ) - -} - -func (i *impressionSyncTaskHelper) synchronize(parallelism int) []error { - - var errs []error - if err := i.iq.RangeAndClear(func(md types.ClientMetadata, q *sss.LockingQueue[dtos.Impression]) { - extracted := make([]dtos.Impression, 0, q.Len()) - n, err := q.Pop(q.Len(), &extracted) - if err != nil && !errors.Is(err, sss.ErrQueueEmpty) { - i.logger.Error("error fetching items from queue: ", err) - return // continue with next one - } - - if n == 0 { - return // nothing to do here - } - - tmp := make(map[string]*dtos.ImpressionsDTO) - for idx := range extracted { - forFeature, ok := tmp[extracted[idx].FeatureName] - if !ok { - forFeature = &dtos.ImpressionsDTO{TestName: extracted[idx].FeatureName} - tmp[extracted[idx].FeatureName] = forFeature - } - - forFeature.KeyImpressions = append(forFeature.KeyImpressions, dtos.ImpressionDTO{ - KeyName: extracted[idx].KeyName, - Treatment: extracted[idx].Treatment, - Time: extracted[idx].Time, - ChangeNumber: extracted[idx].ChangeNumber, - Label: extracted[idx].Label, - BucketingKey: extracted[idx].BucketingKey, - Pt: extracted[idx].Pt, - }) - } - - payload := make([]dtos.ImpressionsDTO, 0, len(tmp)) - for _, v := range tmp { - payload = append(payload, *v) - } - - if err := i.llrec.Record(payload, dtos.Metadata{SDKVersion: md.SdkVersion}, nil); err != nil { - errs = append(errs, err) - } - }); err != nil { - i.logger.Error("error traversing impression queues: ", err) - } - - return errs } diff --git a/splitio/sdk/workers/impressions.go b/splitio/sdk/workers/impressions.go new file mode 100644 index 0000000..a2d3e49 --- /dev/null +++ b/splitio/sdk/workers/impressions.go @@ -0,0 +1,96 @@ +package workers + +import ( + "errors" + + "github.com/splitio/go-split-commons/v4/dtos" + "github.com/splitio/go-split-commons/v4/service" + "github.com/splitio/go-split-commons/v4/storage" + "github.com/splitio/go-split-commons/v4/synchronizer/worker/impression" + "github.com/splitio/go-toolkit/v5/logging" + sdkconf "github.com/splitio/splitd/splitio/sdk/conf" + sss "github.com/splitio/splitd/splitio/sdk/storage" + "github.com/splitio/splitd/splitio/sdk/types" +) + +type MultiMetaImpressionWorker struct { + logger logging.LoggerInterface + telemetry storage.TelemetryRuntimeProducer + llrec service.ImpressionsRecorder + iq *sss.ImpressionsStorage + cfg *sdkconf.Impressions +} + +func NewImpressionsWorker( + logger logging.LoggerInterface, + telemetry storage.TelemetryRuntimeProducer, + llrec service.ImpressionsRecorder, + iq *sss.ImpressionsStorage, + cfg *sdkconf.Impressions, +) *MultiMetaImpressionWorker { + return &MultiMetaImpressionWorker{ + logger: logger, + telemetry: telemetry, + llrec: llrec, + iq: iq, + cfg: cfg, + } +} + +// FlushImpressions implements impression.ImpressionRecorder +func (m *MultiMetaImpressionWorker) FlushImpressions(bulkSize int64) error { + + // TODO(mredolatti): take `bulkSize` into account + var errs []error + if err := m.iq.RangeAndClear(func(md types.ClientMetadata, q *sss.LockingQueue[dtos.Impression]) { + extracted := make([]dtos.Impression, 0, q.Len()) + n, err := q.Pop(q.Len(), &extracted) + if err != nil && !errors.Is(err, sss.ErrQueueEmpty) { + m.logger.Error("error fetching items from queue: ", err) + return // continue with next one + } + + if n == 0 { + return // nothing to do here + } + + tmp := make(map[string]*dtos.ImpressionsDTO) + for idx := range extracted { + forFeature, ok := tmp[extracted[idx].FeatureName] + if !ok { + forFeature = &dtos.ImpressionsDTO{TestName: extracted[idx].FeatureName} + tmp[extracted[idx].FeatureName] = forFeature + } + + forFeature.KeyImpressions = append(forFeature.KeyImpressions, dtos.ImpressionDTO{ + KeyName: extracted[idx].KeyName, + Treatment: extracted[idx].Treatment, + Time: extracted[idx].Time, + ChangeNumber: extracted[idx].ChangeNumber, + Label: extracted[idx].Label, + BucketingKey: extracted[idx].BucketingKey, + Pt: extracted[idx].Pt, + }) + } + + payload := make([]dtos.ImpressionsDTO, 0, len(tmp)) + for _, v := range tmp { + payload = append(payload, *v) + } + + if err := m.llrec.Record(payload, dtos.Metadata{SDKVersion: md.SdkVersion}, nil); err != nil { + errs = append(errs, err) + } + }); err != nil { + m.logger.Error("error traversing impression queues: ", err) + } + + return errors.Join(errs...) +} + +// SynchronizeImpressions implements impression.ImpressionRecorder +func (m *MultiMetaImpressionWorker) SynchronizeImpressions(bulkSize int64) error { + return m.FlushImpressions(bulkSize) +} + +var _ impression.ImpressionRecorder = (*MultiMetaImpressionWorker)(nil) diff --git a/splitio/sdk/tasks/impressions_test.go b/splitio/sdk/workers/impressions_test.go similarity index 88% rename from splitio/sdk/tasks/impressions_test.go rename to splitio/sdk/workers/impressions_test.go index 73389fa..fb718b0 100644 --- a/splitio/sdk/tasks/impressions_test.go +++ b/splitio/sdk/workers/impressions_test.go @@ -1,10 +1,9 @@ -package tasks +package workers import ( "reflect" "sort" "testing" - "time" "github.com/splitio/go-split-commons/v4/dtos" "github.com/splitio/go-split-commons/v4/service" @@ -22,7 +21,7 @@ func TestImpressionsTask(t *testing.T) { logger := logging.NewLogger(nil) rec := &RecorderMock{} - task := NewImpressionSyncTask(rec, is, logger, ts, &conf.Impressions{SyncPeriod: 1 * time.Second}) + worker := NewImpressionsWorker(logger, ts, rec, is, &conf.Impressions{}) var emptyMap map[string]string rec.On("Record", []dtos.ImpressionsDTO{{ @@ -39,8 +38,8 @@ func TestImpressionsTask(t *testing.T) { Return(nil). Once() - // ImpressionsDTO are built from the contents of a map so the ordering is undefined. - // to solve this, we sort the input by feature name (& provided an already sorted expected value) + // ImpressionsDTO are built from the contents of a map so the ordering is undefined. + // to solve this, we sort the input by feature name (& provided an already sorted expected value) rec.On("Record", mock.MatchedBy(func(imps []dtos.ImpressionsDTO) bool { sort.Slice(imps, func(i, j int) bool { return imps[i].TestName < imps[j].TestName }) @@ -63,16 +62,13 @@ func TestImpressionsTask(t *testing.T) { is.Push(types.ClientMetadata{ID: "i2", SdkVersion: "go-1.2.3"}, dtos.Impression{KeyName: "k2", FeatureName: "f2", Treatment: "off", Label: "l2", ChangeNumber: 456, Time: 123457}) - task.Start() - time.Sleep(1500 * time.Millisecond) - + worker.SynchronizeImpressions(5000) is.Push(types.ClientMetadata{ID: "i3", SdkVersion: "python-1.2.3"}, dtos.Impression{KeyName: "k3", FeatureName: "f3", Treatment: "on", Label: "l3", ChangeNumber: 789, Time: 123458}, dtos.Impression{KeyName: "k3", FeatureName: "f4", Treatment: "on", Label: "l4", ChangeNumber: 890, Time: 123459}, ) - time.Sleep(1500 * time.Millisecond) - task.Stop(true) + worker.SynchronizeImpressions(5000) rec.AssertExpectations(t) } From 9bf84cfeca107b8ff669d4d1e1778603376cabe0 Mon Sep 17 00:00:00 2001 From: Martin Redolatti Date: Fri, 4 Aug 2023 17:52:14 -0300 Subject: [PATCH 15/42] bump go version in dockerfile --- infra/sidecar.Dockerfile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/infra/sidecar.Dockerfile b/infra/sidecar.Dockerfile index 05db927..2effa1e 100644 --- a/infra/sidecar.Dockerfile +++ b/infra/sidecar.Dockerfile @@ -1,5 +1,5 @@ # ----- Builder image -FROM golang:1.19.5-alpine3.17 AS builder +FROM golang:1.20.7-alpine3.18 AS builder RUN apk add git build-base bash @@ -8,7 +8,7 @@ COPY . . RUN make clean splitd # ----- Runner image -FROM alpine:3.17 AS runner +FROM alpine:3.18 AS runner RUN apk add gettext yq bash RUN mkdir -p /opt/splitd From a8cc645bb9d6b390fc8b0c661cb9e29362d717e3 Mon Sep 17 00:00:00 2001 From: Martin Redolatti Date: Fri, 4 Aug 2023 18:08:07 -0300 Subject: [PATCH 16/42] sonarqube feedback --- splitio/link/client/v1/impl.go | 12 +++---- splitio/sdk/workers/impressions.go | 57 ++++++++++++++++-------------- 2 files changed, 37 insertions(+), 32 deletions(-) diff --git a/splitio/link/client/v1/impl.go b/splitio/link/client/v1/impl.go index 7c1bea8..f4b23d9 100644 --- a/splitio/link/client/v1/impl.go +++ b/splitio/link/client/v1/impl.go @@ -56,11 +56,11 @@ func (c *Impl) Treatment(key string, bucketingKey string, feature string, attrs resp, err := doRPC[protov1.ResponseWrapper[protov1.TreatmentPayload]](c, &rpc) if err != nil { - return &types.Result{Treatment: Control}, fmt.Errorf("error executing rpc: %w", err) + return &types.Result{Treatment: Control}, fmt.Errorf("error executing treatment rpc: %w", err) } if resp.Status != protov1.ResultOk { - return &types.Result{Treatment: Control}, fmt.Errorf("server responded with error %d", resp.Status) + return &types.Result{Treatment: Control}, fmt.Errorf("server responded treatment rpc with error %d", resp.Status) } var imp *dtos.Impression @@ -93,11 +93,11 @@ func (c *Impl) Treatments(key string, bucketingKey string, features []string, at resp, err := doRPC[protov1.ResponseWrapper[protov1.TreatmentsPayload]](c, &rpc) if err != nil { - return nil, fmt.Errorf("error executing rpc: %w", err) + return nil, fmt.Errorf("error executing treatments rpc: %w", err) } if resp.Status != protov1.ResultOk { - return nil, fmt.Errorf("server responded with error %d", resp.Status) + return nil, fmt.Errorf("server responded treatments rpc with error %d", resp.Status) } results := make(types.Results) @@ -133,11 +133,11 @@ func (c *Impl) register(impressionsFeedback bool) error { resp, err := doRPC[protov1.ResponseWrapper[protov1.RegisterPayload]](c, &rpc) if err != nil { - return fmt.Errorf("error executing rpc: %w", err) + return fmt.Errorf("error executing register rpc: %w", err) } if resp.Status != protov1.ResultOk { - return fmt.Errorf("server responded with error %d", resp.Status) + return fmt.Errorf("server responded register rpc with error %d", resp.Status) } return nil diff --git a/splitio/sdk/workers/impressions.go b/splitio/sdk/workers/impressions.go index a2d3e49..e600f01 100644 --- a/splitio/sdk/workers/impressions.go +++ b/splitio/sdk/workers/impressions.go @@ -40,7 +40,7 @@ func NewImpressionsWorker( // FlushImpressions implements impression.ImpressionRecorder func (m *MultiMetaImpressionWorker) FlushImpressions(bulkSize int64) error { - // TODO(mredolatti): take `bulkSize` into account + // TODO(mredolatti): take `bulkSize` into account var errs []error if err := m.iq.RangeAndClear(func(md types.ClientMetadata, q *sss.LockingQueue[dtos.Impression]) { extracted := make([]dtos.Impression, 0, q.Len()) @@ -54,31 +54,8 @@ func (m *MultiMetaImpressionWorker) FlushImpressions(bulkSize int64) error { return // nothing to do here } - tmp := make(map[string]*dtos.ImpressionsDTO) - for idx := range extracted { - forFeature, ok := tmp[extracted[idx].FeatureName] - if !ok { - forFeature = &dtos.ImpressionsDTO{TestName: extracted[idx].FeatureName} - tmp[extracted[idx].FeatureName] = forFeature - } - - forFeature.KeyImpressions = append(forFeature.KeyImpressions, dtos.ImpressionDTO{ - KeyName: extracted[idx].KeyName, - Treatment: extracted[idx].Treatment, - Time: extracted[idx].Time, - ChangeNumber: extracted[idx].ChangeNumber, - Label: extracted[idx].Label, - BucketingKey: extracted[idx].BucketingKey, - Pt: extracted[idx].Pt, - }) - } - - payload := make([]dtos.ImpressionsDTO, 0, len(tmp)) - for _, v := range tmp { - payload = append(payload, *v) - } - - if err := m.llrec.Record(payload, dtos.Metadata{SDKVersion: md.SdkVersion}, nil); err != nil { + formatted := formatImpressions(extracted) + if err := m.llrec.Record(formatted, dtos.Metadata{SDKVersion: md.SdkVersion}, nil); err != nil { errs = append(errs, err) } }); err != nil { @@ -93,4 +70,32 @@ func (m *MultiMetaImpressionWorker) SynchronizeImpressions(bulkSize int64) error return m.FlushImpressions(bulkSize) } +func formatImpressions(imps []dtos.Impression) []dtos.ImpressionsDTO { + tmp := make(map[string]*dtos.ImpressionsDTO) + for idx := range imps { + forFeature, ok := tmp[imps[idx].FeatureName] + if !ok { + forFeature = &dtos.ImpressionsDTO{TestName: imps[idx].FeatureName} + tmp[imps[idx].FeatureName] = forFeature + } + + forFeature.KeyImpressions = append(forFeature.KeyImpressions, dtos.ImpressionDTO{ + KeyName: imps[idx].KeyName, + Treatment: imps[idx].Treatment, + Time: imps[idx].Time, + ChangeNumber: imps[idx].ChangeNumber, + Label: imps[idx].Label, + BucketingKey: imps[idx].BucketingKey, + Pt: imps[idx].Pt, + }) + } + + formatted := make([]dtos.ImpressionsDTO, 0, len(tmp)) + for _, v := range tmp { + formatted = append(formatted, *v) + } + + return formatted +} + var _ impression.ImpressionRecorder = (*MultiMetaImpressionWorker)(nil) From 5e13efec51124fef5cfb42011f7d129c00302d58 Mon Sep 17 00:00:00 2001 From: Martin Redolatti Date: Fri, 4 Aug 2023 18:40:33 -0300 Subject: [PATCH 17/42] more tests --- splitio/link/protocol/v1/rpcs_test.go | 67 ++++++++++++++++++++------ splitio/link/transfer/acceptor_test.go | 23 +++++++++ splitio/link/transfer/setup.go | 45 +++++------------ 3 files changed, 88 insertions(+), 47 deletions(-) diff --git a/splitio/link/protocol/v1/rpcs_test.go b/splitio/link/protocol/v1/rpcs_test.go index 41a5762..df73f91 100644 --- a/splitio/link/protocol/v1/rpcs_test.go +++ b/splitio/link/protocol/v1/rpcs_test.go @@ -76,7 +76,7 @@ func TestTreatmentRPCParsing(t *testing.T) { assert.Equal(t, "feat1", r.Feature) assert.Equal(t, map[string]interface{}{"a": int64(1)}, r.Attributes) - // nil bucketing key + // nil bucketing key err = r.PopulateFromRPC(&RPC{ RPCBase: protocol.RPCBase{Version: protocol.V1}, OpCode: OCTreatment, @@ -87,8 +87,8 @@ func TestTreatmentRPCParsing(t *testing.T) { assert.Equal(t, "feat1", r.Feature) assert.Equal(t, map[string]interface{}{"a": int64(1)}, r.Attributes) - // nil attributes - r = TreatmentArgs{} + // nil attributes + r = TreatmentArgs{} err = r.PopulateFromRPC(&RPC{ RPCBase: protocol.RPCBase{Version: protocol.V1}, OpCode: OCTreatment, @@ -140,8 +140,8 @@ func TestTreatmentsRPCParsing(t *testing.T) { assert.Equal(t, []string{"feat1", "feat2"}, r.Features) assert.Equal(t, map[string]interface{}{"a": int64(1)}, r.Attributes) - // nil bucketing key - err = r.PopulateFromRPC(&RPC{ + // nil bucketing key + err = r.PopulateFromRPC(&RPC{ RPCBase: protocol.RPCBase{Version: protocol.V1}, OpCode: OCTreatments, Args: []interface{}{"key", nil, []interface{}{"feat1", "feat2"}, map[string]interface{}{"a": 1}}}) @@ -200,7 +200,7 @@ func TestTrackRPCParsing(t *testing.T) { Args: []interface{}{"key", "tt", "et", 2.8, map[string]interface{}{"a": 1}, nil}, })) - now := time.Now() + now := time.Now() err := r.PopulateFromRPC(&RPC{ RPCBase: protocol.RPCBase{Version: protocol.V1}, OpCode: OCTrack, @@ -210,11 +210,11 @@ func TestTrackRPCParsing(t *testing.T) { assert.Equal(t, "key", r.Key) assert.Equal(t, "tt", r.TrafficType) assert.Equal(t, "et", r.EventType) - assert.Equal(t, ref(float64(2.8)), r.Value) + assert.Equal(t, ref(float64(2.8)), r.Value) assert.Equal(t, map[string]interface{}{"a": int64(1)}, r.Properties) - assert.Equal(t, now.UnixMilli(), r.Timestamp) + assert.Equal(t, now.UnixMilli(), r.Timestamp) - // nil properties + // nil properties err = r.PopulateFromRPC(&RPC{ RPCBase: protocol.RPCBase{Version: protocol.V1}, OpCode: OCTrack, @@ -224,12 +224,12 @@ func TestTrackRPCParsing(t *testing.T) { assert.Equal(t, "key", r.Key) assert.Equal(t, "tt", r.TrafficType) assert.Equal(t, "et", r.EventType) - assert.Equal(t, ref(float64(2.8)), r.Value) + assert.Equal(t, ref(float64(2.8)), r.Value) assert.Nil(t, r.Properties) - assert.Equal(t, now.UnixMilli(), r.Timestamp) + assert.Equal(t, now.UnixMilli(), r.Timestamp) - // nil value - r = TrackArgs{} + // nil value + r = TrackArgs{} err = r.PopulateFromRPC(&RPC{ RPCBase: protocol.RPCBase{Version: protocol.V1}, OpCode: OCTrack, @@ -239,9 +239,9 @@ func TestTrackRPCParsing(t *testing.T) { assert.Equal(t, "key", r.Key) assert.Equal(t, "tt", r.TrafficType) assert.Equal(t, "et", r.EventType) - assert.Nil(t, r.Value) + assert.Nil(t, r.Value) assert.Equal(t, map[string]interface{}{"a": int64(1)}, r.Properties) - assert.Equal(t, now.UnixMilli(), r.Timestamp) + assert.Equal(t, now.UnixMilli(), r.Timestamp) } @@ -279,6 +279,43 @@ func TestSanitizeAttributes(t *testing.T) { assert.Equal(t, now.Unix(), attrs["time"]) } +func TestRPCEncoding(t *testing.T) { + ra := RegisterArgs{ + ID: "someID", + SDKVersion: "some-1.2.3", + Flags: 0, + } + encodedRA := ra.Encode() + assert.Equal(t, ra.ID, encodedRA[RegisterArgIDIdx].(string)) + assert.Equal(t, ra.SDKVersion, encodedRA[RegisterArgSDKVersionIdx].(string)) + assert.Equal(t, ra.Flags, encodedRA[RegisterArgFlagsIdx].(RegisterFlags)) + + ta := TreatmentArgs{ + Key: "someKey", + BucketingKey: ref("someBucketing"), + Feature: "someFeature", + Attributes: map[string]interface{}{"some": "attribute"}, + } + encodedTA := ta.Encode() + assert.Equal(t, ta.Key, encodedTA[TreatmentArgKeyIdx].(string)) + assert.Equal(t, *ta.BucketingKey, encodedTA[TreatmentArgBucketingKeyIdx].(string)) + assert.Equal(t, ta.Feature, encodedTA[TreatmentArgFeatureIdx].(string)) + assert.Equal(t, ta.Attributes, encodedTA[TreatmentArgAttributesIdx].(map[string]interface{})) + + tsa := TreatmentsArgs{ + Key: "someKey", + BucketingKey: ref("someBucketing"), + Features: []string{"someFeature", "someFeature2"}, + Attributes: map[string]interface{}{"some": "attribute"}, + } + encodedTsA := tsa.Encode() + assert.Equal(t, tsa.Key, encodedTsA[TreatmentsArgKeyIdx].(string)) + assert.Equal(t, *tsa.BucketingKey, encodedTsA[TreatmentsArgBucketingKeyIdx].(string)) + assert.Equal(t, tsa.Features, encodedTsA[TreatmentsArgFeaturesIdx].([]string)) + assert.Equal(t, tsa.Attributes, encodedTsA[TreatmentsArgAttributesIdx].(map[string]interface{})) + +} + func ref[T any](t T) *T { return &t } diff --git a/splitio/link/transfer/acceptor_test.go b/splitio/link/transfer/acceptor_test.go index 10a7e06..fced743 100644 --- a/splitio/link/transfer/acceptor_test.go +++ b/splitio/link/transfer/acceptor_test.go @@ -76,3 +76,26 @@ func TestAcceptor(t *testing.T) { assert.Nil(t, recv) assert.ErrorIs(t, err, io.EOF) } + +func TestNewAcceptorInstantiation(t *testing.T) { + logger := logging.NewLogger(nil) + + opts := DefaultOpts() + accCfg := DefaultAcceptorConfig() + acc, err := NewAcceptor(logger, &opts, &accCfg) + assert.Nil(t, err) + + assert.Equal(t, opts.Address, acc.address.(*net.UnixAddr).Name) + assert.Equal(t, "unixpacket", acc.address.(*net.UnixAddr).Network()) + assert.Nil(t, acc.Shutdown()) + + opts.Address = "/var/another/don/ga" + opts.ConnType = ConnTypeUnixStream + acc, err = NewAcceptor(logger, &opts, &accCfg) + assert.Nil(t, err) + + assert.Equal(t, opts.Address, acc.address.(*net.UnixAddr).Name) + assert.Equal(t, "unix", acc.address.(*net.UnixAddr).Network()) + assert.Nil(t, acc.Shutdown()) + +} diff --git a/splitio/link/transfer/setup.go b/splitio/link/transfer/setup.go index d26c9ce..15e7645 100644 --- a/splitio/link/transfer/setup.go +++ b/splitio/link/transfer/setup.go @@ -61,41 +61,22 @@ func NewClientConn(o *Options) (RawConn, error) { return newConnWrapper(c, framer, o), nil } -type Option func(*Options) - -/* -func WithAddress(address string) Option { return func(o *Options) { o.Address = address } } -func WithType(t ConnType) Option { return func(o *Options) { o.ConnType = t } } -func WithLogger(logger logging.LoggerInterface) Option { return func(o *Options) { o.Logger = logger } } -func WithBufSize(s int) Option { return func(o *Options) { o.BufferSize = s } } -func WithMaxConns(m int) Option { return func(o *Options) { o.MaxSimultaneousConnections = m } } -func WithReadTimeout(d time.Duration) Option { return func(o *Options) { o.ReadTimeout = d } } -func WithWriteTimeout(d time.Duration) Option { return func(o *Options) { o.WriteTimeout = d } } -func WithAcceptTimeout(d time.Duration) Option { return func(o *Options) { o.AcceptTimeout = d } } -*/ - type Options struct { - ConnType ConnType - Address string - Logger logging.LoggerInterface - BufferSize int - ReadTimeout time.Duration - WriteTimeout time.Duration + ConnType ConnType + Address string + Logger logging.LoggerInterface + BufferSize int + ReadTimeout time.Duration + WriteTimeout time.Duration } -/* -func (o *Options) Parse(opts []Option) { - for _, configure := range opts { - configure(o) - } -} -*/ + func DefaultOpts() Options { return Options{ - ConnType: ConnTypeUnixSeqPacket, - Address: "/var/run/splitd.sock", - Logger: logging.NewLogger(nil), - BufferSize: 1024, - ReadTimeout: 1 * time.Second, - WriteTimeout: 1 * time.Second, + ConnType: ConnTypeUnixSeqPacket, + Address: "/var/run/splitd.sock", + Logger: logging.NewLogger(nil), + BufferSize: 1024, + ReadTimeout: 1 * time.Second, + WriteTimeout: 1 * time.Second, } } From 76ab7adbcdae8d6a8f59413007cbf7916f8fce6b Mon Sep 17 00:00:00 2001 From: Martin Redolatti Date: Fri, 4 Aug 2023 19:04:14 -0300 Subject: [PATCH 18/42] more tests --- splitio/link/protocol/v1/rpcs_test.go | 36 +++++++++++++++------ splitio/util/conf/helpers_test.go | 34 ++++++++++++++++++++ splitio/util/conf/parsers_test.go | 46 +++++++++++++++++++++++++++ 3 files changed, 106 insertions(+), 10 deletions(-) create mode 100644 splitio/util/conf/helpers_test.go create mode 100644 splitio/util/conf/parsers_test.go diff --git a/splitio/link/protocol/v1/rpcs_test.go b/splitio/link/protocol/v1/rpcs_test.go index df73f91..d7194cf 100644 --- a/splitio/link/protocol/v1/rpcs_test.go +++ b/splitio/link/protocol/v1/rpcs_test.go @@ -280,15 +280,15 @@ func TestSanitizeAttributes(t *testing.T) { } func TestRPCEncoding(t *testing.T) { - ra := RegisterArgs{ - ID: "someID", - SDKVersion: "some-1.2.3", - Flags: 0, - } - encodedRA := ra.Encode() - assert.Equal(t, ra.ID, encodedRA[RegisterArgIDIdx].(string)) - assert.Equal(t, ra.SDKVersion, encodedRA[RegisterArgSDKVersionIdx].(string)) - assert.Equal(t, ra.Flags, encodedRA[RegisterArgFlagsIdx].(RegisterFlags)) + ra := RegisterArgs{ + ID: "someID", + SDKVersion: "some-1.2.3", + Flags: 0, + } + encodedRA := ra.Encode() + assert.Equal(t, ra.ID, encodedRA[RegisterArgIDIdx].(string)) + assert.Equal(t, ra.SDKVersion, encodedRA[RegisterArgSDKVersionIdx].(string)) + assert.Equal(t, ra.Flags, encodedRA[RegisterArgFlagsIdx].(RegisterFlags)) ta := TreatmentArgs{ Key: "someKey", @@ -305,7 +305,7 @@ func TestRPCEncoding(t *testing.T) { tsa := TreatmentsArgs{ Key: "someKey", BucketingKey: ref("someBucketing"), - Features: []string{"someFeature", "someFeature2"}, + Features: []string{"someFeature", "someFeature2"}, Attributes: map[string]interface{}{"some": "attribute"}, } encodedTsA := tsa.Encode() @@ -314,6 +314,22 @@ func TestRPCEncoding(t *testing.T) { assert.Equal(t, tsa.Features, encodedTsA[TreatmentsArgFeaturesIdx].([]string)) assert.Equal(t, tsa.Attributes, encodedTsA[TreatmentsArgAttributesIdx].(map[string]interface{})) + tra := TrackArgs{ + Key: "someKey", + TrafficType: "someTrafficType", + EventType: "someEventType", + Value: ref(123.), + Properties: map[string]interface{}{"a": 1}, + Timestamp: 123456, + } + encodedTrA := tra.Encode() + assert.Equal(t, tra.Key, encodedTrA[TrackArgKeyIdx].(string)) + assert.Equal(t, tra.TrafficType, encodedTrA[TrackArgTrafficTypeIdx].(string)) + assert.Equal(t, tra.EventType, encodedTrA[TrackArgEventTypeIdx].(string)) + assert.Equal(t, *tra.Value, *encodedTrA[TrackArgValueIdx].(*float64)) + assert.Equal(t, tra.Properties, encodedTrA[TrackArgPropertiesIdx].(map[string]interface{})) + assert.Equal(t, tra.Timestamp, encodedTrA[TrackArgTimestampIdx].(int64)) + } func ref[T any](t T) *T { diff --git a/splitio/util/conf/helpers_test.go b/splitio/util/conf/helpers_test.go new file mode 100644 index 0000000..5930e40 --- /dev/null +++ b/splitio/util/conf/helpers_test.go @@ -0,0 +1,34 @@ +package conf + +import ( + "github.com/stretchr/testify/assert" + "testing" +) + +func TestConfHelpers(t *testing.T) { + var x int + SetIfNotEmpty(&x, ref(0)) + assert.Equal(t, 0, x) + SetIfNotNil(&x, nil) + assert.Equal(t, 0, x) + + SetIfNotEmpty(&x, ref(5)) + assert.Equal(t, 5, x) + SetIfNotNil(&x, ref(25)) + assert.Equal(t, 25, x) + + x = 0 + MapIfNotEmpty(&x, ref(0), func(z int) int { return z + 1 }) + assert.Equal(t, 0, x) + MapIfNotNil(&x, nil, func(z int) int { return z + 1 }) + assert.Equal(t, 0, x) + + MapIfNotEmpty(&x, ref(1), func(z int) int { return z + 1 }) + assert.Equal(t, 2, x) + MapIfNotEmpty(&x, ref(2), func(z int) int { return z + 1 }) + assert.Equal(t, 3, x) +} + +func ref[T any](t T) *T { + return &t +} diff --git a/splitio/util/conf/parsers_test.go b/splitio/util/conf/parsers_test.go new file mode 100644 index 0000000..3b1a664 --- /dev/null +++ b/splitio/util/conf/parsers_test.go @@ -0,0 +1,46 @@ +package conf + +import ( + "testing" + + "github.com/splitio/splitd/splitio/link/protocol" + "github.com/splitio/splitd/splitio/link/serializer" + "github.com/splitio/splitd/splitio/link/transfer" + "github.com/stretchr/testify/assert" +) + +func TestParseProtocol(t *testing.T) { + pv, err := ParseProtocolVersion("v1") + assert.Nil(t, err) + assert.Equal(t, protocol.V1, pv) + + pv, err = ParseProtocolVersion("v2") + assert.NotNil(t, err) + assert.NotEqual(t, pv, protocol.V1) +} + +func TestParseConnType(t *testing.T) { + ct, err := ParseConnType("unix-stream") + assert.Nil(t, err) + assert.Equal(t, transfer.ConnTypeUnixStream, ct) + + ct, err = ParseConnType("unix-seqpacket") + assert.Nil(t, err) + assert.Equal(t, transfer.ConnTypeUnixSeqPacket, ct) + + ct, err = ParseConnType("something-else") + assert.NotNil(t, err) + assert.NotEqual(t, transfer.ConnTypeUnixSeqPacket, ct) + assert.NotEqual(t, transfer.ConnTypeUnixStream, ct) +} + +func TestParseSerializer(t *testing.T) { + sm, err := ParseSerializer("msgpack") + assert.Nil(t, err) + assert.Equal(t, serializer.MsgPack, sm) + + sm, err = ParseSerializer("something_esle") + assert.NotNil(t, err) + assert.NotEqual(t, serializer.MsgPack, sm) + +} From 522862bb3a7843aee86230abd033029396c108cb Mon Sep 17 00:00:00 2001 From: Martin Redolatti Date: Tue, 8 Aug 2023 16:33:04 -0300 Subject: [PATCH 19/42] more tests --- splitio/conf/splitd_test.go | 43 ++++++++++++++++++++++++++++ splitio/link/protocol/v1/errors.go | 35 ++-------------------- splitio/sdk/storage/multi.go | 1 - splitio/sdk/storage/multi_test.go | 18 +++++++++--- splitio/sdk/storage/storages_test.go | 24 +++++++++++----- 5 files changed, 76 insertions(+), 45 deletions(-) diff --git a/splitio/conf/splitd_test.go b/splitio/conf/splitd_test.go index 7283ff7..09acfcc 100644 --- a/splitio/conf/splitd_test.go +++ b/splitio/conf/splitd_test.go @@ -1,6 +1,10 @@ package conf import ( + "os" + "path/filepath" + "runtime" + "strings" "testing" "time" @@ -12,6 +16,45 @@ import ( "github.com/stretchr/testify/assert" ) +func TestConfig(t *testing.T) { + cfg := Config{SDK: SDK{Apikey: "someVeryLongApikey"}} + assert.Contains(t, cfg.String(), "somexxxxxxx") + + _, filename, _, _ := runtime.Caller(0) + parts := strings.Split(filename, string(filepath.Separator)) + dir := strings.Join(parts[:len(parts)-3], string(filepath.Separator)) + + cfg = Config{} + assert.Nil(t, cfg.parse(dir+string(filepath.Separator)+"splitd.yaml.tpl")) + assert.Equal(t, Config{ + Logger: Logger{Level: ref("ERROR")}, + SDK: SDK{ + Apikey: "YOUR_API_KEY", + URLs: URLs{ + Auth: ref("https://auth.split.io"), + SDK: ref("https://sdk.split.io/api"), + Events: ref("https://events.split.io/api"), + Streaming: ref("https://streaming.split.io/sse"), + Telemetry: ref("https://telemetry.split.io/api/v1"), + }, + }, + Link: Link{ + Type: ref("unix-seqpacket"), + Address: ref("/var/run/splitd.sock"), + Serialization: ref("msgpack"), + }, + }, cfg) + + assert.Error(t, cfg.parse("someNonexistantFile")) + assert.Error(t, cfg.parse(dir+string(filepath.Separator)+"Makefile")) + + os.Setenv("SPLITD_CONF_FILE", dir+string(filepath.Separator)+"splitd.yaml.tpl") + newCfg, err := ReadConfig() + assert.Nil(t, err) + assert.NotNil(t, newCfg) + +} + func TestLink(t *testing.T) { linkCFG := &Link{ diff --git a/splitio/link/protocol/v1/errors.go b/splitio/link/protocol/v1/errors.go index ed73e62..5c2d3d5 100644 --- a/splitio/link/protocol/v1/errors.go +++ b/splitio/link/protocol/v1/errors.go @@ -3,8 +3,6 @@ package v1 import ( "errors" "strconv" - - "github.com/vmihailenco/msgpack/v5" ) var ( @@ -13,32 +11,6 @@ var ( ErrIncorrectArgCount = errors.New("invalid argument count") ) -type InvocationErrorCode int - -const ( - InvocationErrorInvalidArgs InvocationErrorCode = 0 -) - -type ErrorWithResponse interface { - ToResponse() []byte -} - -type InvocationError struct { - code InvocationErrorCode - message string -} - -func (e *InvocationError) Error() string { - return e.message -} - -func (e *InvocationError) ToResponse() []byte { - serialized, _ := msgpack.Marshal(e) - return serialized -} - -// --------------- - type RPCParseErrorCode int const ( @@ -56,7 +28,7 @@ func (c RPCParseErrorCode) formatWithData(data int64) string { case PECInvalidArgType: return "wrong argument type at index " + strconv.Itoa(int(data)) default: - return "unknown error" + return "unknown error" } } @@ -67,10 +39,7 @@ type RPCParseError struct { // Error implements error func (e RPCParseError) Error() string { - return e.Code.formatWithData(e.Data) + return e.Code.formatWithData(e.Data) } var _ error = RPCParseError{} - -// ---------------- -var _ ErrorWithResponse = (*InvocationError)(nil) diff --git a/splitio/sdk/storage/multi.go b/splitio/sdk/storage/multi.go index 56928ae..98c4520 100644 --- a/splitio/sdk/storage/multi.go +++ b/splitio/sdk/storage/multi.go @@ -6,7 +6,6 @@ import ( type MultiMetaQueues[T elemConstraint, U comparable, Q BackingQueue[T]] struct { m sync.Map - qpowSize int cFactory func() Q } diff --git a/splitio/sdk/storage/multi_test.go b/splitio/sdk/storage/multi_test.go index c9fc6ac..aa361bf 100644 --- a/splitio/sdk/storage/multi_test.go +++ b/splitio/sdk/storage/multi_test.go @@ -25,7 +25,7 @@ func TestMultiStorageBasic(t *testing.T) { assert.Equal(t, 3, n) assert.Nil(t, err) - mq.RangeAndClear(func(cm types.ClientMetadata, q *LockingQueue[dtos.EventDTO]) { + mq.Range(func(cm types.ClientMetadata, q *LockingQueue[dtos.EventDTO]) { switch cm.SdkVersion { case "go-1.2.3": assert.Equal(t, 4, q.Len()) @@ -33,6 +33,17 @@ func TestMultiStorageBasic(t *testing.T) { n, err := q.Pop(5, &buf) assert.Equal(t, ErrQueueEmpty, err) assert.Equal(t, 4, n) + case "php-1.2.3": + // do nothing, to be used in the next assertions + default: + assert.Fail(t, "unexpected metadata: "+cm.SdkVersion) + } + }) + + mq.RangeAndClear(func(cm types.ClientMetadata, q *LockingQueue[dtos.EventDTO]) { + switch cm.SdkVersion { + case "go-1.2.3": + // used previously case "php-1.2.3": assert.Equal(t, 3, q.Len()) var buf []dtos.EventDTO @@ -44,7 +55,6 @@ func TestMultiStorageBasic(t *testing.T) { } }) - mq.RangeAndClear(func(cm types.ClientMetadata, q *LockingQueue[dtos.EventDTO]) { - assert.Fail(t, "should not execute") - }) + mq.Range(func(cm types.ClientMetadata, q *LockingQueue[dtos.EventDTO]) { assert.Fail(t, "should not execute") }) + mq.RangeAndClear(func(cm types.ClientMetadata, q *LockingQueue[dtos.EventDTO]) { assert.Fail(t, "should not execute") }) } diff --git a/splitio/sdk/storage/storages_test.go b/splitio/sdk/storage/storages_test.go index e004c7e..ded200e 100644 --- a/splitio/sdk/storage/storages_test.go +++ b/splitio/sdk/storage/storages_test.go @@ -7,11 +7,21 @@ import ( ) func TestGetNearestSizePowerOf2(t *testing.T) { - assert.Equal(t, 1, getNearestSizePowerOf2(2)) - assert.Equal(t, 2, getNearestSizePowerOf2(3)) - assert.Equal(t, 2, getNearestSizePowerOf2(4)) - assert.Equal(t, 3, getNearestSizePowerOf2(5)) - assert.Equal(t, 3, getNearestSizePowerOf2(8)) - assert.Equal(t, 10, getNearestSizePowerOf2(1024)) - assert.Equal(t, 11, getNearestSizePowerOf2(1025)) + assert.Equal(t, 1, getNearestSizePowerOf2(2)) + assert.Equal(t, 2, getNearestSizePowerOf2(3)) + assert.Equal(t, 2, getNearestSizePowerOf2(4)) + assert.Equal(t, 3, getNearestSizePowerOf2(5)) + assert.Equal(t, 3, getNearestSizePowerOf2(8)) + assert.Equal(t, 10, getNearestSizePowerOf2(1024)) + assert.Equal(t, 11, getNearestSizePowerOf2(1025)) +} + +func TestStorageConstruction(t *testing.T) { + ist, size := NewImpressionsQueue(1024) + assert.Equal(t, 1024, size) + assert.Equal(t, 1024, len(ist.cFactory().data)) + + est, size := NewEventsQueue(1024) + assert.Equal(t, 1024, size) + assert.Equal(t, 1024, len(est.cFactory().data)) } From 0f60317d480d944e951b94c303df2e0943b7b0ff Mon Sep 17 00:00:00 2001 From: Martin Redolatti Date: Tue, 8 Aug 2023 18:17:59 -0300 Subject: [PATCH 20/42] more coverage --- splitio/link/client/client_test.go | 39 +++++++++++++++++++ splitio/link/link_test.go | 60 ++++++++++++++++++++++++++++++ 2 files changed, 99 insertions(+) create mode 100644 splitio/link/client/client_test.go create mode 100644 splitio/link/link_test.go diff --git a/splitio/link/client/client_test.go b/splitio/link/client/client_test.go new file mode 100644 index 0000000..388a9f2 --- /dev/null +++ b/splitio/link/client/client_test.go @@ -0,0 +1,39 @@ +package client + +import ( + "testing" + + "github.com/splitio/go-toolkit/v5/logging" + "github.com/splitio/splitd/splitio/link/protocol" + v1 "github.com/splitio/splitd/splitio/link/protocol/v1" + smocks "github.com/splitio/splitd/splitio/link/serializer/mocks" + "github.com/splitio/splitd/splitio/link/transfer/mocks" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/mock" +) + +func TestClientConstruction(t *testing.T) { + connMock := &mocks.RawConnMock{} + connMock.On("SendMessage", mock.Anything).Once().Return(nil) + connMock.On("ReceiveMessage").Once().Return([]byte("registerResp"), nil) + connMock.On("Shutdown").Once().Return(nil) + serializeMock := &smocks.SerializerMock{} + serializeMock.On("Serialize", mock.Anything).Once().Return([]byte("serializedRegister"), nil) + serializeMock.On("Parse", mock.Anything, mock.Anything).Run(func(args mock.Arguments) { + args.Get(1).(*v1.ResponseWrapper[v1.RegisterPayload]).Status = v1.ResultOk + }).Return(nil) + logger := logging.NewLogger(nil) + c, err := New(logger, connMock, serializeMock, Options{Protocol: protocol.V1, ImpressionsFeedback: false}) + assert.Nil(t, err) + assert.NotNil(t, c) + assert.Nil(t, c.Shutdown()) +} + +func TestClientUnknownProtocol(t *testing.T) { + connMock := &mocks.RawConnMock{} + serializeMock := &smocks.SerializerMock{} + logger := logging.NewLogger(nil) + c, err := New(logger, connMock, serializeMock, Options{Protocol: protocol.Version(252), ImpressionsFeedback: false}) + assert.Nil(t, c) + assert.ErrorContains(t, err, "unknown protocol") +} diff --git a/splitio/link/link_test.go b/splitio/link/link_test.go new file mode 100644 index 0000000..39f0a85 --- /dev/null +++ b/splitio/link/link_test.go @@ -0,0 +1,60 @@ +package link + +import ( + "testing" + + "github.com/splitio/go-toolkit/v5/logging" + "github.com/splitio/splitd/splitio/link/client" + "github.com/splitio/splitd/splitio/link/protocol" + "github.com/splitio/splitd/splitio/link/serializer" + "github.com/splitio/splitd/splitio/link/transfer" + "github.com/splitio/splitd/splitio/sdk/mocks" + "github.com/stretchr/testify/assert" +) + +func TestOptions(t *testing.T) { + assert.Equal(t, ListenerOptions{ + Transfer: transfer.DefaultOpts(), + Acceptor: transfer.DefaultAcceptorConfig(), + Serialization: serializer.MsgPack, + Protocol: protocol.V1, + }, + DefaultListenerOptions()) + + assert.Equal(t, ConsumerOptions{ + Transfer: transfer.DefaultOpts(), + Consumer: client.DefaultOptions(), + Serialization: serializer.MsgPack, + }, DefaultConsumerOptions()) +} + +func TestListenErrors(t *testing.T) { + lo := DefaultListenerOptions() + lo.Protocol = protocol.Version(123) + acc, shutdown, err := Listen(logging.NewLogger(nil), &mocks.SDKMock{}, &lo) + assert.Nil(t, acc) + assert.Nil(t, shutdown) + assert.ErrorContains(t, err, "protocol") + + lo = DefaultListenerOptions() + lo.Serialization = serializer.Mechanism(123) + acc, shutdown, err = Listen(logging.NewLogger(nil), &mocks.SDKMock{}, &lo) + assert.Nil(t, acc) + assert.Nil(t, shutdown) + assert.ErrorContains(t, err, "serializer") +} + +func TestConsumerErrors(t *testing.T) { + co := DefaultConsumerOptions() + co.Consumer.Protocol = protocol.Version(123) + client, err := Consumer(logging.NewLogger(nil), &co) + assert.Nil(t, client) + assert.ErrorContains(t, err, "protocol") + + co = DefaultConsumerOptions() + co.Serialization = serializer.Mechanism(123) + client, err = Consumer(logging.NewLogger(nil), &co) + assert.Nil(t, client) + assert.ErrorContains(t, err, "serializer") + +} From 2e3a01ffa737fe1e69281e8785ef3dce80307a85 Mon Sep 17 00:00:00 2001 From: Martin Redolatti Date: Tue, 8 Aug 2023 18:23:36 -0300 Subject: [PATCH 21/42] fix test --- splitio/link/link_test.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/splitio/link/link_test.go b/splitio/link/link_test.go index 39f0a85..c7f857b 100644 --- a/splitio/link/link_test.go +++ b/splitio/link/link_test.go @@ -46,10 +46,10 @@ func TestListenErrors(t *testing.T) { func TestConsumerErrors(t *testing.T) { co := DefaultConsumerOptions() - co.Consumer.Protocol = protocol.Version(123) + co.Transfer.ConnType = transfer.ConnType(123) client, err := Consumer(logging.NewLogger(nil), &co) assert.Nil(t, client) - assert.ErrorContains(t, err, "protocol") + assert.ErrorContains(t, err, "invalid listener type") co = DefaultConsumerOptions() co.Serialization = serializer.Mechanism(123) From 9cafb6bb7e8c70e664087c36e9f7eac873ab3820 Mon Sep 17 00:00:00 2001 From: Martin Redolatti Date: Tue, 8 Aug 2023 19:16:46 -0300 Subject: [PATCH 22/42] more coveraege --- splitio/link/link_test.go | 11 ++- splitio/link/service/v1/clientmgr.go | 2 +- splitio/link/service/v1/clientmgr_test.go | 104 +++++++++++++++++++++- splitio/link/transfer/setup.go | 2 +- 4 files changed, 114 insertions(+), 5 deletions(-) diff --git a/splitio/link/link_test.go b/splitio/link/link_test.go index c7f857b..5e48527 100644 --- a/splitio/link/link_test.go +++ b/splitio/link/link_test.go @@ -30,10 +30,17 @@ func TestOptions(t *testing.T) { func TestListenErrors(t *testing.T) { lo := DefaultListenerOptions() - lo.Protocol = protocol.Version(123) + lo.Transfer.ConnType = transfer.ConnType(222) acc, shutdown, err := Listen(logging.NewLogger(nil), &mocks.SDKMock{}, &lo) assert.Nil(t, acc) assert.Nil(t, shutdown) + assert.ErrorContains(t, err, "invalid conn type") + + lo = DefaultListenerOptions() + lo.Protocol = protocol.Version(123) + acc, shutdown, err = Listen(logging.NewLogger(nil), &mocks.SDKMock{}, &lo) + assert.Nil(t, acc) + assert.Nil(t, shutdown) assert.ErrorContains(t, err, "protocol") lo = DefaultListenerOptions() @@ -49,7 +56,7 @@ func TestConsumerErrors(t *testing.T) { co.Transfer.ConnType = transfer.ConnType(123) client, err := Consumer(logging.NewLogger(nil), &co) assert.Nil(t, client) - assert.ErrorContains(t, err, "invalid listener type") + assert.ErrorContains(t, err, "invalid conn type") co = DefaultConsumerOptions() co.Serialization = serializer.Mechanism(123) diff --git a/splitio/link/service/v1/clientmgr.go b/splitio/link/service/v1/clientmgr.go index 8655956..b74f679 100644 --- a/splitio/link/service/v1/clientmgr.go +++ b/splitio/link/service/v1/clientmgr.go @@ -40,7 +40,7 @@ func NewClientManager( func (m *ClientManager) Manage() { defer func() { if r := recover(); r != nil { - m.logger.Error("CRITICAL - connection handlers are panicking: ", r) + m.logger.Error("CRITICAL - connection handler is panicking: ", r) } }() err := m.handleClientInteractions() diff --git a/splitio/link/service/v1/clientmgr_test.go b/splitio/link/service/v1/clientmgr_test.go index 769e3e4..30970c7 100644 --- a/splitio/link/service/v1/clientmgr_test.go +++ b/splitio/link/service/v1/clientmgr_test.go @@ -9,9 +9,9 @@ import ( "github.com/splitio/go-toolkit/v5/logging" "github.com/splitio/splitd/splitio/link/protocol" v1 "github.com/splitio/splitd/splitio/link/protocol/v1" + proto1Mocks "github.com/splitio/splitd/splitio/link/protocol/v1/mocks" serializerMocks "github.com/splitio/splitd/splitio/link/serializer/mocks" transferMocks "github.com/splitio/splitd/splitio/link/transfer/mocks" - proto1Mocks "github.com/splitio/splitd/splitio/link/protocol/v1/mocks" "github.com/splitio/splitd/splitio/sdk" sdkMocks "github.com/splitio/splitd/splitio/sdk/mocks" "github.com/splitio/splitd/splitio/sdk/types" @@ -187,4 +187,106 @@ func TestConnectionFailureWhenReading(t *testing.T) { rawConnMock.AssertNumberOfCalls(t, "Shutdown", 1) } +func TestManagePanicRecovers(t *testing.T) { + rawConnMock := &transferMocks.RawConnMock{} + rawConnMock.On("ReceiveMessage").Panic("some panic") + rawConnMock.On("Shutdown", mock.Anything).Return(nil) + + logger := &loggerMock{} + logger.On("Error", "CRITICAL - connection handler is panicking: ", "some panic").Once() + + serializerMock := &serializerMocks.SerializerMock{} + sdkMock := &sdkMocks.SDKMock{} + + cm := NewClientManager(rawConnMock, logger, sdkMock, serializerMock) + cm.Manage() + rawConnMock.AssertNumberOfCalls(t, "Shutdown", 1) + + logger.AssertExpectations(t) +} + +func TestFetchRPC(t *testing.T) { + // error reading from conn + someErr := errors.New("someConnErr") + rawConnMock := &transferMocks.RawConnMock{} + rawConnMock.On("ReceiveMessage").Return([]byte(nil), someErr) + rawConnMock.On("Shutdown", mock.Anything).Return(nil) + serializerMock := &serializerMocks.SerializerMock{} + logger := logging.NewLogger(nil) + cm := NewClientManager(rawConnMock, logger, nil, serializerMock) + rpc, err := cm.fetchRPC() + assert.Nil(t, rpc) + assert.ErrorContains(t, err, "someConnErr") + + // error parsing message + someErr = errors.New("someSerializationErr") + rawConnMock = &transferMocks.RawConnMock{} + rawConnMock.On("ReceiveMessage").Return([]byte{}, nil) + rawConnMock.On("Shutdown", mock.Anything).Return(nil) + serializerMock = &serializerMocks.SerializerMock{} + serializerMock.On("Parse", mock.Anything, mock.Anything).Return(someErr) + cm = NewClientManager(rawConnMock, logger, nil, serializerMock) + rpc, err = cm.fetchRPC() + assert.Nil(t, rpc) + assert.ErrorContains(t, err, "someSerializationErr") +} + +func TestSendResponse(t *testing.T) { + // error parsing message + someErr := errors.New("someSerializationErr") + rawConnMock := &transferMocks.RawConnMock{} + rawConnMock.On("Shutdown", mock.Anything).Return(nil) + serializerMock := &serializerMocks.SerializerMock{} + serializerMock.On("Serialize", mock.Anything).Return([]byte(nil), someErr) + logger := logging.NewLogger(nil) + cm := NewClientManager(rawConnMock, logger, nil, serializerMock) + err := cm.sendResponse(nil) + assert.ErrorContains(t, err, "someSerializationErr") + + // error reading from conn + someErr = errors.New("someConnErr") + rawConnMock = &transferMocks.RawConnMock{} + rawConnMock.On("SendMessage", mock.Anything).Return(someErr) + rawConnMock.On("Shutdown", mock.Anything).Return(nil) + serializerMock = &serializerMocks.SerializerMock{} + serializerMock.On("Serialize", mock.Anything).Return([]byte{}, nil) + cm = NewClientManager(rawConnMock, logger, nil, serializerMock) + err = cm.sendResponse(nil) + assert.ErrorContains(t, err, "someConnErr") +} + +func TestHandleRPCErrors(t *testing.T) { + logger := logging.NewLogger(nil) + cm := NewClientManager(nil, logger, nil, nil) + res, err := cm.handleRPC(&v1.RPC{RPCBase: protocol.RPCBase{Version: protocol.V1}, OpCode: v1.OCTreatment}) + assert.Nil(t, res) + assert.ErrorContains(t, err, "first call must be 'register'") + + // register wrong args + res, err = cm.handleRPC(&v1.RPC{RPCBase: protocol.RPCBase{Version: protocol.V1}, OpCode: v1.OCRegister, Args: []interface{}{1, "hola"}}) + assert.Nil(t, res) + assert.ErrorContains(t, err, "error parsing register arguments") + + // set the config to allow other rpcs to be handled + cm.clientConfig = &types.ClientConfig{ReturnImpressionData: true} + + // treatment wrong args + res, err = cm.handleRPC(&v1.RPC{RPCBase: protocol.RPCBase{Version: protocol.V1}, OpCode: v1.OCTreatment, Args: []interface{}{1, "hola"}}) + assert.Nil(t, res) + assert.ErrorContains(t, err, "error parsing treatment arguments") + + // register wrong args + res, err = cm.handleRPC(&v1.RPC{RPCBase: protocol.RPCBase{Version: protocol.V1}, OpCode: v1.OCTreatments, Args: []interface{}{1, "hola"}}) + assert.Nil(t, res) + assert.ErrorContains(t, err, "error parsing treatments arguments") +} + +type loggerMock struct{ mock.Mock } + +func (m *loggerMock) Debug(msg ...interface{}) { m.Called(msg...) } +func (m *loggerMock) Error(msg ...interface{}) { m.Called(msg...) } +func (m *loggerMock) Info(msg ...interface{}) { m.Called(msg...) } +func (m *loggerMock) Verbose(msg ...interface{}) { m.Called(msg...) } +func (m *loggerMock) Warning(msg ...interface{}) { m.Called(msg...) } +var _ logging.LoggerInterface = (*loggerMock)(nil) diff --git a/splitio/link/transfer/setup.go b/splitio/link/transfer/setup.go index 15e7645..fd83f8e 100644 --- a/splitio/link/transfer/setup.go +++ b/splitio/link/transfer/setup.go @@ -18,7 +18,7 @@ const ( ) var ( - ErrInvalidConnType = errors.New("invalid listener type") + ErrInvalidConnType = errors.New("invalid conn type") ) func NewAcceptor(logger logging.LoggerInterface, o *Options, listenerConfig *AcceptorConfig) (*Acceptor, error) { From eb33eebbdc884fe610ba65473a7809e59e1a8e60 Mon Sep 17 00:00:00 2001 From: Martin Redolatti Date: Thu, 10 Aug 2023 13:56:48 -0300 Subject: [PATCH 23/42] changes in config & more testing [WIP] - yaml update pending --- cmd/splitd/main.go | 36 +++++++------ splitio/conf/splitd.go | 62 +++++++++++++++++---- splitio/conf/splitd_test.go | 28 +++++++++- splitio/logging/helpers.go | 53 ++++++++++++++++++ splitio/logging/helpers_test.go | 95 +++++++++++++++++++++++++++++++++ splitio/sdk/conf/conf.go | 33 +++++++++++- splitio/sdk/conf/conf_test.go | 41 ++++++++++++++ splitio/sdk/sdk.go | 6 +++ 8 files changed, 325 insertions(+), 29 deletions(-) create mode 100644 splitio/logging/helpers.go create mode 100644 splitio/logging/helpers_test.go create mode 100644 splitio/sdk/conf/conf_test.go diff --git a/cmd/splitd/main.go b/cmd/splitd/main.go index da34b7f..9ff0625 100644 --- a/cmd/splitd/main.go +++ b/cmd/splitd/main.go @@ -15,25 +15,27 @@ import ( func main() { - printHeader() + printHeader() cfg, err := conf.ReadConfig() if err != nil { fmt.Println("error reading config: ", err.Error()) os.Exit(1) } - handleFlags(cfg) + handleFlags(cfg) - logger := logging.NewLogger(cfg.Logger.ToLoggerOptions()) + loggerCfg, err := cfg.Logger.ToLoggerOptions() + exitOnErr("logging setup", err) + logger := logging.NewLogger(loggerCfg) splitSDK, err := sdk.New(logger, cfg.SDK.Apikey, cfg.SDK.ToSDKConf()) - exitOnErr("sdk initialization", err) + exitOnErr("sdk initialization", err) - linkCFG, err := cfg.Link.ToListenerOpts() - exitOnErr("link config", err) + linkCFG, err := cfg.Link.ToListenerOpts() + exitOnErr("link config", err) errc, lShutdown, err := link.Listen(logger, splitSDK, linkCFG) - exitOnErr("rpc listener setup", err) + exitOnErr("rpc listener setup", err) shutdown := util.NewShutdownHandler() shutdown.RegisterHook(func() { @@ -46,26 +48,26 @@ func main() { // Wait for connection to end (either gracefully of because of an error) err = <-errc - exitOnErr("shutdown: ", err) + exitOnErr("shutdown: ", err) } func printHeader() { - fmt.Println(splitio.ASCILogo) - fmt.Printf("Splitd Agent - Version %s. (2023)\n\n", splitio.Version) + fmt.Println(splitio.ASCILogo) + fmt.Printf("Splitd Agent - Version %s. (2023)\n\n", splitio.Version) } func handleFlags(cfg *conf.Config) { - printConf := flag.Bool("outputConfig", false, "print config (with partially obfuscated apikey)") - flag.Parse() - if *printConf { - fmt.Printf("\nConfig: %s\n", cfg) - os.Exit(0) - } + printConf := flag.Bool("outputConfig", false, "print config (with partially obfuscated apikey)") + flag.Parse() + if *printConf { + fmt.Printf("\nConfig: %s\n", cfg) + os.Exit(0) + } } func exitOnErr(ctxStr string, err error) { if err != nil { - fmt.Printf("%s: startup error: %s\n", ctxStr, err.Error()) + fmt.Printf("%s: startup error: %s\n", ctxStr, err.Error()) os.Exit(1) } diff --git a/splitio/conf/splitd.go b/splitio/conf/splitd.go index 615edc0..e1fc39e 100644 --- a/splitio/conf/splitd.go +++ b/splitio/conf/splitd.go @@ -11,6 +11,7 @@ import ( "github.com/splitio/go-toolkit/v5/logging" "github.com/splitio/splitd/splitio/link" + sdlogging "github.com/splitio/splitd/splitio/logging" sdkConf "github.com/splitio/splitd/splitio/sdk/conf" cc "github.com/splitio/splitd/splitio/util/conf" "gopkg.in/yaml.v3" @@ -94,20 +95,50 @@ func (l *Link) ToListenerOpts() (*link.ListenerOptions, error) { } type SDK struct { - Apikey string `yaml:"apikey"` - LabelsEnabled *bool `yaml:"labelsEnabled"` - StreamingEnabled *bool `yaml:"streamingEnabled"` - URLs URLs `yaml:"urls"` + Apikey string `yaml:"apikey"` + LabelsEnabled *bool `yaml:"labelsEnabled"` + StreamingEnabled *bool `yaml:"streamingEnabled"` + URLs URLs `yaml:"urls"` + FeatureFlags FeatureFlags `yaml:"featureFlags"` + Impressions Impressions `yaml:"impressions"` } -func (s *SDK) ToSDKConf() *sdkConf.Config { +type FeatureFlags struct { + SplitNotificationQueueSize *int `yaml:"splitNotificationQueueSize"` + SplitRefreshRateSeconds *int `yaml:"splitRefreshSeconds"` + SegmentNotificationQueueSize *int `yaml:"segmentNotificationQueueSize"` + SegmentRefreshRateSeconds *int `yaml:"segmentRefreshSeconds"` + SegmentUpdateWorkers *int `yaml:"segmentUpdateWorkers"` + SegmentUpdateQueueSize *int `yaml:"segmentUpdateQueueSize"` +} +type Impressions struct { + Mode *string `yaml:"mode"` + RefreshRateSeconds *int `yaml:"refreshRateSeconds"` + CountRefreshRateSeconds *int `yaml:"countRefreshRateSeconds"` + QueueSize *int `yaml:"queueSize"` + ObserverSize *int `yaml:"observerSize"` + Watermark *int `yaml:"watermark"` +} + +func (s *SDK) ToSDKConf() *sdkConf.Config { cfg := sdkConf.DefaultConfig() + durationFromSeconds := func(seconds int) time.Duration { return time.Duration(seconds) * time.Second } cc.SetIfNotNil(&cfg.LabelsEnabled, s.LabelsEnabled) cc.SetIfNotNil(&cfg.StreamingEnabled, s.StreamingEnabled) + cc.SetIfNotEmpty(&cfg.Splits.UpdateBufferSize, s.FeatureFlags.SplitNotificationQueueSize) + cc.MapIfNotNil(&cfg.Splits.SyncPeriod, s.FeatureFlags.SplitRefreshRateSeconds, durationFromSeconds) + cc.SetIfNotEmpty(&cfg.Segments.UpdateBufferSize, s.FeatureFlags.SegmentNotificationQueueSize) + cc.SetIfNotEmpty(&cfg.Segments.QueueSize, s.FeatureFlags.SegmentUpdateQueueSize) + cc.SetIfNotEmpty(&cfg.Segments.WorkerCount, s.FeatureFlags.SegmentUpdateWorkers) + cc.MapIfNotNil(&cfg.Segments.SyncPeriod, s.FeatureFlags.SegmentRefreshRateSeconds, durationFromSeconds) + cc.SetIfNotEmpty(&cfg.Impressions.Mode, s.Impressions.Mode) + cc.SetIfNotEmpty(&cfg.Impressions.ObserverSize, s.Impressions.ObserverSize) + cc.SetIfNotEmpty(&cfg.Impressions.QueueSize, s.Impressions.QueueSize) + cc.MapIfNotNil(&cfg.Impressions.SyncPeriod, s.Impressions.RefreshRateSeconds, durationFromSeconds) + cc.MapIfNotNil(&cfg.Impressions.CountSyncPeriod, s.Impressions.CountRefreshRateSeconds, durationFromSeconds) s.URLs.updateSDKConfURLs(&cfg.URLs) return cfg - } type URLs struct { @@ -127,21 +158,34 @@ func (u *URLs) updateSDKConfURLs(dst *sdkConf.URLs) { } type Logger struct { - Level *string `yaml:"level"` + Level *string `yaml:"level"` + Output *string `yaml:"file"` + RotationMaxFiles *int `yaml:"rotationMaxFiles"` + RotationMaxBytesPerFile *int `yaml:"rotationMaxBytesPerFile"` } -func (l *Logger) ToLoggerOptions() *logging.LoggerOptions { +func (l *Logger) ToLoggerOptions() (*logging.LoggerOptions, error) { + + writer, err := sdlogging.GetWriter(l.Output, l.RotationMaxFiles, l.RotationMaxBytesPerFile) + if err != nil { + return nil, fmt.Errorf("error parsing logger options: %w", err) + } opts := &logging.LoggerOptions{ LogLevel: logging.LevelError, StandardLoggerFlags: log.Ltime | log.Lshortfile, + ErrorWriter: writer, + WarningWriter: writer, + InfoWriter: writer, + DebugWriter: writer, + VerboseWriter: writer, } if l.Level != nil { opts.LogLevel = logging.Level(strings.ToUpper(*l.Level)) } - return opts + return opts, nil } func ReadConfig() (*Config, error) { diff --git a/splitio/conf/splitd_test.go b/splitio/conf/splitd_test.go index 09acfcc..9683d05 100644 --- a/splitio/conf/splitd_test.go +++ b/splitio/conf/splitd_test.go @@ -117,6 +117,22 @@ func TestSDK(t *testing.T) { Streaming: ref("streamingURL"), Telemetry: ref("telemetryURL"), }, + FeatureFlags: FeatureFlags{ + SplitNotificationQueueSize: ref(1), + SplitRefreshRateSeconds: ref(2), + SegmentNotificationQueueSize: ref(3), + SegmentRefreshRateSeconds: ref(4), + SegmentUpdateWorkers: ref(5), + SegmentUpdateQueueSize: ref(6), + }, + Impressions: Impressions{ + Mode: ref("optimized"), + RefreshRateSeconds: ref(1), + CountRefreshRateSeconds: ref(2), + QueueSize: ref(3), + ObserverSize: ref(4), + Watermark: ref(5), + }, } expected := conf.DefaultConfig() @@ -127,7 +143,17 @@ func TestSDK(t *testing.T) { expected.URLs.Events = "eventsURL" expected.URLs.Streaming = "streamingURL" expected.URLs.Telemetry = "telemetryURL" - + expected.Splits.UpdateBufferSize = 1 + expected.Splits.SyncPeriod = 2 * time.Second + expected.Segments.UpdateBufferSize = 3 + expected.Segments.SyncPeriod = 4 * time.Second + expected.Segments.WorkerCount = 5 + expected.Segments.QueueSize = 6 + expected.Impressions.Mode = "optimized" + expected.Impressions.SyncPeriod = 1 * time.Second + expected.Impressions.CountSyncPeriod = 2 * time.Second + expected.Impressions.QueueSize = 3 + expected.Impressions.ObserverSize = 4 assert.Equal(t, expected, sdkCFG.ToSDKConf()) } diff --git a/splitio/logging/helpers.go b/splitio/logging/helpers.go new file mode 100644 index 0000000..b872edc --- /dev/null +++ b/splitio/logging/helpers.go @@ -0,0 +1,53 @@ +package logging + +import ( + "fmt" + "io" + "os" + + "github.com/splitio/go-toolkit/v5/logging" +) + +const ( + defaultMaxFiles = 10 + defaultMaxFileSize = 1024 * 1024 // 1M +) + +var defaultWriter = os.Stdout + +func GetWriter(source *string, maxFiles *int, maxFileSize *int) (io.Writer, error) { + if source == nil { + return defaultWriter, nil + } + + switch *source { + case "stdout", "/dev/stdout": + return os.Stdout, nil + case "stderr", "/dev/stderr": + return os.Stderr, nil + default: + // assume it's a regular file + if maxFiles == nil && maxFileSize == nil { + fileWriter, err := os.OpenFile(*source, os.O_CREATE|os.O_WRONLY|os.O_APPEND, 0644) + if err != nil { + return nil, fmt.Errorf("error creating log-output file: %w", err) + } + return fileWriter, nil + } + + mf := valueOr(maxFiles, defaultMaxFiles) + mfs := valueOr(maxFileSize, defaultMaxFileSize) + return logging.NewFileRotate(&logging.FileRotateOptions{ + MaxBytes: int64(mfs), + BackupCount: mf, + Path: *source, + }) + } +} + +func valueOr[T any](t *T, fallback T) T { + if t == nil { + return fallback + } + return *t +} diff --git a/splitio/logging/helpers_test.go b/splitio/logging/helpers_test.go new file mode 100644 index 0000000..64016fb --- /dev/null +++ b/splitio/logging/helpers_test.go @@ -0,0 +1,95 @@ +package logging + +import ( + "io" + "io/ioutil" + "os" + "strings" + "testing" + "time" + + "github.com/stretchr/testify/assert" +) + +func TestGetWriterStd(t *testing.T) { + // stdout/stderr + writer, err := GetWriter(ref("stdout"), nil, nil) + assert.Nil(t, err) + assert.Equal(t, os.Stdout, writer) + writer, err = GetWriter(ref("/dev/stdout"), nil, nil) + assert.Nil(t, err) + assert.Equal(t, os.Stdout, writer) + writer, err = GetWriter(ref("stderr"), nil, nil) + assert.Nil(t, err) + assert.Equal(t, os.Stderr, writer) + writer, err = GetWriter(ref("/dev/stderr"), nil, nil) + assert.Nil(t, err) + assert.Equal(t, os.Stderr, writer) +} + +func TestGetWriterFileNoRotation(t *testing.T) { + // regular file, no rotation + fn := strings.Join([]string{os.TempDir(), "someLogFile2"}, string(os.PathSeparator)) + defer func() { + if err := os.Remove(fn); err != nil { + t.Error("error deleting file: ", err) + } + }() + writer, err := GetWriter(&fn, nil, nil) + assert.Nil(t, err) + + writer.Write([]byte("hola que tal")) + writer.(io.Closer).Close() + + assertFileContents(t, "hola que tal", fn) +} + +func TestGetWriterFileRotation(t *testing.T) { + + // remove any old file + fl, err := os.ReadDir(os.TempDir()) + assert.Nil(t, err) + for _, fe := range fl { + if strings.HasPrefix(fe.Name(), "someLogFile") { + os.Remove(fe.Name()) + } + } + + + fn := strings.Join([]string{os.TempDir(), "someLogFile"}, string(os.PathSeparator)) + writer, err := GetWriter(&fn, ref(2), ref(5)) + assert.Nil(t, err) + + writer.Write([]byte("12345")) + writer.Write([]byte("67890")) + writer.Write([]byte("qwert")) + writer.Write([]byte("asdfg")) + writer.Write([]byte("zxcvb")) + writer.Write([]byte("hjaiu")) + + + time.Sleep(1*time.Second) // file rotate writer is async.. give it a second before looking at the fs + fl, err = os.ReadDir(os.TempDir()) + assert.Nil(t, err) + names := make([]string, 0, 3) + for _, fe := range fl { + if strings.HasPrefix(fe.Name(), "someLogFile") { + names = append(names, fe.Name()) + } + } + + assert.Contains(t, names, "someLogFile") + assert.Contains(t, names, "someLogFile.1") + assert.Contains(t, names, "someLogFile.2") + assert.NotContains(t, names, "someLogFile.3") +} + +func assertFileContents(t *testing.T, expected string, fn string) { + t.Helper() + contents, err := ioutil.ReadFile(fn) + assert.Nil(t, err) + assert.Equal(t, expected, string(contents)) +} +func ref[T any](t T) *T { + return &t +} diff --git a/splitio/sdk/conf/conf.go b/splitio/sdk/conf/conf.go index 2f1bbe8..4d56346 100644 --- a/splitio/sdk/conf/conf.go +++ b/splitio/sdk/conf/conf.go @@ -6,6 +6,11 @@ import ( "github.com/splitio/go-split-commons/v4/conf" ) +const ( + defaultImpressionsMode = "optimized" + minimumImpressionsRefreshRate = 30 * time.Minute +) + type Config struct { LabelsEnabled bool StreamingEnabled bool @@ -46,14 +51,23 @@ type URLs struct { func (c *Config) ToAdvancedConfig() *conf.AdvancedConfig { d := conf.GetDefaultAdvancedConfig() + d.SplitsRefreshRate = int(c.Splits.SyncPeriod.Seconds()) + d.SplitUpdateQueueSize = int64(c.Splits.UpdateBufferSize) d.SegmentsRefreshRate = int(c.Segments.SyncPeriod.Seconds()) + d.SegmentQueueSize = c.Segments.QueueSize + d.SegmentUpdateQueueSize = int64(c.Segments.UpdateBufferSize) + d.SegmentWorkers = c.Segments.WorkerCount d.StreamingEnabled = c.StreamingEnabled + d.AuthServiceURL = c.URLs.Auth d.SdkURL = c.URLs.SDK d.EventsURL = c.URLs.Events d.StreamingServiceURL = c.URLs.Streaming d.TelemetryServiceURL = c.URLs.Telemetry + + d.ImpressionsQueueSize = c.Impressions.QueueSize + return &d } @@ -75,8 +89,8 @@ func DefaultConfig() *Config { Mode: "optimized", ObserverSize: 500000, QueueSize: 8192, - SyncPeriod: 5 * time.Second, - CountSyncPeriod: 5 * time.Second, + SyncPeriod: 30 * time.Minute, + CountSyncPeriod: 60 * time.Minute, PostConcurrency: 1, }, URLs: URLs{ @@ -88,3 +102,18 @@ func DefaultConfig() *Config { }, } } + +func (c *Config) Normalize() []string { + var warnings []string + if c.Impressions.Mode != defaultImpressionsMode { + warnings = append(warnings, "only `optimized` impressions mode supported currently. ignoring user config") + c.Impressions.Mode = defaultImpressionsMode + } + + if c.Impressions.SyncPeriod < minimumImpressionsRefreshRate { + warnings = append(warnings, "minimum impressions refresh rate is 30 min. ignoring user config") + c.Impressions.SyncPeriod = minimumImpressionsRefreshRate + } + + return warnings +} diff --git a/splitio/sdk/conf/conf_test.go b/splitio/sdk/conf/conf_test.go new file mode 100644 index 0000000..6dc7e62 --- /dev/null +++ b/splitio/sdk/conf/conf_test.go @@ -0,0 +1,41 @@ +package conf + +import ( + "testing" + "time" + + "github.com/stretchr/testify/assert" +) + +func TestSDKConf(t *testing.T) { + dc := DefaultConfig() + dc.Impressions.Mode = "debug" + dc.Impressions.SyncPeriod = 1 * time.Minute + warns := dc.Normalize() + assert.Equal(t, warns, []string{ + "only `optimized` impressions mode supported currently. ignoring user config", + "minimum impressions refresh rate is 30 min. ignoring user config", + }) + assert.Equal(t, "optimized", dc.Impressions.Mode) + assert.Equal(t, 30*time.Minute, dc.Impressions.SyncPeriod) + + + adv := dc.ToAdvancedConfig() + assert.Equal(t, 30, adv.HTTPTimeout) + assert.Equal(t, dc.Segments.QueueSize, adv.SegmentQueueSize) + assert.Equal(t, dc.Segments.WorkerCount, adv.SegmentWorkers) + assert.Equal(t, dc.URLs.SDK, adv.SdkURL) + assert.Equal(t, dc.URLs.Events, adv.EventsURL) + assert.Equal(t, dc.URLs.Telemetry, adv.TelemetryServiceURL) + // assert.Equal(t, TODO, adv.EventsBulkSize) + // assert.Equal(t, TODO, adv.EventsQueueSize) + assert.Equal(t, dc.Impressions.QueueSize, adv.ImpressionsQueueSize) + // assert.Equal(t, TODO, adv.ImpressionsBulkSize) + assert.Equal(t, dc.StreamingEnabled, adv.StreamingEnabled) + assert.Equal(t, dc.URLs.Auth, adv.AuthServiceURL) + assert.Equal(t, dc.URLs.Streaming, adv.StreamingServiceURL) + assert.Equal(t, int64(dc.Splits.UpdateBufferSize), adv.SplitUpdateQueueSize) + assert.Equal(t, int64(dc.Segments.UpdateBufferSize), adv.SegmentUpdateQueueSize) + assert.Equal(t, int(dc.Splits.SyncPeriod.Seconds()), adv.SplitsRefreshRate) + assert.Equal(t, int(dc.Segments.SyncPeriod.Seconds()), adv.SegmentsRefreshRate) +} diff --git a/splitio/sdk/sdk.go b/splitio/sdk/sdk.go index 056b633..7b0936f 100644 --- a/splitio/sdk/sdk.go +++ b/splitio/sdk/sdk.go @@ -47,6 +47,12 @@ type Impl struct { func New(logger logging.LoggerInterface, apikey string, c *conf.Config) (*Impl, error) { + if warnings := c.Normalize(); len(warnings) > 0 { + for _, w := range warnings { + logger.Warning(w) + } + } + md := dtos.Metadata{SDKVersion: fmt.Sprintf("splitd-%s", splitio.Version)} advCfg := c.ToAdvancedConfig() From 0d8fe2f537dee93eed1735d2f13c4af1b4e60785 Mon Sep 17 00:00:00 2001 From: Martin Redolatti Date: Fri, 18 Aug 2023 17:12:12 -0300 Subject: [PATCH 24/42] support event track ops --- cmd/splitcli/main.go | 16 +++- splitio/conf/splitcli.go | 49 +++++++----- splitio/conf/splitcli_test.go | 18 ++--- splitio/link/client/client.go | 6 +- splitio/link/client/types/interfaces.go | 1 + splitio/link/client/v1/impl.go | 31 ++++++-- splitio/link/client/v1/impl_test.go | 16 ++-- splitio/link/link.go | 28 +++---- splitio/link/protocol/v1/mocks/mocks.go | 8 +- splitio/link/protocol/v1/responses.go | 5 ++ splitio/link/protocol/v1/rpcs.go | 11 +-- splitio/link/protocol/v1/rpcs_test.go | 31 ++------ splitio/link/service/v1/clientmgr.go | 25 ++++++ splitio/link/service/v1/clientmgr_test.go | 9 ++- splitio/sdk/conf/conf.go | 12 +++ splitio/sdk/helpers.go | 12 ++- splitio/sdk/mocks/sdk.go | 13 +++- splitio/sdk/results.go | 2 +- splitio/sdk/sdk.go | 58 +++++++++++--- splitio/sdk/tasks/events.go | 29 +++++++ splitio/sdk/tasks/impressions.go | 6 +- splitio/sdk/workers/events.go | 94 +++++++++++++++++++++++ splitio/sdk/workers/impressions.go | 43 ++++++++--- splitio/sdk/workers/impressions_test.go | 44 +++++++++++ splitio/util/errors/concurrent.go | 23 ++++++ splitio/util/errors/concurrent_test.go | 40 ++++++++++ 26 files changed, 498 insertions(+), 132 deletions(-) create mode 100644 splitio/sdk/tasks/events.go create mode 100644 splitio/sdk/workers/events.go create mode 100644 splitio/util/errors/concurrent.go create mode 100644 splitio/util/errors/concurrent_test.go diff --git a/cmd/splitcli/main.go b/cmd/splitcli/main.go index a55d520..ecbed11 100644 --- a/cmd/splitcli/main.go +++ b/cmd/splitcli/main.go @@ -3,6 +3,7 @@ package main import ( "fmt" "os" + "strings" "time" "github.com/splitio/go-toolkit/v5/logging" @@ -67,7 +68,20 @@ func executeCall(c types.ClientInterface, a *conf.CliArgs) (string, error) { case "treatment": res, err := c.Treatment(a.Key, a.BucketingKey, a.Feature, a.Attributes) return res.Treatment, err - case "treatments", "treatmentWithConfig", "treatmentsWithConfig", "track": + case "treatments": + res, err := c.Treatments(a.Key, a.BucketingKey, a.Features, a.Attributes) + var sb strings.Builder + for _, result := range res { + if sb.Len() == 0 { // first item doesn't require a leading ',' + sb.WriteString(result.Treatment) + } else { + sb.WriteString("," + result.Treatment) + } + } + return sb.String(), err + case "track": + return "", c.Track(a.Key, a.TrafficType, a.EventType, a.EventVal, nil) + case "treatmentWithConfig", "treatmentsWithConfig": return "", fmt.Errorf("method '%s' is not yet implemented", a.Method) default: return "", fmt.Errorf("unknwon method '%s'", a.Method) diff --git a/splitio/conf/splitcli.go b/splitio/conf/splitcli.go index 9555cdd..5a50fcf 100644 --- a/splitio/conf/splitcli.go +++ b/splitio/conf/splitcli.go @@ -14,6 +14,7 @@ import ( ) type CliArgs struct { + ID string LogLevel string Protocol string Serialization string @@ -31,13 +32,15 @@ type CliArgs struct { Features []string TrafficType string EventType string - EventVal float64 + EventVal *float64 Attributes map[string]interface{} } func (a *CliArgs) LinkOpts() (*link.ConsumerOptions, error) { opts := link.DefaultConsumerOptions() + cc.SetIfNotEmpty(&opts.Consumer.ID, &a.ID) + var err error if a.Protocol != "" { @@ -45,13 +48,11 @@ func (a *CliArgs) LinkOpts() (*link.ConsumerOptions, error) { return nil, fmt.Errorf("invalid protocol version %s", a.Protocol) } } - if a.ConnType != "" { if opts.Transfer.ConnType, err = cc.ParseConnType(a.ConnType); err != nil { return nil, fmt.Errorf("invalid connection type %s", a.ConnType) } } - if a.Serialization != "" { if opts.Serialization, err = cc.ParseSerializer(a.Serialization); err != nil { return nil, fmt.Errorf("invalid serialization %s", a.Serialization) @@ -69,6 +70,7 @@ func (a *CliArgs) LinkOpts() (*link.ConsumerOptions, error) { func ParseCliArgs() (*CliArgs, error) { cliFlags := flag.NewFlagSet(os.Args[0], flag.ContinueOnError) + id := cliFlags.String("id", "", "ID to use for internal event queuing separation. Defaults to app's pid") p := cliFlags.String("protocol", "", "Protocol version [v1]") s := cliFlags.String("serialization", "", "Client-Daemon communication serialization mechanism [msgpack]") ll := cliFlags.String("log-level", "INFO", "log level [ERROR,WARNING,INFO,DEBUG]") @@ -89,9 +91,13 @@ func ParseCliArgs() (*CliArgs, error) { return nil, fmt.Errorf("error parsing arguments: %w", err) } - val, err := strconv.ParseFloat(*ev, 64) - if *ev != "" && err != nil { - return nil, fmt.Errorf("error parsing event value") + var eventVal *float64 + if *ev != "" { + val, err := strconv.ParseFloat(*ev, 64) + if err != nil { + return nil, fmt.Errorf("error parsing event value") + } + eventVal = &val } if *at == "" { @@ -103,20 +109,21 @@ func ParseCliArgs() (*CliArgs, error) { } return &CliArgs{ - Serialization: *s, - Protocol: *p, - LogLevel: *ll, - ConnType: *ct, - ConnAddr: *ca, - BufSize: *bs, - Method: *m, - Key: *k, - BucketingKey: *bk, - Feature: *f, - Features: strings.Split(*fs, ","), - TrafficType: *tt, - EventType: *et, - EventVal: val, - Attributes: attrs, + ID: *id, + Serialization: *s, + Protocol: *p, + LogLevel: *ll, + ConnType: *ct, + ConnAddr: *ca, + BufSize: *bs, + Method: *m, + Key: *k, + BucketingKey: *bk, + Feature: *f, + Features: strings.Split(*fs, ","), + TrafficType: *tt, + EventType: *et, + EventVal: eventVal, + Attributes: attrs, }, nil } diff --git a/splitio/conf/splitcli_test.go b/splitio/conf/splitcli_test.go index e935a7e..31b9290 100644 --- a/splitio/conf/splitcli_test.go +++ b/splitio/conf/splitcli_test.go @@ -39,7 +39,7 @@ func TestCliConfig(t *testing.T) { assert.Equal(t, []string{"someFeature1", "someFeature2"}, parsed.Features) assert.Equal(t, "someTrafficType", parsed.TrafficType) assert.Equal(t, "someEventType", parsed.EventType) - assert.Equal(t, 0.123, parsed.EventVal) + assert.Equal(t, ref(float64(0.123)), parsed.EventVal) assert.Equal(t, map[string]interface{}{"some": "attribute"}, parsed.Attributes) // test bad buffer size @@ -63,32 +63,32 @@ func TestCliConfig(t *testing.T) { } func TestLinkOptions(t *testing.T) { - // test defaults - os.Args = os.Args[:1] + // test defaults + os.Args = os.Args[:1] parsed, err := ParseCliArgs() assert.Nil(t, err) lo, err := parsed.LinkOpts() assert.Nil(t, err) assert.Equal(t, link.DefaultConsumerOptions(), *lo) - // test bad protocol - os.Args = []string{os.Args[0], "-protocol=sarasa"} + // test bad protocol + os.Args = []string{os.Args[0], "-protocol=sarasa"} parsed, err = ParseCliArgs() assert.Nil(t, err) lo, err = parsed.LinkOpts() assert.NotNil(t, err) assert.ErrorContains(t, err, "protocol") - // test bad conn type - os.Args = []string{os.Args[0], "-conn-type=sarasa"} + // test bad conn type + os.Args = []string{os.Args[0], "-conn-type=sarasa"} parsed, err = ParseCliArgs() assert.Nil(t, err) lo, err = parsed.LinkOpts() assert.NotNil(t, err) assert.ErrorContains(t, err, "connection type") - // test bad serialization - os.Args = []string{os.Args[0], "-serialization=pinpin"} + // test bad serialization + os.Args = []string{os.Args[0], "-serialization=pinpin"} parsed, err = ParseCliArgs() assert.Nil(t, err) lo, err = parsed.LinkOpts() diff --git a/splitio/link/client/client.go b/splitio/link/client/client.go index 0f9d45f..9102372 100644 --- a/splitio/link/client/client.go +++ b/splitio/link/client/client.go @@ -2,6 +2,8 @@ package client import ( "fmt" + "os" + "strconv" "github.com/splitio/go-toolkit/v5/logging" "github.com/splitio/splitd/splitio/link/client/types" @@ -14,18 +16,20 @@ import ( func New(logger logging.LoggerInterface, conn transfer.RawConn, serial serializer.Interface, opts Options) (types.ClientInterface, error) { switch opts.Protocol { case protocol.V1: - return clientv1.New(logger, conn, serial, opts.ImpressionsFeedback) + return clientv1.New(opts.ID, logger, conn, serial, opts.ImpressionsFeedback) } return nil, fmt.Errorf("unknown protocol version: '%d'", opts.Protocol) } type Options struct { + ID string Protocol protocol.Version ImpressionsFeedback bool } func DefaultOptions() Options { return Options{ + ID: strconv.Itoa(os.Getpid()), Protocol: protocol.V1, ImpressionsFeedback: false, } diff --git a/splitio/link/client/types/interfaces.go b/splitio/link/client/types/interfaces.go index 638e9b5..67f3487 100644 --- a/splitio/link/client/types/interfaces.go +++ b/splitio/link/client/types/interfaces.go @@ -5,6 +5,7 @@ import "github.com/splitio/go-split-commons/v4/dtos" type ClientInterface interface { Treatment(key string, bucketingKey string, feature string, attrs map[string]interface{}) (*Result, error) Treatments(key string, bucketingKey string, features []string, attrs map[string]interface{}) (Results, error) + Track(key string, trafficType string, eventType string, value *float64, properties map[string]interface{}) error Shutdown() error } diff --git a/splitio/link/client/v1/impl.go b/splitio/link/client/v1/impl.go index f4b23d9..1d9a701 100644 --- a/splitio/link/client/v1/impl.go +++ b/splitio/link/client/v1/impl.go @@ -2,8 +2,6 @@ package v1 import ( "fmt" - "os" - "strconv" "github.com/splitio/go-split-commons/v4/dtos" "github.com/splitio/go-toolkit/v5/logging" @@ -26,7 +24,7 @@ type Impl struct { listenerFeedback bool } -func New(logger logging.LoggerInterface, conn transfer.RawConn, serializer serializer.Interface, listenerFeedback bool) (*Impl, error) { +func New(id string, logger logging.LoggerInterface, conn transfer.RawConn, serializer serializer.Interface, listenerFeedback bool) (*Impl, error) { i := &Impl{ logger: logger, conn: conn, @@ -34,7 +32,7 @@ func New(logger logging.LoggerInterface, conn transfer.RawConn, serializer seria listenerFeedback: listenerFeedback, } - if err := i.register(listenerFeedback); err != nil { + if err := i.register(id, listenerFeedback); err != nil { i.conn.Shutdown() return nil, fmt.Errorf("error during client registration: %w", err) } @@ -120,7 +118,28 @@ func (c *Impl) Treatments(key string, bucketingKey string, features []string, at return results, nil } -func (c *Impl) register(impressionsFeedback bool) error { +// Track implements types.ClientInterface +func (c *Impl) Track(key string, trafficType string, eventType string, value *float64, properties map[string]interface{}) error { + + rpc := protov1.RPC{ + RPCBase: protocol.RPCBase{Version: protocol.V1}, + OpCode: protov1.OCTrack, + Args: protov1.TrackArgs{Key: key, TrafficType: trafficType, EventType: eventType, Value: value, Properties: properties}.Encode(), + } + + resp, err := doRPC[protov1.ResponseWrapper[protov1.TrackPayload]](c, &rpc) + if err != nil { + return fmt.Errorf("error executing treatment rpc: %w", err) + } + + if resp.Status != protov1.ResultOk { + return fmt.Errorf("server responded treatment rpc with error %d", resp.Status) + } + + return nil +} + +func (c *Impl) register(id string, impressionsFeedback bool) error { var flags protov1.RegisterFlags if impressionsFeedback { flags |= 1 << protov1.RegisterFlagReturnImpressionData @@ -128,7 +147,7 @@ func (c *Impl) register(impressionsFeedback bool) error { rpc := protov1.RPC{ RPCBase: protocol.RPCBase{Version: protocol.V1}, OpCode: protov1.OCRegister, - Args: protov1.RegisterArgs{ID: strconv.Itoa(os.Getpid()), SDKVersion: fmt.Sprintf("splitd-%s", splitio.Version), Flags: flags}.Encode(), + Args: protov1.RegisterArgs{ID: id, SDKVersion: fmt.Sprintf("splitd-%s", splitio.Version), Flags: flags}.Encode(), } resp, err := doRPC[protov1.ResponseWrapper[protov1.RegisterPayload]](c, &rpc) diff --git a/splitio/link/client/v1/impl_test.go b/splitio/link/client/v1/impl_test.go index e365ab7..c90be6c 100644 --- a/splitio/link/client/v1/impl_test.go +++ b/splitio/link/client/v1/impl_test.go @@ -24,7 +24,7 @@ func TestClientGetTreatmentNoImpression(t *testing.T) { rawConnMock.On("ReceiveMessage").Return([]byte("treatmentResult"), nil).Once() serializerMock := &serializerMocks.SerializerMock{} - serializerMock.On("Serialize", proto1Mocks.NewRegisterRPC(false)).Return([]byte("registrationMessage"), nil).Once() + serializerMock.On("Serialize", proto1Mocks.NewRegisterRPC("some", false)).Return([]byte("registrationMessage"), nil).Once() serializerMock.On("Parse", []byte("registrationSuccess"), mock.Anything).Return(nil).Run(func(args mock.Arguments) { *args.Get(1).(*v1.ResponseWrapper[v1.RegisterPayload]) = v1.ResponseWrapper[v1.RegisterPayload]{Status: v1.ResultOk} }).Once() @@ -37,7 +37,7 @@ func TestClientGetTreatmentNoImpression(t *testing.T) { Payload: v1.TreatmentPayload{Treatment: "on"}, } }).Once() - client, err := New(logger, rawConnMock, serializerMock, false) + client, err := New("some", logger, rawConnMock, serializerMock, false) assert.NotNil(t, client) assert.Nil(t, err) @@ -58,7 +58,7 @@ func TestClientGetTreatmentWithImpression(t *testing.T) { rawConnMock.On("ReceiveMessage").Return([]byte("treatmentResult"), nil).Once() serializerMock := &serializerMocks.SerializerMock{} - serializerMock.On("Serialize", proto1Mocks.NewRegisterRPC(true)).Return([]byte("registrationMessage"), nil).Once() + serializerMock.On("Serialize", proto1Mocks.NewRegisterRPC("some", true)).Return([]byte("registrationMessage"), nil).Once() serializerMock.On("Parse", []byte("registrationSuccess"), mock.Anything).Return(nil).Run(func(args mock.Arguments) { *args.Get(1).(*v1.ResponseWrapper[v1.RegisterPayload]) = v1.ResponseWrapper[v1.RegisterPayload]{Status: v1.ResultOk} }).Once() @@ -74,7 +74,7 @@ func TestClientGetTreatmentWithImpression(t *testing.T) { }, } }).Once() - client, err := New(logger, rawConnMock, serializerMock, true) + client, err := New("some", logger, rawConnMock, serializerMock, true) assert.NotNil(t, client) assert.Nil(t, err) @@ -104,7 +104,7 @@ func TestClientGetTreatmentsNoImpression(t *testing.T) { rawConnMock.On("ReceiveMessage").Return([]byte("treatmentsResult"), nil).Once() serializerMock := &serializerMocks.SerializerMock{} - serializerMock.On("Serialize", proto1Mocks.NewRegisterRPC(false)).Return([]byte("registrationMessage"), nil).Once() + serializerMock.On("Serialize", proto1Mocks.NewRegisterRPC("some", false)).Return([]byte("registrationMessage"), nil).Once() serializerMock.On("Parse", []byte("registrationSuccess"), mock.Anything).Return(nil).Run(func(args mock.Arguments) { *args.Get(1).(*v1.ResponseWrapper[v1.RegisterPayload]) = v1.ResponseWrapper[v1.RegisterPayload]{Status: v1.ResultOk} }).Once() @@ -116,7 +116,7 @@ func TestClientGetTreatmentsNoImpression(t *testing.T) { Status: v1.ResultOk, Payload: v1.TreatmentsPayload{Results: []v1.TreatmentPayload{{Treatment: "on"}, {Treatment: "off"}, {Treatment: "na"}}}} }).Once() - client, err := New(logger, rawConnMock, serializerMock, false) + client, err := New("some", logger, rawConnMock, serializerMock, false) assert.NotNil(t, client) assert.Nil(t, err) @@ -142,7 +142,7 @@ func TestClientGetTreatmentsWithImpression(t *testing.T) { rawConnMock.On("ReceiveMessage").Return([]byte("treatmentsResult"), nil).Once() serializerMock := &serializerMocks.SerializerMock{} - serializerMock.On("Serialize", proto1Mocks.NewRegisterRPC(true)).Return([]byte("registrationMessage"), nil).Once() + serializerMock.On("Serialize", proto1Mocks.NewRegisterRPC("some", true)).Return([]byte("registrationMessage"), nil).Once() serializerMock.On("Parse", []byte("registrationSuccess"), mock.Anything).Return(nil).Run(func(args mock.Arguments) { *args.Get(1).(*v1.ResponseWrapper[v1.RegisterPayload]) = v1.ResponseWrapper[v1.RegisterPayload]{Status: v1.ResultOk} }).Once() @@ -158,7 +158,7 @@ func TestClientGetTreatmentsWithImpression(t *testing.T) { {Treatment: "na", ListenerData: &v1.ListenerExtraData{Label: "l3", Timestamp: 3, ChangeNumber: 7}}, }}} }).Once() - client, err := New(logger, rawConnMock, serializerMock, true) + client, err := New("some", logger, rawConnMock, serializerMock, true) assert.NotNil(t, client) assert.Nil(t, err) diff --git a/splitio/link/link.go b/splitio/link/link.go index 8e35efb..6b45eeb 100644 --- a/splitio/link/link.go +++ b/splitio/link/link.go @@ -61,24 +61,24 @@ type ListenerOptions struct { } func DefaultListenerOptions() ListenerOptions { - return ListenerOptions{ - Transfer: transfer.DefaultOpts(), - Acceptor: transfer.DefaultAcceptorConfig(), - Serialization: serializer.MsgPack, - Protocol: protocol.V1, - } + return ListenerOptions{ + Transfer: transfer.DefaultOpts(), + Acceptor: transfer.DefaultAcceptorConfig(), + Serialization: serializer.MsgPack, + Protocol: protocol.V1, + } } type ConsumerOptions struct { - Transfer transfer.Options - Consumer client.Options - Serialization serializer.Mechanism + Transfer transfer.Options + Consumer client.Options + Serialization serializer.Mechanism } func DefaultConsumerOptions() ConsumerOptions { - return ConsumerOptions{ - Transfer: transfer.DefaultOpts(), - Consumer: client.DefaultOptions(), - Serialization: serializer.MsgPack, - } + return ConsumerOptions{ + Transfer: transfer.DefaultOpts(), + Consumer: client.DefaultOptions(), + Serialization: serializer.MsgPack, + } } diff --git a/splitio/link/protocol/v1/mocks/mocks.go b/splitio/link/protocol/v1/mocks/mocks.go index 0787802..a412c8f 100644 --- a/splitio/link/protocol/v1/mocks/mocks.go +++ b/splitio/link/protocol/v1/mocks/mocks.go @@ -2,8 +2,6 @@ package mocks import ( "fmt" - "os" - "strconv" "github.com/splitio/splitd/splitio" "github.com/splitio/splitd/splitio/link/protocol" @@ -11,7 +9,7 @@ import ( "github.com/splitio/splitd/splitio/sdk" ) -func NewRegisterRPC(listener bool) *v1.RPC { +func NewRegisterRPC(id string, listener bool) *v1.RPC { var flags v1.RegisterFlags if listener { flags = 1 << v1.RegisterFlagReturnImpressionData @@ -19,7 +17,7 @@ func NewRegisterRPC(listener bool) *v1.RPC { return &v1.RPC{ RPCBase: protocol.RPCBase{Version: protocol.V1}, OpCode: v1.OCRegister, - Args: []interface{}{strconv.Itoa(os.Getpid()), fmt.Sprintf("splitd-%s", splitio.Version), flags}, + Args: []interface{}{id, fmt.Sprintf("splitd-%s", splitio.Version), flags}, } } @@ -64,7 +62,7 @@ func NewTreatmentResp(ok bool, treatment string, ilData *v1.ListenerExtraData) * } } -func NewTreatmentsResp(ok bool, data []sdk.Result) *v1.ResponseWrapper[v1.TreatmentsPayload] { +func NewTreatmentsResp(ok bool, data []sdk.EvaluationResult) *v1.ResponseWrapper[v1.TreatmentsPayload] { res := v1.ResultOk if !ok { res = v1.ResultInternalError diff --git a/splitio/link/protocol/v1/responses.go b/splitio/link/protocol/v1/responses.go index 968be85..7436931 100644 --- a/splitio/link/protocol/v1/responses.go +++ b/splitio/link/protocol/v1/responses.go @@ -33,6 +33,10 @@ type TreatmentsWithConfigPayload struct { Results []TreatmentWithConfigPayload `msgpack:"r"` } +type TrackPayload struct { + Success bool +} + type ListenerExtraData struct { Label string `msgpack:"l"` Timestamp int64 `msgpack:"m"` @@ -44,5 +48,6 @@ type validPayloadsConstraint interface { TreatmentsPayload | TreatmentWithConfigPayload | TreatmentsWithConfigPayload | + TrackPayload | RegisterPayload } diff --git a/splitio/link/protocol/v1/rpcs.go b/splitio/link/protocol/v1/rpcs.go index d0bed80..0184c65 100644 --- a/splitio/link/protocol/v1/rpcs.go +++ b/splitio/link/protocol/v1/rpcs.go @@ -206,7 +206,6 @@ const ( TrackArgEventTypeIdx int = 2 TrackArgValueIdx int = 3 TrackArgPropertiesIdx int = 4 - TrackArgTimestampIdx int = 5 ) type TrackArgs struct { @@ -215,18 +214,17 @@ type TrackArgs struct { EventType string `msgpack:"e"` Value *float64 `msgpack:"v"` Properties map[string]interface{} `msgpack:"p"` - Timestamp int64 `msgpack:"m"` } func (r TrackArgs) Encode() []interface{} { - return []interface{}{r.Key, r.TrafficType, r.EventType, r.Value, r.Properties, r.Timestamp} + return []interface{}{r.Key, r.TrafficType, r.EventType, r.Value, r.Properties} } func (t *TrackArgs) PopulateFromRPC(rpc *RPC) error { if rpc.OpCode != OCTrack { return RPCParseError{Code: PECOpCodeMismatch} } - if len(rpc.Args) != 6 { + if len(rpc.Args) != 5 { return RPCParseError{Code: PECWrongArgCount} } @@ -257,11 +255,6 @@ func (t *TrackArgs) PopulateFromRPC(rpc *RPC) error { return RPCParseError{Code: PECInvalidArgType, Data: int64(TrackArgPropertiesIdx)} } - if t.Timestamp, ok = rpc.Args[TrackArgTimestampIdx].(int64); !ok { - return RPCParseError{Code: PECInvalidArgType, Data: int64(TrackArgTimestampIdx)} - - } - return nil } diff --git a/splitio/link/protocol/v1/rpcs_test.go b/splitio/link/protocol/v1/rpcs_test.go index d7194cf..eef58c6 100644 --- a/splitio/link/protocol/v1/rpcs_test.go +++ b/splitio/link/protocol/v1/rpcs_test.go @@ -174,37 +174,28 @@ func TestTrackRPCParsing(t *testing.T) { ) assert.Equal(t, RPCParseError{Code: PECInvalidArgType, Data: int64(TrackArgKeyIdx)}, - r.PopulateFromRPC(&RPC{RPCBase: protocol.RPCBase{Version: protocol.V1}, OpCode: OCTrack, Args: []interface{}{nil, nil, nil, 123, 123, nil}}), + r.PopulateFromRPC(&RPC{RPCBase: protocol.RPCBase{Version: protocol.V1}, OpCode: OCTrack, Args: []interface{}{nil, nil, nil, 123, 123}}), ) assert.Equal(t, RPCParseError{Code: PECInvalidArgType, Data: int64(TrackArgTrafficTypeIdx)}, - r.PopulateFromRPC(&RPC{RPCBase: protocol.RPCBase{Version: protocol.V1}, OpCode: OCTrack, Args: []interface{}{"key", nil, nil, "asd", 123, nil}}), + r.PopulateFromRPC(&RPC{RPCBase: protocol.RPCBase{Version: protocol.V1}, OpCode: OCTrack, Args: []interface{}{"key", nil, nil, "asd", 123}}), ) assert.Equal(t, RPCParseError{Code: PECInvalidArgType, Data: int64(TrackArgEventTypeIdx)}, - r.PopulateFromRPC(&RPC{RPCBase: protocol.RPCBase{Version: protocol.V1}, OpCode: OCTrack, Args: []interface{}{"key", "tt", nil, "asd", 123, nil}}), + r.PopulateFromRPC(&RPC{RPCBase: protocol.RPCBase{Version: protocol.V1}, OpCode: OCTrack, Args: []interface{}{"key", "tt", nil, "asd", 123}}), ) assert.Equal(t, RPCParseError{Code: PECInvalidArgType, Data: int64(TrackArgValueIdx)}, - r.PopulateFromRPC(&RPC{RPCBase: protocol.RPCBase{Version: protocol.V1}, OpCode: OCTrack, Args: []interface{}{"key", "tt", "et", "asd", 123, nil}})) + r.PopulateFromRPC(&RPC{RPCBase: protocol.RPCBase{Version: protocol.V1}, OpCode: OCTrack, Args: []interface{}{"key", "tt", "et", "asd", 123}})) assert.Equal(t, RPCParseError{Code: PECInvalidArgType, Data: int64(TrackArgPropertiesIdx)}, - r.PopulateFromRPC(&RPC{RPCBase: protocol.RPCBase{Version: protocol.V1}, OpCode: OCTrack, Args: []interface{}{"key", "tt", "et", 2.8, 123, nil}})) + r.PopulateFromRPC(&RPC{RPCBase: protocol.RPCBase{Version: protocol.V1}, OpCode: OCTrack, Args: []interface{}{"key", "tt", "et", 2.8, 123}})) - assert.Equal(t, - RPCParseError{Code: PECInvalidArgType, Data: int64(TrackArgTimestampIdx)}, - r.PopulateFromRPC(&RPC{ - RPCBase: protocol.RPCBase{Version: protocol.V1}, - OpCode: OCTrack, - Args: []interface{}{"key", "tt", "et", 2.8, map[string]interface{}{"a": 1}, nil}, - })) - - now := time.Now() err := r.PopulateFromRPC(&RPC{ RPCBase: protocol.RPCBase{Version: protocol.V1}, OpCode: OCTrack, - Args: []interface{}{"key", "tt", "et", 2.8, map[string]interface{}{"a": int64(1)}, now.UnixMilli()}, + Args: []interface{}{"key", "tt", "et", 2.8, map[string]interface{}{"a": int64(1)}}, }) assert.Nil(t, err) assert.Equal(t, "key", r.Key) @@ -212,13 +203,12 @@ func TestTrackRPCParsing(t *testing.T) { assert.Equal(t, "et", r.EventType) assert.Equal(t, ref(float64(2.8)), r.Value) assert.Equal(t, map[string]interface{}{"a": int64(1)}, r.Properties) - assert.Equal(t, now.UnixMilli(), r.Timestamp) // nil properties err = r.PopulateFromRPC(&RPC{ RPCBase: protocol.RPCBase{Version: protocol.V1}, OpCode: OCTrack, - Args: []interface{}{"key", "tt", "et", 2.8, nil, now.UnixMilli()}, + Args: []interface{}{"key", "tt", "et", 2.8, nil}, }) assert.Nil(t, err) assert.Equal(t, "key", r.Key) @@ -226,14 +216,13 @@ func TestTrackRPCParsing(t *testing.T) { assert.Equal(t, "et", r.EventType) assert.Equal(t, ref(float64(2.8)), r.Value) assert.Nil(t, r.Properties) - assert.Equal(t, now.UnixMilli(), r.Timestamp) // nil value r = TrackArgs{} err = r.PopulateFromRPC(&RPC{ RPCBase: protocol.RPCBase{Version: protocol.V1}, OpCode: OCTrack, - Args: []interface{}{"key", "tt", "et", nil, map[string]interface{}{"a": int64(1)}, now.UnixMilli()}, + Args: []interface{}{"key", "tt", "et", nil, map[string]interface{}{"a": int64(1)}}, }) assert.Nil(t, err) assert.Equal(t, "key", r.Key) @@ -241,7 +230,6 @@ func TestTrackRPCParsing(t *testing.T) { assert.Equal(t, "et", r.EventType) assert.Nil(t, r.Value) assert.Equal(t, map[string]interface{}{"a": int64(1)}, r.Properties) - assert.Equal(t, now.UnixMilli(), r.Timestamp) } @@ -320,7 +308,6 @@ func TestRPCEncoding(t *testing.T) { EventType: "someEventType", Value: ref(123.), Properties: map[string]interface{}{"a": 1}, - Timestamp: 123456, } encodedTrA := tra.Encode() assert.Equal(t, tra.Key, encodedTrA[TrackArgKeyIdx].(string)) @@ -328,8 +315,6 @@ func TestRPCEncoding(t *testing.T) { assert.Equal(t, tra.EventType, encodedTrA[TrackArgEventTypeIdx].(string)) assert.Equal(t, *tra.Value, *encodedTrA[TrackArgValueIdx].(*float64)) assert.Equal(t, tra.Properties, encodedTrA[TrackArgPropertiesIdx].(map[string]interface{})) - assert.Equal(t, tra.Timestamp, encodedTrA[TrackArgTimestampIdx].(int64)) - } func ref[T any](t T) *T { diff --git a/splitio/link/service/v1/clientmgr.go b/splitio/link/service/v1/clientmgr.go index b74f679..db55e2e 100644 --- a/splitio/link/service/v1/clientmgr.go +++ b/splitio/link/service/v1/clientmgr.go @@ -5,6 +5,7 @@ import ( "fmt" "io" "os" + "runtime/debug" "github.com/splitio/go-toolkit/v5/logging" @@ -41,6 +42,7 @@ func (m *ClientManager) Manage() { defer func() { if r := recover(); r != nil { m.logger.Error("CRITICAL - connection handler is panicking: ", r) + m.logger.Error(string(debug.Stack())) } }() err := m.handleClientInteractions() @@ -131,6 +133,13 @@ func (m *ClientManager) handleRPC(rpc *protov1.RPC) (interface{}, error) { return nil, fmt.Errorf("error parsing treatments arguments: %w", err) } return m.handleGetTreatments(&args) + case protov1.OCTrack: + var args protov1.TrackArgs + if err := args.PopulateFromRPC(rpc); err != nil { + return nil, fmt.Errorf("error parsing track argumentts: %w", err) + } + return m.handleTrack(&args) + } return nil, fmt.Errorf("RPC not implemented") } @@ -199,3 +208,19 @@ func (m *ClientManager) handleGetTreatments(args *protov1.TreatmentsArgs) (inter return response, nil } + +func (m *ClientManager) handleTrack(args *protov1.TrackArgs) (interface{}, error) { + err := m.splitSDK.Track(m.clientConfig, args.Key, args.TrafficType, args.EventType, args.Value, args.Properties) + if err != nil && !errors.Is(err, sdk.ErrEventsQueueFull) { + return &protov1.ResponseWrapper[protov1.TreatmentPayload]{Status: protov1.ResultInternalError}, err + } + + response := &protov1.ResponseWrapper[protov1.TrackPayload]{ + Status: protov1.ResultOk, + Payload: protov1.TrackPayload{Success: err != nil}, // can only be events queue full at this point + } + + return response, nil +} + + diff --git a/splitio/link/service/v1/clientmgr_test.go b/splitio/link/service/v1/clientmgr_test.go index 30970c7..3d0d403 100644 --- a/splitio/link/service/v1/clientmgr_test.go +++ b/splitio/link/service/v1/clientmgr_test.go @@ -49,7 +49,7 @@ func TestRegisterAndTreatmentHappyPath(t *testing.T) { sdkMock := &sdkMocks.SDKMock{} sdkMock. On("Treatment", &types.ClientConfig{Metadata: types.ClientMetadata{ID: "someID", SdkVersion: "some_sdk-1.2.3"}}, "key", (*string)(nil), "someFeature", map[string]interface{}(nil)). - Return(&sdk.Result{Treatment: "on"}, nil).Once() + Return(&sdk.EvaluationResult{Treatment: "on"}, nil).Once() logger := logging.NewLogger(nil) cm := NewClientManager(rawConnMock, logger, sdkMock, serializerMock) @@ -83,7 +83,7 @@ func TestRegisterAndTreatmentsHappyPath(t *testing.T) { Args: []interface{}{"key", nil, []interface{}{"feat1", "feat2", "feat3"}, map[string]interface{}(nil)}, } }).Once() - serializerMock.On("Serialize", proto1Mocks.NewTreatmentsResp(true, []sdk.Result{ + serializerMock.On("Serialize", proto1Mocks.NewTreatmentsResp(true, []sdk.EvaluationResult{ {Treatment: "on"}, {Treatment: "off"}, {Treatment: "control"}, })).Return([]byte("successPayload"), nil).Once() @@ -96,7 +96,7 @@ func TestRegisterAndTreatmentsHappyPath(t *testing.T) { (*string)(nil), []string{"feat1", "feat2", "feat3"}, map[string]interface{}(nil), - ).Return(map[string]sdk.Result{ + ).Return(map[string]sdk.EvaluationResult{ "feat1": {Treatment: "on"}, "feat2": {Treatment: "off"}, "feat3": {Treatment: "control"}, @@ -142,7 +142,7 @@ func TestRegisterWithImpsAndTreatmentHappyPath(t *testing.T) { On("Treatment", &types.ClientConfig{Metadata: types.ClientMetadata{ID: "someID", SdkVersion: "some_sdk-1.2.3"}, ReturnImpressionData: true}, "key", (*string)(nil), "someFeature", map[string]interface{}(nil)). - Return(&sdk.Result{Treatment: "on", Impression: &dtos.Impression{Label: "l1", Time: 1234556}}, nil).Once() + Return(&sdk.EvaluationResult{Treatment: "on", Impression: &dtos.Impression{Label: "l1", Time: 1234556}}, nil).Once() logger := logging.NewLogger(nil) cm := NewClientManager(rawConnMock, logger, sdkMock, serializerMock) @@ -194,6 +194,7 @@ func TestManagePanicRecovers(t *testing.T) { logger := &loggerMock{} logger.On("Error", "CRITICAL - connection handler is panicking: ", "some panic").Once() + logger.On("Error", mock.AnythingOfType("string")).Once() serializerMock := &serializerMocks.SerializerMock{} sdkMock := &sdkMocks.SDKMock{} diff --git a/splitio/sdk/conf/conf.go b/splitio/sdk/conf/conf.go index 4d56346..4ba0fa6 100644 --- a/splitio/sdk/conf/conf.go +++ b/splitio/sdk/conf/conf.go @@ -17,6 +17,7 @@ type Config struct { Splits Splits Segments Segments Impressions Impressions + Events Events URLs URLs } @@ -41,6 +42,12 @@ type Impressions struct { PostConcurrency int } +type Events struct { + QueueSize int + SyncPeriod time.Duration + PostConcurrency int +} + type URLs struct { Auth string SDK string @@ -93,6 +100,11 @@ func DefaultConfig() *Config { CountSyncPeriod: 60 * time.Minute, PostConcurrency: 1, }, + Events: Events{ + QueueSize: 8192, + SyncPeriod: 1*time.Minute, + PostConcurrency: 1, + }, URLs: URLs{ Auth: "https://auth.split.io", SDK: "https://sdk.split.io/api", diff --git a/splitio/sdk/helpers.go b/splitio/sdk/helpers.go index 2f4de7d..deb3541 100644 --- a/splitio/sdk/helpers.go +++ b/splitio/sdk/helpers.go @@ -31,13 +31,14 @@ func setupWorkers( api *api.SplitAPI, str *storages, hc application.MonitorProducerInterface, - cfg *sdkConf.Impressions, + cfg *sdkConf.Config, ) *synchronizer.Workers { return &synchronizer.Workers{ SplitFetcher: split.NewSplitFetcher(str.splits, api.SplitFetcher, logger, str.telemetry, hc), SegmentFetcher: segment.NewSegmentFetcher(str.splits, str.segments, api.SegmentFetcher, logger, str.telemetry, hc), - ImpressionRecorder: workers.NewImpressionsWorker(logger, str.telemetry, api.ImpressionRecorder, str.impressions, cfg), + ImpressionRecorder: workers.NewImpressionsWorker(logger, str.telemetry, api.ImpressionRecorder, str.impressions, &cfg.Impressions), + EventRecorder: workers.NewEventsWorker(logger, str.telemetry, api.EventRecorder, str.events, &cfg.Events), } } @@ -51,18 +52,18 @@ func setupTasks( api *api.SplitAPI, ) *synchronizer.SplitTasks { impCfg := cfg.Impressions + evCfg := cfg.Events return &synchronizer.SplitTasks{ SplitSyncTask: tasks.NewFetchSplitsTask(workers.SplitFetcher, int(cfg.Splits.SyncPeriod.Seconds()), logger), SegmentSyncTask: tasks.NewFetchSegmentsTask(workers.SegmentFetcher, int(cfg.Segments.SyncPeriod.Seconds()), cfg.Segments.WorkerCount, cfg.Segments.QueueSize, logger), ImpressionSyncTask: tasks.NewRecordImpressionsTask(workers.ImpressionRecorder, int(impCfg.SyncPeriod.Seconds()), logger, 5000), - //ImpressionSyncTask: tss.NewImpressionSyncTask(workers.ImpressionRecorder, logger, cfg.Impressions), ImpressionsCountSyncTask: tasks.NewRecordImpressionsCountTask( impressionscount.NewRecorderSingle(impComponents.counter, api.ImpressionRecorder, md, logger, str.telemetry), logger, int(impCfg.CountSyncPeriod.Seconds()), ), + EventSyncTask: tasks.NewRecordEventsTask(workers.EventRecorder, 5000, int(evCfg.SyncPeriod.Seconds()), logger), TelemetrySyncTask: &NoOpTask{}, - EventSyncTask: &NoOpTask{}, UniqueKeysTask: &NoOpTask{}, CleanFilterTask: &NoOpTask{}, ImpsCountConsumerTask: &NoOpTask{}, @@ -103,16 +104,19 @@ type storages struct { segments storage.SegmentStorage telemetry storage.TelemetryStorage impressions *sss.ImpressionsStorage + events *sss.EventsStorage } func setupStorages(cfg *sdkConf.Config) *storages { ts, _ := inmemory.NewTelemetryStorage() iq, _ := sss.NewImpressionsQueue(cfg.Impressions.QueueSize) + eq, _ := sss.NewEventsQueue(cfg.Events.QueueSize) return &storages{ splits: mutexmap.NewMMSplitStorage(), segments: mutexmap.NewMMSegmentStorage(), impressions: iq, + events: eq, telemetry: ts, } } diff --git a/splitio/sdk/mocks/sdk.go b/splitio/sdk/mocks/sdk.go index 8d3059e..e53a180 100644 --- a/splitio/sdk/mocks/sdk.go +++ b/splitio/sdk/mocks/sdk.go @@ -17,9 +17,9 @@ func (m *SDKMock) Treatment( bucketingKey *string, feature string, attributes map[string]interface{}, -) (*sdk.Result, error) { +) (*sdk.EvaluationResult, error) { args := m.Called(md, key, bucketingKey, feature, attributes) - return args.Get(0).(*sdk.Result), nil + return args.Get(0).(*sdk.EvaluationResult), args.Error(1) } // Treatments implements sdk.Interface @@ -29,10 +29,15 @@ func (m *SDKMock) Treatments( bucketingKey *string, features []string, attributes map[string]interface{}, -) (map[string]sdk.Result, error) { +) (map[string]sdk.EvaluationResult, error) { args := m.Called(md, key, bucketingKey, features, attributes) - return args.Get(0).(map[string]sdk.Result), nil + return args.Get(0).(map[string]sdk.EvaluationResult), args.Error(1) } +// Track implements sdk.Interface +func (m *SDKMock) Track(cfg *types.ClientConfig, key string, trafficType string, eventType string, value *float64, properties map[string]interface{}) error { + args := m.Called(cfg, key, trafficType, eventType, value, properties) + return args.Error(0) +} var _ sdk.Interface = (*SDKMock)(nil) diff --git a/splitio/sdk/results.go b/splitio/sdk/results.go index c697566..546a454 100644 --- a/splitio/sdk/results.go +++ b/splitio/sdk/results.go @@ -2,7 +2,7 @@ package sdk import "github.com/splitio/go-split-commons/v4/dtos" -type Result struct { +type EvaluationResult struct { Treatment string Impression *dtos.Impression Config *string diff --git a/splitio/sdk/sdk.go b/splitio/sdk/sdk.go index 7b0936f..47a7ed7 100644 --- a/splitio/sdk/sdk.go +++ b/splitio/sdk/sdk.go @@ -1,6 +1,7 @@ package sdk import ( + "errors" "fmt" "time" @@ -24,13 +25,19 @@ import ( const ( impressionsFullNotif = "IMPRESSIONS_FULL" + eventsFullNotif = "EVENTS_FULL" +) + +var ( + ErrEventsQueueFull = errors.New("events queue full") ) type Attributes = map[string]interface{} type Interface interface { - Treatment(cfg *types.ClientConfig, Key string, BucketingKey *string, Feature string, attributes map[string]interface{}) (*Result, error) - Treatments(cfg *types.ClientConfig, Key string, BucketingKey *string, Features []string, attributes map[string]interface{}) (map[string]Result, error) + Treatment(cfg *types.ClientConfig, key string, bucketingKey *string, feature string, attributes map[string]interface{}) (*EvaluationResult, error) + Treatments(cfg *types.ClientConfig, key string, bucketingKey *string, features []string, attributes map[string]interface{}) (map[string]EvaluationResult, error) + Track(cfg *types.ClientConfig, key string, trafficType string, eventType string, value *float64, properties map[string]interface{}) error } type Impl struct { @@ -39,6 +46,7 @@ type Impl struct { sm synchronizer.Manager ss synchronizer.Synchronizer is *storage.ImpressionsStorage + es *storage.EventsStorage iq provisional.ImpressionManager cfg sdkConf.Config status chan int @@ -64,9 +72,9 @@ func New(logger logging.LoggerInterface, apikey string, c *conf.Config) (*Impl, hc := &application.Dummy{} - queueFullChan := make(chan string, 1) // Only one item so that we don't queue N flushes (which makes no sense) if we're getting hit too hard + queueFullChan := make(chan string, 2) splitApi := api.NewSplitAPI(apikey, *advCfg, logger, md) - workers := setupWorkers(logger, splitApi, stores, hc, &c.Impressions) + workers := setupWorkers(logger, splitApi, stores, hc, c) tasks := setupTasks(c, stores, logger, workers, impc, md, splitApi) sync := synchronizer.NewSynchronizer(*advCfg, *tasks, *workers, logger, queueFullChan, nil) @@ -89,6 +97,7 @@ func New(logger logging.LoggerInterface, apikey string, c *conf.Config) (*Impl, ss: sync, ev: evaluator.NewEvaluator(stores.splits, stores.segments, engine.NewEngine(logger), logger), is: stores.impressions, + es: stores.events, iq: impc.manager, cfg: *c, queueFullChan: queueFullChan, @@ -96,14 +105,14 @@ func New(logger logging.LoggerInterface, apikey string, c *conf.Config) (*Impl, } // Treatment implements Interface -func (i *Impl) Treatment(cfg *types.ClientConfig, key string, bk *string, feature string, attributes Attributes) (*Result, error) { +func (i *Impl) Treatment(cfg *types.ClientConfig, key string, bk *string, feature string, attributes Attributes) (*EvaluationResult, error) { res := i.ev.EvaluateFeature(key, bk, feature, attributes) if res == nil { return nil, fmt.Errorf("nil result") } imp := i.handleImpression(key, bk, feature, res, cfg.Metadata) - return &Result{ + return &EvaluationResult{ Treatment: res.Treatment, Impression: imp, Config: res.Config, @@ -111,19 +120,19 @@ func (i *Impl) Treatment(cfg *types.ClientConfig, key string, bk *string, featur } // Treatment implements Interface -func (i *Impl) Treatments(cfg *types.ClientConfig, key string, bk *string, features []string, attributes Attributes) (map[string]Result, error) { +func (i *Impl) Treatments(cfg *types.ClientConfig, key string, bk *string, features []string, attributes Attributes) (map[string]EvaluationResult, error) { res := i.ev.EvaluateFeatures(key, bk, features, attributes) - toRet := make(map[string]Result, len(res.Evaluations)) + toRet := make(map[string]EvaluationResult, len(res.Evaluations)) for _, feature := range features { curr, ok := res.Evaluations[feature] if !ok { - toRet[feature] = Result{Treatment: "control"} + toRet[feature] = EvaluationResult{Treatment: "control"} continue } - var eres Result + var eres EvaluationResult eres.Treatment = curr.Treatment eres.Impression = i.handleImpression(key, bk, feature, &curr, cfg.Metadata) eres.Config = curr.Config @@ -133,6 +142,35 @@ func (i *Impl) Treatments(cfg *types.ClientConfig, key string, bk *string, featu return toRet, nil } +func (i *Impl) Track(cfg *types.ClientConfig, key string, trafficType string, eventType string, value *float64, properties map[string]interface{}) error { + + // TODO(mredolatti): validate traffic type & truncate properties if needed + + event := &dtos.EventDTO{ + Key: key, + TrafficTypeName: trafficType, + EventTypeID: eventType, + Value: value, + Timestamp: timeMillis(), + Properties: properties, + } + + _, err := i.es.Push(cfg.Metadata, *event) + if err != nil { + if err == storage.ErrQueueFull { + select { + case i.queueFullChan <- eventsFullNotif: + default: + i.logger.Warning("events queue has filled up and is currently performing a flush. Current event will be dropped") + } + return ErrEventsQueueFull + } + i.logger.Error("error handling event: ", err) + return err + } + return nil +} + func (i *Impl) handleImpression(key string, bk *string, f string, r *evaluator.Result, cm types.ClientMetadata) *dtos.Impression { var label string if i.cfg.LabelsEnabled { diff --git a/splitio/sdk/tasks/events.go b/splitio/sdk/tasks/events.go new file mode 100644 index 0000000..53e5278 --- /dev/null +++ b/splitio/sdk/tasks/events.go @@ -0,0 +1,29 @@ +package tasks + +import ( + "github.com/splitio/go-toolkit/v5/asynctask" + "github.com/splitio/go-toolkit/v5/logging" + sdkconf "github.com/splitio/splitd/splitio/sdk/conf" + "github.com/splitio/splitd/splitio/sdk/workers" +) + +const ( + defaultEventsBulkSize = 5000 +) + +func NewEventsSyncTask( + worker *workers.MultiMetaEventsWorker, + logger logging.LoggerInterface, + cfg *sdkconf.Impressions, +) *asynctask.AsyncTask { + + // TODO(mredolatti): pass a proper bulk size (currently ignored, everything is flushed) + return asynctask.NewAsyncTask( + "events-sender", + func(logging.LoggerInterface) error { worker.SynchronizeEvents(defaultEventsBulkSize); return nil }, + int(cfg.SyncPeriod.Seconds()), + nil, + func(logging.LoggerInterface) { worker.SynchronizeEvents(defaultEventsBulkSize) }, + logger, + ) +} diff --git a/splitio/sdk/tasks/impressions.go b/splitio/sdk/tasks/impressions.go index 71bb17a..e37f876 100644 --- a/splitio/sdk/tasks/impressions.go +++ b/splitio/sdk/tasks/impressions.go @@ -8,7 +8,7 @@ import ( ) const ( - defaultBulkSize = 5000 + defaultImpressionsBulkSize = 5000 ) func NewImpressionSyncTask( @@ -20,10 +20,10 @@ func NewImpressionSyncTask( // TODO(mredolatti): pass a proper bulk size (currently ignored, everything is flushed) return asynctask.NewAsyncTask( "impressions-sender", - func(logging.LoggerInterface) error { worker.SynchronizeImpressions(defaultBulkSize); return nil }, + func(logging.LoggerInterface) error { worker.SynchronizeImpressions(defaultImpressionsBulkSize); return nil }, int(cfg.SyncPeriod.Seconds()), nil, - func(logging.LoggerInterface) { worker.SynchronizeImpressions(defaultBulkSize) }, + func(logging.LoggerInterface) { worker.SynchronizeImpressions(defaultImpressionsBulkSize) }, logger, ) } diff --git a/splitio/sdk/workers/events.go b/splitio/sdk/workers/events.go new file mode 100644 index 0000000..d5c95b0 --- /dev/null +++ b/splitio/sdk/workers/events.go @@ -0,0 +1,94 @@ +package workers + +import ( + "errors" + "sync" + + sdkconf "github.com/splitio/splitd/splitio/sdk/conf" + sss "github.com/splitio/splitd/splitio/sdk/storage" + "github.com/splitio/splitd/splitio/sdk/types" + serrors "github.com/splitio/splitd/splitio/util/errors" + + "github.com/splitio/go-split-commons/v4/dtos" + "github.com/splitio/go-split-commons/v4/service" + "github.com/splitio/go-split-commons/v4/storage" + "github.com/splitio/go-split-commons/v4/synchronizer/worker/event" + "github.com/splitio/go-toolkit/v5/logging" + gtsync "github.com/splitio/go-toolkit/v5/sync" +) + +type MultiMetaEventsWorker struct { + logger logging.LoggerInterface + telemetry storage.TelemetryRuntimeProducer + llrec service.EventsRecorder + iq *sss.EventsStorage + cfg *sdkconf.Events + runnning gtsync.AtomicBool +} + +func NewEventsWorker( + logger logging.LoggerInterface, + telemetry storage.TelemetryRuntimeProducer, + llrec service.EventsRecorder, + iq *sss.EventsStorage, + cfg *sdkconf.Events, +) *MultiMetaEventsWorker { + return &MultiMetaEventsWorker{ + logger: logger, + telemetry: telemetry, + llrec: llrec, + iq: iq, + cfg: cfg, + } +} + +// FlushImpressions implements impression.ImpressionRecorder +// TODO(mredolatti): take `bulkSize` into account +func (m *MultiMetaEventsWorker) FlushEvents(bulkSize int64) error { + + // prevent 2 evictions from happening at the same time. we don't want a sync.Mutex since that would only cause 43928729 + // function calls to pile up and get called after each mutex release. + if !m.runnning.TestAndSet() { + m.logger.Warning("flush/sync requested while another one is in progress. ignoring") + return nil + } + defer m.runnning.Unset() + + var errs serrors.ConcurrentErrorCollector + var wg sync.WaitGroup + + // same logic as impressions workers, without the need for formatting. check impressions.go for a better + // description of what's being done + if err := m.iq.RangeAndClear(func(md types.ClientMetadata, q *sss.LockingQueue[dtos.EventDTO]) { + extracted := make([]dtos.EventDTO, 0, q.Len()) + n, err := q.Pop(q.Len(), &extracted) + if err != nil && !errors.Is(err, sss.ErrQueueEmpty) { + m.logger.Error("error fetching items from queue: ", err) + return // continue with queue + } + + if n == 0 { + return // nothing to do here + } + + wg.Add(1) + go func(events []dtos.EventDTO, cc types.ClientMetadata) { + defer wg.Done() + if err := m.llrec.Record(events, dtos.Metadata{SDKVersion: cc.SdkVersion}); err != nil { + errs.Append(err) + } + }(extracted, md) + }); err != nil { + m.logger.Error("error traversing event queues: ", err) + } + + wg.Wait() + return errs.Join() +} + +// SynchronizeImpressions implements impression.ImpressionRecorder +func (m *MultiMetaEventsWorker) SynchronizeEvents(bulkSize int64) error { + return m.FlushEvents(bulkSize) +} + +var _ event.EventRecorder = (*MultiMetaEventsWorker)(nil) diff --git a/splitio/sdk/workers/impressions.go b/splitio/sdk/workers/impressions.go index e600f01..6b97f73 100644 --- a/splitio/sdk/workers/impressions.go +++ b/splitio/sdk/workers/impressions.go @@ -2,15 +2,19 @@ package workers import ( "errors" + "sync" + + sdkconf "github.com/splitio/splitd/splitio/sdk/conf" + sss "github.com/splitio/splitd/splitio/sdk/storage" + "github.com/splitio/splitd/splitio/sdk/types" + serrors "github.com/splitio/splitd/splitio/util/errors" "github.com/splitio/go-split-commons/v4/dtos" "github.com/splitio/go-split-commons/v4/service" "github.com/splitio/go-split-commons/v4/storage" "github.com/splitio/go-split-commons/v4/synchronizer/worker/impression" "github.com/splitio/go-toolkit/v5/logging" - sdkconf "github.com/splitio/splitd/splitio/sdk/conf" - sss "github.com/splitio/splitd/splitio/sdk/storage" - "github.com/splitio/splitd/splitio/sdk/types" + gtsync "github.com/splitio/go-toolkit/v5/sync" ) type MultiMetaImpressionWorker struct { @@ -19,6 +23,7 @@ type MultiMetaImpressionWorker struct { llrec service.ImpressionsRecorder iq *sss.ImpressionsStorage cfg *sdkconf.Impressions + runnning gtsync.AtomicBool } func NewImpressionsWorker( @@ -38,10 +43,24 @@ func NewImpressionsWorker( } // FlushImpressions implements impression.ImpressionRecorder +// TODO(mredolatti): take `bulkSize` into account func (m *MultiMetaImpressionWorker) FlushImpressions(bulkSize int64) error { - // TODO(mredolatti): take `bulkSize` into account - var errs []error + // prevent 2 evictions from happening at the same time. we don't want a sync.Mutex since that would only cause 43928729 + // function calls to pile up and get called after each mutex release. + if !m.runnning.TestAndSet() { + m.logger.Warning("flush/sync requested while another one is in progress. ignoring") + return nil + } + defer m.runnning.Unset() + + var errs serrors.ConcurrentErrorCollector + var wg sync.WaitGroup + + // iterate all internal queues (one per thin-client associate-data) + // for each [metadata, impressions] tuple, format impressions accordingly, and create a goroutine to post them in BG. + // after all impressions posting-goroutines have been created, wait for all of them to complete, collect errors, + // and unset the `running` flag so that this func can be called again if err := m.iq.RangeAndClear(func(md types.ClientMetadata, q *sss.LockingQueue[dtos.Impression]) { extracted := make([]dtos.Impression, 0, q.Len()) n, err := q.Pop(q.Len(), &extracted) @@ -55,14 +74,20 @@ func (m *MultiMetaImpressionWorker) FlushImpressions(bulkSize int64) error { } formatted := formatImpressions(extracted) - if err := m.llrec.Record(formatted, dtos.Metadata{SDKVersion: md.SdkVersion}, nil); err != nil { - errs = append(errs, err) - } + + wg.Add(1) + go func(imps []dtos.ImpressionsDTO, md dtos.Metadata) { + defer wg.Done() + if err := m.llrec.Record(imps, md, nil); err != nil { + errs.Append(err) + } + }(formatted, dtos.Metadata{SDKVersion: md.SdkVersion}) }); err != nil { m.logger.Error("error traversing impression queues: ", err) } - return errors.Join(errs...) + wg.Wait() + return errs.Join() } // SynchronizeImpressions implements impression.ImpressionRecorder diff --git a/splitio/sdk/workers/impressions_test.go b/splitio/sdk/workers/impressions_test.go index fb718b0..1285ddf 100644 --- a/splitio/sdk/workers/impressions_test.go +++ b/splitio/sdk/workers/impressions_test.go @@ -4,6 +4,7 @@ import ( "reflect" "sort" "testing" + "time" "github.com/splitio/go-split-commons/v4/dtos" "github.com/splitio/go-split-commons/v4/service" @@ -12,6 +13,7 @@ import ( "github.com/splitio/splitd/splitio/sdk/conf" sss "github.com/splitio/splitd/splitio/sdk/storage" "github.com/splitio/splitd/splitio/sdk/types" + "github.com/stretchr/testify/assert" "github.com/stretchr/testify/mock" ) @@ -73,6 +75,48 @@ func TestImpressionsTask(t *testing.T) { rec.AssertExpectations(t) } +func TestImpressionsTaskNoParallelism(t *testing.T) { + + // to test this, we set up a Recorder that sleeps for 1 second and returns (no err). + // we one call to `SyncrhonizeImpressions()` wait for 500ms, and fire another one. + // the second one should finish immediately, (becase it does nothing). The second one + // should finish after 2 seconds + + is, _ := sss.NewImpressionsQueue(100) + ts, _ := inmemory.NewTelemetryStorage() + logger := logging.NewLogger(nil) + rec := &RecorderMock{} + + worker := NewImpressionsWorker(logger, ts, rec, is, &conf.Impressions{}) + + rec.On("Record", mock.Anything, mock.Anything, mock.Anything).Run(func(mock.Arguments) { time.Sleep(1 * time.Second) }).Return(nil).Twice() + + is.Push(types.ClientMetadata{ID: "i1", SdkVersion: "php-1.2.3"}, + dtos.Impression{KeyName: "k1", FeatureName: "f1", Treatment: "on", Label: "l1", ChangeNumber: 123, Time: 123456}) + is.Push(types.ClientMetadata{ID: "i2", SdkVersion: "go-1.2.3"}, + dtos.Impression{KeyName: "k2", FeatureName: "f2", Treatment: "off", Label: "l2", ChangeNumber: 456, Time: 123457}) + + done := make(chan struct{}) + + go func() { + worker.SynchronizeImpressions(5000) + done <- struct{}{} + }() + + time.Sleep(500 * time.Millisecond) + assert.Nil(t, worker.SynchronizeImpressions(5000)) + + // 2nd call has finished, assert that the first one hasn't: + select { + case <-done: // first call has finished, fail the test + assert.Fail(t, "first call shouldn't have finished yet") + default: + } + + <-done // blocking wait for 1st to finish + +} + type RecorderMock struct { mock.Mock } diff --git a/splitio/util/errors/concurrent.go b/splitio/util/errors/concurrent.go new file mode 100644 index 0000000..02dcca8 --- /dev/null +++ b/splitio/util/errors/concurrent.go @@ -0,0 +1,23 @@ +package errors + +import ( + "errors" + "sync" +) + +type ConcurrentErrorCollector struct { + errors []error + mutex sync.Mutex +} + +func (c *ConcurrentErrorCollector) Append(err error) { + c.mutex.Lock() + c.errors = append(c.errors, err) + c.mutex.Unlock() +} + +func (c *ConcurrentErrorCollector) Join() error { + c.mutex.Lock() + defer c.mutex.Unlock() + return errors.Join(c.errors...) +} diff --git a/splitio/util/errors/concurrent_test.go b/splitio/util/errors/concurrent_test.go new file mode 100644 index 0000000..874efb9 --- /dev/null +++ b/splitio/util/errors/concurrent_test.go @@ -0,0 +1,40 @@ +package errors + +import ( + "errors" + "fmt" + "sync" + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestConcurrentErrors(t *testing.T) { + + // test setup: + // errors are interfaces containing pointers, in order for them to be compared with errors.Is, it MUST be the same instance. + // to do the test we create a slice with many vectors, use them and then compare against the original + original := make([]error, 100) + for idx := range original { + original[idx] = errors.New(fmt.Sprintf("err_%d", idx)) + } + + var c ConcurrentErrorCollector + + var wg sync.WaitGroup + wg.Add(100) + for idx := 0; idx < 100; idx++ { + go func(i int) { + c.Append(original[i]) + wg.Done() + }(idx) + } + + wg.Wait() + + je := c.Join() + for idx := 0; idx < 100; idx++ { + assert.ErrorIs(t, je, original[idx]) + } + +} From 841be6cb82469e16982d709a9dc9fc9813b1390d Mon Sep 17 00:00:00 2001 From: Martin Redolatti Date: Sat, 19 Aug 2023 08:50:30 -0300 Subject: [PATCH 25/42] enable imps debug mode --- splitio/link/protocol/v1/responses.go | 2 +- splitio/sdk/conf/conf.go | 7 +------ splitio/sdk/conf/conf_test.go | 8 +------- splitio/sdk/helpers.go | 27 +++++++++++++++++++-------- 4 files changed, 22 insertions(+), 22 deletions(-) diff --git a/splitio/link/protocol/v1/responses.go b/splitio/link/protocol/v1/responses.go index 7436931..88aee19 100644 --- a/splitio/link/protocol/v1/responses.go +++ b/splitio/link/protocol/v1/responses.go @@ -34,7 +34,7 @@ type TreatmentsWithConfigPayload struct { } type TrackPayload struct { - Success bool + Success bool `msgpack:"s"` } type ListenerExtraData struct { diff --git a/splitio/sdk/conf/conf.go b/splitio/sdk/conf/conf.go index 4ba0fa6..411fc30 100644 --- a/splitio/sdk/conf/conf.go +++ b/splitio/sdk/conf/conf.go @@ -117,12 +117,7 @@ func DefaultConfig() *Config { func (c *Config) Normalize() []string { var warnings []string - if c.Impressions.Mode != defaultImpressionsMode { - warnings = append(warnings, "only `optimized` impressions mode supported currently. ignoring user config") - c.Impressions.Mode = defaultImpressionsMode - } - - if c.Impressions.SyncPeriod < minimumImpressionsRefreshRate { + if c.Impressions.Mode == "optimized" && c.Impressions.SyncPeriod < minimumImpressionsRefreshRate { warnings = append(warnings, "minimum impressions refresh rate is 30 min. ignoring user config") c.Impressions.SyncPeriod = minimumImpressionsRefreshRate } diff --git a/splitio/sdk/conf/conf_test.go b/splitio/sdk/conf/conf_test.go index 6dc7e62..f3e87ea 100644 --- a/splitio/sdk/conf/conf_test.go +++ b/splitio/sdk/conf/conf_test.go @@ -9,17 +9,11 @@ import ( func TestSDKConf(t *testing.T) { dc := DefaultConfig() - dc.Impressions.Mode = "debug" dc.Impressions.SyncPeriod = 1 * time.Minute warns := dc.Normalize() - assert.Equal(t, warns, []string{ - "only `optimized` impressions mode supported currently. ignoring user config", - "minimum impressions refresh rate is 30 min. ignoring user config", - }) - assert.Equal(t, "optimized", dc.Impressions.Mode) + assert.Equal(t, warns, []string{"minimum impressions refresh rate is 30 min. ignoring user config"}) assert.Equal(t, 30*time.Minute, dc.Impressions.SyncPeriod) - adv := dc.ToAdvancedConfig() assert.Equal(t, 30, adv.HTTPTimeout) assert.Equal(t, dc.Segments.QueueSize, adv.SegmentQueueSize) diff --git a/splitio/sdk/helpers.go b/splitio/sdk/helpers.go index deb3541..17b2332 100644 --- a/splitio/sdk/helpers.go +++ b/splitio/sdk/helpers.go @@ -38,7 +38,7 @@ func setupWorkers( SplitFetcher: split.NewSplitFetcher(str.splits, api.SplitFetcher, logger, str.telemetry, hc), SegmentFetcher: segment.NewSegmentFetcher(str.splits, str.segments, api.SegmentFetcher, logger, str.telemetry, hc), ImpressionRecorder: workers.NewImpressionsWorker(logger, str.telemetry, api.ImpressionRecorder, str.impressions, &cfg.Impressions), - EventRecorder: workers.NewEventsWorker(logger, str.telemetry, api.EventRecorder, str.events, &cfg.Events), + EventRecorder: workers.NewEventsWorker(logger, str.telemetry, api.EventRecorder, str.events, &cfg.Events), } } @@ -53,21 +53,32 @@ func setupTasks( ) *synchronizer.SplitTasks { impCfg := cfg.Impressions evCfg := cfg.Events - return &synchronizer.SplitTasks{ - SplitSyncTask: tasks.NewFetchSplitsTask(workers.SplitFetcher, int(cfg.Splits.SyncPeriod.Seconds()), logger), - SegmentSyncTask: tasks.NewFetchSegmentsTask(workers.SegmentFetcher, int(cfg.Segments.SyncPeriod.Seconds()), cfg.Segments.WorkerCount, cfg.Segments.QueueSize, logger), - ImpressionSyncTask: tasks.NewRecordImpressionsTask(workers.ImpressionRecorder, int(impCfg.SyncPeriod.Seconds()), logger, 5000), - ImpressionsCountSyncTask: tasks.NewRecordImpressionsCountTask( - impressionscount.NewRecorderSingle(impComponents.counter, api.ImpressionRecorder, md, logger, str.telemetry), + tg := &synchronizer.SplitTasks{ + SplitSyncTask: tasks.NewFetchSplitsTask(workers.SplitFetcher, int(cfg.Splits.SyncPeriod.Seconds()), logger), + SegmentSyncTask: tasks.NewFetchSegmentsTask( + workers.SegmentFetcher, + int(cfg.Segments.SyncPeriod.Seconds()), + cfg.Segments.WorkerCount, + cfg.Segments.QueueSize, logger, - int(impCfg.CountSyncPeriod.Seconds()), ), + ImpressionSyncTask: tasks.NewRecordImpressionsTask(workers.ImpressionRecorder, int(impCfg.SyncPeriod.Seconds()), logger, 5000), EventSyncTask: tasks.NewRecordEventsTask(workers.EventRecorder, 5000, int(evCfg.SyncPeriod.Seconds()), logger), TelemetrySyncTask: &NoOpTask{}, UniqueKeysTask: &NoOpTask{}, CleanFilterTask: &NoOpTask{}, ImpsCountConsumerTask: &NoOpTask{}, } + + if impCfg.Mode == "optimized" { + tg.ImpressionsCountSyncTask = tasks.NewRecordImpressionsCountTask( + impressionscount.NewRecorderSingle(impComponents.counter, api.ImpressionRecorder, md, logger, str.telemetry), + logger, + int(impCfg.CountSyncPeriod.Seconds()), + ) + } + + return tg } type impComponents struct { From 58b38d4dbd209d1c934c10e3a4c3e17fde3fdf6e Mon Sep 17 00:00:00 2001 From: Martin Redolatti Date: Sat, 19 Aug 2023 11:31:28 -0300 Subject: [PATCH 26/42] fix track return value in rpc --- splitio/link/service/v1/clientmgr.go | 18 ++++++++---------- 1 file changed, 8 insertions(+), 10 deletions(-) diff --git a/splitio/link/service/v1/clientmgr.go b/splitio/link/service/v1/clientmgr.go index db55e2e..f19cd67 100644 --- a/splitio/link/service/v1/clientmgr.go +++ b/splitio/link/service/v1/clientmgr.go @@ -42,7 +42,7 @@ func (m *ClientManager) Manage() { defer func() { if r := recover(); r != nil { m.logger.Error("CRITICAL - connection handler is panicking: ", r) - m.logger.Error(string(debug.Stack())) + m.logger.Error(string(debug.Stack())) } }() err := m.handleClientInteractions() @@ -133,12 +133,12 @@ func (m *ClientManager) handleRPC(rpc *protov1.RPC) (interface{}, error) { return nil, fmt.Errorf("error parsing treatments arguments: %w", err) } return m.handleGetTreatments(&args) - case protov1.OCTrack: - var args protov1.TrackArgs - if err := args.PopulateFromRPC(rpc); err != nil { - return nil, fmt.Errorf("error parsing track argumentts: %w", err) - } - return m.handleTrack(&args) + case protov1.OCTrack: + var args protov1.TrackArgs + if err := args.PopulateFromRPC(rpc); err != nil { + return nil, fmt.Errorf("error parsing track argumentts: %w", err) + } + return m.handleTrack(&args) } return nil, fmt.Errorf("RPC not implemented") @@ -217,10 +217,8 @@ func (m *ClientManager) handleTrack(args *protov1.TrackArgs) (interface{}, error response := &protov1.ResponseWrapper[protov1.TrackPayload]{ Status: protov1.ResultOk, - Payload: protov1.TrackPayload{Success: err != nil}, // can only be events queue full at this point + Payload: protov1.TrackPayload{Success: err == nil}, // if err != nil it can only be ErrEventsQueueFull at this point } return response, nil } - - From 0c50f5cfb1698893cafb7d31686d1fbc218e8768 Mon Sep 17 00:00:00 2001 From: Martin Redolatti Date: Sun, 20 Aug 2023 18:47:46 -0300 Subject: [PATCH 27/42] add evenet properties validation --- splitio/sdk/sdk.go | 11 ++++++++- splitio/sdk/validators.go | 49 +++++++++++++++++++++++++++++++++++++++ 2 files changed, 59 insertions(+), 1 deletion(-) create mode 100644 splitio/sdk/validators.go diff --git a/splitio/sdk/sdk.go b/splitio/sdk/sdk.go index 47a7ed7..55560dd 100644 --- a/splitio/sdk/sdk.go +++ b/splitio/sdk/sdk.go @@ -51,6 +51,7 @@ type Impl struct { cfg sdkConf.Config status chan int queueFullChan chan string + validator Validator } func New(logger logging.LoggerInterface, apikey string, c *conf.Config) (*Impl, error) { @@ -101,6 +102,7 @@ func New(logger logging.LoggerInterface, apikey string, c *conf.Config) (*Impl, iq: impc.manager, cfg: *c, queueFullChan: queueFullChan, + validator: Validator{logger}, }, nil } @@ -146,6 +148,11 @@ func (i *Impl) Track(cfg *types.ClientConfig, key string, trafficType string, ev // TODO(mredolatti): validate traffic type & truncate properties if needed + properties, _, err := i.validator.validateTrackProperties(properties) + if err != nil { + return err + } + event := &dtos.EventDTO{ Key: key, TrafficTypeName: trafficType, @@ -155,7 +162,9 @@ func (i *Impl) Track(cfg *types.ClientConfig, key string, trafficType string, ev Properties: properties, } - _, err := i.es.Push(cfg.Metadata, *event) + fmt.Printf("EVENTO GENERADO: %+v\n", event) + + _, err = i.es.Push(cfg.Metadata, *event) if err != nil { if err == storage.ErrQueueFull { select { diff --git a/splitio/sdk/validators.go b/splitio/sdk/validators.go new file mode 100644 index 0000000..c60bdef --- /dev/null +++ b/splitio/sdk/validators.go @@ -0,0 +1,49 @@ +package sdk + +import ( + "errors" + + "github.com/splitio/go-toolkit/v5/logging" +) + +// MaxEventLength constant to limit the event size +const MaxEventLength = 32768 + +var ErrEventTooBig = errors.New("The maximum size allowed for the properties is 32kb. Event not queued") + +type Validator struct { + logger logging.LoggerInterface +} + +func (i *Validator) validateTrackProperties(properties map[string]interface{}) (map[string]interface{}, int, error) { + if len(properties) == 0 { + return nil, 0, nil + } + + if len(properties) > 300 { + i.logger.Warning("Track: Event has more than 300 properties. Some of them will be trimmed when processed") + } + + processed := make(map[string]interface{}) + size := 1024 // Average event size is ~750 bytes. Using 1kbyte as a starting point. + for name, value := range properties { + size += len(name) + switch value.(type) { + case int, int8, int16, int32, int64, uint, uint8, uint16, uint32, uint64, float32, float64, bool, nil: + processed[name] = value + case string: + asStr := value.(string) + size += len(asStr) + processed[name] = value + default: + i.logger.Warning("Property %s is of invalid type. Setting value to nil") + processed[name] = nil + } + + if size > MaxEventLength { + i.logger.Error("The maximum size allowed for the properties is 32kb. Event not queued") + return nil, size, ErrEventTooBig + } + } + return processed, size, nil +} From f2047c3470f1cfce2478afee4951ac5826f680ce Mon Sep 17 00:00:00 2001 From: Martin Redolatti Date: Wed, 30 Aug 2023 16:57:27 -0300 Subject: [PATCH 28/42] add test coverage for track --- splitio/link/client/v1/impl_test.go | 29 +++++ splitio/link/protocol/v1/mocks/mocks.go | 26 ++++ splitio/link/protocol/v1/rpcs.go | 9 +- splitio/link/protocol/v1/rpcs_test.go | 2 +- splitio/link/service/v1/clientmgr_test.go | 45 ++++++- splitio/sdk/sdk.go | 18 +-- splitio/sdk/sdk_test.go | 144 ++++++++++++++++++++++ splitio/sdk/validators.go | 22 ++++ splitio/sdk/workers/events_test.go | 115 +++++++++++++++++ 9 files changed, 398 insertions(+), 12 deletions(-) create mode 100644 splitio/sdk/workers/events_test.go diff --git a/splitio/link/client/v1/impl_test.go b/splitio/link/client/v1/impl_test.go index c90be6c..743fcce 100644 --- a/splitio/link/client/v1/impl_test.go +++ b/splitio/link/client/v1/impl_test.go @@ -47,6 +47,35 @@ func TestClientGetTreatmentNoImpression(t *testing.T) { assert.Nil(t, res.Impression) } +func TestTrack(t *testing.T) { + + logger := logging.NewLogger(nil) + + rawConnMock := &transferMocks.RawConnMock{} + rawConnMock.On("SendMessage", []byte("registrationMessage")).Return(nil).Once() + rawConnMock.On("ReceiveMessage").Return([]byte("registrationSuccess"), nil).Once() + rawConnMock.On("SendMessage", []byte("trackMessage")).Return(nil).Once() + rawConnMock.On("ReceiveMessage").Return([]byte("trackResult"), nil).Once() + + serializerMock := &serializerMocks.SerializerMock{} + serializerMock.On("Serialize", proto1Mocks.NewRegisterRPC("some", false)).Return([]byte("registrationMessage"), nil).Once() + serializerMock.On("Parse", []byte("registrationSuccess"), mock.Anything).Return(nil).Run(func(args mock.Arguments) { + *args.Get(1).(*v1.ResponseWrapper[v1.RegisterPayload]) = v1.ResponseWrapper[v1.RegisterPayload]{Status: v1.ResultOk} + }).Once() + + serializerMock.On("Serialize", proto1Mocks.NewTrackRPC("key1", "user", "checkin", ref(2.74), map[string]interface{}{"p1": 123})). + Return([]byte("trackMessage"), nil).Once() + serializerMock.On("Parse", []byte("trackResult"), mock.Anything).Return(nil).Run(func(args mock.Arguments) { + *args.Get(1).(*v1.ResponseWrapper[v1.TrackPayload]) = *proto1Mocks.NewTrackResp(true) + }).Once() + client, err := New("some", logger, rawConnMock, serializerMock, false) + assert.NotNil(t, client) + assert.Nil(t, err) + + err = client.Track("key1", "user", "checkin", ref(2.74), map[string]interface{}{"p1": 123}) + assert.Nil(t, err) +} + func TestClientGetTreatmentWithImpression(t *testing.T) { logger := logging.NewLogger(nil) diff --git a/splitio/link/protocol/v1/mocks/mocks.go b/splitio/link/protocol/v1/mocks/mocks.go index a412c8f..abf4034 100644 --- a/splitio/link/protocol/v1/mocks/mocks.go +++ b/splitio/link/protocol/v1/mocks/mocks.go @@ -37,6 +37,14 @@ func NewTreatmentsRPC(key string, bucketing string, features []string, attrs map } } +func NewTrackRPC(key string, trafficType string, eventType string, eventVal *float64, props map[string]interface{}) *v1.RPC { + return &v1.RPC{ + RPCBase: protocol.RPCBase{Version: protocol.V1}, + OpCode: v1.OCTrack, + Args: []interface{}{key, trafficType, eventType, nilOrVal(eventVal), props}, + } +} + func NewRegisterResp(ok bool) *v1.ResponseWrapper[v1.RegisterPayload] { res := v1.ResultOk if !ok { @@ -86,3 +94,21 @@ func NewTreatmentsResp(ok bool, data []sdk.EvaluationResult) *v1.ResponseWrapper Payload: v1.TreatmentsPayload{Results: payload}, } } + +func NewTrackResp(ok bool) *v1.ResponseWrapper[v1.TrackPayload] { + res := v1.ResultOk + if !ok { + res = v1.ResultInternalError + } + return &v1.ResponseWrapper[v1.TrackPayload]{ + Status: res, + Payload: v1.TrackPayload{Success: ok}, + } +} + +func nilOrVal(v *float64) interface{} { + if v == nil { + return nil + } + return *v +} diff --git a/splitio/link/protocol/v1/rpcs.go b/splitio/link/protocol/v1/rpcs.go index 0184c65..dba2d60 100644 --- a/splitio/link/protocol/v1/rpcs.go +++ b/splitio/link/protocol/v1/rpcs.go @@ -217,7 +217,14 @@ type TrackArgs struct { } func (r TrackArgs) Encode() []interface{} { - return []interface{}{r.Key, r.TrafficType, r.EventType, r.Value, r.Properties} + asInterface := make([]interface{}, 0, 5) + asInterface = append(asInterface, r.Key, r.TrafficType, r.EventType) + if r.Value == nil { + asInterface = append(asInterface, nil) + } + asInterface = append(asInterface, *r.Value) + asInterface = append(asInterface, r.Properties) + return asInterface } func (t *TrackArgs) PopulateFromRPC(rpc *RPC) error { diff --git a/splitio/link/protocol/v1/rpcs_test.go b/splitio/link/protocol/v1/rpcs_test.go index eef58c6..d27fedf 100644 --- a/splitio/link/protocol/v1/rpcs_test.go +++ b/splitio/link/protocol/v1/rpcs_test.go @@ -313,7 +313,7 @@ func TestRPCEncoding(t *testing.T) { assert.Equal(t, tra.Key, encodedTrA[TrackArgKeyIdx].(string)) assert.Equal(t, tra.TrafficType, encodedTrA[TrackArgTrafficTypeIdx].(string)) assert.Equal(t, tra.EventType, encodedTrA[TrackArgEventTypeIdx].(string)) - assert.Equal(t, *tra.Value, *encodedTrA[TrackArgValueIdx].(*float64)) + assert.Equal(t, *tra.Value, encodedTrA[TrackArgValueIdx].(float64)) assert.Equal(t, tra.Properties, encodedTrA[TrackArgPropertiesIdx].(map[string]interface{})) } diff --git a/splitio/link/service/v1/clientmgr_test.go b/splitio/link/service/v1/clientmgr_test.go index 3d0d403..b22a892 100644 --- a/splitio/link/service/v1/clientmgr_test.go +++ b/splitio/link/service/v1/clientmgr_test.go @@ -151,6 +151,43 @@ func TestRegisterWithImpsAndTreatmentHappyPath(t *testing.T) { rawConnMock.AssertNumberOfCalls(t, "Shutdown", 1) } +func TestTrack(t *testing.T) { + rawConnMock := &transferMocks.RawConnMock{} + rawConnMock.On("ReceiveMessage").Return([]byte("registrationMessage"), nil).Once() + rawConnMock.On("SendMessage", []byte("successRegistration")).Return(nil).Once() + rawConnMock.On("ReceiveMessage").Return([]byte("trackMessage"), nil).Once() + rawConnMock.On("SendMessage", []byte("successPayload")).Return(nil).Once() + rawConnMock.On("ReceiveMessage").Return([]byte(nil), io.EOF).Once() + rawConnMock.On("Shutdown").Return(nil).Once() + + serializerMock := &serializerMocks.SerializerMock{} + serializerMock.On("Parse", []byte("registrationMessage"), mock.Anything).Return(nil).Run(func(args mock.Arguments) { + *args.Get(1).(*v1.RPC) = v1.RPC{ + RPCBase: protocol.RPCBase{Version: protocol.V1}, + OpCode: v1.OCRegister, + Args: []interface{}{"someID", "some_sdk-1.2.3", uint64(0)}, + } + }).Once() + serializerMock.On("Serialize", proto1Mocks.NewRegisterResp(true)).Return([]byte("successRegistration"), nil).Once() + serializerMock.On("Parse", []byte("trackMessage"), mock.Anything).Return(nil).Run(func(args mock.Arguments) { + *args.Get(1).(*v1.RPC) = *proto1Mocks.NewTrackRPC("key1", "user", "checkin", ref(2.75), map[string]interface{}{"a": 1}) + }).Once() + serializerMock.On("Serialize", proto1Mocks.NewTrackResp(true)).Return([]byte("successPayload"), nil).Once() + + sdkMock := &sdkMocks.SDKMock{} + sdkMock. + On("Track", + &types.ClientConfig{Metadata: types.ClientMetadata{ID: "someID", SdkVersion: "some_sdk-1.2.3"}}, + "key1", "user", "checkin", ref(float64(2.75)), map[string]interface{}{"a": 1}). + Return((error)(nil)).Once() + + logger := logging.NewLogger(nil) + cm := NewClientManager(rawConnMock, logger, sdkMock, serializerMock) + err := cm.handleClientInteractions() + assert.Nil(t, err) + rawConnMock.AssertNumberOfCalls(t, "Shutdown", 1) +} + func TestTreatmentWithoutRegister(t *testing.T) { rawConnMock := &transferMocks.RawConnMock{} rawConnMock.On("ReceiveMessage").Return([]byte("treatmentMessage"), nil).Once() @@ -268,8 +305,8 @@ func TestHandleRPCErrors(t *testing.T) { assert.Nil(t, res) assert.ErrorContains(t, err, "error parsing register arguments") - // set the config to allow other rpcs to be handled - cm.clientConfig = &types.ClientConfig{ReturnImpressionData: true} + // set the config to allow other rpcs to be handled + cm.clientConfig = &types.ClientConfig{ReturnImpressionData: true} // treatment wrong args res, err = cm.handleRPC(&v1.RPC{RPCBase: protocol.RPCBase{Version: protocol.V1}, OpCode: v1.OCTreatment, Args: []interface{}{1, "hola"}}) @@ -290,4 +327,8 @@ func (m *loggerMock) Info(msg ...interface{}) { m.Called(msg...) } func (m *loggerMock) Verbose(msg ...interface{}) { m.Called(msg...) } func (m *loggerMock) Warning(msg ...interface{}) { m.Called(msg...) } +func ref[T any](t T) *T { + return &t +} + var _ logging.LoggerInterface = (*loggerMock)(nil) diff --git a/splitio/sdk/sdk.go b/splitio/sdk/sdk.go index 55560dd..e9fd470 100644 --- a/splitio/sdk/sdk.go +++ b/splitio/sdk/sdk.go @@ -102,7 +102,7 @@ func New(logger logging.LoggerInterface, apikey string, c *conf.Config) (*Impl, iq: impc.manager, cfg: *c, queueFullChan: queueFullChan, - validator: Validator{logger}, + validator: Validator{logger: logger, splits: stores.splits}, }, nil } @@ -147,11 +147,15 @@ func (i *Impl) Treatments(cfg *types.ClientConfig, key string, bk *string, featu func (i *Impl) Track(cfg *types.ClientConfig, key string, trafficType string, eventType string, value *float64, properties map[string]interface{}) error { // TODO(mredolatti): validate traffic type & truncate properties if needed + trafficType, err := i.validator.validateTrafficType(trafficType) + if err != nil { + return err + } - properties, _, err := i.validator.validateTrackProperties(properties) - if err != nil { - return err - } + properties, _, err = i.validator.validateTrackProperties(properties) + if err != nil { + return err + } event := &dtos.EventDTO{ Key: key, @@ -162,9 +166,7 @@ func (i *Impl) Track(cfg *types.ClientConfig, key string, trafficType string, ev Properties: properties, } - fmt.Printf("EVENTO GENERADO: %+v\n", event) - - _, err = i.es.Push(cfg.Metadata, *event) + _, err = i.es.Push(cfg.Metadata, *event) if err != nil { if err == storage.ErrQueueFull { select { diff --git a/splitio/sdk/sdk_test.go b/splitio/sdk/sdk_test.go index 1001c77..6ef20f6 100644 --- a/splitio/sdk/sdk_test.go +++ b/splitio/sdk/sdk_test.go @@ -2,6 +2,7 @@ package sdk import ( "fmt" + "strings" "testing" "time" @@ -9,8 +10,10 @@ import ( "github.com/splitio/go-split-commons/v4/dtos" "github.com/splitio/go-split-commons/v4/provisional" "github.com/splitio/go-split-commons/v4/service" + scstorage "github.com/splitio/go-split-commons/v4/storage" "github.com/splitio/go-split-commons/v4/storage/inmemory" "github.com/splitio/go-split-commons/v4/synchronizer" + "github.com/splitio/go-toolkit/v5/datastructures/set" "github.com/splitio/go-toolkit/v5/logging" "github.com/splitio/splitd/splitio/sdk/conf" "github.com/splitio/splitd/splitio/sdk/storage" @@ -279,6 +282,116 @@ func TestImpressionsQueueFull(t *testing.T) { assert.Equal(t, 1, totalSize) // assert no more impressions in queue } +func TestTrack(t *testing.T) { + + es, _ := storage.NewEventsQueue(1000) + logger := logging.NewLogger(nil) + + ss := &SplitStorageMock{} + ss.On("TrafficTypeExists", "user").Return(true) + + client := &Impl{ + logger: logging.NewLogger(nil), + es: es, + cfg: conf.Config{LabelsEnabled: false}, + validator: Validator{logger, ss}, + } + + md := types.ClientConfig{Metadata: types.ClientMetadata{ID: "some", SdkVersion: "go-1.2.3"}} + + err := client.Track(&md, "key1", "user", "checkin", ref(123.4), map[string]interface{}{"a": 123}) + assert.Nil(t, err) + + err = es.RangeAndClear(func(md types.ClientMetadata, st *storage.LockingQueue[dtos.EventDTO]) { + assert.Equal(t, types.ClientMetadata{ID: "some", SdkVersion: "go-1.2.3"}, md) + assert.Equal(t, 1, st.Len()) + + var evs []dtos.EventDTO + n, err := st.Pop(1, &evs) + assert.Nil(t, nil) + assert.Equal(t, 1, n) + assert.Equal(t, 1, len(evs)) + assertEventEq(t, &dtos.EventDTO{ + Key: "key1", + TrafficTypeName: "user", + EventTypeID: "checkin", + Value: ref(123.4), + Properties: map[string]interface{}{"a": 123}, + }, &evs[0]) + n, err = st.Pop(1, &evs) + assert.ErrorIs(t, err, storage.ErrQueueEmpty) + + }) + assert.Nil(t, err) + + err = client.Track(&md, "key1", "", "checkin", ref(123.4), map[string]interface{}{"a": 123}) + assert.ErrorIs(t, err, ErrEmtpyTrafficType) + + err = client.Track(&md, "key1", "user", "checkin", ref(123.4), map[string]interface{}{"a": strings.Repeat("qwertyui", 100000)}) + assert.ErrorIs(t, err, ErrEventTooBig) + +} + +func TestTrackEventsFlush(t *testing.T) { + + es, _ := storage.NewEventsQueue(4) + logger := logging.NewLogger(nil) + + ss := &SplitStorageMock{} + ss.On("TrafficTypeExists", "user").Return(true) + + client := &Impl{ + logger: logging.NewLogger(nil), + queueFullChan: make(chan string, 2), + es: es, + cfg: conf.Config{LabelsEnabled: false}, + validator: Validator{logger, ss}, + } + + md := types.ClientConfig{Metadata: types.ClientMetadata{ID: "some", SdkVersion: "go-1.2.3"}} + + err := client.Track(&md, "key1", "user", "checkin", ref(123.4), map[string]interface{}{"a": 123}) + assert.Nil(t, err) + err = client.Track(&md, "key2", "user", "checkin", ref(123.4), map[string]interface{}{"a": 123}) + assert.Nil(t, err) + err = client.Track(&md, "key3", "user", "checkin", ref(123.4), map[string]interface{}{"a": 123}) + assert.Nil(t, err) + err = client.Track(&md, "key4", "user", "checkin", ref(123.4), map[string]interface{}{"a": 123}) + assert.ErrorIs(t, err, ErrEventsQueueFull) + + assert.Equal(t, "EVENTS_FULL", <-client.queueFullChan) + /* + err = es.RangeAndClear(func(md types.ClientMetadata, st *storage.LockingQueue[dtos.EventDTO]) { + assert.Equal(t, types.ClientMetadata{ID: "some", SdkVersion: "go-1.2.3"}, md) + assert.Equal(t, 1, st.Len()) + + var evs []dtos.EventDTO + n, err := st.Pop(1, &evs) + assert.Nil(t, nil) + assert.Equal(t, 1, n) + assert.Equal(t, 1, len(evs)) + assertEventEq(t, &dtos.EventDTO{ + Key: "key1", + TrafficTypeName: "user", + EventTypeID: "checkin", + Value: ref(123.4), + Properties: map[string]interface{}{"a": 123}, + }, &evs[0]) + n, err = st.Pop(1, &evs) + assert.ErrorIs(t, err, storage.ErrQueueEmpty) + + }) + assert.Nil(t, err) + + err = client.Track(&md, "key1", "", "checkin", ref(123.4), map[string]interface{}{"a": 123}) + assert.ErrorIs(t, err, ErrEmtpyTrafficType) + + err = client.Track(&md, "key1", "user", "checkin", ref(123.4), map[string]interface{}{"a": strings.Repeat("qwertyui", 100000)}) + assert.ErrorIs(t, err, ErrEventTooBig) + */ + +} + func assertImpEq(t *testing.T, i1, i2 *dtos.Impression) { t.Helper() assert.Equal(t, i1.KeyName, i2.KeyName) @@ -289,6 +402,15 @@ func assertImpEq(t *testing.T, i1, i2 *dtos.Impression) { assert.Equal(t, i1.ChangeNumber, i2.ChangeNumber) } +func assertEventEq(t *testing.T, e1, e2 *dtos.EventDTO) { + t.Helper() + assert.Equal(t, e1.Key, e2.Key) + assert.Equal(t, e1.TrafficTypeName, e2.TrafficTypeName) + assert.Equal(t, e1.EventTypeID, e2.EventTypeID) + assert.Equal(t, e1.Value, e2.Value) + assert.Equal(t, e1.Properties, e2.Properties) +} + // mocks type EvaluatorMock struct { @@ -339,6 +461,28 @@ func (m *ImpressionRecorderMock) RecordImpressionsCount(pf dtos.ImpressionsCount return args.Error(0) } +type SplitStorageMock struct{ mock.Mock } + +func (m *SplitStorageMock) All() []dtos.SplitDTO { panic("unimplemented") } +func (m *SplitStorageMock) ChangeNumber() (int64, error) { panic("unimplemented") } +func (m *SplitStorageMock) FetchMany([]string) map[string]*dtos.SplitDTO { panic("unimplemented") } +func (m *SplitStorageMock) KillLocally(string, string, int64) { panic("unimplemented") } +func (m *SplitStorageMock) SegmentNames() *set.ThreadUnsafeSet { panic("unimplemented") } +func (m *SplitStorageMock) SetChangeNumber(changeNumber int64) error { panic("unimplemented") } +func (m *SplitStorageMock) Split(splitName string) *dtos.SplitDTO { panic("unimplemented") } +func (m *SplitStorageMock) SplitNames() []string { panic("unimplemented") } +func (m *SplitStorageMock) Update([]dtos.SplitDTO, []dtos.SplitDTO, int64) { panic("unimplemented") } + +func (m *SplitStorageMock) TrafficTypeExists(trafficType string) bool { + args := m.Called(trafficType) + return args.Bool(0) +} + +func ref[T any](t T) *T { + return &t +} + +var _ scstorage.SplitStorage = (*SplitStorageMock)(nil) var _ evaluator.Interface = (*EvaluatorMock)(nil) var _ provisional.ImpressionManager = (*ImpressionManagerMock)(nil) var _ service.ImpressionsRecorder = (*ImpressionRecorderMock)(nil) diff --git a/splitio/sdk/validators.go b/splitio/sdk/validators.go index c60bdef..529343f 100644 --- a/splitio/sdk/validators.go +++ b/splitio/sdk/validators.go @@ -2,7 +2,9 @@ package sdk import ( "errors" + "strings" + "github.com/splitio/go-split-commons/v4/storage" "github.com/splitio/go-toolkit/v5/logging" ) @@ -10,9 +12,29 @@ import ( const MaxEventLength = 32768 var ErrEventTooBig = errors.New("The maximum size allowed for the properties is 32kb. Event not queued") +var ErrEmtpyTrafficType = errors.New("Traffic type cannot be empty") type Validator struct { logger logging.LoggerInterface + splits storage.SplitStorage +} + +func (i *Validator) validateTrafficType(trafficType string) (string, error) { + if len(trafficType) == 0 { + return "", ErrEmtpyTrafficType + } + + toLower := strings.ToLower(trafficType) + if toLower != trafficType { + i.logger.Warning("Track: traffic type should be all lowercase - converting string to lowercase") + } + + if !i.splits.TrafficTypeExists(toLower) { + i.logger.Warning("Track: traffic type " + toLower + " does not have any corresponding feature flags in this environment, " + + "make sure you’re tracking your events to a valid traffic type defined in the Split user interface") + } + + return toLower, nil } func (i *Validator) validateTrackProperties(properties map[string]interface{}) (map[string]interface{}, int, error) { diff --git a/splitio/sdk/workers/events_test.go b/splitio/sdk/workers/events_test.go new file mode 100644 index 0000000..3bc83ea --- /dev/null +++ b/splitio/sdk/workers/events_test.go @@ -0,0 +1,115 @@ +package workers + +import ( + "testing" + "time" + + "github.com/splitio/go-split-commons/v4/dtos" + "github.com/splitio/go-split-commons/v4/service" + "github.com/splitio/go-split-commons/v4/storage/inmemory" + "github.com/splitio/go-toolkit/v5/logging" + "github.com/splitio/splitd/splitio/sdk/conf" + sss "github.com/splitio/splitd/splitio/sdk/storage" + "github.com/splitio/splitd/splitio/sdk/types" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/mock" +) + +func TestEventsTask(t *testing.T) { + is, _ := sss.NewEventsQueue(100) + ts, _ := inmemory.NewTelemetryStorage() + logger := logging.NewLogger(nil) + rec := &EventsRecorderMock{} + + worker := NewEventsWorker(logger, ts, rec, is, &conf.Events{}) + + rec.On("Record", []dtos.EventDTO{ + {Key: "key1", TrafficTypeName: "user", EventTypeID: "checkin", Value: nil, Timestamp: 123, Properties: map[string]interface{}{"a": 2}}, + }, dtos.Metadata{SDKVersion: "php-1.2.3", MachineIP: "", MachineName: ""}). + Return(nil). + Once() + + rec.On("Record", []dtos.EventDTO{ + {Key: "key2", TrafficTypeName: "user", EventTypeID: "checkin", Value: nil, Timestamp: 123, Properties: map[string]interface{}{"a": 2}}, + }, dtos.Metadata{SDKVersion: "go-1.2.3", MachineIP: "", MachineName: ""}). + Return(nil). + Once() + + rec.On("Record", []dtos.EventDTO{ + {Key: "key3", TrafficTypeName: "user", EventTypeID: "checkout", Value: nil, Timestamp: 123, Properties: map[string]interface{}{"a": 2}}, + {Key: "key4", TrafficTypeName: "user", EventTypeID: "checkout", Value: nil, Timestamp: 123, Properties: map[string]interface{}{"a": 2}}, + }, dtos.Metadata{SDKVersion: "python-1.2.3", MachineIP: "", MachineName: ""}).Return(nil).Once() + + is.Push(types.ClientMetadata{ID: "i1", SdkVersion: "php-1.2.3"}, + dtos.EventDTO{Key: "key1", TrafficTypeName: "user", EventTypeID: "checkin", Value: nil, Timestamp: 123, Properties: map[string]interface{}{"a": 2}}, + ) + is.Push(types.ClientMetadata{ID: "i2", SdkVersion: "go-1.2.3"}, + dtos.EventDTO{Key: "key2", TrafficTypeName: "user", EventTypeID: "checkin", Value: nil, Timestamp: 123, Properties: map[string]interface{}{"a": 2}}, + ) + + worker.SynchronizeEvents(5000) + is.Push(types.ClientMetadata{ID: "i3", SdkVersion: "python-1.2.3"}, + dtos.EventDTO{Key: "key3", TrafficTypeName: "user", EventTypeID: "checkout", Value: nil, Timestamp: 123, Properties: map[string]interface{}{"a": 2}}, + dtos.EventDTO{Key: "key4", TrafficTypeName: "user", EventTypeID: "checkout", Value: nil, Timestamp: 123, Properties: map[string]interface{}{"a": 2}}, + ) + + worker.SynchronizeEvents(5000) + + rec.AssertExpectations(t) +} + +func TestEventsTaskNoParallelism(t *testing.T) { + + // to test this, we set up a Recorder that sleeps for 1 second and returns (no err). + // we one call to `SyncrhonizeImpressions()` wait for 500ms, and fire another one. + // the second one should finish immediately, (becase it does nothing). The second one + // should finish after 2 seconds + + es, _ := sss.NewEventsQueue(100) + ts, _ := inmemory.NewTelemetryStorage() + logger := logging.NewLogger(nil) + rec := &EventsRecorderMock{} + + worker := NewEventsWorker(logger, ts, rec, es, &conf.Events{}) + + rec.On("Record", mock.Anything, mock.Anything).Run(func(mock.Arguments) { time.Sleep(1 * time.Second) }).Return(nil).Twice() + + es.Push(types.ClientMetadata{ID: "i1", SdkVersion: "php-1.2.3"}, + dtos.EventDTO{Key: "key1", TrafficTypeName: "user", EventTypeID: "checkout", Value: nil, Timestamp: 123, Properties: map[string]interface{}{"a": 2}}, + ) + es.Push(types.ClientMetadata{ID: "i2", SdkVersion: "go-1.2.3"}, + dtos.EventDTO{Key: "key2", TrafficTypeName: "user", EventTypeID: "checkout", Value: nil, Timestamp: 123, Properties: map[string]interface{}{"a": 2}}, + ) + + done := make(chan struct{}) + + go func() { + worker.SynchronizeEvents(5000) + done <- struct{}{} + }() + + time.Sleep(500 * time.Millisecond) + assert.Nil(t, worker.SynchronizeEvents(5000)) + + // 2nd call has finished, assert that the first one hasn't: + select { + case <-done: // first call has finished, fail the test + assert.Fail(t, "first call shouldn't have finished yet") + default: + } + + <-done // blocking wait for 1st to finish + +} + +type EventsRecorderMock struct { + mock.Mock +} + +// Record implements service.EventsRecorder +func (m *EventsRecorderMock) Record(events []dtos.EventDTO, metadata dtos.Metadata) error { + args := m.Called(events, metadata) + return args.Error(0) +} + +var _ service.EventsRecorder = (*EventsRecorderMock)(nil) From e7ec4cca0904b9937cbfefe2ec0a2c77a7a24d63 Mon Sep 17 00:00:00 2001 From: Martin Redolatti Date: Fri, 1 Sep 2023 15:53:04 -0300 Subject: [PATCH 29/42] fix umask in docker image --- infra/entrypoint.sh | 4 ++++ splitio/link/link.go | 2 +- splitio/link/transfer/acceptor_test.go | 4 ++-- splitio/link/transfer/setup.go | 4 +--- 4 files changed, 8 insertions(+), 6 deletions(-) diff --git a/infra/entrypoint.sh b/infra/entrypoint.sh index e68e6a7..26d5920 100755 --- a/infra/entrypoint.sh +++ b/infra/entrypoint.sh @@ -34,6 +34,10 @@ accum=$(yq '.sdk.apikey = env(SPLITD_APIKEY) | .link.address = env(SPLITD_LINK_A # logger configs [ ! -z ${SPLITD_LOG_LEVEL+x} ] && accum=$(echo "${accum}" | yq '.logging.level = env(SPLITD_LOG_LEVEL)') # @} + +# Ensure that the socket-file is read-writable by anyone +umask 000 + # Output final config and start daemon echo "${accum}" > ${SPLITD_CFG_OUTPUT} exec env SPLITD_CONF_FILE="${SPLITD_CFG_OUTPUT}" "${SPLITD_EXEC}" $@ diff --git a/splitio/link/link.go b/splitio/link/link.go index 6b45eeb..3e8c09b 100644 --- a/splitio/link/link.go +++ b/splitio/link/link.go @@ -45,7 +45,7 @@ func Consumer(logger logging.LoggerInterface, opts *ConsumerOptions) (types.Clie return nil, fmt.Errorf("error building serializer") } - conn, err := transfer.NewClientConn(&opts.Transfer) + conn, err := transfer.NewClientConn(logger, &opts.Transfer) if err != nil { return nil, fmt.Errorf("errpr creating connection: %w", err) } diff --git a/splitio/link/transfer/acceptor_test.go b/splitio/link/transfer/acceptor_test.go index fced743..b6df647 100644 --- a/splitio/link/transfer/acceptor_test.go +++ b/splitio/link/transfer/acceptor_test.go @@ -59,7 +59,7 @@ func TestAcceptor(t *testing.T) { clientOpts.Address = serverSockFN clientOpts.ConnType = ConnTypeUnixStream - client1, err := NewClientConn(&clientOpts) + client1, err := NewClientConn(logger, &clientOpts) assert.Nil(t, err) assert.NotNil(t, client1) err = client1.SendMessage([]byte("some")) @@ -68,7 +68,7 @@ func TestAcceptor(t *testing.T) { assert.Nil(t, err) assert.Equal(t, []byte("thing"), recv) - client2, err := NewClientConn(&clientOpts) + client2, err := NewClientConn(logger, &clientOpts) assert.Nil(t, err) err = client2.SendMessage([]byte("some")) assert.Nil(t, err) // write doesn't fail. instead causes the transition of the socket to EOF state diff --git a/splitio/link/transfer/setup.go b/splitio/link/transfer/setup.go index fd83f8e..d827b65 100644 --- a/splitio/link/transfer/setup.go +++ b/splitio/link/transfer/setup.go @@ -39,7 +39,7 @@ func NewAcceptor(logger logging.LoggerInterface, o *Options, listenerConfig *Acc return newAcceptor(address, connFactory, logger, listenerConfig), nil } -func NewClientConn(o *Options) (RawConn, error) { +func NewClientConn(logger logging.LoggerInterface, o *Options) (RawConn, error) { var address net.Addr var framer framing.Interface @@ -64,7 +64,6 @@ func NewClientConn(o *Options) (RawConn, error) { type Options struct { ConnType ConnType Address string - Logger logging.LoggerInterface BufferSize int ReadTimeout time.Duration WriteTimeout time.Duration @@ -74,7 +73,6 @@ func DefaultOpts() Options { return Options{ ConnType: ConnTypeUnixSeqPacket, Address: "/var/run/splitd.sock", - Logger: logging.NewLogger(nil), BufferSize: 1024, ReadTimeout: 1 * time.Second, WriteTimeout: 1 * time.Second, From df57747b76cb1df56cb3d0901b5c6d0c522a7fa1 Mon Sep 17 00:00:00 2001 From: Martin Redolatti Date: Fri, 1 Sep 2023 16:39:46 -0300 Subject: [PATCH 30/42] more coverage --- Makefile | 10 +++++++++- splitio/link/client/client_test.go | 9 +++++++++ splitio/link/protocol/v1/errors.go | 7 ------- splitio/link/protocol/v1/errors_test.go | 22 ++++++++++++++++++++++ 4 files changed, 40 insertions(+), 8 deletions(-) create mode 100644 splitio/link/protocol/v1/errors_test.go diff --git a/Makefile b/Makefile index 45e924f..c405f28 100644 --- a/Makefile +++ b/Makefile @@ -9,6 +9,8 @@ SHELL = /usr/bin/env bash -o pipefail PLATFORM ?= PLATFORM_STR := $(if $(PLATFORM),--platform=$(PLATFORM),) +COVERAGE_FILE ?= coverage.out + VERSION := $(shell cat splitio/version.go | grep 'const Version' | sed 's/const Version = //' | tr -d '"') GO_FILES := $(shell find . -name "*.go") go.sum @@ -35,7 +37,11 @@ test: unit-tests entrypoint-test ## run go unit tests unit-tests: - $(GO) test ./... -count=1 -race -coverprofile=coverage.out + $(GO) test ./... -count=1 -race -coverprofile=$(COVERAGE_FILE) + +## display unit test coverage derived from last test run (use `make test display-coverage` for up-to-date results) +display-coverage: coverage.out + go tool cover -html=coverage.out ## run bash entrypoint tests entrypoint-test: splitd # requires splitd binary to generate a config and validate env var forwarding @@ -59,6 +65,8 @@ images_release: # entrypoints ## build release for binaires binaries_release: splitd-linux-amd64-$(VERSION).bin splitd-darwin-amd64-$(VERSION).bin splitd-linux-arm-$(VERSION).bin splitd-darwin-arm-$(VERSION).bin +$(COVERAGE_FILE): unit-tests + splitd-linux-amd64-$(VERSION).bin: $(GO_FILES) GOARCH=amd64 GOOS=linux $(GO) build -o $@ cmd/splitd/main.go diff --git a/splitio/link/client/client_test.go b/splitio/link/client/client_test.go index 388a9f2..f846da1 100644 --- a/splitio/link/client/client_test.go +++ b/splitio/link/client/client_test.go @@ -1,6 +1,8 @@ package client import ( + "os" + "strconv" "testing" "github.com/splitio/go-toolkit/v5/logging" @@ -37,3 +39,10 @@ func TestClientUnknownProtocol(t *testing.T) { assert.Nil(t, c) assert.ErrorContains(t, err, "unknown protocol") } + +func TestDefaultOpts(t *testing.T) { + do := DefaultOptions() + assert.Equal(t, strconv.Itoa(os.Getpid()), do.ID) + assert.Equal(t, false, do.ImpressionsFeedback) + assert.Equal(t, protocol.V1, do.Protocol) +} diff --git a/splitio/link/protocol/v1/errors.go b/splitio/link/protocol/v1/errors.go index 5c2d3d5..470fba3 100644 --- a/splitio/link/protocol/v1/errors.go +++ b/splitio/link/protocol/v1/errors.go @@ -1,16 +1,9 @@ package v1 import ( - "errors" "strconv" ) -var ( - ErrInternal = errors.New("internal agent error") - ErrOpcodeArgMismatch = errors.New("opcode doesn't match arguments type") - ErrIncorrectArgCount = errors.New("invalid argument count") -) - type RPCParseErrorCode int const ( diff --git a/splitio/link/protocol/v1/errors_test.go b/splitio/link/protocol/v1/errors_test.go new file mode 100644 index 0000000..03630f8 --- /dev/null +++ b/splitio/link/protocol/v1/errors_test.go @@ -0,0 +1,22 @@ +package v1 + +import ( + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestErrors(t *testing.T) { + + e1 := RPCParseError{Code: PECWrongArgCount} + assert.Equal(t, "wrong number of arguments for current opcode", e1.Error()) + + e2 := RPCParseError{Code: PECOpCodeMismatch} + assert.Equal(t, "opcode doesn't match the rpc whose arguments are being parsed", e2.Error()) + + e3 := RPCParseError{Code: PECInvalidArgType, Data: 2} + assert.Equal(t, "wrong argument type at index 2", e3.Error()) + + e4 := RPCParseError{Code: RPCParseErrorCode(777)} + assert.Equal(t, "unknown error", e4.Error()) +} From 87a1c017f7b95d60c8093492eaaebc56f2bf8ffe Mon Sep 17 00:00:00 2001 From: Martin Redolatti Date: Mon, 4 Sep 2023 16:57:05 -0300 Subject: [PATCH 31/42] entrypoint update --- .gitignore | 1 + CHANGES | 3 + Makefile | 13 +++- cmd/sdhelper/main.go | 38 ++++++++++ infra/entrypoint.sh | 39 ++++++++-- infra/sidecar.Dockerfile | 4 +- infra/test/test_entrypoint.sh | 66 +++++++++++++---- splitd.yaml.tpl | 53 ++++++++++---- splitio/conf/splitd.go | 101 ++++++++++++++++++++++++-- splitio/conf/splitd_test.go | 64 ++++++++++++---- splitio/link/protocol/protocol.go | 9 +++ splitio/link/serializer/serializer.go | 10 +++ splitio/link/transfer/setup.go | 11 +++ splitio/version.go | 2 +- 14 files changed, 354 insertions(+), 60 deletions(-) create mode 100644 CHANGES create mode 100644 cmd/sdhelper/main.go diff --git a/.gitignore b/.gitignore index 9057784..45f25e5 100644 --- a/.gitignore +++ b/.gitignore @@ -16,6 +16,7 @@ /splitd /splitcli +/sdhelper /splitd.yaml testcfg shared diff --git a/CHANGES b/CHANGES new file mode 100644 index 0000000..27e4d2e --- /dev/null +++ b/CHANGES @@ -0,0 +1,3 @@ +1.0.1: +- Add support for .track +- Fixed issue where impressions could take too long to evict even with the queue full diff --git a/Makefile b/Makefile index c405f28..67bc192 100644 --- a/Makefile +++ b/Makefile @@ -14,6 +14,8 @@ COVERAGE_FILE ?= coverage.out VERSION := $(shell cat splitio/version.go | grep 'const Version' | sed 's/const Version = //' | tr -d '"') GO_FILES := $(shell find . -name "*.go") go.sum +CONFIG_TEMPLATE ?= splitd.yaml.tpl + default: help ## generate go.sum from go.mod @@ -30,7 +32,9 @@ clean: splitd-darwin-arm-$(VERSION).bin ## build binaries for this platform -build: splitd splitcli +build: splitd splitcli sdhelper + + ## run all tests test: unit-tests entrypoint-test @@ -55,6 +59,13 @@ splitd: $(GO_FILES) splitcli: $(GO_FILES) go build -o splitcli cmd/splitcli/main.go +## regenerate config file template with defaults +$(CONFIG_TEMPLATE): $(SOURCES) sdhelper + ./sdhelper -command="gen-config-template" > $(CONFIG_TEMPLATE) +## build splitd helper (for code/doc generation purposes only) +sdhelper: $(GO_FILES) + go build -o sdhelper cmd/sdhelper/main.go + ## build docker images for sidecar images_release: # entrypoints $(DOCKER) build $(PLATFORM_STR) -t splitsoftware/splitd-sidecar:latest -t splitsoftware/splitd-sidecar:$(VERSION) -f infra/sidecar.Dockerfile . diff --git a/cmd/sdhelper/main.go b/cmd/sdhelper/main.go new file mode 100644 index 0000000..bbc78d4 --- /dev/null +++ b/cmd/sdhelper/main.go @@ -0,0 +1,38 @@ +package main + +import ( + "flag" + "fmt" + + "github.com/splitio/splitd/splitio/conf" + "gopkg.in/yaml.v3" +) + +func main() { + command := flag.String("command", "", "command to execute") + flag.Parse() + switch *command { + case "gen-config-template": + generateTemplateWithDefaults() + default: + fmt.Println("invalid command supplied") + } +} + +func generateTemplateWithDefaults() { + var cfg conf.Config + cfg.PopulateWithDefaults() + + raw, err := yaml.Marshal(cfg) + mustNotFail(err) + + fmt.Println("# vi:ft=yaml") // vim type hint + fmt.Println(string(raw)) + +} + +func mustNotFail(err error) { + if err != nil { + panic(err.Error()) + } +} diff --git a/infra/entrypoint.sh b/infra/entrypoint.sh index 26d5920..3133c23 100755 --- a/infra/entrypoint.sh +++ b/infra/entrypoint.sh @@ -17,13 +17,34 @@ accum=$(yq '.sdk.apikey = env(SPLITD_APIKEY) | .link.address = env(SPLITD_LINK_A # Generate a new yaml file by substituting values on the template with user-provided env-vars # @{ # sdk configs -[ ! -z ${SPLITD_AUTH_URL+x} ] && accum=$(echo "${accum}" | yq '.sdk.urls.auth = env(SPLITD_AUTH_URL)') -[ ! -z ${SPLITD_SDK_URL+x} ] && accum=$(echo "${accum}" | yq '.sdk.urls.sdk = env(SPLITD_SDK_URL)') -[ ! -z ${SPLITD_EVENTS_URL+x} ] && accum=$(echo "${accum}" | yq '.sdk.urls.events = env(SPLITD_EVENTS_URL)') -[ ! -z ${SPLITD_TELEMETRY_URL+x} ] && accum=$(echo "${accum}" | yq '.sdk.urls.telemetry = env(SPLITD_TELEMETRY_URL)') -[ ! -z ${SPLITD_STREAMING_URL+x} ] && accum=$(echo "${accum}" | yq '.sdk.urls.streaming = env(SPLITD_STREAMING_URL)') -[ ! -z ${SPLITD_STREAMING_ENABLED+x} ] && accum=$(echo "${accum}" | yq '.sdk.streamingEnabled = env(SPLITD_STREAMING_ENABLED)') -[ ! -z ${SPLITD_LABELS_ENABLED+x} ] && accum=$(echo "${accum}" | yq '.sdk.labelsEnabled = env(SPLITD_LABELS_ENABLED)') +[ ! -z ${SPLITD_AUTH_URL+x} ] && accum=$(echo "${accum}" | yq '.sdk.urls.auth = env(SPLITD_AUTH_URL)') +[ ! -z ${SPLITD_SDK_URL+x} ] && accum=$(echo "${accum}" | yq '.sdk.urls.sdk = env(SPLITD_SDK_URL)') +[ ! -z ${SPLITD_EVENTS_URL+x} ] && accum=$(echo "${accum}" | yq '.sdk.urls.events = env(SPLITD_EVENTS_URL)') +[ ! -z ${SPLITD_TELEMETRY_URL+x} ] && accum=$(echo "${accum}" | yq '.sdk.urls.telemetry = env(SPLITD_TELEMETRY_URL)') +[ ! -z ${SPLITD_STREAMING_URL+x} ] && accum=$(echo "${accum}" | yq '.sdk.urls.streaming = env(SPLITD_STREAMING_URL)') +[ ! -z ${SPLITD_STREAMING_ENABLED+x} ] && accum=$(echo "${accum}" | yq '.sdk.streamingEnabled = env(SPLITD_STREAMING_ENABLED)') +[ ! -z ${SPLITD_LABELS_ENABLED+x} ] && accum=$(echo "${accum}" | yq '.sdk.labelsEnabled = env(SPLITD_LABELS_ENABLED)') +[ ! -z ${SPLITD_FEATURE_FLAGS_SPLIT_REFRESH_SECS+x} ] && accum=$(\ + echo "${accum}" | yq '.sdk.featureFlags.splitRefreshSeconds = env(SPLITD_FEATURE_FLAGS_SPLIT_REFRESH_SECS)') +[ ! -z ${SPLITD_FEATURE_FLAGS_SPLIT_QUEUE_SIZE+x} ] && accum=$(\ + echo "${accum}" | yq '.sdk.featureFlags.splitNotificationQueueSize = env(SPLITD_FEATURE_FLAGS_SPLIT_QUEUE_SIZE)') +[ ! -z ${SPLITD_FEATURE_FLAGS_SEGMENT_REFRESH_SECS+x} ] && accum=$(\ + echo "${accum}" | yq '.sdk.featureFlags.segmentRefreshSeconds = env(SPLITD_FEATURE_FLAGS_SEGMENT_REFRESH_SECS)') +[ ! -z ${SPLITD_FEATURE_FLAGS_SEGMENT_QUEUE_SIZE+x} ] && accum=$(\ + echo "${accum}" | yq '.sdk.featureFlags.segmentNotificationQueueSize = env(SPLITD_FEATURE_FLAGS_SEGMENT_QUEUE_SIZE)') +[ ! -z ${SPLITD_FEATURE_FLAGS_SEGMENT_WORKER_COUNT+x} ] && accum=$(\ + echo "${accum}" | yq '.sdk.featureFlags.segmentUpdateWorkers = env(SPLITD_FEATURE_FLAGS_SEGMENT_WORKER_COUNT)') +[ ! -z ${SPLITD_FEATURE_FLAGS_SEGMENT_SYNC_BUFFER+x} ] && accum=$(\ + echo "${accum}" | yq '.sdk.featureFlags.segmentUpdateQueueSize = env(SPLITD_FEATURE_FLAGS_SEGMENT_SYNC_BUFFER)') +[ ! -z ${SPLITD_IMPRESSIONS_MODE+x} ] && accum=$(echo "${accum}" | yq '.sdk.impressions.mode = env(SPLITD_IMPRESSIONS_MODE)') +[ ! -z ${SPLITD_IMPRESSIONS_REFRESH_SECS+x} ] && accum=$(echo "${accum}" | yq '.sdk.impressions.refreshRateSeconds = env(SPLITD_IMPRESSIONS_REFRESH_SECS)') +[ ! -z ${SPLITD_IMPRESSIONS_QUEUE_SIZE+x} ] && accum=$(echo "${accum}" | yq '.sdk.impressions.queueSize = env(SPLITD_IMPRESSIONS_QUEUE_SIZE)') +[ ! -z ${SPLITD_IMPRESSIONS_COUNT_REFRESH_SECS+x} ] && accum=$(\ + echo "${accum}" | yq '.sdk.impressions.countRefreshRateSeconds = env(SPLITD_IMPRESSIONS_COUNT_REFRESH_SECS)') +[ ! -z ${SPLITD_IMPRESSIONS_OBSERVER_SIZE+x} ] && accum=$(echo "${accum}" | yq '.sdk.impressions.observerSize = env(SPLITD_IMPRESSIONS_OBSERVER_SIZE)') +[ ! -z ${SPLITD_EVENTS_REFRESH_SECS+x} ] && accum=$(echo "${accum}" | yq '.sdk.events.refreshRateSeconds = env(SPLITD_EVENTS_REFRESH_SECS)') +[ ! -z ${SPLITD_EVENTS_QUEUE_SIZE+x} ] && accum=$(echo "${accum}" | yq '.sdk.events.queueSize = env(SPLITD_EVENTS_QUEUE_SIZE)') + # link configs [ ! -z ${SPLITD_LINK_TYPE+x} ] && accum=$(echo "${accum}" | yq '.link.type = env(SPLITD_LINK_TYPE)') [ ! -z ${SPLITD_LINK_SERIALIZATION+x} ] && accum=$(echo "${accum}" | yq '.link.serialization = env(SPLITD_LINK_SERIALIZATION)') @@ -31,8 +52,10 @@ accum=$(yq '.sdk.apikey = env(SPLITD_APIKEY) | .link.address = env(SPLITD_LINK_A [ ! -z ${SPLITD_LINK_READ_TIMEOUT_MS+x} ] && accum=$(echo "${accum}" | yq '.link.readTimeoutMS = env(SPLITD_LINK_READ_TIMEOUT_MS)') [ ! -z ${SPLITD_LINK_WRITE_TIMEOUT_MS+x} ] && accum=$(echo "${accum}" | yq '.link.writeTimeoutMS = env(SPLITD_LINK_WRITE_TIMEOUT_MS)') [ ! -z ${SPLITD_LINK_ACCEPT_TIMEOUT_MS+x} ] && accum=$(echo "${accum}" | yq '.link.acceptTimeoutMS = env(SPLITD_LINK_ACCEPT_TIMEOUT_MS)') + # logger configs -[ ! -z ${SPLITD_LOG_LEVEL+x} ] && accum=$(echo "${accum}" | yq '.logging.level = env(SPLITD_LOG_LEVEL)') +[ ! -z ${SPLITD_LOG_LEVEL+x} ] && accum=$(echo "${accum}" | yq '.logging.level = env(SPLITD_LOG_LEVEL)') +[ ! -z ${SPLITD_LOG_OUTPUT+x} ] && accum=$(echo "${accum}" | yq '.logging.output = env(SPLITD_LOG_OUTPUT)') # @} # Ensure that the socket-file is read-writable by anyone diff --git a/infra/sidecar.Dockerfile b/infra/sidecar.Dockerfile index 2effa1e..a3450a9 100644 --- a/infra/sidecar.Dockerfile +++ b/infra/sidecar.Dockerfile @@ -5,7 +5,7 @@ RUN apk add git build-base bash WORKDIR /splitd COPY . . -RUN make clean splitd +RUN make clean splitd splitd.yaml.tpl # ----- Runner image FROM alpine:3.18 AS runner @@ -13,7 +13,7 @@ FROM alpine:3.18 AS runner RUN apk add gettext yq bash RUN mkdir -p /opt/splitd COPY --from=builder /splitd/splitd /opt/splitd -COPY splitd.yaml.tpl /opt/splitd +COPY --from=builder /splitd/splitd.yaml.tpl /opt/splitd COPY infra/entrypoint.sh /opt/splitd RUN chmod +x /opt/splitd/entrypoint.sh diff --git a/infra/test/test_entrypoint.sh b/infra/test/test_entrypoint.sh index 21be58e..f8a37fd 100755 --- a/infra/test/test_entrypoint.sh +++ b/infra/test/test_entrypoint.sh @@ -36,6 +36,22 @@ function testAllVars { export SPLITD_LINK_WRITE_TIMEOUT_MS=3 export SPLITD_LINK_ACCEPT_TIMEOUT_MS=4 export SPLITD_LOG_LEVEL="WARNING" + export SPLITD_LOG_OUTPUT="/dev/stderr" + + export SPLITD_FEATURE_FLAGS_SPLIT_REFRESH_SECS="1" + export SPLITD_FEATURE_FLAGS_SPLIT_QUEUE_SIZE="2" + export SPLITD_FEATURE_FLAGS_SEGMENT_REFRESH_SECS="3" + export SPLITD_FEATURE_FLAGS_SEGMENT_QUEUE_SIZE="4" + export SPLITD_FEATURE_FLAGS_SEGMENT_WORKER_COUNT="5" + export SPLITD_FEATURE_FLAGS_SEGMENT_SYNC_BUFFER="6" + export SPLITD_IMPRESSIONS_MODE="anotherMode" + export SPLITD_IMPRESSIONS_REFRESH_SECS="7" + export SPLITD_IMPRESSIONS_QUEUE_SIZE="8" + export SPLITD_IMPRESSIONS_COUNT_REFRESH_SECS="9" + export SPLITD_IMPRESSIONS_OBSERVER_SIZE="10" + export SPLITD_EVENTS_REFRESH_SECS="11" + export SPLITD_EVENTS_QUEUE_SIZE="12" + # Exec entrypoint [ -f "./testcfg" ] && rm ./testcfg @@ -45,23 +61,43 @@ function testAllVars { conf_json=$(bash "${SCRIPT_DIR}/../entrypoint.sh" -outputConfig | awk '/^Config:/ {print $2}') # Validate config output - assert_eq "\"somexxxxxxx\"" $(echo "$conf_json" | jq '.SDK.Apikey') "invalid apikey" - assert_eq "\"someAuthURL\"" $(echo "$conf_json" | jq '.SDK.URLs.Auth') "invalid auth url" - assert_eq "\"someSdkURL\"" $(echo "$conf_json" | jq '.SDK.URLs.SDK') "invalid sdk url" - assert_eq "\"someEventsURL\"" $(echo "$conf_json" | jq '.SDK.URLs.Events') "invalid events url" - assert_eq "\"someTelemetryURL\"" $(echo "$conf_json" | jq '.SDK.URLs.Telemetry') "invalid telemetry url" - assert_eq "\"someStreamingURL\"" $(echo "$conf_json" | jq '.SDK.URLs.Streaming') "invalid streaming url" + assert_eq '"somexxxxxxx"' $(echo "$conf_json" | jq '.SDK.Apikey') "incorrect apikey" + assert_eq '"someAuthURL"' $(echo "$conf_json" | jq '.SDK.URLs.Auth') "incorrect auth url" + assert_eq '"someSdkURL"' $(echo "$conf_json" | jq '.SDK.URLs.SDK') "incorrect sdk url" + assert_eq '"someEventsURL"' $(echo "$conf_json" | jq '.SDK.URLs.Events') "incorrect events url" + assert_eq '"someTelemetryURL"' $(echo "$conf_json" | jq '.SDK.URLs.Telemetry') "incorrect telemetry url" + assert_eq '"someStreamingURL"' $(echo "$conf_json" | jq '.SDK.URLs.Streaming') "incorrect streaming url" assert_eq "false" $(echo "$conf_json" | jq '.SDK.StreamingEnabled') "streaming should be enabled" assert_eq "false" $(echo "$conf_json" | jq '.SDK.LabelsEnabled') "labels should be enabled" - assert_eq "\"someLinkType\"" $(echo "$conf_json" | jq '.Link.Type') "invalid link type" - assert_eq "\"someLinkAddress\"" $(echo "$conf_json" | jq '.Link.Address') "invalid link address" - assert_eq "\"someSerialization\"" $(echo "$conf_json" | jq '.Link.Serialization') "invalid serialization" - assert_eq "1" $(echo "$conf_json" | jq '.Link.MaxSimultaneousConns') "invalid max simultaneous conns" - assert_eq "2" $(echo "$conf_json" | jq '.Link.ReadTimeoutMS') "invalid read timeout" - assert_eq "3" $(echo "$conf_json" | jq '.Link.WriteTimeoutMS') "invalid write timeout" - assert_eq "4" $(echo "$conf_json" | jq '.Link.AcceptTimeoutMS') "invalid accept timeout" - assert_eq "\"WARNING\"" $(echo "$conf_json" | jq '.Logger.Level') "invalid log level" -} + assert_eq "1" $(echo "$conf_json" | jq '.SDK.FeatureFlags.SplitRefreshRateSeconds') "incorrect split refresh rate" + assert_eq "2" $(echo "$conf_json" | jq '.SDK.FeatureFlags.SplitNotificationQueueSize') "incorrect split queue size" + assert_eq "3" $(echo "$conf_json" | jq '.SDK.FeatureFlags.SegmentRefreshRateSeconds') "incorrect segment refresh rate" + assert_eq "4" $(echo "$conf_json" | jq '.SDK.FeatureFlags.SegmentNotificationQueueSize') "incorrect segment queue size" + assert_eq "5" $(echo "$conf_json" | jq '.SDK.FeatureFlags.SegmentWorkerCount') "incorrect segment worker count" + assert_eq "6" $(echo "$conf_json" | jq '.SDK.FeatureFlags.SegmentWorkerBufferSize') "incorrect segment sync buffer" + assert_eq '"anotherMode"' $(echo "$conf_json" | jq '.SDK.Impressions.Mode') "incorrect impressions mode" + assert_eq "7" $(echo "$conf_json" | jq '.SDK.Impressions.RefreshRateSeconds') "incorrect impressions refresh rate" + assert_eq "8" $(echo "$conf_json" | jq '.SDK.Impressions.QueueSize') "incorrect impressions impressions queue size" + assert_eq "9" $(echo "$conf_json" | jq '.SDK.Impressions.CountRefreshRateSeconds') "incorrect impressions count refresh rate" + assert_eq "10" $(echo "$conf_json" | jq '.SDK.Impressions.ObserverSize') "incorrect impressions observer size" + assert_eq "11" $(echo "$conf_json" | jq '.SDK.Events.RefreshRateSeconds') "incorrect events refresh rate" + assert_eq "12" $(echo "$conf_json" | jq '.SDK.Events.QueueSize') "incorrect events queue size" + + # --- + assert_eq '"someLinkType"' $(echo "$conf_json" | jq '.Link.Type') "incorrect link type" + assert_eq '"someLinkAddress"' $(echo "$conf_json" | jq '.Link.Address') "incorrect link address" + assert_eq '"someSerialization"' $(echo "$conf_json" | jq '.Link.Serialization') "incorrect serialization" + assert_eq "1" $(echo "$conf_json" | jq '.Link.MaxSimultaneousConns') "incorrect max simultaneous conns" + assert_eq "2" $(echo "$conf_json" | jq '.Link.ReadTimeoutMS') "incorrect read timeout" + assert_eq "3" $(echo "$conf_json" | jq '.Link.WriteTimeoutMS') "incorrect write timeout" + assert_eq "4" $(echo "$conf_json" | jq '.Link.AcceptTimeoutMS') "incorrect accept timeout" + + # --- + + assert_eq '"WARNING"' $(echo "$conf_json" | jq '.Logger.Level') "incorrect log level" + assert_eq '"/dev/stderr"' $(echo "$conf_json" | jq '.Logger.Output') "incorrect log output" + +} testNoApikeyFails && testNoAddressFails && testAllVars && echo "entrypoint tests success." diff --git a/splitd.yaml.tpl b/splitd.yaml.tpl index 7c067d4..cfdaaf3 100644 --- a/splitd.yaml.tpl +++ b/splitd.yaml.tpl @@ -1,16 +1,43 @@ -# vim:ft=yaml +# vi:ft=yaml logging: - level: "ERROR" + level: error + output: /dev/stdout + rotationMaxFiles: null + rotationMaxBytesPerFile: null sdk: - apikey: "YOUR_API_KEY" - urls: - auth: "https://auth.split.io" - sdk: "https://sdk.split.io/api" - events: "https://events.split.io/api" - streaming: "https://streaming.split.io/sse" - telemetry: "https://telemetry.split.io/api/v1" - + apikey: + labelsEnabled: true + streamingEnabled: true + urls: + auth: https://auth.split.io + sdk: https://sdk.split.io/api + events: https://events.split.io/api + streaming: https://streaming.split.io/sse + telemetry: https://telemetry.split.io/api/v1 + featureFlags: + splitNotificationQueueSize: 5000 + splitRefreshSeconds: 30 + segmentNotificationQueueSize: 5000 + segmentRefreshSeconds: 60 + segmentUpdateWorkers: 20 + segmentUpdateQueueSize: 500 + impressions: + mode: optimized + refreshRateSeconds: 1800 + countRefreshRateSeconds: 3600 + queueSize: 8192 + observerSize: 500000 + events: + refreshRateSeconds: 60 + queueSize: 8192 link: - type: "unix-seqpacket" - address: "/var/run/splitd.sock" - serialization: "msgpack" + type: unix-seqpacket + address: /var/run/splitd.sock + maxSimultaneousConns: 32 + readTimeoutMS: 1000 + writeTimeoutMS: 1000 + acceptTimeoutMS: 1000 + serialization: msgpack + bufferSize: 1024 + protocol: v1 + diff --git a/splitio/conf/splitd.go b/splitio/conf/splitd.go index e1fc39e..9e5f2ed 100644 --- a/splitio/conf/splitd.go +++ b/splitio/conf/splitd.go @@ -17,7 +17,12 @@ import ( "gopkg.in/yaml.v3" ) -const defaultConfigFN = "/etc/splitd.yaml" +const ( + defaultConfigFN = "/etc/splitd.yaml" + apikeyPlaceHolder = "" + defaultLogLevel = "error" + defaultLogOutput = "/dev/stdout" +) type Config struct { Logger Logger `yaml:"logging"` @@ -49,6 +54,12 @@ func (c *Config) parse(fn string) error { return nil } +func (c *Config) PopulateWithDefaults() { + c.SDK.PopulateWithDefaults() + c.Link.PopulateWithDefaults() + c.Logger.PopulateWithDefaults() +} + type Link struct { Type *string `yaml:"type"` Address *string `yaml:"address"` @@ -61,6 +72,19 @@ type Link struct { Protocol *string `yaml:"protocol"` } +func (l *Link) PopulateWithDefaults() { + linkOpts := link.DefaultListenerOptions() + l.Address = ref(linkOpts.Transfer.Address) + l.Type = ref(linkOpts.Transfer.ConnType.String()) + l.ReadTimeoutMS = ref(int(linkOpts.Transfer.ReadTimeout.Milliseconds())) + l.WriteTimeoutMS = ref(int(linkOpts.Transfer.WriteTimeout.Milliseconds())) + l.AcceptTimeoutMS = ref(int(linkOpts.Acceptor.AcceptTimeout.Milliseconds())) + l.BufferSize = ref(linkOpts.Transfer.BufferSize) + l.MaxSimultaneousConns = ref(linkOpts.Acceptor.MaxSimultaneousConnections) + l.Protocol = ref(linkOpts.Protocol.String()) + l.Serialization = ref(linkOpts.Serialization.String()) +} + func (l *Link) ToListenerOpts() (*link.ListenerOptions, error) { opts := link.DefaultListenerOptions() @@ -101,6 +125,18 @@ type SDK struct { URLs URLs `yaml:"urls"` FeatureFlags FeatureFlags `yaml:"featureFlags"` Impressions Impressions `yaml:"impressions"` + Events Events `yank:"events"` +} + +func (s *SDK) PopulateWithDefaults() { + cfg := sdkConf.DefaultConfig() + s.Apikey = apikeyPlaceHolder + s.LabelsEnabled = ref(cfg.LabelsEnabled) + s.StreamingEnabled = ref(cfg.StreamingEnabled) + s.URLs.PopulateWithDefaults() + s.FeatureFlags.PopulateWithDefaults() + s.Impressions.PopulateWithDefaults() + s.Events.PopulateWithDefaults() } type FeatureFlags struct { @@ -108,8 +144,18 @@ type FeatureFlags struct { SplitRefreshRateSeconds *int `yaml:"splitRefreshSeconds"` SegmentNotificationQueueSize *int `yaml:"segmentNotificationQueueSize"` SegmentRefreshRateSeconds *int `yaml:"segmentRefreshSeconds"` - SegmentUpdateWorkers *int `yaml:"segmentUpdateWorkers"` - SegmentUpdateQueueSize *int `yaml:"segmentUpdateQueueSize"` + SegmentWorkerCount *int `yaml:"segmentUpdateWorkers"` + SegmentWorkerBufferSize *int `yaml:"segmentUpdateQueueSize"` +} + +func (f *FeatureFlags) PopulateWithDefaults() { + ffOpts := sdkConf.DefaultConfig() + f.SegmentNotificationQueueSize = ref(ffOpts.Segments.UpdateBufferSize) + f.SegmentRefreshRateSeconds = ref(int(ffOpts.Segments.SyncPeriod.Seconds())) + f.SegmentWorkerBufferSize = ref(ffOpts.Segments.QueueSize) + f.SegmentWorkerCount = ref(ffOpts.Segments.WorkerCount) + f.SplitNotificationQueueSize = ref(ffOpts.Splits.UpdateBufferSize) + f.SplitRefreshRateSeconds = ref(int(ffOpts.Splits.SyncPeriod.Seconds())) } type Impressions struct { @@ -118,7 +164,28 @@ type Impressions struct { CountRefreshRateSeconds *int `yaml:"countRefreshRateSeconds"` QueueSize *int `yaml:"queueSize"` ObserverSize *int `yaml:"observerSize"` - Watermark *int `yaml:"watermark"` + Watermark *int `yaml:"watermark,omitempty"` // TODO(mredolatti) remove omitempty when fully implemented +} + +func (i *Impressions) PopulateWithDefaults() { + cfg := sdkConf.DefaultConfig().Impressions + i.CountRefreshRateSeconds = ref(int(cfg.CountSyncPeriod.Seconds())) + i.Mode = ref(cfg.Mode) + i.ObserverSize = ref(cfg.ObserverSize) + i.RefreshRateSeconds = ref(int(cfg.SyncPeriod.Seconds())) + i.QueueSize = ref(cfg.QueueSize) +} + +type Events struct { + RefreshRateSeconds *int `yaml:"refreshRateSeconds"` + QueueSize *int `yaml:"queueSize"` + Watermark *int `yaml:"watermark,omitempty"` // TODO(mredolatti) remove omitempty when fully implemented +} + +func (e *Events) PopulateWithDefaults() { + cfg := sdkConf.DefaultConfig().Events + e.RefreshRateSeconds = ref(int(cfg.SyncPeriod.Seconds())) + e.QueueSize = ref(cfg.QueueSize) } func (s *SDK) ToSDKConf() *sdkConf.Config { @@ -129,14 +196,16 @@ func (s *SDK) ToSDKConf() *sdkConf.Config { cc.SetIfNotEmpty(&cfg.Splits.UpdateBufferSize, s.FeatureFlags.SplitNotificationQueueSize) cc.MapIfNotNil(&cfg.Splits.SyncPeriod, s.FeatureFlags.SplitRefreshRateSeconds, durationFromSeconds) cc.SetIfNotEmpty(&cfg.Segments.UpdateBufferSize, s.FeatureFlags.SegmentNotificationQueueSize) - cc.SetIfNotEmpty(&cfg.Segments.QueueSize, s.FeatureFlags.SegmentUpdateQueueSize) - cc.SetIfNotEmpty(&cfg.Segments.WorkerCount, s.FeatureFlags.SegmentUpdateWorkers) + cc.SetIfNotEmpty(&cfg.Segments.QueueSize, s.FeatureFlags.SegmentWorkerBufferSize) + cc.SetIfNotEmpty(&cfg.Segments.WorkerCount, s.FeatureFlags.SegmentWorkerCount) cc.MapIfNotNil(&cfg.Segments.SyncPeriod, s.FeatureFlags.SegmentRefreshRateSeconds, durationFromSeconds) cc.SetIfNotEmpty(&cfg.Impressions.Mode, s.Impressions.Mode) cc.SetIfNotEmpty(&cfg.Impressions.ObserverSize, s.Impressions.ObserverSize) cc.SetIfNotEmpty(&cfg.Impressions.QueueSize, s.Impressions.QueueSize) cc.MapIfNotNil(&cfg.Impressions.SyncPeriod, s.Impressions.RefreshRateSeconds, durationFromSeconds) cc.MapIfNotNil(&cfg.Impressions.CountSyncPeriod, s.Impressions.CountRefreshRateSeconds, durationFromSeconds) + cc.SetIfNotEmpty(&cfg.Events.QueueSize, s.Events.QueueSize) + cc.MapIfNotNil(&cfg.Events.SyncPeriod, s.Events.RefreshRateSeconds, durationFromSeconds) s.URLs.updateSDKConfURLs(&cfg.URLs) return cfg } @@ -157,13 +226,27 @@ func (u *URLs) updateSDKConfURLs(dst *sdkConf.URLs) { cc.SetIfNotNil(&dst.Telemetry, u.Telemetry) } +func (u *URLs) PopulateWithDefaults() { + cfg := sdkConf.DefaultConfig().URLs + u.Auth = ref(cfg.Auth) + u.Events = ref(cfg.Events) + u.SDK = ref(cfg.SDK) + u.Streaming = ref(cfg.Streaming) + u.Telemetry = ref(cfg.Telemetry) +} + type Logger struct { Level *string `yaml:"level"` - Output *string `yaml:"file"` + Output *string `yaml:"output"` RotationMaxFiles *int `yaml:"rotationMaxFiles"` RotationMaxBytesPerFile *int `yaml:"rotationMaxBytesPerFile"` } +func (l *Logger) PopulateWithDefaults() { + l.Level = ref(defaultLogLevel) + l.Output = ref(defaultLogOutput) +} + func (l *Logger) ToLoggerOptions() (*logging.LoggerOptions, error) { writer, err := sdlogging.GetWriter(l.Output, l.RotationMaxFiles, l.RotationMaxBytesPerFile) @@ -197,3 +280,7 @@ func ReadConfig() (*Config, error) { var c Config return &c, c.parse(cfgFN) } + +func ref[T any](v T) *T { + return &v +} diff --git a/splitio/conf/splitd_test.go b/splitio/conf/splitd_test.go index 9683d05..b410e18 100644 --- a/splitio/conf/splitd_test.go +++ b/splitio/conf/splitd_test.go @@ -122,8 +122,8 @@ func TestSDK(t *testing.T) { SplitRefreshRateSeconds: ref(2), SegmentNotificationQueueSize: ref(3), SegmentRefreshRateSeconds: ref(4), - SegmentUpdateWorkers: ref(5), - SegmentUpdateQueueSize: ref(6), + SegmentWorkerCount: ref(5), + SegmentWorkerBufferSize: ref(6), }, Impressions: Impressions{ Mode: ref("optimized"), @@ -145,18 +145,56 @@ func TestSDK(t *testing.T) { expected.URLs.Telemetry = "telemetryURL" expected.Splits.UpdateBufferSize = 1 expected.Splits.SyncPeriod = 2 * time.Second - expected.Segments.UpdateBufferSize = 3 - expected.Segments.SyncPeriod = 4 * time.Second - expected.Segments.WorkerCount = 5 - expected.Segments.QueueSize = 6 - expected.Impressions.Mode = "optimized" - expected.Impressions.SyncPeriod = 1 * time.Second - expected.Impressions.CountSyncPeriod = 2 * time.Second - expected.Impressions.QueueSize = 3 - expected.Impressions.ObserverSize = 4 + expected.Segments.UpdateBufferSize = 3 + expected.Segments.SyncPeriod = 4 * time.Second + expected.Segments.WorkerCount = 5 + expected.Segments.QueueSize = 6 + expected.Impressions.Mode = "optimized" + expected.Impressions.SyncPeriod = 1 * time.Second + expected.Impressions.CountSyncPeriod = 2 * time.Second + expected.Impressions.QueueSize = 3 + expected.Impressions.ObserverSize = 4 assert.Equal(t, expected, sdkCFG.ToSDKConf()) } -func ref[T any](v T) *T { - return &v +func TestDefaultConf(t *testing.T) { + var c Config + c.PopulateWithDefaults() + + sdkConf := conf.DefaultConfig() + assert.Equal(t, apikeyPlaceHolder, c.SDK.Apikey) + assert.Equal(t, sdkConf.LabelsEnabled, *c.SDK.LabelsEnabled) + assert.Equal(t, sdkConf.StreamingEnabled, *c.SDK.StreamingEnabled) + assert.Equal(t, sdkConf.URLs.Auth, *c.SDK.URLs.Auth) + assert.Equal(t, sdkConf.URLs.SDK, *c.SDK.URLs.SDK) + assert.Equal(t, sdkConf.URLs.Events, *c.SDK.URLs.Events) + assert.Equal(t, sdkConf.URLs.Telemetry, *c.SDK.URLs.Telemetry) + assert.Equal(t, sdkConf.URLs.Streaming, *c.SDK.URLs.Streaming) + assert.Equal(t, sdkConf.Splits.SyncPeriod.Seconds(), float64(*c.SDK.FeatureFlags.SplitRefreshRateSeconds)) + assert.Equal(t, sdkConf.Splits.UpdateBufferSize, int(*c.SDK.FeatureFlags.SplitNotificationQueueSize)) + assert.Equal(t, sdkConf.Segments.SyncPeriod.Seconds(), float64(*c.SDK.FeatureFlags.SegmentRefreshRateSeconds)) + assert.Equal(t, sdkConf.Segments.UpdateBufferSize, *c.SDK.FeatureFlags.SegmentNotificationQueueSize) + assert.Equal(t, sdkConf.Segments.QueueSize, *c.SDK.FeatureFlags.SegmentWorkerBufferSize) + assert.Equal(t, sdkConf.Segments.WorkerCount, *c.SDK.FeatureFlags.SegmentWorkerCount) + assert.Equal(t, sdkConf.Impressions.Mode, *c.SDK.Impressions.Mode) + assert.Equal(t, sdkConf.Impressions.ObserverSize, *c.SDK.Impressions.ObserverSize) + assert.Equal(t, sdkConf.Impressions.QueueSize, *c.SDK.Impressions.QueueSize) + assert.Equal(t, sdkConf.Impressions.CountSyncPeriod.Seconds(), float64(*c.SDK.Impressions.CountRefreshRateSeconds)) + assert.Equal(t, sdkConf.Impressions.SyncPeriod.Seconds(), float64(*c.SDK.Impressions.RefreshRateSeconds)) + assert.Equal(t, sdkConf.Events.QueueSize, *c.SDK.Events.QueueSize) + assert.Equal(t, sdkConf.Events.SyncPeriod.Seconds(), float64(*c.SDK.Events.RefreshRateSeconds)) + + linkConf := link.DefaultListenerOptions() + assert.Equal(t, linkConf.Protocol.String(), *c.Link.Protocol) + assert.Equal(t, linkConf.Serialization.String(), *c.Link.Serialization) + assert.Equal(t, linkConf.Transfer.ConnType.String(), *c.Link.Type) + assert.Equal(t, linkConf.Transfer.Address, *c.Link.Address) + assert.Equal(t, linkConf.Transfer.BufferSize, *c.Link.BufferSize) + assert.Equal(t, linkConf.Transfer.ReadTimeout.Milliseconds(), int64(*c.Link.ReadTimeoutMS)) + assert.Equal(t, linkConf.Transfer.WriteTimeout.Milliseconds(), int64(*c.Link.WriteTimeoutMS)) + assert.Equal(t, linkConf.Acceptor.AcceptTimeout.Milliseconds(), int64(*c.Link.AcceptTimeoutMS)) + assert.Equal(t, linkConf.Acceptor.MaxSimultaneousConnections, *c.Link.MaxSimultaneousConns) + + assert.Equal(t, defaultLogLevel, *c.Logger.Level) + assert.Equal(t, defaultLogOutput, *c.Logger.Output) } diff --git a/splitio/link/protocol/protocol.go b/splitio/link/protocol/protocol.go index 528c0db..6de255a 100644 --- a/splitio/link/protocol/protocol.go +++ b/splitio/link/protocol/protocol.go @@ -2,6 +2,15 @@ package protocol type Version byte +func (v Version) String() string { + switch v { + case V1: + return "v1" + default: + return "invalid-version" + } +} + const ( V1 Version = 0x01 ) diff --git a/splitio/link/serializer/serializer.go b/splitio/link/serializer/serializer.go index 24c7554..aa63b7c 100644 --- a/splitio/link/serializer/serializer.go +++ b/splitio/link/serializer/serializer.go @@ -3,6 +3,16 @@ package serializer import "fmt" type Mechanism int + +func (m Mechanism) String() string { + switch m { + case MsgPack: + return "msgpack" + default: + return "invalid-serialization" + } +} + const ( MsgPack Mechanism = 1 ) diff --git a/splitio/link/transfer/setup.go b/splitio/link/transfer/setup.go index d827b65..1f84380 100644 --- a/splitio/link/transfer/setup.go +++ b/splitio/link/transfer/setup.go @@ -12,6 +12,17 @@ import ( type ConnType int +func (c ConnType) String() string { + switch c { + case ConnTypeUnixSeqPacket: + return "unix-seqpacket" + case ConnTypeUnixStream: + return "unix-stream" + default: + return "invalid-socket-type" + } +} + const ( ConnTypeUnixSeqPacket ConnType = 1 ConnTypeUnixStream ConnType = 2 diff --git a/splitio/version.go b/splitio/version.go index 8024fb3..2e71213 100644 --- a/splitio/version.go +++ b/splitio/version.go @@ -1,3 +1,3 @@ package splitio -const Version = "1.0.0" +const Version = "1.0.1" From 7a519a56397409bfa609562f3f64fa4ad7f57cdd Mon Sep 17 00:00:00 2001 From: Martin Redolatti Date: Mon, 4 Sep 2023 17:26:01 -0300 Subject: [PATCH 32/42] fix yaml cfg test --- splitio/conf/splitd_test.go | 22 ++++------------------ 1 file changed, 4 insertions(+), 18 deletions(-) diff --git a/splitio/conf/splitd_test.go b/splitio/conf/splitd_test.go index b410e18..5d466ec 100644 --- a/splitio/conf/splitd_test.go +++ b/splitio/conf/splitd_test.go @@ -24,26 +24,12 @@ func TestConfig(t *testing.T) { parts := strings.Split(filename, string(filepath.Separator)) dir := strings.Join(parts[:len(parts)-3], string(filepath.Separator)) + expected := Config{} + expected.PopulateWithDefaults() cfg = Config{} + assert.Nil(t, cfg.parse(dir+string(filepath.Separator)+"splitd.yaml.tpl")) - assert.Equal(t, Config{ - Logger: Logger{Level: ref("ERROR")}, - SDK: SDK{ - Apikey: "YOUR_API_KEY", - URLs: URLs{ - Auth: ref("https://auth.split.io"), - SDK: ref("https://sdk.split.io/api"), - Events: ref("https://events.split.io/api"), - Streaming: ref("https://streaming.split.io/sse"), - Telemetry: ref("https://telemetry.split.io/api/v1"), - }, - }, - Link: Link{ - Type: ref("unix-seqpacket"), - Address: ref("/var/run/splitd.sock"), - Serialization: ref("msgpack"), - }, - }, cfg) + assert.Equal(t, expected, cfg) assert.Error(t, cfg.parse("someNonexistantFile")) assert.Error(t, cfg.parse(dir+string(filepath.Separator)+"Makefile")) From 2ac2b2749cc433d742c23ba0ed35f230273ed9e5 Mon Sep 17 00:00:00 2001 From: Martin Redolatti Date: Mon, 4 Sep 2023 17:31:38 -0300 Subject: [PATCH 33/42] more ci updates --- .github/workflows/ci.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 5dd4698..29830d4 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -43,7 +43,7 @@ jobs: - name: Setup Go version uses: actions/setup-go@v4 with: - go-version: '^1.19.1' + go-version: '^1.20.7' - name: Build binaries for host machine run: make splitd splitcli @@ -55,7 +55,7 @@ jobs: run: make binaries_release - name: SonarQube Scan - uses: SonarSource/sonarcloud-github-action@v1.9.1 + uses: SonarSource/sonarcloud-github-action@v2.0.0 env: SONAR_TOKEN: ${{ secrets.SONARQUBE_TOKEN }} GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} From 4607ab90bb28bb60a4dde0e049e4c6e39959dcaa Mon Sep 17 00:00:00 2001 From: Martin Redolatti Date: Mon, 4 Sep 2023 18:45:56 -0300 Subject: [PATCH 34/42] coverage --- splitio/link/protocol/protocol_test.go | 11 +++++++++++ splitio/link/serializer/serializer_test.go | 19 +++++++++++++++++++ splitio/link/transfer/setup_test.go | 12 ++++++++++++ 3 files changed, 42 insertions(+) create mode 100644 splitio/link/protocol/protocol_test.go create mode 100644 splitio/link/serializer/serializer_test.go create mode 100644 splitio/link/transfer/setup_test.go diff --git a/splitio/link/protocol/protocol_test.go b/splitio/link/protocol/protocol_test.go new file mode 100644 index 0000000..5e70eb0 --- /dev/null +++ b/splitio/link/protocol/protocol_test.go @@ -0,0 +1,11 @@ +package protocol + +import ( + "github.com/stretchr/testify/assert" + "testing" +) + +func TestProtocolVersion(t *testing.T) { + assert.Equal(t, "v1", Version(V1).String()) + assert.Equal(t, "invalid-version", Version(5).String()) +} diff --git a/splitio/link/serializer/serializer_test.go b/splitio/link/serializer/serializer_test.go new file mode 100644 index 0000000..43f29d1 --- /dev/null +++ b/splitio/link/serializer/serializer_test.go @@ -0,0 +1,19 @@ +package serializer + +import ( + "github.com/stretchr/testify/assert" + "testing" +) + +func TestConnType(t *testing.T) { + assert.Equal(t, "msgpack", MsgPack.String()) + assert.Equal(t, "invalid-serialization", Mechanism(123).String()) +} + +func TestSetup(t *testing.T) { + _, err := Setup(MsgPack) + assert.Nil(t, err) + + _, err = Setup(Mechanism(123)) + assert.ErrorContains(t, err, "unknown serialization mechanism") +} diff --git a/splitio/link/transfer/setup_test.go b/splitio/link/transfer/setup_test.go new file mode 100644 index 0000000..14b7152 --- /dev/null +++ b/splitio/link/transfer/setup_test.go @@ -0,0 +1,12 @@ +package transfer + +import ( + "github.com/stretchr/testify/assert" + "testing" +) + +func TestConnType(t *testing.T) { + assert.Equal(t, "unix-seqpacket", ConnTypeUnixSeqPacket.String()) + assert.Equal(t, "unix-stream", ConnTypeUnixStream.String()) + assert.Equal(t, "invalid-socket-type", ConnType(123).String()) +} From c19870d3d9010ebc288a6b4d0e0207eb03c006f7 Mon Sep 17 00:00:00 2001 From: Martin Redolatti Date: Mon, 4 Sep 2023 23:07:16 -0300 Subject: [PATCH 35/42] publish this build in jfrog --- .github/workflows/docker.yml | 16 ++-------------- 1 file changed, 2 insertions(+), 14 deletions(-) diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index 74d8028..96dd454 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -35,18 +35,6 @@ jobs: with: context: . file: "infra/sidecar.Dockerfile" - push: ${{ github.event_name == 'push' }} + push: true + # push: ${{ github.event_name == 'push' }} tags: splitio-docker.jfrog.io/${{ github.event.repository.name }}/sidecar:${{ github.sha }} - - deploy: - name: Deploy to testing - if: ${{ github.event_name == 'push' }} - runs-on: ubuntu-latest - needs: build-docker-image - steps: - - name: Deploy to testing - run: | - curl --header "Content-Type: application/json" \ - --request PUT \ - --data '{ "service": "${{ github.event.repository.name }}", "environment": "testing", "tag": "${{ github.sha }}", "token": "${{ secrets.R2D2_SLACK_TOKEN_STAGE }}"}' \ - https://r2d2.split-stage.io/deployment From c51c4cabfd43ba917ae5245d145fe11d1682d5e6 Mon Sep 17 00:00:00 2001 From: Martin Redolatti Date: Mon, 4 Sep 2023 23:11:28 -0300 Subject: [PATCH 36/42] publish this build in jfrog --- .github/workflows/docker.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index 96dd454..152e583 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -20,7 +20,7 @@ jobs: runs-on: ubuntu-latest steps: - name: Login to Artifactory - if: ${{ github.event_name == 'push' }} + # if: ${{ github.event_name == 'push' }} uses: docker/login-action@v2 with: registry: splitio-docker.jfrog.io From f3bce29579c0e1b3be577863f20f1b2be274abd8 Mon Sep 17 00:00:00 2001 From: Martin Redolatti Date: Tue, 5 Sep 2023 11:58:40 -0300 Subject: [PATCH 37/42] split docker images between stable & unstable --- .github/workflows/docker.yml | 4 ++-- .github/workflows/unstable.yml | 34 ++++++++++++++++++++++++++++++++++ 2 files changed, 36 insertions(+), 2 deletions(-) create mode 100644 .github/workflows/unstable.yml diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index 152e583..50b3dca 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -20,7 +20,7 @@ jobs: runs-on: ubuntu-latest steps: - name: Login to Artifactory - # if: ${{ github.event_name == 'push' }} + if: ${{ github.event_name == 'push' }} uses: docker/login-action@v2 with: registry: splitio-docker.jfrog.io @@ -36,5 +36,5 @@ jobs: context: . file: "infra/sidecar.Dockerfile" push: true - # push: ${{ github.event_name == 'push' }} + push: ${{ github.event_name == 'push' }} tags: splitio-docker.jfrog.io/${{ github.event.repository.name }}/sidecar:${{ github.sha }} diff --git a/.github/workflows/unstable.yml b/.github/workflows/unstable.yml new file mode 100644 index 0000000..b35fee1 --- /dev/null +++ b/.github/workflows/unstable.yml @@ -0,0 +1,34 @@ +name: unstable +on: + push: + branches-ignore: + - main + +jobs: + push-docker-image: + name: Build and Push Docker Image + runs-on: ubuntu-latest + steps: + - name: Login to Artifactory + uses: docker/login-action@v2 + with: + registry: splitio-docker-dev.jfrog.io + username: ${{ secrets.ARTIFACTORY_DOCKER_USER }} + password: ${{ secrets.ARTIFACTORY_DOCKER_PASS }} + + - name: Checkout code + uses: actions/checkout@v3 + + - name: Set VERSION env + run: echo "VERSION=$(cat splitio/version.go | grep 'Version =' | awk '{print $3}' | tr -d '"')" >> $GITHUB_ENV + + - name: Get short hash + run: echo "SHORT_SHA=$(git rev-parse --short HEAD)" >> $GITHUB_ENV + + - name: Docker Build and Push + uses: docker/build-push-action@v4 + with: + context: . + file: "infra/sidecar.Dockerfile" + push: true + tags: splitio-docker-dev.jfrog.io/splitd/sidecar:${{ env.SHORT_SHA }},splitio-docker-dev.jfrog.io/splitd/sidecar:${{ env.VERSION }} From 18fd33a44f51bdb748fd5658b284692bd90e7242 Mon Sep 17 00:00:00 2001 From: Martin Redolatti Date: Tue, 5 Sep 2023 12:08:08 -0300 Subject: [PATCH 38/42] fix tag --- .github/workflows/ci.yml | 2 +- .github/workflows/unstable.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 29830d4..9a92181 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -23,7 +23,7 @@ jobs: fetch-depth: 0 - name: Set VERSION env - run: echo "VERSION=$(cat splitio/version.go | grep 'Version =' | awk '{print $3}' | tr -d '"')" >> $GITHUB_ENV + run: echo "VERSION=$(cat splitio/version.go | grep 'Version =' | awk '{print $4}' | tr -d '"')" >> $GITHUB_ENV - name: Version validation if: ${{ github.event_name == 'pull_request' }} diff --git a/.github/workflows/unstable.yml b/.github/workflows/unstable.yml index b35fee1..717241f 100644 --- a/.github/workflows/unstable.yml +++ b/.github/workflows/unstable.yml @@ -20,7 +20,7 @@ jobs: uses: actions/checkout@v3 - name: Set VERSION env - run: echo "VERSION=$(cat splitio/version.go | grep 'Version =' | awk '{print $3}' | tr -d '"')" >> $GITHUB_ENV + run: echo "VERSION=$(cat splitio/version.go | grep 'Version =' | awk '{print $4}' | tr -d '"')" >> $GITHUB_ENV - name: Get short hash run: echo "SHORT_SHA=$(git rev-parse --short HEAD)" >> $GITHUB_ENV From 827a3591581b9bbaf25f0128f023056a7ce4adbe Mon Sep 17 00:00:00 2001 From: Martin Redolatti Date: Tue, 5 Sep 2023 12:32:03 -0300 Subject: [PATCH 39/42] fix actions --- .github/workflows/ci.yml | 4 ++-- .github/workflows/docker.yml | 10 ++-------- .github/workflows/unstable.yml | 4 ++-- 3 files changed, 6 insertions(+), 12 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 9a92181..0e58962 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -18,7 +18,7 @@ jobs: runs-on: ubuntu-latest steps: - name: Checkout code - uses: actions/checkout@v3 + uses: actions/checkout@v4 with: fetch-depth: 0 @@ -30,7 +30,7 @@ jobs: uses: mukunku/tag-exists-action@v1.2.0 id: checkTag with: - tag: ${{ env.VERSION }} + tag: v${{ env.VERSION }} env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index 50b3dca..f21984c 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -3,11 +3,6 @@ name: docker on: push: branches: - - dev - - main - pull_request: - branches: - - dev - main concurrency: @@ -28,13 +23,12 @@ jobs: password: ${{ secrets.ARTIFACTORY_DOCKER_PASS }} - name: Checkout code - uses: actions/checkout@v3 + uses: actions/checkout@v4 - name: Docker build and push uses: docker/build-push-action@v4 with: context: . file: "infra/sidecar.Dockerfile" - push: true push: ${{ github.event_name == 'push' }} - tags: splitio-docker.jfrog.io/${{ github.event.repository.name }}/sidecar:${{ github.sha }} + tags: splitio-docker.jfrog.io/splitd/sidecar:${{ github.sha }},splitio-docker.jfrog.io/splitd/sidecar:${{ env.VERSION }} diff --git a/.github/workflows/unstable.yml b/.github/workflows/unstable.yml index 717241f..9b03154 100644 --- a/.github/workflows/unstable.yml +++ b/.github/workflows/unstable.yml @@ -17,7 +17,7 @@ jobs: password: ${{ secrets.ARTIFACTORY_DOCKER_PASS }} - name: Checkout code - uses: actions/checkout@v3 + uses: actions/checkout@v4 - name: Set VERSION env run: echo "VERSION=$(cat splitio/version.go | grep 'Version =' | awk '{print $4}' | tr -d '"')" >> $GITHUB_ENV @@ -31,4 +31,4 @@ jobs: context: . file: "infra/sidecar.Dockerfile" push: true - tags: splitio-docker-dev.jfrog.io/splitd/sidecar:${{ env.SHORT_SHA }},splitio-docker-dev.jfrog.io/splitd/sidecar:${{ env.VERSION }} + tags: splitio-docker-dev.jfrog.io/splitd/sidecar:${{ env.SHORT_SHA }} From e29cb7fcf8bac0a3619dab536d6727ba4cdbf115 Mon Sep 17 00:00:00 2001 From: Martin Redolatti Date: Tue, 5 Sep 2023 12:58:02 -0300 Subject: [PATCH 40/42] more feedback --- .github/workflows/docker.yml | 12 +++++------- .github/workflows/unstable.yml | 3 --- 2 files changed, 5 insertions(+), 10 deletions(-) diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index f21984c..76ccd05 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -5,17 +5,12 @@ on: branches: - main -concurrency: - group: ${{ github.workflow }}-${{ github.event_name == 'push' && github.run_number || github.event.pull_request.number }} - cancel-in-progress: true - jobs: build-docker-image: name: Build and push Docker image runs-on: ubuntu-latest steps: - name: Login to Artifactory - if: ${{ github.event_name == 'push' }} uses: docker/login-action@v2 with: registry: splitio-docker.jfrog.io @@ -25,10 +20,13 @@ jobs: - name: Checkout code uses: actions/checkout@v4 + - name: Set VERSION env + run: echo "VERSION=$(cat splitio/version.go | grep 'Version =' | awk '{print $4}' | tr -d '"')" >> $GITHUB_ENV + - name: Docker build and push uses: docker/build-push-action@v4 with: context: . file: "infra/sidecar.Dockerfile" - push: ${{ github.event_name == 'push' }} - tags: splitio-docker.jfrog.io/splitd/sidecar:${{ github.sha }},splitio-docker.jfrog.io/splitd/sidecar:${{ env.VERSION }} + push: true + tags: splitio-docker.jfrog.io/splitd/sidecar:latest,splitio-docker.jfrog.io/splitd/sidecar:${{ env.VERSION }} diff --git a/.github/workflows/unstable.yml b/.github/workflows/unstable.yml index 9b03154..f0c0c7d 100644 --- a/.github/workflows/unstable.yml +++ b/.github/workflows/unstable.yml @@ -19,9 +19,6 @@ jobs: - name: Checkout code uses: actions/checkout@v4 - - name: Set VERSION env - run: echo "VERSION=$(cat splitio/version.go | grep 'Version =' | awk '{print $4}' | tr -d '"')" >> $GITHUB_ENV - - name: Get short hash run: echo "SHORT_SHA=$(git rev-parse --short HEAD)" >> $GITHUB_ENV From e1ecfc91100630f46569d799d9f258079bc83103 Mon Sep 17 00:00:00 2001 From: Martin Redolatti Date: Tue, 5 Sep 2023 14:03:14 -0300 Subject: [PATCH 41/42] update changes --- CHANGES | 3 ++- splitio/sdk/helpers_test.go | 30 ++++++++++++++++++++++++++++++ 2 files changed, 32 insertions(+), 1 deletion(-) create mode 100644 splitio/sdk/helpers_test.go diff --git a/CHANGES b/CHANGES index 27e4d2e..fe3587d 100644 --- a/CHANGES +++ b/CHANGES @@ -1,3 +1,4 @@ -1.0.1: +1.0.1 (Sep 5, 2023): - Add support for .track - Fixed issue where impressions could take too long to evict even with the queue full +- Internal: bumped test covereage, bumped CI configuration & automated release logic, module configuration cleanup diff --git a/splitio/sdk/helpers_test.go b/splitio/sdk/helpers_test.go new file mode 100644 index 0000000..8cf4441 --- /dev/null +++ b/splitio/sdk/helpers_test.go @@ -0,0 +1,30 @@ +package sdk + +import ( + "testing" + + sdkConf "github.com/splitio/splitd/splitio/sdk/conf" + "github.com/stretchr/testify/assert" +) + +func TestSetupImpressionsComponents(t *testing.T) { + + sdkCfg := sdkConf.DefaultConfig() + storages := setupStorages(sdkCfg) + + ic, err := setupImpressionsComponents(&sdkCfg.Impressions, storages.telemetry) + assert.Nil(t, err) + assert.NotNil(t, ic.counter) + + sdkCfg.Impressions.Mode = "debug" + ic, err = setupImpressionsComponents(&sdkCfg.Impressions, storages.telemetry) + assert.Nil(t, err) + assert.Nil(t, ic.counter) +} + +func TestNoOpTask(t *testing.T) { + var task NoOpTask + assert.Equal(t, false, task.IsRunning()) + task.Start() + assert.Nil(t, task.Stop(true)) +} From 4be9a8d96f940337d9f4c657412c805ad3a039db Mon Sep 17 00:00:00 2001 From: Martin Redolatti Date: Tue, 5 Sep 2023 17:16:55 -0300 Subject: [PATCH 42/42] prepare for release --- CHANGES | 1 + Makefile | 16 +++-- cmd/splitd/main.go | 5 +- splitio/commitsha.go | 3 + splitio/sdk/integration_test.go | 111 ++++++++++++++++++++++++++++++++ splitio/sdk/mocks/sdk.go | 9 ++- splitio/sdk/sdk.go | 6 ++ 7 files changed, 143 insertions(+), 8 deletions(-) create mode 100644 splitio/commitsha.go create mode 100644 splitio/sdk/integration_test.go diff --git a/CHANGES b/CHANGES index fe3587d..f119870 100644 --- a/CHANGES +++ b/CHANGES @@ -1,4 +1,5 @@ 1.0.1 (Sep 5, 2023): - Add support for .track - Fixed issue where impressions could take too long to evict even with the queue full +- Fixed eviction on shutdown - Internal: bumped test covereage, bumped CI configuration & automated release logic, module configuration cleanup diff --git a/Makefile b/Makefile index 67bc192..2106c2d 100644 --- a/Makefile +++ b/Makefile @@ -1,4 +1,4 @@ -.PHONY: clean build test sidecar_image unit-tests entrypoint-test +.PHONY: clean build test sidecar_image unit-tests entrypoint-test splitio/commitsha.go # Setup defaults GO ?=go @@ -9,12 +9,15 @@ SHELL = /usr/bin/env bash -o pipefail PLATFORM ?= PLATFORM_STR := $(if $(PLATFORM),--platform=$(PLATFORM),) -COVERAGE_FILE ?= coverage.out - VERSION := $(shell cat splitio/version.go | grep 'const Version' | sed 's/const Version = //' | tr -d '"') -GO_FILES := $(shell find . -name "*.go") go.sum +COMMIT_SHA := $(shell git rev-parse --short HEAD) +COMMIT_SHA_FILE := splitio/commitsha.go + +GO_FILES := $(shell find . -name "*.go" -not -name "$(COMMIT_SHA_FILE)") go.sum CONFIG_TEMPLATE ?= splitd.yaml.tpl +COVERAGE_FILE ?= coverage.out + default: help @@ -78,6 +81,11 @@ binaries_release: splitd-linux-amd64-$(VERSION).bin splitd-darwin-amd64-$(VERSIO $(COVERAGE_FILE): unit-tests +$(COMMIT_SHA_FILE): + @echo "package splitio" > $(COMMIT_SHA_FILE) + @echo "" >> $(COMMIT_SHA_FILE) + @echo "const CommitSHA = \"$(COMMIT_SHA)\"" >> $(COMMIT_SHA_FILE) + splitd-linux-amd64-$(VERSION).bin: $(GO_FILES) GOARCH=amd64 GOOS=linux $(GO) build -o $@ cmd/splitd/main.go diff --git a/cmd/splitd/main.go b/cmd/splitd/main.go index 9ff0625..c94c4d1 100644 --- a/cmd/splitd/main.go +++ b/cmd/splitd/main.go @@ -41,8 +41,9 @@ func main() { shutdown.RegisterHook(func() { err := lShutdown() if err != nil { - logger.Error(err) + logger.Error("error shutting down listener: ", err.Error()) } + splitSDK.Shutdown() // evict pending impressions & events }) defer shutdown.Wait() @@ -53,7 +54,7 @@ func main() { func printHeader() { fmt.Println(splitio.ASCILogo) - fmt.Printf("Splitd Agent - Version %s. (2023)\n\n", splitio.Version) + fmt.Printf("Splitd Agent - Version %s - build [%s] (2023)\n\n", splitio.Version, splitio.CommitSHA) } func handleFlags(cfg *conf.Config) { diff --git a/splitio/commitsha.go b/splitio/commitsha.go new file mode 100644 index 0000000..3c7c97b --- /dev/null +++ b/splitio/commitsha.go @@ -0,0 +1,3 @@ +package splitio + +const CommitSHA = "e1ecfc9" diff --git a/splitio/sdk/integration_test.go b/splitio/sdk/integration_test.go new file mode 100644 index 0000000..e6a3221 --- /dev/null +++ b/splitio/sdk/integration_test.go @@ -0,0 +1,111 @@ +package sdk + +import ( + "encoding/json" + "fmt" + "io" + "io/ioutil" + "net/http" + "net/http/httptest" + "sync/atomic" + "testing" + + "github.com/splitio/go-split-commons/v4/dtos" + "github.com/splitio/go-toolkit/v5/logging" + "github.com/splitio/splitd/splitio/sdk/conf" + "github.com/splitio/splitd/splitio/sdk/types" + "github.com/stretchr/testify/assert" +) + +func TestInstantiationAndGetTreatmentE2E(t *testing.T) { + metricsInitCalled := 0 + mockedSplit1 := dtos.SplitDTO{ + Algo: 2, + ChangeNumber: 123, + DefaultTreatment: "default", + Killed: false, + Name: "split", + Seed: 1234, + Status: "ACTIVE", + TrafficAllocation: 1, + TrafficAllocationSeed: -1667452163, + TrafficTypeName: "tt1", + Conditions: []dtos.ConditionDTO{ + { + ConditionType: "ROLLOUT", + Label: "in segment all", + MatcherGroup: dtos.MatcherGroupDTO{ + Combiner: "AND", + Matchers: []dtos.MatcherDTO{{MatcherType: "ALL_KEYS"}}, + }, + Partitions: []dtos.PartitionDTO{{Size: 100, Treatment: "on"}}, + }, + }, + } + mockedSplit2 := dtos.SplitDTO{Name: "split2", Killed: true, Status: "ACTIVE"} + mockedSplit3 := dtos.SplitDTO{Name: "split3", Killed: true, Status: "INACTIVE"} + + sdkServer := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + assert.Equal(t, "GET", r.Method) + assert.Equal(t, "/splitChanges", r.URL.Path) + + splitChanges := dtos.SplitChangesDTO{ + Splits: []dtos.SplitDTO{mockedSplit1, mockedSplit2, mockedSplit3}, + Since: 3, + Till: 3, + } + + raw, err := json.Marshal(splitChanges) + assert.Nil(t, err) + + w.Write(raw) + })) + defer sdkServer.Close() + + var eventsCalls int32 + eventsServer := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + atomic.AddInt32(&eventsCalls, 1) + assert.Equal(t, "POST", r.Method) + assert.Equal(t, "/testImpressions/bulk", r.URL.Path) + + body, err := io.ReadAll(r.Body) + assert.Nil(t, err) + + var imps []dtos.ImpressionsDTO + assert.Nil(t, json.Unmarshal(body, &imps)) + + assert.Equal(t, "split", imps[0].TestName) + assert.Equal(t, 1, len(imps[0].KeyImpressions)) + w.WriteHeader(200) + })) + defer eventsServer.Close() + + telemetryServer := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + switch r.URL.Path { + case "/metrics/config": + metricsInitCalled++ + rBody, _ := ioutil.ReadAll(r.Body) + var dataInPost dtos.Config + err := json.Unmarshal(rBody, &dataInPost) + assert.Nil(t, err) + } + fmt.Fprintln(w, "ok") + })) + defer telemetryServer.Close() + + sdkConf := conf.DefaultConfig() + sdkConf.URLs.Events = eventsServer.URL + sdkConf.URLs.SDK = sdkServer.URL + sdkConf.URLs.Telemetry = telemetryServer.URL + sdkConf.StreamingEnabled = false + + logger := logging.NewLogger(nil) + client, err := New(logger, "someApikey", sdkConf) + assert.Nil(t, err) + + res, err := client.Treatment(&types.ClientConfig{}, "aaaaaaklmnbv", nil, "split", nil) + assert.Equal(t, "on", res.Treatment) + + assert.Nil(t, client.Shutdown()) + assert.Equal(t, int32(1), atomic.LoadInt32(&eventsCalls)) +} diff --git a/splitio/sdk/mocks/sdk.go b/splitio/sdk/mocks/sdk.go index e53a180..5679349 100644 --- a/splitio/sdk/mocks/sdk.go +++ b/splitio/sdk/mocks/sdk.go @@ -36,8 +36,13 @@ func (m *SDKMock) Treatments( // Track implements sdk.Interface func (m *SDKMock) Track(cfg *types.ClientConfig, key string, trafficType string, eventType string, value *float64, properties map[string]interface{}) error { - args := m.Called(cfg, key, trafficType, eventType, value, properties) - return args.Error(0) + args := m.Called(cfg, key, trafficType, eventType, value, properties) + return args.Error(0) +} + +func (m *SDKMock) Shutdown() error { + args := m.Called() + return args.Error(0) } var _ sdk.Interface = (*SDKMock)(nil) diff --git a/splitio/sdk/sdk.go b/splitio/sdk/sdk.go index e9fd470..1dd18b6 100644 --- a/splitio/sdk/sdk.go +++ b/splitio/sdk/sdk.go @@ -38,6 +38,7 @@ type Interface interface { Treatment(cfg *types.ClientConfig, key string, bucketingKey *string, feature string, attributes map[string]interface{}) (*EvaluationResult, error) Treatments(cfg *types.ClientConfig, key string, bucketingKey *string, features []string, attributes map[string]interface{}) (map[string]EvaluationResult, error) Track(cfg *types.ClientConfig, key string, trafficType string, eventType string, value *float64, properties map[string]interface{}) error + Shutdown() error } type Impl struct { @@ -182,6 +183,11 @@ func (i *Impl) Track(cfg *types.ClientConfig, key string, trafficType string, ev return nil } +func (i *Impl) Shutdown() error { + i.sm.Stop() + return nil +} + func (i *Impl) handleImpression(key string, bk *string, f string, r *evaluator.Result, cm types.ClientMetadata) *dtos.Impression { var label string if i.cfg.LabelsEnabled {