From 7e8df0250ec1c7ebbe861a3464565de5345ba404 Mon Sep 17 00:00:00 2001 From: Cody Constine Date: Wed, 7 Aug 2024 13:34:48 -0600 Subject: [PATCH 1/3] Modified tests to make all work for performance testing --- .../uid2-operator/k6-uid2-operator.js | 437 ++++++++---------- 1 file changed, 186 insertions(+), 251 deletions(-) diff --git a/performance-testing/uid2-operator/k6-uid2-operator.js b/performance-testing/uid2-operator/k6-uid2-operator.js index 0606631..bae29c6 100644 --- a/performance-testing/uid2-operator/k6-uid2-operator.js +++ b/performance-testing/uid2-operator/k6-uid2-operator.js @@ -1,11 +1,14 @@ -import { crypto } from "k6/experimental/webcrypto"; +import {crypto} from "k6/experimental/webcrypto"; import encoding from 'k6/encoding'; -import { check } from 'k6'; +import {check} from 'k6'; import http from 'k6/http'; -import exec from 'k6/execution'; -const testDurationInSeconds = 4300; -const targetVUCount = 100; +const testDurationInSeconds = 2000; +const tokenGenerateTests = true; +const tokenRefreshTests = true; +const identityMapTests = true; +const identityBucketTests = true; + //30 warm up on each // 5 min each @@ -16,238 +19,82 @@ export const options = { noConnectionReuse: false, scenarios: { // Warmup scenarios - identityMapWarmup12: { - executor: 'ramping-vus', - startVUs: 0, - stages: [ - { duration: '30s', target: 12 } - ], - exec: 'identityMapLargeBatch', - gracefulStop: '0s', - }, - identityMapLargeBatch12: { - executor: 'constant-vus', - exec: 'identityMapLargeBatch', - vus: 12, - duration: '300s', - gracefulStop: '0s', - startTime: '30s', - }, - identityMapWarmup50: { - executor: 'ramping-vus', - startVUs: 12, - stages: [ - { duration: '30s', target: 50 } - ], - exec: 'identityMapLargeBatch', - gracefulStop: '0s', - startTime: '330s' - }, - identityMapLargeBatch50: { + tokenGenerateWarmup: { executor: 'constant-vus', - exec: 'identityMapLargeBatch', - vus: 50, - duration: '300s', - gracefulStop: '0s', - startTime: '360s', - }, - identityMapWarmup100: { - executor: 'ramping-vus', - startVUs: 50, - stages: [ - { duration: '30s', target: 100 } - ], - exec: 'identityMapLargeBatch', + exec: 'tokenGenerate', + vus: 300, + duration: '30s', gracefulStop: '0s', - startTime: '660s', }, - identityMapLargeBatch100: { + tokenRefreshWarmup: { executor: 'constant-vus', - exec: 'identityMapLargeBatch', - vus: 100, - duration: '300s', - gracefulStop: '0s', - startTime: '690s', - }, - identityMapWarmup150: { - executor: 'ramping-vus', - startVUs: 100, - stages: [ - { duration: '30s', target: 150 } - ], - exec: 'identityMapLargeBatch', + exec: 'tokenRefresh', + vus: 300, + duration: '30s', gracefulStop: '0s', - startTime: '990s', }, - identityMapLargeBatch150: { + identityMapWarmup: { executor: 'constant-vus', - exec: 'identityMapLargeBatch', - vus: 150, - duration: '300s', - gracefulStop: '0s', - startTime: '1020s', - }, - identityMapWarmup200: { - executor: 'ramping-vus', - startVUs: 150, - stages: [ - { duration: '30s', target: 200 } - ], - exec: 'identityMapLargeBatch', + exec: 'identityMap', + vus: 300, + duration: '30s', gracefulStop: '0s', - startTime: '1320s', }, - identityMapLargeBatch200: { + identityBucketsWarmup: { executor: 'constant-vus', - exec: 'identityMapLargeBatch', - vus: 200, - duration: '300s', - gracefulStop: '0s', - startTime: '1350s', - }, - identityMapWarmup250: { - executor: 'ramping-vus', - startVUs: 200, - stages: [ - { duration: '30s', target: 250 } - ], - exec: 'identityMapLargeBatch', + exec: 'identityBuckets', + vus: 2, + duration: '30s', gracefulStop: '0s', - startTime: '1650s', }, - identityMapLargeBatch250: { + // Actual testing scenarios + tokenGenerate: { executor: 'constant-vus', - exec: 'identityMapLargeBatch', - vus: 250, - duration: '300s', - gracefulStop: '0s', - startTime: '1680s', - }, - identityMapWarmup300: { - executor: 'ramping-vus', - startVUs: 250, - stages: [ - { duration: '30s', target: 300 } - ], - exec: 'identityMapLargeBatch', - gracefulStop: '0s', - startTime: '1980s', - }, - identityMapLargeBatch300: { - executor: 'constant-vus', - exec: 'identityMapLargeBatch', + exec: 'tokenGenerate', vus: 300, duration: '300s', gracefulStop: '0s', - startTime: '2010s', - }, - identityMapWarmup350: { - executor: 'ramping-vus', - startVUs: 300, - stages: [ - { duration: '30s', target: 350 } - ], - exec: 'identityMapLargeBatch', - gracefulStop: '0s', - startTime: '2310s', - }, - identityMapLargeBatch350: { - executor: 'constant-vus', - exec: 'identityMapLargeBatch', - vus: 350, - duration: '300s', - gracefulStop: '0s', - startTime: '2340s', - }, - identityMapWarmup400: { - executor: 'ramping-vus', - startVUs: 350, - stages: [ - { duration: '30s', target: 400 } - ], - exec: 'identityMapLargeBatch', - gracefulStop: '0s', - startTime: '2640s', + startTime: '40s', }, - identityMapLargeBatch400: { + tokenRefresh: { executor: 'constant-vus', - exec: 'identityMapLargeBatch', - vus: 400, + exec: 'tokenRefresh', + vus: 300, duration: '300s', gracefulStop: '0s', - startTime: '2670s', + startTime: '350s', }, - identityMapWarmup450: { - executor: 'ramping-vus', - startVUs: 400, - stages: [ - { duration: '30s', target: 450 } - ], - exec: 'identityMapLargeBatch', - gracefulStop: '0s', - startTime: '2970s', - }, - identityMapLargeBatch450: { + identityMap: { executor: 'constant-vus', - exec: 'identityMapLargeBatch', - vus: 450, + exec: 'identityMap', + vus: 300, duration: '300s', gracefulStop: '0s', - startTime: '3000s', - }, - identityMapWarmup500: { - executor: 'ramping-vus', - startVUs: 450, - stages: [ - { duration: '30s', target: 500 } - ], - exec: 'identityMapLargeBatch', - gracefulStop: '0s', - startTime: '3300s', + startTime: '660s', }, - identityMapLargeBatch500: { + identityMapLargeBatchSequential: { executor: 'constant-vus', exec: 'identityMapLargeBatch', - vus: 500, + vus: 1, duration: '300s', gracefulStop: '0s', - startTime: '3330s', + startTime: '970s', }, - identityMapWarmup550: { - executor: 'ramping-vus', - startVUs: 500, - stages: [ - { duration: '30s', target: 550 } - ], - exec: 'identityMapLargeBatch', - gracefulStop: '0s', - startTime: '3630s', - }, - identityMapLargeBatch550: { + identityMapLargeBatch: { executor: 'constant-vus', exec: 'identityMapLargeBatch', - vus: 550, + vus: 16, duration: '300s', gracefulStop: '0s', - startTime: '3660s', + startTime: '1280s', }, - identityMapWarmup600: { - executor: 'ramping-vus', - startVUs: 550, - stages: [ - { duration: '30s', target: 600 } - ], - exec: 'identityMapLargeBatch', - gracefulStop: '0s', - startTime: '3960s', - }, - identityMapLargeBatch600: { + identityBuckets: { executor: 'constant-vus', - exec: 'identityMapLargeBatch', - vus: 600, + exec: 'identityBuckets', + vus: 2, duration: '300s', gracefulStop: '0s', - startTime: '3990s', + startTime: '1590s', }, }, // So we get count in the summary, to demonstrate different metrics are different @@ -279,57 +126,76 @@ const baseUrl = __ENV.BASE_URL; export async function setup() { // pregenerate the envelopes so they don't expire, but can be reused. Means the load test is not constrained by the client // Each is used fopr 45 sec. Add 2 to ensure we have enough - const numberOfRequestsToGenerate = Math.round(testDurationInSeconds / 45) + 2; + const numberOfRequestsToGenerate = Math.round(testDurationInSeconds / 45) + 2; - const generateRefreshRequest = (data) => { - // TODO: call tokenGenerate to get the refresh token from response - const refreshToken = __ENV.REFRESH_TOKEN; - return refreshToken; + async function generateRefreshRequest(data) { + let request = await createReq( {'optout_check': 1, 'email': 'test5000@example.com'}); + var requestData = { + endpoint: '/v2/token/generate', + requestBody: request, + } + let response = await send(requestData, clientKey); + let decrypt = await decryptEnvelope(response.body, clientSecret) + return decrypt.refresh_token; }; - const generateSinceTimestampStr = () => { - var date = new Date(Date.now() - 2 * 24 * 60 * 60 * 1000 /* 2 days ago */); - var year = date.getFullYear(); - var month = (date.getMonth() + 1).toString().padStart(2, '0'); - var day = date.getDate().toString().padStart(2, '0'); - return `${year}-${month}-${day}T00:00:00`; - }; - - const tokenGenerateData = { - endpoint: '/v2/token/generate', - requestBody: await createReq({ - 'optout_check': 1, - 'email': 'test@example.com', - }), - }; - return { - tokenGenerate: tokenGenerateData, - tokenRefresh: { + let tokenGenerateData = {}; + if (tokenGenerateTests) { + tokenGenerateData = { + endpoint: '/v2/token/generate', + requestData: await generateFutureGenerateRequests(numberOfRequestsToGenerate), + }; + } + let tokenRefreshData = {}; + if(tokenRefreshTests) { + tokenRefreshData = { endpoint: '/v2/token/refresh', - requestBody: generateRefreshRequest(tokenGenerateData), - }, - identityMap: { + requestBody: await generateRefreshRequest(tokenGenerateData), + }; + } + + let identityMapData = {}; + let identityMapLargeBatchData = {}; + if(identityMapTests) { + identityMapData = { endpoint: '/v2/identity/map', - requestBody: await createReq(generateIdentityMapRequest(2)), - }, - identityMapLargeBatch: { - requestData: await generateFutureMapRequest(numberOfRequestsToGenerate) - }, - identityBuckets: { + requestData: await generateFutureMapRequest(numberOfRequestsToGenerate, 2) + }; + identityMapLargeBatchData = { + requestData: await generateFutureMapRequest(numberOfRequestsToGenerate, 5000) + }; + } + + let identityBucketData = {} + if(identityBucketTests) { + identityBucketData = { endpoint: '/v2/identity/buckets', - requestBody: await createReq({ - "since_timestamp": generateSinceTimestampStr(), - }), - } + requestData: await generateFutureBucketRequests(numberOfRequestsToGenerate) + }; + } + + return { + tokenGenerate: tokenGenerateData, + tokenRefresh: tokenRefreshData, + identityMap: identityMapData, + identityMapLargeBatch: identityMapLargeBatchData, + identityBuckets: identityBucketData }; } // Scenarios export function tokenGenerate(data) { - execute(data.tokenGenerate, true); + var requestData = data.tokenGenerate.requestData; + var elementToUse = selectRequestData(requestData); + + var tokenGenerateData = { + endpoint: data.tokenGenerate.endpoint, + requestBody: elementToUse.requestBody, + } + execute(tokenGenerateData, true); } export function tokenRefresh(data) { @@ -337,20 +203,19 @@ export function tokenRefresh(data) { } export function identityMap(data) { - execute(data.identityMap, true); + var requestData = data.identityMap.requestData; + var elementToUse = selectRequestData(requestData); + + var identityData = { + endpoint: '/v2/identity/map', + requestBody: elementToUse.requestBody, + } + execute(identityData, true); } export function identityMapLargeBatch(data) { var requestData = data.identityMapLargeBatch.requestData; - var elementToUse = requestData[0]; - for (var i = 0; i < requestData.length; i++) { - var currentTime = Date.now() + 5000; - if (currentTime > requestData[i].time && currentTime < requestData[i + 1].time) { - elementToUse = requestData[i]; - //console.log("VU: " + exec.vu.idInTest + ", item: " + i); - break; - } - } + var elementToUse = selectRequestData(requestData); var identityData = { endpoint: '/v2/identity/map', @@ -361,10 +226,30 @@ export function identityMapLargeBatch(data) { } export function identityBuckets(data) { - execute(data.identityBuckets, true); + var requestData = data.identityBuckets.requestData; + var elementToUse = selectRequestData(requestData); + + var bucketData = { + endpoint: data.identityBuckets.endpoint, + requestBody: elementToUse.requestBody, + } + execute(bucketData, true); } // Helpers +function selectRequestData(requestData) { + var elementToUse = requestData[0]; + for (var i = 0; i < requestData.length; i++) { + var currentTime = Date.now() + 5000; + if (currentTime > requestData[i].time && currentTime < requestData[i + 1].time) { + elementToUse = requestData[i]; + //console.log("VU: " + exec.vu.idInTest + ", item: " + i); + break; + } + } + return elementToUse; +} + async function createReq(obj) { var envelope = getEnvelope(obj); return encoding.b64encode((await encryptEnvelope(envelope, clientSecret)).buffer); @@ -438,6 +323,33 @@ async function encryptEnvelope(envelope, clientSecret) { return result; } +async function decryptEnvelope(envelope, clientSecret) { + const rawKey = encoding.b64decode(clientSecret); + const rawData = encoding.b64decode(envelope); + const key = await crypto.subtle.importKey("raw", rawKey, "AES-GCM", true, [ + "encrypt", + "decrypt", + ]); + const length = rawData.byteLength; + const iv = rawData.slice(0, 12); + + const decrypted = await crypto.subtle.decrypt( + { + name: "AES-GCM", + iv: iv, + tagLength: 128 + }, + key, + rawData.slice(12) + ); + + + const decryptedResponse = String.fromCharCode.apply(String, new Uint8Array(decrypted.slice(16))); + const response = JSON.parse(decryptedResponse); + + return response; +} + function getEnvelopeWithTimestamp(timestampArray, obj) { var randomBytes = new Uint8Array(8); crypto.getRandomValues(randomBytes); @@ -502,13 +414,12 @@ function stringToUint8Array(str) { return view; } -async function generateFutureMapRequest(count) { +async function generateFutureRequests(count, obj) { const result = []; for (var i = 0; i < count; i++) { var time = Date.now() + (i * 45000) var timestampArr = new Uint8Array(getTimestampFromTime(time)); - var emails = generateIdentityMapRequest(5000); - var requestBody = await createReqWithTimestamp(timestampArr, emails); + var requestBody = await createReqWithTimestamp(timestampArr, obj); var element = { time: time, requestBody: requestBody @@ -516,4 +427,28 @@ async function generateFutureMapRequest(count) { result.push(element); } return result; -} \ No newline at end of file +} + +async function generateFutureMapRequest(count, emailCount) { + let emails = generateIdentityMapRequest(emailCount); + return await generateFutureRequests(count, emails); +} + +async function generateFutureGenerateRequests(count) { + let obj = {'optout_check': 1, 'email': 'test@example.com'}; + return await generateFutureRequests(count, obj) +} + +const generateSinceTimestampStr = () => { + var date = new Date(Date.now() - 2 * 24 * 60 * 60 * 1000 /* 2 days ago */); + var year = date.getFullYear(); + var month = (date.getMonth() + 1).toString().padStart(2, '0'); + var day = date.getDate().toString().padStart(2, '0'); + + return `${year}-${month}-${day}T00:00:00`; +}; + +async function generateFutureBucketRequests(count) { + let obj = {"since_timestamp": generateSinceTimestampStr()}; + return await generateFutureRequests(count, obj) +} From d3341b0446460941ba4fe230657e631c9cf93320 Mon Sep 17 00:00:00 2001 From: Cody Constine Date: Thu, 8 Aug 2024 15:14:38 -0600 Subject: [PATCH 2/3] Fixed a small issue with refresh --- performance-testing/uid2-operator/k6-uid2-operator.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/performance-testing/uid2-operator/k6-uid2-operator.js b/performance-testing/uid2-operator/k6-uid2-operator.js index bae29c6..bde58ac 100644 --- a/performance-testing/uid2-operator/k6-uid2-operator.js +++ b/performance-testing/uid2-operator/k6-uid2-operator.js @@ -137,7 +137,7 @@ export async function setup() { } let response = await send(requestData, clientKey); let decrypt = await decryptEnvelope(response.body, clientSecret) - return decrypt.refresh_token; + return decrypt.body.refresh_token; }; From 2e83cd7758d92e59948fe35df64fd745c93ca45b Mon Sep 17 00:00:00 2001 From: Cody Constine Date: Thu, 8 Aug 2024 15:24:59 -0600 Subject: [PATCH 3/3] Fixed the README and fixed some issues with the default tests --- performance-testing/uid2-operator/README.md | 65 ++++--------------- .../uid2-operator/k6-uid2-operator.js | 4 +- 2 files changed, 14 insertions(+), 55 deletions(-) diff --git a/performance-testing/uid2-operator/README.md b/performance-testing/uid2-operator/README.md index d1a1cf4..8d08ed5 100644 --- a/performance-testing/uid2-operator/README.md +++ b/performance-testing/uid2-operator/README.md @@ -1,77 +1,36 @@ # UID2 Operator Performance Testing Tool +The following instructions will work for any operator. ## Steps +### Step 1 - Configure the K6 script +The script as checked in has a basic config that will run a check against all endpoints. -### Step 1 - Deploy Operator Instance -In order to reduce overhead on client side, this tool generates an encrypted request first, then uses it for all load testing requests. Therefore we need to disable the timestamp check of operator, use https://github.com/IABTechLab/uid2-operator/commit/979d43076ce94b80622f9c7d9eed87d97fc71a7b as an example. -You can deploy an operator instance as a Docker container by providing required environment variables. +All the manual config can be changed at the top of the script. -Save and modify following environment variables as `env.txt` file. -``` -optout_api_token= -core_api_token= -service_verbose=true -service_instances=16 -storage_mock=false -optout_s3_path_compat=false -optout_data_dir=/opt/uid2/operator-optout/ -optout_bloom_filter_size=8192 -optout_delta_rotate_interval=300 -optout_delta_backtrack_in_days=1 -optout_partition_interval=86400 -optout_max_partitions=30 -optout_heap_default_capacity=8192 -cloud_download_threads=8 -cloud_upload_threads=2 -cloud_refresh_interval=60 -optout_inmem_cache=true -identity_token_expires_after_seconds=86400 -refresh_token_expires_after_seconds=2592000 -refresh_identity_token_after_seconds=3600 -enforce_https=true -allow_legacy_api=false -clients_metadata_path=https://core-integ.uidapi.com/clients/refresh -keys_metadata_path=https://core-integ.uidapi.com/key/refresh -keys_acl_metadata_path=https://core-integ.uidapi.com/key/acl/refresh -salts_metadata_path=https://core-integ.uidapi.com/salt/refresh -optout_metadata_path=https://optout-integ.uidapi.com/optout/refresh -core_attest_url=https://core-integ.uidapi.com/attest -optout_api_uri=https://optout-integ.uidapi.com/optout/replicate -optout_s3_folder=uid-optout-integ/ -KUBERNETES_SERVICE_HOST=mock -``` - -Run following command to start an operator container, remember to change image tag accordingly. -```shell -docker run -d --env-file env.txt -p 8080:8080 ghcr.io/iabtechlab/uid2-operator:4.3.6-SNAPSHOT-default -``` +To change/remove scenarios modify the options.scenarios. For information on configuring k6 scenarios see https://k6.io/docs/using-k6/scenarios/. +The variable `testDurationInSeconds` must be greater than the total duration of the test you are running. +Any of the `*Tests` booleans can be set false if you are not running tests for that endpoint, this will save memory and start up time. ### Step 2 - Execute K6 Script -You can manually modify the scenarios in the script to test on a single endpoint with different settings. - -For the following options, you can get a valid `REFRESH_TOKEN` from a recent successful `/token/generate` response. -In the future, we should upgrade the k6 script to get refresh token automatically. - #### Option 2a - Execute K6 Script Locally (uid2-dev-workspace) If you would like to test locally, follow these steps: 1. Pull the K6 Docker image: `docker pull grafana/k6` 2. Execute the K6 script - * PowerShell: + * PowerShell/Bash: ``` cat k6-uid2-operator.js ` | docker run --network="host" --rm -i ` - -e CLIENT_KEY="UID2-C-L-999-fCXrMM.fsR3mDqAXELtWWMS+xG1s7RdgRTMqdOH2qaAo=" ` - -e CLIENT_SECRET="DzBzbjTJcYL0swDtFs2krRNu+g1Eokm2tBU4dEuD0Wk=" ` - -e BASE_URL="http://localhost:8080" ` - -e REFRESH_TOKEN="" ` + -e CLIENT_KEY="" ` + -e CLIENT_SECRET="" ` + -e BASE_URL="" ` grafana/k6 run - ``` #### Option 2b - Execute K6 Script in K8s In order to reduce network latency, we should deploy k6 and its script with the same zone of UID2 operator. -Set environment variables `CLIENT_KEY`, `CLIENT_SECRET`, `BASE_URL` and `REFRESH_TOKEN`, then use k6 to execute the testing by following command. +Set environment variables `CLIENT_KEY`, `CLIENT_SECRET`, `BASE_URL` and then use k6 to execute the testing by following command. ``` k6 run k6-uid2-operator.js -e CLIENT_KEY=$CLIENT_KEY -e CLIENT_SECRET=$CLIENT_SECRET -e BASE_URL=$BASE_URL -e REFRESH_TOKEN=$REFRESH_TOKEN ``` diff --git a/performance-testing/uid2-operator/k6-uid2-operator.js b/performance-testing/uid2-operator/k6-uid2-operator.js index bde58ac..4f3668e 100644 --- a/performance-testing/uid2-operator/k6-uid2-operator.js +++ b/performance-testing/uid2-operator/k6-uid2-operator.js @@ -3,7 +3,7 @@ import encoding from 'k6/encoding'; import {check} from 'k6'; import http from 'k6/http'; -const testDurationInSeconds = 2000; +const testDurationInSeconds = 2500; const tokenGenerateTests = true; const tokenRefreshTests = true; const identityMapTests = true; @@ -435,7 +435,7 @@ async function generateFutureMapRequest(count, emailCount) { } async function generateFutureGenerateRequests(count) { - let obj = {'optout_check': 1, 'email': 'test@example.com'}; + let obj = {'optout_check': 1, 'email': 'test500@example.com'}; return await generateFutureRequests(count, obj) }