From 4bc9d6b8d7bc1f66a3775fbaee1ce629807fe38d Mon Sep 17 00:00:00 2001 From: Cassidy Scheffer Date: Wed, 16 Oct 2024 16:45:02 -0400 Subject: [PATCH] Update docs and use small buffer --- .github/workflows/integration.yml | 30 ++++----- README.md | 11 ++-- k6/boot-servers.sh | 43 ++++++++---- k6/graphql-api/app.rb | 2 +- k6/graphql-api/schema.rb | 2 +- k6/integration-test.js | 104 +++++++++++++++--------------- 6 files changed, 104 insertions(+), 88 deletions(-) diff --git a/.github/workflows/integration.yml b/.github/workflows/integration.yml index e49c6b7..58f7ba7 100644 --- a/.github/workflows/integration.yml +++ b/.github/workflows/integration.yml @@ -1,11 +1,12 @@ -name: Benchmark +name: Integration on: + push: pull_request: branches: - master jobs: - benchmarks: + run-k6-tests: runs-on: ubuntu-latest steps: - name: Checkout Repository @@ -25,25 +26,20 @@ jobs: ruby-version: "3.3" bundler-cache: true - - name: Install k6 - working-directory: ./k6 - env: - K6_RELEASE_ARTIFACT_URL: https://github.com/grafana/k6/releases/download/v0.37.0/k6-v0.37.0-linux-amd64.tar.gz - - run: curl "${K6_RELEASE_ARTIFACT_URL}" -L | tar xvz --strip-components 1 - name: Start Servers - working-directory: ./k6/graphql-api + working-directory: ./k6 run: | ./boot-servers.sh & npx wait-on http://localhost:9292 --timeout 5s npx wait-on http://localhost:9291 --timeout 5s npx wait-on http://localhost:8888 --timeout 5s - - name: Run Integration Test - working-directory: ./k6 - run: | - ./k6 \ - -e GITHUB_PR=${{ github.event.number }} \ - -e GITHUB_SHA=${{ github.sha }} \ - -e GITHUB_TOKEN=${{secrets.GH_PA_TOKEN}} \ - run integration-test.js + - uses: grafana/setup-k6-action@v1 + - uses: grafana/run-k6-action@v1 + env: + GITHUB_PR: ${{ github.event.number }} + GITHUB_SHA: ${{ github.sha }} + GITHUB_TOKEN: ${{secrets.GH_PA_TOKEN}} + with: + path: | + ./k6/integration-test.js diff --git a/README.md b/README.md index 70a8565..950fc74 100644 --- a/README.md +++ b/README.md @@ -167,12 +167,15 @@ bundle exec rake test 5. **Run Integration Tests**: +In one terminal, start the servers: + ```sh cd k6 -yarn install -cd graphql-api -bundle install -cd ../ ./boot-servers.sh +``` + +In another terminal, run the tests: + +```sh k6 run integration-tests.js ``` diff --git a/k6/boot-servers.sh b/k6/boot-servers.sh index f76f5c7..f0463e8 100755 --- a/k6/boot-servers.sh +++ b/k6/boot-servers.sh @@ -1,10 +1,10 @@ #!/bin/bash log_with_prefix() { - local prefix="$1" - while IFS= read -r line; do - echo "[$prefix] $line" - done + local prefix="$1" + while IFS= read -r line; do + echo "[$prefix] $line" + done } start_puma_server() { @@ -14,14 +14,23 @@ start_puma_server() { echo "Starting Puma server with Hive ${hive_enabled}..." HIVE_ENABLED=$hive_enabled \ - PORT=$port \ - LOG_LEVEL=$LOG_LEVEL \ - bundle exec puma -C puma.rb | log_with_prefix "$prefix" & + PORT=$port \ + LOG_LEVEL=$LOG_LEVEL \ + bundle exec puma -C puma.rb | log_with_prefix "$prefix" & } +echo "Installing dependencies..." +yarn install +cd graphql-api || { + echo "Could not find graphql-api" && exit 1 +} +bundle install +cd .. + # Start Node.js server echo "Starting usage-api server..." LOG_LEVEL=$LOG_LEVEL node usage-api.js | log_with_prefix "usage-api" & + # Start first Puma server cd graphql-api || { echo "Could not find graphql-api" && exit 1 @@ -31,13 +40,19 @@ start_puma_server false 9292 "hive-disabled" # Function to handle shutdown shutdown_servers() { - echo "Received shutdown signal. Shutting down servers..." - kill $(lsof -t -i:9291) - kill $(lsof -t -i:9292) - kill $(lsof -t -i:8888) - wait - echo "Servers shut down gracefully." - exit 0 + echo "Received shutdown signal. Shutting down servers..." + + for port in 9291 9292 8888; do + pid=$(lsof -t -i:$port) + if [ -n "$pid" ]; then + kill "$pid" + else + echo "No process found on port $port" + fi + done + + echo "Servers shut down gracefully." + exit 0 } # Listen for kill signals diff --git a/k6/graphql-api/app.rb b/k6/graphql-api/app.rb index c72e4f1..d8fcbdc 100644 --- a/k6/graphql-api/app.rb +++ b/k6/graphql-api/app.rb @@ -9,7 +9,7 @@ class DemoApp < Sinatra::Base use Rack::JSONBodyParser configure do - set :logger, Logger.new(STDOUT) + set :logger, Logger.new($stdout) log_level = ENV.fetch("LOG_LEVEL", "INFO").upcase logger.level = begin Logger.const_get(log_level) diff --git a/k6/graphql-api/schema.rb b/k6/graphql-api/schema.rb index 9b9d8b2..659cbe9 100644 --- a/k6/graphql-api/schema.rb +++ b/k6/graphql-api/schema.rb @@ -34,7 +34,7 @@ class Schema < GraphQL::Schema enabled: ENV["HIVE_ENABLED"] === "true", endpoint: "localhost", debug: false, - buffer_size: 10, + buffer_size: 1, port: 8888, token: "stress-token", report_schema: false diff --git a/k6/integration-test.js b/k6/integration-test.js index 58d2197..ec44fa2 100644 --- a/k6/integration-test.js +++ b/k6/integration-test.js @@ -25,8 +25,24 @@ export const options = { }, }, thresholds: { - "http_req_duration{hive:enabled}": ["p(95)<15"], - "http_req_duration{hive:disabled}": ["p(95)<15"], + "http_req_duration{hive:enabled}": [ + { + threshold: "p(95)<25", + abortOnFail: true, + }, + ], + "http_req_duration{hive:disabled}": [ + { + threshold: "p(95)<25", + abortOnFail: true, + }, + ], + checks: [ + { + threshold: "rate===1", + abortOnFail: true, + }, + ], }, }; @@ -39,13 +55,11 @@ const QUERY = /* GraphQL */ ` } `; export function setup() { - // Ensure usage counter is at 0 const response = http.post("http://localhost:8888/reset"); const { count } = JSON.parse(response.body); check(count, { "usage-api starts with 0 operations": (count) => count === 0, }); - return { count }; } export default function () { @@ -72,73 +86,61 @@ export default function () { "response body is not a GraphQL error": (res) => !res.body.includes("errors"), }); + return res; } -export function teardown(_data) { - const res = http.get("http://localhost:8888/count"); - const count = JSON.parse(res.body).count; - console.log(`📊 Total operations: ${count}`); +function sleep(seconds) { + return new Promise((resolve) => setTimeout(resolve, seconds * 1000)); +} + +export async function teardown(data) { + let count = 0; + for (let i = 0; i < 10; i++) { + const res = http.get("http://localhost:8888/count"); + count = JSON.parse(res.body).count; + console.log(`📊 Total operations: ${count}`); + if (count === REQUEST_COUNT) { + break; + } + await sleep(1); + } check(count, { - "usage-api received 200 operations": (count) => count === REQUEST_COUNT, + "usage-api received correct number of operations": (count) => + count === REQUEST_COUNT, }); + const response = http.post("http://localhost:8888/reset"); + const { count: newCount } = JSON.parse(response.body); + check(newCount, { + "usage-api is reset": (c) => c === 0, + }); + return data; } export function handleSummary(data) { - const overhead = getOverheadPercentage(data); - const didPass = check(overhead, { - "overhead is less than 1%": (p) => p >= REGRESSION_THRESHOLD, - }); - - postGithubComment(didPass); - - console.log(`⏰ Overhead percentage: ${overhead.toFixed(2)}%`); - - if (!didPass) { - fail("❌❌ Performance regression detected ❌❌"); - } + const checks = data.metrics.checks; + const didPass = checks.values.fails.length === 0; + postGithubComment(data, didPass); return { stdout: textSummary(data, { indent: " ", enableColors: true }), }; } -function postGithubComment(didPass) { +function postGithubComment(data, didPass) { if (!__ENV.GITHUB_TOKEN) { return; } - githubComment(data, { token: __ENV.GITHUB_TOKEN, commit: __ENV.GITHUB_SHA, pr: __ENV.GITHUB_PR, - org: "charlypoly", + org: "rperryng", repo: "graphql-ruby-hive", - renderTitle: () => { - return didPass ? "✅ Benchmark Results" : "❌ Benchmark Failed"; - }, - renderMessage: () => { - const result = []; - if (didPass) { - result.push( - "**Performance regression detected**: it seems like your Pull Request adds some extra latency to GraphQL Hive operations processing", - ); - } else { - result.push("Overhead < 5%"); - } - return result.join("\n"); - }, + renderTitle: () => + didPass ? "✅ Integration Test Passed" : "❌ Integration Test Failed", + renderMessage: () => + didPass + ? "" + : "The integration test failed. Please check the action logs for more information.", }); } - -function getOverheadPercentage(data) { - const enabledMetric = data.metrics["http_req_duration{hive:enabled}"]; - const disabledMetric = data.metrics["http_req_duration{hive:disabled}"]; - - if (enabledMetric && disabledMetric) { - const withHive = enabledMetric.values["avg"]; - const withoutHive = disabledMetric.values["avg"]; - return 100 - (withHive * 100.0) / withoutHive; - } else { - throw new Error("Could not calculate overhead. Missing metrics."); - } -}