diff --git a/.github/workflows/benchmark.yml b/.github/workflows/benchmark.yml index c8bce14..e11b3e7 100644 --- a/.github/workflows/benchmark.yml +++ b/.github/workflows/benchmark.yml @@ -15,33 +15,35 @@ jobs: uses: actions/setup-node@v4 with: node-version: 22 - cache: 'yarn' - cache-dependency-path: 'k6/yarn.lock' + cache: "yarn" + cache-dependency-path: "k6/yarn.lock" - name: Use Ruby uses: ruby/setup-ruby@2a9a743e19810b9f3c38060637daf594dbd7b37f with: working-directory: ./k6/graphql-api - ruby-version: '3.3' + ruby-version: "3.3" bundler-cache: true - name: Start GraphQL API with hive enabled working-directory: ./k6/graphql-api run: | bundle - bundle exec puma -t 0:1 -p 9292 & + bundle exec puma npx wait-on http://localhost:9292 --timeout 5s env: - HIVE_ENABLED: 'true' + HIVE_ENABLED: "true" + PORT: 9292 - name: Start GraphQL API with hive disabled working-directory: ./k6/graphql-api run: | bundle - bundle exec puma -t 0:1 -p 9291 & + bundle exec puma npx wait-on http://localhost:9291 --timeout 5s env: - HIVE_ENABLED: 'false' + HIVE_ENABLED: "false" + PORT: 9291 - name: Start Fake Usage API working-directory: ./k6/ @@ -53,8 +55,7 @@ jobs: - name: Install k6 working-directory: ./k6 env: - K6_RELEASE_ARTIFACT_URL: - https://github.com/grafana/k6/releases/download/v0.37.0/k6-v0.37.0-linux-amd64.tar.gz + K6_RELEASE_ARTIFACT_URL: https://github.com/grafana/k6/releases/download/v0.37.0/k6-v0.37.0-linux-amd64.tar.gz run: curl "${K6_RELEASE_ARTIFACT_URL}" -L | tar xvz --strip-components 1 - name: Run Benchmark diff --git a/k6/graphql-api/Gemfile.lock b/k6/graphql-api/Gemfile.lock index aae4968..735035e 100644 --- a/k6/graphql-api/Gemfile.lock +++ b/k6/graphql-api/Gemfile.lock @@ -2,17 +2,28 @@ PATH remote: ../.. specs: graphql-hive (0.4.4) + faraday (< 3) graphql (>= 2.3, < 3) GEM remote: https://rubygems.org/ specs: base64 (0.2.0) + faraday (2.12.0) + faraday-net_http (>= 2.0, < 3.4) + json + logger + faraday-net_http (3.3.0) + net-http graphql (2.3.7) base64 + json (2.7.2) + logger (1.6.1) multi_json (1.15.0) mustermann (2.0.2) ruby2_keywords (~> 0.0.1) + net-http (0.4.1) + uri nio4r (2.7.3) puma (6.4.3) nio4r (~> 2.0) @@ -35,6 +46,7 @@ GEM sinatra (= 2.2.4) tilt (~> 2.0) tilt (2.4.0) + uri (0.13.1) PLATFORMS ruby diff --git a/k6/graphql-api/puma.rb b/k6/graphql-api/puma.rb new file mode 100644 index 0000000..539e624 --- /dev/null +++ b/k6/graphql-api/puma.rb @@ -0,0 +1,13 @@ +port ENV.fetch("PORT") { 9291 } +threads_count = 5 +threads threads_count, threads_count +workers 2 +preload_app! + +on_worker_boot do + GraphQL::Hive.instance.on_start +end + +on_worker_shutdown do + GraphQL::Hive.instance.on_exit +end diff --git a/k6/k6.js b/k6/k6.js index 170c221..24fd21f 100644 --- a/k6/k6.js +++ b/k6/k6.js @@ -3,36 +3,44 @@ import http from "k6/http"; import { textSummary } from "https://jslib.k6.io/k6-summary/0.0.1/index.js"; import { githubComment } from "https://raw.githubusercontent.com/dotansimha/k6-github-pr-comment/master/lib.js"; +const VIRTUAL_USER_COUNT = 10; +const ITERATIONS = 100; +const P_95_SUCCESS_THRESHOLD = 50; + export const options = { discardResponseBodies: true, scenarios: { hiveEnabled: { executor: "shared-iterations", - vus: 120, - iterations: 500, - maxDuration: "30s", + vus: VIRTUAL_USER_COUNT, + iterations: ITERATIONS, + maxDuration: "5s", env: { GQL_API_PORT: "9292", HIVE_ENABLED: "true" }, }, hiveDisabled: { executor: "shared-iterations", - vus: 120, - iterations: 500, - maxDuration: "30s", - startTime: "30s", + vus: VIRTUAL_USER_COUNT, + iterations: ITERATIONS, + maxDuration: "5s", env: { GQL_API_PORT: "9291", HIVE_ENABLED: "false" }, }, }, thresholds: { - "http_req_duration{hive:enabled}": ["avg<4500"], - "http_req_duration{hive:disabled}": ["avg<4500"], + "http_req_duration{hive:enabled}": [ + { threshold: `p(95)<${P_95_SUCCESS_THRESHOLD}`, abortOnFail: true }, + ], + "http_req_duration{hive:disabled}": [ + { threshold: `p(95)<${P_95_SUCCESS_THRESHOLD}`, abortOnFail: true }, + ], + http_req_failed: ["rate<0.1"], }, }; const QUERY = /* GraphQL */ ` query GetPost { post(id: 1) { + id title - myId: id } } `; @@ -49,51 +57,58 @@ export default function () { tags: { hive: __ENV.HIVE_ENABLED === "true" ? "enabled" : "disabled" }, }; - return http.post( + const response = http.post( `http://localhost:${__ENV.GQL_API_PORT}/graphql`, payload, - params + params, ); + check(response, { + "is status 200": (r) => r.status === 200, + "body is graphql success": (r) => + r.body === + '{"data":{"post":{"id": 1, "title": "GraphQL Hive with `graphql-ruby`","truncated_preview": "Monitor operations, inspect your queries and publish your GraphQL schema with GraphQL Hive"}}}', + }); } export function handleSummary(data) { - let overheadPercentage = null; - if ( - data.metrics["http_req_duration{hive:enabled}"] && - data.metrics["http_req_duration{hive:disabled}"] - ) { - const withHive = - data.metrics["http_req_duration{hive:enabled}"].values["avg"]; - const withoutHive = - data.metrics["http_req_duration{hive:disabled}"].values["avg"]; - overheadPercentage = 100 - (withHive * 100.0) / withoutHive; - } - if (__ENV.GITHUB_TOKEN) { - githubComment(data, { - token: __ENV.GITHUB_TOKEN, - commit: __ENV.GITHUB_SHA, - pr: __ENV.GITHUB_PR, - org: "charlypoly", - repo: "graphql-ruby-hive", - renderTitle: () => { - return overheadPercentage < 5 - ? "✅ Benchmark Results" - : "❌ Benchmark Failed"; - }, - renderMessage: () => { - const result = []; - if (overheadPercentage > 5) { - result.push( - "**Performance regression detected**: it seems like your Pull Request adds some extra latency to GraphQL Hive operations processing" - ); - } else { - result.push("Overhead <= 1%"); - } - return result.join("\n"); - }, - }); - } + postToGithub(data); + return { stdout: textSummary(data, { indent: " ", enableColors: true }), }; } + +const postToGithub = (data) => { + if (!__ENV.GITHUB_TOKEN) { + return; + } + + const message = { + title: "✅ Benchmark Results: Overhead <= 1%", + text: "Overhead <= 5%", + }; + + const withHive = + data.metrics["http_req_duration{hive:enabled}"].values["avg"]; + const withoutHive = + data.metrics["http_req_duration{hive:disabled}"].values["avg"]; + const overheadPercentage = 100 - (withHive * 100.0) / withoutHive; + + if (overheadPercentage > 5) { + message.title = "❌ Benchmark Failed: Performance regression detected"; + message.text = + "**Performance regression detected**: this Pull Request adds extra latency to GraphQL Server"; + } + + const githubCommentOptions = { + token: __ENV.GITHUB_TOKEN, + commit: __ENV.GITHUB_SHA, + pr: __ENV.GITHUB_PR, + org: "charlypoly", + repo: "graphql-ruby-hive", + renderTitle: () => message.title, + renderMessage: () => message.text, + }; + + githubComment(data, githubCommentOptions); +};