refactor benchmark

rperryng · Oct 9, 2024 · 75d96b6 · 75d96b6
1 parent 44a52ec
commit 75d96b6
Show file tree

Hide file tree

Showing 4 changed files with 98 additions and 57 deletions.
diff --git a/.github/workflows/benchmark.yml b/.github/workflows/benchmark.yml
@@ -15,33 +15,35 @@ jobs:
         uses: actions/setup-node@v4
         with:
           node-version: 22
-          cache: 'yarn'
-          cache-dependency-path: 'k6/yarn.lock'
+          cache: "yarn"
+          cache-dependency-path: "k6/yarn.lock"
 
       - name: Use Ruby
         uses: ruby/setup-ruby@2a9a743e19810b9f3c38060637daf594dbd7b37f
         with:
           working-directory: ./k6/graphql-api
-          ruby-version: '3.3'
+          ruby-version: "3.3"
           bundler-cache: true
 
       - name: Start GraphQL API with hive enabled
         working-directory: ./k6/graphql-api
         run: |
           bundle
-          bundle exec puma -t 0:1 -p 9292 &
+          bundle exec puma
           npx wait-on http://localhost:9292 --timeout 5s
         env:
-          HIVE_ENABLED: 'true'
+          HIVE_ENABLED: "true"
+          PORT: 9292
 
       - name: Start GraphQL API with hive disabled
         working-directory: ./k6/graphql-api
         run: |
           bundle
-          bundle exec puma -t 0:1 -p 9291 &
+          bundle exec puma
           npx wait-on http://localhost:9291 --timeout 5s
         env:
-          HIVE_ENABLED: 'false'
+          HIVE_ENABLED: "false"
+          PORT: 9291
 
       - name: Start Fake Usage API
         working-directory: ./k6/
@@ -53,8 +55,7 @@ jobs:
       - name: Install k6
         working-directory: ./k6
         env:
-          K6_RELEASE_ARTIFACT_URL:
-            https://github.com/grafana/k6/releases/download/v0.37.0/k6-v0.37.0-linux-amd64.tar.gz
+          K6_RELEASE_ARTIFACT_URL: https://github.com/grafana/k6/releases/download/v0.37.0/k6-v0.37.0-linux-amd64.tar.gz
         run: curl "${K6_RELEASE_ARTIFACT_URL}" -L | tar xvz --strip-components 1
 
       - name: Run Benchmark

diff --git a/k6/graphql-api/Gemfile.lock b/k6/graphql-api/Gemfile.lock
@@ -2,17 +2,28 @@ PATH
   remote: ../..
   specs:
     graphql-hive (0.4.4)
+      faraday (< 3)
       graphql (>= 2.3, < 3)
 
 GEM
   remote: https://rubygems.org/
   specs:
     base64 (0.2.0)
+    faraday (2.12.0)
+      faraday-net_http (>= 2.0, < 3.4)
+      json
+      logger
+    faraday-net_http (3.3.0)
+      net-http
     graphql (2.3.7)
       base64
+    json (2.7.2)
+    logger (1.6.1)
     multi_json (1.15.0)
     mustermann (2.0.2)
       ruby2_keywords (~> 0.0.1)
+    net-http (0.4.1)
+      uri
     nio4r (2.7.3)
     puma (6.4.3)
       nio4r (~> 2.0)
@@ -35,6 +46,7 @@ GEM
       sinatra (= 2.2.4)
       tilt (~> 2.0)
     tilt (2.4.0)
+    uri (0.13.1)
 
 PLATFORMS
   ruby

diff --git a/k6/graphql-api/puma.rb b/k6/graphql-api/puma.rb
@@ -0,0 +1,13 @@
+port ENV.fetch("PORT") { 9291 }
+threads_count = 5
+threads threads_count, threads_count
+workers 2
+preload_app!
+
+on_worker_boot do
+  GraphQL::Hive.instance.on_start
+end
+
+on_worker_shutdown do
+  GraphQL::Hive.instance.on_exit
+end
diff --git a/k6/k6.js b/k6/k6.js
@@ -3,36 +3,44 @@ import http from "k6/http";
 import { textSummary } from "https://jslib.k6.io/k6-summary/0.0.1/index.js";
 import { githubComment } from "https://raw.githubusercontent.com/dotansimha/k6-github-pr-comment/master/lib.js";
 
+const VIRTUAL_USER_COUNT = 10;
+const ITERATIONS = 100;
+const P_95_SUCCESS_THRESHOLD = 50;
+
 export const options = {
   discardResponseBodies: true,
   scenarios: {
     hiveEnabled: {
       executor: "shared-iterations",
-      vus: 120,
-      iterations: 500,
-      maxDuration: "30s",
+      vus: VIRTUAL_USER_COUNT,
+      iterations: ITERATIONS,
+      maxDuration: "5s",
       env: { GQL_API_PORT: "9292", HIVE_ENABLED: "true" },
     },
     hiveDisabled: {
       executor: "shared-iterations",
-      vus: 120,
-      iterations: 500,
-      maxDuration: "30s",
-      startTime: "30s",
+      vus: VIRTUAL_USER_COUNT,
+      iterations: ITERATIONS,
+      maxDuration: "5s",
       env: { GQL_API_PORT: "9291", HIVE_ENABLED: "false" },
     },
   },
   thresholds: {
-    "http_req_duration{hive:enabled}": ["avg<4500"],
-    "http_req_duration{hive:disabled}": ["avg<4500"],
+    "http_req_duration{hive:enabled}": [
+      { threshold: `p(95)<${P_95_SUCCESS_THRESHOLD}`, abortOnFail: true },
+    ],
+    "http_req_duration{hive:disabled}": [
+      { threshold: `p(95)<${P_95_SUCCESS_THRESHOLD}`, abortOnFail: true },
+    ],
+    http_req_failed: ["rate<0.1"],
   },
 };
 
 const QUERY = /* GraphQL */ `
   query GetPost {
     post(id: 1) {
+      id
       title
-      myId: id
     }
   }
 `;
@@ -49,51 +57,58 @@ export default function () {
     tags: { hive: __ENV.HIVE_ENABLED === "true" ? "enabled" : "disabled" },
   };
 
-  return http.post(
+  const response = http.post(
     `http://localhost:${__ENV.GQL_API_PORT}/graphql`,
     payload,
-    params
+    params,
   );
+  check(response, {
+    "is status 200": (r) => r.status === 200,
+    "body is graphql success": (r) =>
+      r.body ===
+      '{"data":{"post":{"id": 1, "title": "GraphQL Hive with `graphql-ruby`","truncated_preview": "Monitor operations, inspect your queries and publish your GraphQL schema with GraphQL Hive"}}}',
+  });
 }
 
 export function handleSummary(data) {
-  let overheadPercentage = null;
-  if (
-    data.metrics["http_req_duration{hive:enabled}"] &&
-    data.metrics["http_req_duration{hive:disabled}"]
-  ) {
-    const withHive =
-      data.metrics["http_req_duration{hive:enabled}"].values["avg"];
-    const withoutHive =
-      data.metrics["http_req_duration{hive:disabled}"].values["avg"];
-    overheadPercentage = 100 - (withHive * 100.0) / withoutHive;
-  }
-  if (__ENV.GITHUB_TOKEN) {
-    githubComment(data, {
-      token: __ENV.GITHUB_TOKEN,
-      commit: __ENV.GITHUB_SHA,
-      pr: __ENV.GITHUB_PR,
-      org: "charlypoly",
-      repo: "graphql-ruby-hive",
-      renderTitle: () => {
-        return overheadPercentage < 5
-          ? "✅ Benchmark Results"
-          : "❌ Benchmark Failed";
-      },
-      renderMessage: () => {
-        const result = [];
-        if (overheadPercentage > 5) {
-          result.push(
-            "**Performance regression detected**: it seems like your Pull Request adds some extra latency to GraphQL Hive operations processing"
-          );
-        } else {
-          result.push("Overhead <= 1%");
-        }
-        return result.join("\n");
-      },
-    });
-  }
+  postToGithub(data);
+
   return {
     stdout: textSummary(data, { indent: " ", enableColors: true }),
   };
 }
+
+const postToGithub = (data) => {
+  if (!__ENV.GITHUB_TOKEN) {
+    return;
+  }
+
+  const message = {
+    title: "✅ Benchmark Results: Overhead <= 1%",
+    text: "Overhead <= 5%",
+  };
+
+  const withHive =
+    data.metrics["http_req_duration{hive:enabled}"].values["avg"];
+  const withoutHive =
+    data.metrics["http_req_duration{hive:disabled}"].values["avg"];
+  const overheadPercentage = 100 - (withHive * 100.0) / withoutHive;
+
+  if (overheadPercentage > 5) {
+    message.title = "❌ Benchmark Failed: Performance regression detected";
+    message.text =
+      "**Performance regression detected**: this Pull Request adds extra latency to GraphQL Server";
+  }
+
+  const githubCommentOptions = {
+    token: __ENV.GITHUB_TOKEN,
+    commit: __ENV.GITHUB_SHA,
+    pr: __ENV.GITHUB_PR,
+    org: "charlypoly",
+    repo: "graphql-ruby-hive",
+    renderTitle: () => message.title,
+    renderMessage: () => message.text,
+  };
+
+  githubComment(data, githubCommentOptions);
+};