Update docs and use small buffer

.github/workflows/integration.yml

@@ -1,11 +1,12 @@
-name: Benchmark
+name: Integration
 on:
+  push:
   pull_request:
     branches:
       - master
 
 jobs:
-  benchmarks:
+  run-k6-tests:
     runs-on: ubuntu-latest
     steps:
       - name: Checkout Repository
@@ -25,25 +26,20 @@ jobs:
           ruby-version: "3.3"
           bundler-cache: true
 
-      - name: Install k6
-        working-directory: ./k6
-        env:
-          K6_RELEASE_ARTIFACT_URL: https://github.com/grafana/k6/releases/download/v0.37.0/k6-v0.37.0-linux-amd64.tar.gz
-
-        run: curl "${K6_RELEASE_ARTIFACT_URL}" -L | tar xvz --strip-components 1
       - name: Start Servers
-        working-directory: ./k6/graphql-api
+        working-directory: ./k6
         run: |
           ./boot-servers.sh &
           npx wait-on http://localhost:9292 --timeout 5s
           npx wait-on http://localhost:9291 --timeout 5s
           npx wait-on http://localhost:8888 --timeout 5s
 
-      - name: Run Integration Test
-        working-directory: ./k6
-        run: |
-          ./k6 \
-            -e GITHUB_PR=${{ github.event.number }} \
-            -e GITHUB_SHA=${{ github.sha }} \
-            -e GITHUB_TOKEN=${{secrets.GH_PA_TOKEN}} \
-            run integration-test.js
+      - uses: grafana/setup-k6-action@v1
+      - uses: grafana/run-k6-action@v1
+        env:
+          GITHUB_PR: ${{ github.event.number }}
+          GITHUB_SHA: ${{ github.sha }}
+          GITHUB_TOKEN: ${{secrets.GH_PA_TOKEN}}
+        with:
+          path: |
+            ./k6/integration-test.js

README.md

@@ -167,12 +167,15 @@ bundle exec rake test
 
 5. **Run Integration Tests**:
 
+In one terminal, start the servers:
+
 ```sh
 cd k6
-yarn install
-cd graphql-api
-bundle install
-cd ../
 ./boot-servers.sh
+```
+
+In another terminal, run the tests:
+
+```sh
 k6 run integration-tests.js
 ```

k6/boot-servers.sh

@@ -1,10 +1,10 @@
 #!/bin/bash
 
 log_with_prefix() {
-    local prefix="$1"
-    while IFS= read -r line; do
-        echo "[$prefix] $line"
-    done
+  local prefix="$1"
+  while IFS= read -r line; do
+    echo "[$prefix] $line"
+  done
 }
 
 start_puma_server() {
@@ -14,14 +14,23 @@ start_puma_server() {
 
   echo "Starting Puma server with Hive ${hive_enabled}..."
   HIVE_ENABLED=$hive_enabled \
-  PORT=$port \
-  LOG_LEVEL=$LOG_LEVEL \
-  bundle exec puma -C puma.rb | log_with_prefix "$prefix" &
+    PORT=$port \
+    LOG_LEVEL=$LOG_LEVEL \
+    bundle exec puma -C puma.rb | log_with_prefix "$prefix" &
 }
 
+echo "Installing dependencies..."
+yarn install
+cd graphql-api || {
+  echo "Could not find graphql-api" && exit 1
+}
+bundle install
+cd ..
+
 # Start Node.js server
 echo "Starting usage-api server..."
 LOG_LEVEL=$LOG_LEVEL node usage-api.js | log_with_prefix "usage-api" &
+
 # Start first Puma server
 cd graphql-api || {
   echo "Could not find graphql-api" && exit 1
@@ -31,13 +40,19 @@ start_puma_server false 9292 "hive-disabled"
 
 # Function to handle shutdown
 shutdown_servers() {
-    echo "Received shutdown signal. Shutting down servers..."
-    kill $(lsof -t -i:9291)
-    kill $(lsof -t -i:9292)
-    kill $(lsof -t -i:8888)
-    wait
-    echo "Servers shut down gracefully."
-    exit 0
+  echo "Received shutdown signal. Shutting down servers..."
+
+  for port in 9291 9292 8888; do
+    pid=$(lsof -t -i:$port)
+    if [ -n "$pid" ]; then
+      kill "$pid"
+    else
+      echo "No process found on port $port"
+    fi
+  done
+
+  echo "Servers shut down gracefully."
+  exit 0
 }
 
 # Listen for kill signals

k6/graphql-api/app.rb

@@ -9,7 +9,7 @@ class DemoApp < Sinatra::Base
   use Rack::JSONBodyParser
 
   configure do
-    set :logger, Logger.new(STDOUT)
+    set :logger, Logger.new($stdout)
     log_level = ENV.fetch("LOG_LEVEL", "INFO").upcase
     logger.level = begin
       Logger.const_get(log_level)

k6/graphql-api/schema.rb

@@ -34,7 +34,7 @@ class Schema < GraphQL::Schema
     enabled: ENV["HIVE_ENABLED"] === "true",
     endpoint: "localhost",
     debug: false,
-    buffer_size: 10,
+    buffer_size: 1,
     port: 8888,
     token: "stress-token",
     report_schema: false

k6/integration-test.js

@@ -25,8 +25,24 @@ export const options = {
     },
   },
   thresholds: {
-    "http_req_duration{hive:enabled}": ["p(95)<15"],
-    "http_req_duration{hive:disabled}": ["p(95)<15"],
+    "http_req_duration{hive:enabled}": [
+      {
+        threshold: "p(95)<25",
+        abortOnFail: true,
+      },
+    ],
+    "http_req_duration{hive:disabled}": [
+      {
+        threshold: "p(95)<25",
+        abortOnFail: true,
+      },
+    ],
+    checks: [
+      {
+        threshold: "rate===1",
+        abortOnFail: true,
+      },
+    ],
   },
 };
 
@@ -39,13 +55,11 @@ const QUERY = /* GraphQL */ `
   }
 `;
 export function setup() {
-  // Ensure usage counter is at 0
   const response = http.post("http://localhost:8888/reset");
   const { count } = JSON.parse(response.body);
   check(count, {
     "usage-api starts with 0 operations": (count) => count === 0,
   });
-  return { count };
 }
 
 export default function () {
@@ -72,73 +86,61 @@ export default function () {
     "response body is not a GraphQL error": (res) =>
       !res.body.includes("errors"),
   });
+  return res;
 }
 
-export function teardown(_data) {
-  const res = http.get("http://localhost:8888/count");
-  const count = JSON.parse(res.body).count;
-  console.log(`📊 Total operations: ${count}`);
+function sleep(seconds) {
+  return new Promise((resolve) => setTimeout(resolve, seconds * 1000));
+}
+
+export async function teardown(data) {
+  let count = 0;
+  for (let i = 0; i < 10; i++) {
+    const res = http.get("http://localhost:8888/count");
+    count = JSON.parse(res.body).count;
+    console.log(`📊 Total operations: ${count}`);
+    if (count === REQUEST_COUNT) {
+      break;
+    }
+    await sleep(1);
+  }
   check(count, {
-    "usage-api received 200 operations": (count) => count === REQUEST_COUNT,
+    "usage-api received correct number of operations": (count) =>
+      count === REQUEST_COUNT,
   });
+  const response = http.post("http://localhost:8888/reset");
+  const { count: newCount } = JSON.parse(response.body);
+  check(newCount, {
+    "usage-api is reset": (c) => c === 0,
+  });
+  return data;
 }
 
 export function handleSummary(data) {
-  const overhead = getOverheadPercentage(data);
-  const didPass = check(overhead, {
-    "overhead is less than 1%": (p) => p >= REGRESSION_THRESHOLD,
-  });
-
-  postGithubComment(didPass);
-
-  console.log(`⏰ Overhead percentage: ${overhead.toFixed(2)}%`);
-
-  if (!didPass) {
-    fail("❌❌ Performance regression detected ❌❌");
-  }
+  const checks = data.metrics.checks;
+  const didPass = checks.values.fails.length === 0;
+  postGithubComment(data, didPass);
 
   return {
     stdout: textSummary(data, { indent: " ", enableColors: true }),
   };
 }
 
-function postGithubComment(didPass) {
+function postGithubComment(data, didPass) {
   if (!__ENV.GITHUB_TOKEN) {
     return;
   }
-
   githubComment(data, {
     token: __ENV.GITHUB_TOKEN,
     commit: __ENV.GITHUB_SHA,
     pr: __ENV.GITHUB_PR,
-    org: "charlypoly",
+    org: "rperryng",
     repo: "graphql-ruby-hive",
-    renderTitle: () => {
-      return didPass ? "✅ Benchmark Results" : "❌ Benchmark Failed";
-    },
-    renderMessage: () => {
-      const result = [];
-      if (didPass) {
-        result.push(
-          "**Performance regression detected**: it seems like your Pull Request adds some extra latency to GraphQL Hive operations processing",
-        );
-      } else {
-        result.push("Overhead < 5%");
-      }
-      return result.join("\n");
-    },
+    renderTitle: () =>
+      didPass ? "✅ Integration Test Passed" : "❌ Integration Test Failed",
+    renderMessage: () =>
+      didPass
+        ? ""
+        : "The integration test failed. Please check the action logs for more information.",
   });
 }
-
-function getOverheadPercentage(data) {
-  const enabledMetric = data.metrics["http_req_duration{hive:enabled}"];
-  const disabledMetric = data.metrics["http_req_duration{hive:disabled}"];
-
-  if (enabledMetric && disabledMetric) {
-    const withHive = enabledMetric.values["avg"];
-    const withoutHive = disabledMetric.values["avg"];
-    return 100 - (withHive * 100.0) / withoutHive;
-  } else {
-    throw new Error("Could not calculate overhead. Missing metrics.");
-  }
-}