remove regression calculation

k6/boot-servers.sh

@@ -1,10 +1,10 @@
 #!/bin/bash
 
 log_with_prefix() {
-    local prefix="$1"
-    while IFS= read -r line; do
-        echo "[$prefix] $line"
-    done
+  local prefix="$1"
+  while IFS= read -r line; do
+    echo "[$prefix] $line"
+  done
 }
 
 start_puma_server() {
@@ -14,9 +14,9 @@ start_puma_server() {
 
   echo "Starting Puma server with Hive ${hive_enabled}..."
   HIVE_ENABLED=$hive_enabled \
-  PORT=$port \
-  LOG_LEVEL=$LOG_LEVEL \
-  bundle exec puma -C puma.rb | log_with_prefix "$prefix" &
+    PORT=$port \
+    LOG_LEVEL=$LOG_LEVEL \
+    bundle exec puma -C puma.rb | log_with_prefix "$prefix" &
 }
 
 # Start Node.js server
@@ -31,13 +31,13 @@ start_puma_server false 9292 "hive-disabled"
 
 # Function to handle shutdown
 shutdown_servers() {
-    echo "Received shutdown signal. Shutting down servers..."
-    kill $(lsof -t -i:9291)
-    kill $(lsof -t -i:9292)
-    kill $(lsof -t -i:8888)
-    wait
-    echo "Servers shut down gracefully."
-    exit 0
+  echo "Received shutdown signal. Shutting down servers..."
+  kill $(lsof -t -i:9291)
+  kill $(lsof -t -i:9292)
+  kill $(lsof -t -i:8888)
+  wait
+  echo "Servers shut down gracefully."
+  exit 0
 }
 
 # Listen for kill signals

k6/integration-test.js

@@ -39,13 +39,11 @@ const QUERY = /* GraphQL */ `
   }
 `;
 export function setup() {
-  // Ensure usage counter is at 0
   const response = http.post("http://localhost:8888/reset");
   const { count } = JSON.parse(response.body);
   check(count, {
     "usage-api starts with 0 operations": (count) => count === 0,
   });
-  return { count };
 }
 
 export default function () {
@@ -72,73 +70,69 @@ export default function () {
     "response body is not a GraphQL error": (res) =>
       !res.body.includes("errors"),
   });
+  return res;
 }
 
-export function teardown(_data) {
-  const res = http.get("http://localhost:8888/count");
-  const count = JSON.parse(res.body).count;
+function sleep(seconds) {
+  return new Promise((resolve) => setTimeout(resolve, seconds * 1000));
+}
+
+function checkCount(count) {
   console.log(`📊 Total operations: ${count}`);
   check(count, {
     "usage-api received 1000 operations": (count) => count === REQUEST_COUNT,
   });
+  const response = http.post("http://localhost:8888/reset");
+  const { count: newCount } = JSON.parse(response.body);
+  check(newCount, {
+    "usage-api is reset": (c) => c === 0,
+  });
+}
+export function teardown(data) {
+  let count;
+  const res = http.get("http://localhost:8888/count");
+  count = JSON.parse(res.body).count;
+  if (count !== REQUEST_COUNT) {
+    sleep(1).then(() => {
+      console.log(`⁉️ Count was ${count}, retrying...`);
+      const res = http.get("http://localhost:8888/count");
+      count = JSON.parse(res.body).count;
+      checkCount(count);
+      return data;
+    });
+  } else {
+    checkCount(count);
+    return data;
+  }
 }
 
 export function handleSummary(data) {
-  const overhead = getOverheadPercentage(data);
-  const didPass = check(overhead, {
-    "overhead is less than 1%": (p) => p >= REGRESSION_THRESHOLD,
-  });
-
-  postGithubComment(didPass);
-
-  console.log(`⏰ Overhead percentage: ${overhead.toFixed(2)}%`);
-
-  if (!didPass) {
-    fail("❌❌ Performance regression detected ❌❌");
-  }
+  postGithubComment(data);
 
   return {
     stdout: textSummary(data, { indent: " ", enableColors: true }),
   };
 }
 
-function postGithubComment(didPass) {
+function postGithubComment(data) {
   if (!__ENV.GITHUB_TOKEN) {
     return;
   }
 
+  const checks = data.metrics.checks;
+  const didPass = checks.failed === 0;
+
   githubComment(data, {
     token: __ENV.GITHUB_TOKEN,
     commit: __ENV.GITHUB_SHA,
     pr: __ENV.GITHUB_PR,
     org: "charlypoly",
     repo: "graphql-ruby-hive",
-    renderTitle: () => {
-      return didPass ? "✅ Benchmark Results" : "❌ Benchmark Failed";
-    },
-    renderMessage: () => {
-      const result = [];
-      if (didPass) {
-        result.push(
-          "**Performance regression detected**: it seems like your Pull Request adds some extra latency to GraphQL Hive operations processing",
-        );
-      } else {
-        result.push("Overhead < 5%");
-      }
-      return result.join("\n");
-    },
+    renderTitle: () =>
+      didPass ? "✅ Integration Test Passed" : "❌ Integration Test Failed",
+    renderMessage: () =>
+      didPass
+        ? ""
+        : "The integration test failed. Please check the action logs for more information.",
   });
 }
-
-function getOverheadPercentage(data) {
-  const enabledMetric = data.metrics["http_req_duration{hive:enabled}"];
-  const disabledMetric = data.metrics["http_req_duration{hive:disabled}"];
-
-  if (enabledMetric && disabledMetric) {
-    const withHive = enabledMetric.values["avg"];
-    const withoutHive = disabledMetric.values["avg"];
-    return 100 - (withHive * 100.0) / withoutHive;
-  } else {
-    throw new Error("Could not calculate overhead. Missing metrics.");
-  }
-}