feat: ai failure analysis (#176)

test-results-reporter · May 5, 2024 · 2ec86fd · 2ec86fd
1 parent a7bea71
commit 2ec86fd
Show file tree

Hide file tree

Showing 8 changed files with 269 additions and 2 deletions.
diff --git a/src/beats/index.js b/src/beats/index.js
@@ -1,6 +1,7 @@
 const request = require('phin-retry');
 const TestResult = require('test-results-parser/src/models/TestResult');
 const { getCIInformation } = require('../helpers/ci');
+const { HOOK } = require('../helpers/constants');
 
 function get_base_url() {
   return process.env.TEST_BEATS_URL || "https://app.testbeats.com";
@@ -16,6 +17,7 @@ async function run(config, result) {
     const run_id = await publishTestResults(config, result);
     if (run_id) {
       attachTestBeatsReportHyperLink(config, run_id);
+      await attachTestBeatsFailureSummary(config, result, run_id);
     }
   } else {
     console.warn('Missing testbeats config parameters');
@@ -84,6 +86,82 @@ function attachTestBeatsReportHyperLink(config, run_id) {
   }
 }
 
+/**
+ * @param {import('../index').PublishReport} config
+ * @param {TestResult} result
+ * @param {string} run_id
+ */
+async function attachTestBeatsFailureSummary(config, result, run_id) {
+  if (result.status !== 'FAIL') {
+    return;
+  }
+  if (config.show_failure_summary === false) {
+    return;
+  }
+  try {
+    await processFailureSummary(config, run_id);
+  } catch (error) {
+    console.log(error);
+    console.log("error processing failure summary");
+  }
+}
+
+async function processFailureSummary(config, run_id) {
+  let retry = 3;
+  while (retry > 0) {
+    const test_run = await getTestRun(config, run_id);
+    if (test_run && test_run.failure_summary_status) {
+      if (test_run.failure_summary_status === 'COMPLETED') {
+        addAIFailureSummaryExtension(config, test_run);
+        return;
+      } else if (test_run.failure_summary_status === 'FAILED') {
+        console.log(`Test run failure summary failed`);
+        return;
+      } else if (test_run.failure_summary_status === 'SKIPPED') {
+        console.log(`Test run failure summary failed`);
+        return;
+      }
+    }
+    console.log(`Test run failure summary not completed, retrying...`);
+    await new Promise(resolve => setTimeout(resolve, 3000));
+    retry = retry - 1;
+  }
+}
+
+/**
+ * @param {import('../index').PublishReport} config
+ * @param {string} run_id
+ */
+function getTestRun(config, run_id) {
+  return request.get({
+    url: `${get_base_url()}/api/core/v1/test-runs/key?id=${run_id}`,
+    headers: {
+      'x-api-key': config.api_key
+    }
+  });
+}
+
+function getAIFailureSummaryExtension(test_run) {
+  const execution_metric = test_run.execution_metrics[0];
+  return {
+    name: 'ai-failure-summary',
+    hook: HOOK.AFTER_SUMMARY,
+    inputs: {
+      failure_summary: execution_metric.failure_summary
+    }
+  };
+}
+
+function addAIFailureSummaryExtension(config, test_run) {
+  const extension = getAIFailureSummaryExtension(test_run);
+  if (config.targets) {
+    for (const target of config.targets) {
+      target.extensions = target.extensions || [];
+      target.extensions.push(extension);
+    }
+  }
+}
+
 /**
  *
  * @param {string} run_id

diff --git a/src/extensions/ai-failure-summary.js b/src/extensions/ai-failure-summary.js
@@ -0,0 +1,72 @@
+const { STATUS, HOOK } = require("../helpers/constants");
+const { addChatExtension, addSlackExtension, addTeamsExtension } = require('../helpers/extension.helper');
+
+/**
+ * @param {object} param0
+ * @param {import('..').Target} param0.target
+ * @param {import('..').MetadataExtension} param0.extension
+ */
+async function run({ target, extension, result, payload, root_payload }) {
+  extension.inputs = Object.assign({}, default_inputs, extension.inputs);
+  if (target.name === 'teams') {
+    extension.inputs = Object.assign({}, default_inputs_teams, extension.inputs);
+    await attachForTeams({ target, extension, payload, result });
+  } else if (target.name === 'slack') {
+    extension.inputs = Object.assign({}, default_inputs_slack, extension.inputs);
+    await attachForSlack({ target, extension, payload, result });
+  } else if (target.name === 'chat') {
+    extension.inputs = Object.assign({}, default_inputs_chat, extension.inputs);
+    await attachForChat({ target, extension, payload, result });
+  }
+}
+
+/**
+ * @param {object} param0
+ * @param {import('..').MetadataExtension} param0.extension
+ */
+async function attachForTeams({ target, extension, payload, result }) {
+  const text = extension.inputs.failure_summary
+  if (text) {
+    addTeamsExtension({ payload, extension, text });
+  }
+}
+
+async function attachForSlack({ target, extension, payload, result }) {
+  const text = extension.inputs.failure_summary
+  if (text) {
+    addSlackExtension({ payload, extension, text });
+  }
+}
+
+async function attachForChat({ target, extension, payload, result }) {
+  const text = extension.inputs.failure_summary
+  if (text) {
+    addChatExtension({ payload, extension, text });
+  }
+}
+
+const default_options = {
+  hook: HOOK.AFTER_SUMMARY,
+  condition: STATUS.FAIL,
+}
+
+const default_inputs = {
+  title: 'AI Failure Summary ✨'
+}
+
+const default_inputs_teams = {
+  separator: true
+}
+
+const default_inputs_slack = {
+  separator: false
+}
+
+const default_inputs_chat = {
+  separator: true
+}
+
+module.exports = {
+  run,
+  default_options
+}
diff --git a/src/extensions/index.js b/src/extensions/index.js
@@ -7,6 +7,7 @@ const percy_analysis = require('./percy-analysis');
 const custom = require('./custom');
 const metadata = require('./metadata');
 const ci_info = require('./ci-info');
+const ai_failure_summary = require('./ai-failure-summary');
 const { EXTENSION } = require('../helpers/constants');
 const { checkCondition } = require('../helpers/helper');
 
@@ -53,6 +54,8 @@ function getExtensionRunner(extension) {
       return metadata;
     case EXTENSION.CI_INFO:
       return ci_info;
+    case EXTENSION.AI_FAILURE_SUMMARY:
+      return ai_failure_summary;
     default:
       return require(extension.name);
   }

diff --git a/src/helpers/constants.js b/src/helpers/constants.js
@@ -20,6 +20,7 @@ const TARGET = Object.freeze({
 });
 
 const EXTENSION = Object.freeze({
+  AI_FAILURE_SUMMARY: 'ai-failure-summary',
   HYPERLINKS: 'hyperlinks',
   MENTIONS: 'mentions',
   REPORT_PORTAL_ANALYSIS: 'report-portal-analysis',

diff --git a/src/index.d.ts b/src/index.d.ts
@@ -4,7 +4,7 @@ import { Schedule, User } from 'rosters';
 import { ParseOptions } from 'test-results-parser';
 import TestResult from 'test-results-parser/src/models/TestResult';
 
-export type ExtensionName = 'report-portal-analysis' | 'hyperlinks' | 'mentions' | 'report-portal-history' | 'quick-chart-test-summary' | 'metadata' | 'ci-info' | 'custom';
+export type ExtensionName = 'report-portal-analysis' | 'hyperlinks' | 'mentions' | 'report-portal-history' | 'quick-chart-test-summary' | 'metadata' | 'ci-info' | 'custom' | 'ai-failure-summary';
 export type Hook = 'start' | 'end' | 'after-summary';
 export type TargetName = 'slack' | 'teams' | 'chat' | 'custom' | 'delay';
 export type PublishReportType = 'test-summary' | 'test-summary-slim' | 'failure-details';
@@ -61,11 +61,15 @@ export interface CIInfoInputs extends ExtensionInputs {
   data?: Metadata[];
 }
 
+export interface AIFailureSummaryInputs extends ExtensionInputs {
+  failure_summary: string;
+}
+
 export interface Extension {
   name: ExtensionName;
   condition?: Condition;
   hook?: Hook;
-  inputs?: ReportPortalAnalysisInputs | ReportPortalHistoryInputs | HyperlinkInputs | MentionInputs | QuickChartTestSummaryInputs | PercyAnalysisInputs | CustomExtensionInputs | MetadataInputs | CIInfoInputs;
+  inputs?: ReportPortalAnalysisInputs | ReportPortalHistoryInputs | HyperlinkInputs | MentionInputs | QuickChartTestSummaryInputs | PercyAnalysisInputs | CustomExtensionInputs | MetadataInputs | CIInfoInputs | AIFailureSummaryInputs;
 }
 
 export interface PercyAnalysisInputs extends ExtensionInputs {
@@ -222,6 +226,7 @@ export interface PublishReport {
   api_key?: string;
   project?: string;
   run?: string;
+  show_failure_summary?: boolean;
   targets?: Target[];
   results?: ParseOptions[] | PerformanceParseOptions[] | CustomResultOptions[];
 }

diff --git a/test/beats.spec.js b/test/beats.spec.js
@@ -34,6 +34,38 @@ describe('TestBeats', () => {
     assert.equal(mock.getInteraction(id2).exercised, true);
   });
 
+  it('should send results with failures to beats', async () => {
+    const id1 = mock.addInteraction('post test results to beats');
+    const id2 = mock.addInteraction('get test results from beats');
+    const id3 = mock.addInteraction('post test-summary with beats to teams with ai failure summary');
+    await publish({
+      config: {
+        api_key: 'api-key',
+        project: 'project-name',
+        run: 'build-name',
+        targets: [
+          {
+            name: 'teams',
+            inputs: {
+              url: 'http://localhost:9393/message'
+            }
+          }
+        ],
+        results: [
+          {
+            type: 'testng',
+            files: [
+              'test/data/testng/single-suite-failures.xml'
+            ]
+          }
+        ]
+      }
+    });
+    assert.equal(mock.getInteraction(id1).exercised, true);
+    assert.equal(mock.getInteraction(id2).exercised, true);
+    assert.equal(mock.getInteraction(id3).exercised, true);
+  });
+
   afterEach(() => {
     mock.clearInteractions();
   });

diff --git a/test/mocks/beats.mock.js b/test/mocks/beats.mock.js
@@ -14,4 +14,29 @@ addInteractionHandler('post test results to beats', () => {
       }
     }
   }
+});
+
+addInteractionHandler('get test results from beats', () => {
+  return {
+    strict: false,
+    request: {
+      method: 'GET',
+      path: '/api/core/v1/test-runs/key',
+      queryParams: {
+        "id": "test-run-id"
+      }
+    },
+    response: {
+      status: 200,
+      body: {
+        id: 'test-run-id',
+        "failure_summary_status": "COMPLETED",
+        "execution_metrics": [
+          {
+            "failure_summary": "test failure summary"
+          }
+        ]
+      }
+    }
+  }
 });
diff --git a/test/mocks/teams.mock.js b/test/mocks/teams.mock.js
@@ -1523,4 +1523,55 @@ addInteractionHandler('post test-summary with ci-info to teams', () => {
       status: 200
     }
   }
+});
+
+addInteractionHandler('post test-summary with beats to teams with ai failure summary', () => {
+  return {
+    request: {
+      method: 'POST',
+      path: '/message',
+      body: {
+        "type": "message",
+        "attachments": [
+          {
+            "contentType": "application/vnd.microsoft.card.adaptive",
+            "content": {
+              "$schema": "http://adaptivecards.io/schemas/adaptive-card.json",
+              "type": "AdaptiveCard",
+              "version": "1.0",
+              "body": [
+                {
+                  "type": "TextBlock",
+                  "text": "[❌ Default suite](http://localhost:9393/reports/test-run-id)",
+                  "size": "medium",
+                  "weight": "bolder",
+                  "wrap": true
+                },
+                {
+                  "@DATA:TEMPLATE@": "TEAMS_ROOT_RESULTS_SINGLE_SUITE_FAILURES",
+                },
+                {
+                  "type": "TextBlock",
+                  "text": "AI Failure Summary ✨",
+                  "isSubtle": true,
+                  "weight": "bolder",
+                  "separator": true,
+                  "wrap": true
+                },
+                {
+                  "type": "TextBlock",
+                  "text": "test failure summary",
+                  "wrap": true
+                }
+              ],
+              "actions": []
+            }
+          }
+        ]
+      }
+    },
+    response: {
+      status: 200
+    }
+  }
 });