From 2ec86fd1f063d5aee35342e5402c346fc132683e Mon Sep 17 00:00:00 2001 From: Anudeep Date: Sun, 5 May 2024 22:08:52 +0530 Subject: [PATCH] feat: ai failure analysis (#176) --- src/beats/index.js | 78 ++++++++++++++++++++++++++++ src/extensions/ai-failure-summary.js | 72 +++++++++++++++++++++++++ src/extensions/index.js | 3 ++ src/helpers/constants.js | 1 + src/index.d.ts | 9 +++- test/beats.spec.js | 32 ++++++++++++ test/mocks/beats.mock.js | 25 +++++++++ test/mocks/teams.mock.js | 51 ++++++++++++++++++ 8 files changed, 269 insertions(+), 2 deletions(-) create mode 100644 src/extensions/ai-failure-summary.js diff --git a/src/beats/index.js b/src/beats/index.js index 9b4c3ac..1c900a7 100644 --- a/src/beats/index.js +++ b/src/beats/index.js @@ -1,6 +1,7 @@ const request = require('phin-retry'); const TestResult = require('test-results-parser/src/models/TestResult'); const { getCIInformation } = require('../helpers/ci'); +const { HOOK } = require('../helpers/constants'); function get_base_url() { return process.env.TEST_BEATS_URL || "https://app.testbeats.com"; @@ -16,6 +17,7 @@ async function run(config, result) { const run_id = await publishTestResults(config, result); if (run_id) { attachTestBeatsReportHyperLink(config, run_id); + await attachTestBeatsFailureSummary(config, result, run_id); } } else { console.warn('Missing testbeats config parameters'); @@ -84,6 +86,82 @@ function attachTestBeatsReportHyperLink(config, run_id) { } } +/** + * @param {import('../index').PublishReport} config + * @param {TestResult} result + * @param {string} run_id + */ +async function attachTestBeatsFailureSummary(config, result, run_id) { + if (result.status !== 'FAIL') { + return; + } + if (config.show_failure_summary === false) { + return; + } + try { + await processFailureSummary(config, run_id); + } catch (error) { + console.log(error); + console.log("error processing failure summary"); + } +} + +async function processFailureSummary(config, run_id) { + let retry = 3; + while (retry > 0) { + const test_run = await getTestRun(config, run_id); + if (test_run && test_run.failure_summary_status) { + if (test_run.failure_summary_status === 'COMPLETED') { + addAIFailureSummaryExtension(config, test_run); + return; + } else if (test_run.failure_summary_status === 'FAILED') { + console.log(`Test run failure summary failed`); + return; + } else if (test_run.failure_summary_status === 'SKIPPED') { + console.log(`Test run failure summary failed`); + return; + } + } + console.log(`Test run failure summary not completed, retrying...`); + await new Promise(resolve => setTimeout(resolve, 3000)); + retry = retry - 1; + } +} + +/** + * @param {import('../index').PublishReport} config + * @param {string} run_id + */ +function getTestRun(config, run_id) { + return request.get({ + url: `${get_base_url()}/api/core/v1/test-runs/key?id=${run_id}`, + headers: { + 'x-api-key': config.api_key + } + }); +} + +function getAIFailureSummaryExtension(test_run) { + const execution_metric = test_run.execution_metrics[0]; + return { + name: 'ai-failure-summary', + hook: HOOK.AFTER_SUMMARY, + inputs: { + failure_summary: execution_metric.failure_summary + } + }; +} + +function addAIFailureSummaryExtension(config, test_run) { + const extension = getAIFailureSummaryExtension(test_run); + if (config.targets) { + for (const target of config.targets) { + target.extensions = target.extensions || []; + target.extensions.push(extension); + } + } +} + /** * * @param {string} run_id diff --git a/src/extensions/ai-failure-summary.js b/src/extensions/ai-failure-summary.js new file mode 100644 index 0000000..7226750 --- /dev/null +++ b/src/extensions/ai-failure-summary.js @@ -0,0 +1,72 @@ +const { STATUS, HOOK } = require("../helpers/constants"); +const { addChatExtension, addSlackExtension, addTeamsExtension } = require('../helpers/extension.helper'); + +/** + * @param {object} param0 + * @param {import('..').Target} param0.target + * @param {import('..').MetadataExtension} param0.extension + */ +async function run({ target, extension, result, payload, root_payload }) { + extension.inputs = Object.assign({}, default_inputs, extension.inputs); + if (target.name === 'teams') { + extension.inputs = Object.assign({}, default_inputs_teams, extension.inputs); + await attachForTeams({ target, extension, payload, result }); + } else if (target.name === 'slack') { + extension.inputs = Object.assign({}, default_inputs_slack, extension.inputs); + await attachForSlack({ target, extension, payload, result }); + } else if (target.name === 'chat') { + extension.inputs = Object.assign({}, default_inputs_chat, extension.inputs); + await attachForChat({ target, extension, payload, result }); + } +} + +/** + * @param {object} param0 + * @param {import('..').MetadataExtension} param0.extension + */ +async function attachForTeams({ target, extension, payload, result }) { + const text = extension.inputs.failure_summary + if (text) { + addTeamsExtension({ payload, extension, text }); + } +} + +async function attachForSlack({ target, extension, payload, result }) { + const text = extension.inputs.failure_summary + if (text) { + addSlackExtension({ payload, extension, text }); + } +} + +async function attachForChat({ target, extension, payload, result }) { + const text = extension.inputs.failure_summary + if (text) { + addChatExtension({ payload, extension, text }); + } +} + +const default_options = { + hook: HOOK.AFTER_SUMMARY, + condition: STATUS.FAIL, +} + +const default_inputs = { + title: 'AI Failure Summary ✨' +} + +const default_inputs_teams = { + separator: true +} + +const default_inputs_slack = { + separator: false +} + +const default_inputs_chat = { + separator: true +} + +module.exports = { + run, + default_options +} \ No newline at end of file diff --git a/src/extensions/index.js b/src/extensions/index.js index 4e174e7..ea4b485 100644 --- a/src/extensions/index.js +++ b/src/extensions/index.js @@ -7,6 +7,7 @@ const percy_analysis = require('./percy-analysis'); const custom = require('./custom'); const metadata = require('./metadata'); const ci_info = require('./ci-info'); +const ai_failure_summary = require('./ai-failure-summary'); const { EXTENSION } = require('../helpers/constants'); const { checkCondition } = require('../helpers/helper'); @@ -53,6 +54,8 @@ function getExtensionRunner(extension) { return metadata; case EXTENSION.CI_INFO: return ci_info; + case EXTENSION.AI_FAILURE_SUMMARY: + return ai_failure_summary; default: return require(extension.name); } diff --git a/src/helpers/constants.js b/src/helpers/constants.js index b9f5dab..c2bf4fe 100644 --- a/src/helpers/constants.js +++ b/src/helpers/constants.js @@ -20,6 +20,7 @@ const TARGET = Object.freeze({ }); const EXTENSION = Object.freeze({ + AI_FAILURE_SUMMARY: 'ai-failure-summary', HYPERLINKS: 'hyperlinks', MENTIONS: 'mentions', REPORT_PORTAL_ANALYSIS: 'report-portal-analysis', diff --git a/src/index.d.ts b/src/index.d.ts index 9213440..3c0be10 100644 --- a/src/index.d.ts +++ b/src/index.d.ts @@ -4,7 +4,7 @@ import { Schedule, User } from 'rosters'; import { ParseOptions } from 'test-results-parser'; import TestResult from 'test-results-parser/src/models/TestResult'; -export type ExtensionName = 'report-portal-analysis' | 'hyperlinks' | 'mentions' | 'report-portal-history' | 'quick-chart-test-summary' | 'metadata' | 'ci-info' | 'custom'; +export type ExtensionName = 'report-portal-analysis' | 'hyperlinks' | 'mentions' | 'report-portal-history' | 'quick-chart-test-summary' | 'metadata' | 'ci-info' | 'custom' | 'ai-failure-summary'; export type Hook = 'start' | 'end' | 'after-summary'; export type TargetName = 'slack' | 'teams' | 'chat' | 'custom' | 'delay'; export type PublishReportType = 'test-summary' | 'test-summary-slim' | 'failure-details'; @@ -61,11 +61,15 @@ export interface CIInfoInputs extends ExtensionInputs { data?: Metadata[]; } +export interface AIFailureSummaryInputs extends ExtensionInputs { + failure_summary: string; +} + export interface Extension { name: ExtensionName; condition?: Condition; hook?: Hook; - inputs?: ReportPortalAnalysisInputs | ReportPortalHistoryInputs | HyperlinkInputs | MentionInputs | QuickChartTestSummaryInputs | PercyAnalysisInputs | CustomExtensionInputs | MetadataInputs | CIInfoInputs; + inputs?: ReportPortalAnalysisInputs | ReportPortalHistoryInputs | HyperlinkInputs | MentionInputs | QuickChartTestSummaryInputs | PercyAnalysisInputs | CustomExtensionInputs | MetadataInputs | CIInfoInputs | AIFailureSummaryInputs; } export interface PercyAnalysisInputs extends ExtensionInputs { @@ -222,6 +226,7 @@ export interface PublishReport { api_key?: string; project?: string; run?: string; + show_failure_summary?: boolean; targets?: Target[]; results?: ParseOptions[] | PerformanceParseOptions[] | CustomResultOptions[]; } diff --git a/test/beats.spec.js b/test/beats.spec.js index 36e1940..f48ac28 100644 --- a/test/beats.spec.js +++ b/test/beats.spec.js @@ -34,6 +34,38 @@ describe('TestBeats', () => { assert.equal(mock.getInteraction(id2).exercised, true); }); + it('should send results with failures to beats', async () => { + const id1 = mock.addInteraction('post test results to beats'); + const id2 = mock.addInteraction('get test results from beats'); + const id3 = mock.addInteraction('post test-summary with beats to teams with ai failure summary'); + await publish({ + config: { + api_key: 'api-key', + project: 'project-name', + run: 'build-name', + targets: [ + { + name: 'teams', + inputs: { + url: 'http://localhost:9393/message' + } + } + ], + results: [ + { + type: 'testng', + files: [ + 'test/data/testng/single-suite-failures.xml' + ] + } + ] + } + }); + assert.equal(mock.getInteraction(id1).exercised, true); + assert.equal(mock.getInteraction(id2).exercised, true); + assert.equal(mock.getInteraction(id3).exercised, true); + }); + afterEach(() => { mock.clearInteractions(); }); diff --git a/test/mocks/beats.mock.js b/test/mocks/beats.mock.js index ef7c777..89f9665 100644 --- a/test/mocks/beats.mock.js +++ b/test/mocks/beats.mock.js @@ -14,4 +14,29 @@ addInteractionHandler('post test results to beats', () => { } } } +}); + +addInteractionHandler('get test results from beats', () => { + return { + strict: false, + request: { + method: 'GET', + path: '/api/core/v1/test-runs/key', + queryParams: { + "id": "test-run-id" + } + }, + response: { + status: 200, + body: { + id: 'test-run-id', + "failure_summary_status": "COMPLETED", + "execution_metrics": [ + { + "failure_summary": "test failure summary" + } + ] + } + } + } }); \ No newline at end of file diff --git a/test/mocks/teams.mock.js b/test/mocks/teams.mock.js index 488bcca..990e077 100644 --- a/test/mocks/teams.mock.js +++ b/test/mocks/teams.mock.js @@ -1523,4 +1523,55 @@ addInteractionHandler('post test-summary with ci-info to teams', () => { status: 200 } } +}); + +addInteractionHandler('post test-summary with beats to teams with ai failure summary', () => { + return { + request: { + method: 'POST', + path: '/message', + body: { + "type": "message", + "attachments": [ + { + "contentType": "application/vnd.microsoft.card.adaptive", + "content": { + "$schema": "http://adaptivecards.io/schemas/adaptive-card.json", + "type": "AdaptiveCard", + "version": "1.0", + "body": [ + { + "type": "TextBlock", + "text": "[❌ Default suite](http://localhost:9393/reports/test-run-id)", + "size": "medium", + "weight": "bolder", + "wrap": true + }, + { + "@DATA:TEMPLATE@": "TEAMS_ROOT_RESULTS_SINGLE_SUITE_FAILURES", + }, + { + "type": "TextBlock", + "text": "AI Failure Summary ✨", + "isSubtle": true, + "weight": "bolder", + "separator": true, + "wrap": true + }, + { + "type": "TextBlock", + "text": "test failure summary", + "wrap": true + } + ], + "actions": [] + } + } + ] + } + }, + response: { + status: 200 + } + } }); \ No newline at end of file