From 622e10dbda736aee23491d3fa329ee07fd3a1eb8 Mon Sep 17 00:00:00 2001
From: bryan cook <3217452+bryanbcook@users.noreply.github.com>
Date: Sat, 11 Nov 2023 13:31:44 -0500
Subject: [PATCH 1/4] implementation for nunit #4
---
src/helpers/helper.js | 15 ++-
src/parsers/index.js | 3 +
src/parsers/nunit.js | 150 ++++++++++++++++++++++++++++
tests/data/nunit/sample.xml | 190 ++++++++++++++++++++++++++++++++++++
tests/parser.nunit.spec.js | 113 +++++++++++++++++++++
5 files changed, 470 insertions(+), 1 deletion(-)
create mode 100644 src/parsers/nunit.js
create mode 100644 tests/data/nunit/sample.xml
create mode 100644 tests/parser.nunit.spec.js
diff --git a/src/helpers/helper.js b/src/helpers/helper.js
index 7416c0e..f5c43eb 100644
--- a/src/helpers/helper.js
+++ b/src/helpers/helper.js
@@ -31,13 +31,26 @@ const FORCED_ARRAY_KEYS = [
"testng-results.suite.test",
"testng-results.suite.test.class",
"testng-results.suite.test.class.test-method",
- "testng-results.suite.test.class.test-method.exception",
+ "testng-results.suite.test.class.test-method.exception"
];
const configured_parser = new XMLParser({
isArray: (name, jpath, isLeafNode, isAttribute) => {
if( FORCED_ARRAY_KEYS.indexOf(jpath) !== -1) {
return true;
+ }
+ // handle nunit deep hierarchy
+ else if (jpath.startsWith("test-results")) {
+ let parts = jpath.split(".");
+ switch(parts[parts.length - 1]) {
+ case "category":
+ case "property":
+ case "test-suite":
+ case "test-case":
+ return true;
+ default:
+ return false;
+ }
}
},
ignoreAttributes: false,
diff --git a/src/parsers/index.js b/src/parsers/index.js
index 6cacfc4..edf4d6c 100644
--- a/src/parsers/index.js
+++ b/src/parsers/index.js
@@ -1,5 +1,6 @@
const testng = require('./testng');
const junit = require('./junit');
+const nunit = require('./nunit');
const xunit = require('./xunit');
const mocha = require('./mocha');
const cucumber = require('./cucumber');
@@ -37,6 +38,8 @@ function getParser(type) {
return junit;
case 'xunit':
return xunit;
+ case 'nunit':
+ return nunit;
case 'mocha':
return mocha;
case 'cucumber':
diff --git a/src/parsers/nunit.js b/src/parsers/nunit.js
new file mode 100644
index 0000000..2aee0f9
--- /dev/null
+++ b/src/parsers/nunit.js
@@ -0,0 +1,150 @@
+const { getJsonFromXMLFile } = require('../helpers/helper');
+
+const TestResult = require('../models/TestResult');
+const TestSuite = require('../models/TestSuite');
+const TestCase = require('../models/TestCase');
+
+const SUITE_TYPES_WITH_TESTCASES = [
+ "TestFixture",
+ "ParameterizedTest",
+ "GenericFixture"
+]
+
+const RESULTMAP = {
+ Success: "PASS",
+ Failure: "FAIL",
+ Ignored: "SKIP",
+ NotRunnable: "SKIP",
+ Error: "ERROR",
+ Inconclusive: "FAIL"
+}
+
+function populateMetaData(raw, map) {
+ if (raw.categories) {
+ let categories = raw.categories.category;
+ for (let i = 0; i < categories.length; i++) {
+ let categoryName = categories[i]["@_name"];
+ map.set(categoryName, "");
+
+ // create comma-delimited list of categories
+ if (map.has("Categories")) {
+ map.set("Categories", map.get("Categories").concat(",", categoryName));
+ } else {
+ map.set("Categories", categoryName);
+ }
+ }
+ }
+ if (raw.properties) {
+ let properties = raw.properties.property;
+ for (let i = 0; i < properties.length; i++) {
+ let property = properties[i];
+ map.set(property["@_name"], property["@_value"]);
+ }
+ }
+}
+
+function getTestCases(rawSuite, parent_meta) {
+ var cases = [];
+
+ let rawTestCases = rawSuite.results["test-case"];
+ if (rawTestCases) {
+ for (let i = 0; i < rawTestCases.length; i++) {
+ let rawCase = rawTestCases[i];
+ let testCase = new TestCase();
+ let result = rawCase["@_result"]
+ testCase.name = rawCase["@_name"];
+ testCase.duration = rawCase["@_time"] * 1000; // in milliseconds
+ testCase.status = RESULTMAP[result];
+ if (rawCase["@_executed"] == "False") {
+ testCase.status = "SKIP"; // exclude failures that weren't executed.
+ }
+ let errorDetails = rawCase.reason ?? rawCase.failure;
+ if (errorDetails !== undefined) {
+ testCase.setFailure(errorDetails.message);
+ if (errorDetails["stack-trace"]) {
+ testCase.stack_trace = errorDetails["stack-trace"]
+ }
+ }
+ // copy parent_meta data to test case
+ for( let kvp of parent_meta.entries()) {
+ testCase.meta_data.set(kvp[0], kvp[1]);
+ }
+ populateMetaData(rawCase, testCase.meta_data);
+
+ cases.push( testCase );
+ }
+ }
+
+ return cases;
+}
+
+function getTestSuites(rawSuites) {
+ var suites = [];
+
+ for(let i = 0; i < rawSuites.length; i++) {
+ let rawSuite = rawSuites[i];
+
+ if (rawSuite.results["test-suite"]) {
+ // handle nested test-suites
+ suites.push(...getTestSuites(rawSuite.results["test-suite"]));
+ } else if (SUITE_TYPES_WITH_TESTCASES.indexOf(rawSuite["@_type"]) !== -1) {
+
+ let suite = new TestSuite();
+ suite.duration = rawSuite["@_time"] * 1000; // in milliseconds
+ suite.status = RESULTMAP[rawSuite["@_result"]];
+
+ var meta_data = new Map();
+ populateMetaData(rawSuite, meta_data);
+ suite.cases.push(...getTestCases(rawSuite, meta_data));
+
+ // calculate totals
+ suite.total = suite.cases.length;
+ suite.passed = suite.cases.filter(i => i.status == "PASS").length;
+ suite.failed = suite.cases.filter(i => i.status == "FAIL").length;
+ suite.errors = suite.cases.filter(i => i.status == "ERROR").length;
+ suite.skipped = suite.cases.filter(i => i.status == "SKIP").length;
+
+ suites.push(suite);
+ }
+ }
+
+ return suites;
+}
+
+
+function getTestResult(json) {
+ const rawResult = json["test-results"];
+ const rawSuite = rawResult["test-suite"][0];
+
+ const result = new TestResult();
+ result.name = rawResult["@_name"];
+ result.duration = rawSuite["@_time"] * 1000; // in milliseconds
+ // test-results attributes related to totals
+ // total = executed=True
+ // errors = result="Error"
+ // failures = result="Failure"
+ // not-run = executed=False
+ // inconclusive = result="Inconclusive"
+ // ignored = result="Ignored"
+ // skipped = sample has zero?
+ // invalid = result="NotRunable"
+ result.total = rawResult["@_total"] + rawResult["@_not-run"]; // total executed and not executed
+ result.errors = rawResult["@_errors"];
+ result.failed = rawResult["@_failures"];
+ result.skipped = rawResult["@_not-run"]; // Ignored, NotRunnable
+ // assume inconclusive is neither a pass or failure to prevent religious wars, total's won't match as a result.
+ result.passed = rawResult["@_total"] - (result.errors + result.failed + rawResult["@_inconclusive"]);
+
+ result.suites.push(getTestSuites( [ rawSuite ]) );
+
+ return result;
+}
+
+function parse(file) {
+ const json = getJsonFromXMLFile(file);
+ return getTestResult(json);
+}
+
+module.exports = {
+ parse
+}
\ No newline at end of file
diff --git a/tests/data/nunit/sample.xml b/tests/data/nunit/sample.xml
new file mode 100644
index 0000000..7a645c6
--- /dev/null
+++ b/tests/data/nunit/sample.xml
@@ -0,0 +1,190 @@
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
\ No newline at end of file
diff --git a/tests/parser.nunit.spec.js b/tests/parser.nunit.spec.js
new file mode 100644
index 0000000..955cdd9
--- /dev/null
+++ b/tests/parser.nunit.spec.js
@@ -0,0 +1,113 @@
+const { parse } = require('../src');
+const assert = require('assert');
+const path = require('path');
+
+describe('Parser - NUnit', () => {
+
+ const testDataPath = "tests/data/nunit";
+ var result;
+
+ before( () => {
+ result = parse({ type: 'nunit', files: [`${testDataPath}/sample.xml`] });
+ });
+
+ it('Should calculate totals', () => {
+ // evaluate totals on the testresult
+ assert.equal(result.total, 28);
+ assert.equal(result.passed, 18);
+ assert.equal(result.failed, 1);
+ assert.equal(result.errors, 1);
+ assert.equal(result.skipped, 7);
+ // inconclusive is excluded from results so total is not accurate
+ assert.equal(result.total - (result.passed + result.failed + result.errors + result.skipped), 1 /* not zero because inconclusive */);
+
+ // compare sum of suite totals to testresult
+ assert.equal( result.suites.reduce( (total, suite) => { return total + suite.total} ,0), result.total);
+ assert.equal( result.suites.reduce( (total, suite) => { return total + suite.passed} ,0), result.passed);
+ assert.equal( result.suites.reduce( (total, suite) => { return total + suite.failed} ,0) - /*inconclusive*/ 1, result.failed);
+ assert.equal( result.suites.reduce( (total, suite) => { return total + suite.errors} ,0), result.errors);
+ assert.equal( result.suites.reduce( (total, suite) => { return total + suite.skipped} ,0), result.skipped);
+
+ // compare sum results of test cases to testresult totals
+ assert.equal( sumCases(result, (testCase) => { return 1 /* count of testcase */ }), result.total);
+ assert.equal( sumCases(result, (testCase) => { return testCase.status == "PASS" ? 1 : 0 }), result.passed);
+ assert.equal( sumCases(result, (testCase) => { return testCase.status == "FAIL" ? 1 : 0 }) - 1 /*remove inconclusive */, result.failed);
+ assert.equal( sumCases(result, (testCase) => { return testCase.status == "ERROR" }), result.errors);
+ assert.equal( sumCases(result, (testCase) => { return testCase.status == "SKIP" ? 1 : 0 }), result.skipped);
+ });
+
+ it('Should express durations in milliseconds', () => {
+ let totalDuration = result.duration;
+ let suiteDuration = result.suites[0].duration; // sample
+ let testDuration = result.suites[0].cases[0].duration; // sample
+ assert.equal(totalDuration > 100, true, `TestResult duration should be more than 100 millisecond (${totalDuration}`);
+ assert.equal(suiteDuration > 100, true, `Suite duration should be more than 100 milliseconds (${suiteDuration})`);
+ assert.equal(testDuration > 10, true, `Test duration should be more than 10 milliseconds (${testDuration})`);
+ });
+
+ it('Should include names for all tests', () => {
+ let count = 0;
+ result.suites.forEach( s => {
+ s.cases.forEach( c => {
+ count++;
+ assert.equal( c.name !== '' && c.name !== undefined, true);
+ });
+ });
+ assert.equal( count > 0, true, "Should have evaluated multiple test cases.");
+ });
+
+ it('Should map results correctly', () => {
+ assert.equal(result.suites[0].status, "FAIL");
+ assert.equal(result.suites[0].cases[0].status, "FAIL"); // Failure
+ assert.equal(result.suites[0].cases[8].status, "ERROR"); // Error
+ assert.equal(result.suites[1].status, "SKIP");
+ assert.equal(result.suites[1].cases[0].status, "SKIP"); // NotRunnable
+ assert.equal(result.suites[2].status, "PASS");
+ assert.equal(result.suites[2].cases[0].status, "PASS"); // Success
+ assert.equal(result.suites[6].status, "SKIP");
+ assert.equal(result.suites[6].cases[0].status, "SKIP"); // Ignored
+ });
+
+ it('Should get reason for inconclusive', () => {
+ assert.equal(result.suites[0].cases[1].status, 'FAIL');
+ assert.notEqual(result.suites[0].cases[1].failure, undefined);
+ });
+
+ it('Should get failure message and stack trace for failure', () => {
+ assert.equal(result.suites[0].cases[0].status, 'FAIL');
+ assert.notEqual(result.suites[0].cases[0].failure, '');
+ assert.notEqual(result.suites[0].cases[0].stack_trace, '');
+ });
+
+ it('Should map suite categories to testcases', () => {
+ let count = 0;
+ result.suites[0].cases.forEach( c => {
+ count++;
+ assert.equal(c.meta_data.has("FixtureCategory"), true);
+ assert.equal(c.meta_data.get("Categories").includes("FixtureCategory"), true);
+ });
+ assert.equal( count > 0, true);
+ });
+
+ it('Should map test-case categories and properties to testcases', () => {
+ // case 3 has additional category + properties
+ let testcase = result.suites[0].cases[3];
+ // categories
+ assert.equal(testcase.meta_data.has("MockCategory"), true);
+ assert.equal(testcase.meta_data.get("Categories").includes("MockCategory"), true);
+ assert.equal(testcase.meta_data.get("Categories"), "FixtureCategory,MockCategory"); // combined
+
+ // properties
+ assert.equal(testcase.meta_data.get("Severity"),"Critical");
+ })
+
+ function sumCases(result, predicate) {
+ return result.suites.reduce( (total, suite) => {
+ return total + suite.cases.reduce( (testcaseTotal, testcase) => {
+ let output = predicate(testcase);
+ return testcaseTotal + output;
+ }, 0);
+ }, 0);
+ }
+
+});
\ No newline at end of file
From 2fcc95fce7fd83902e1ca1723be8732882587cfc Mon Sep 17 00:00:00 2001
From: bryan cook <3217452+bryanbcook@users.noreply.github.com>
Date: Sat, 11 Nov 2023 15:06:13 -0500
Subject: [PATCH 2/4] reorganized tests to support v2 + v3
---
tests/data/nunit/{sample.xml => nunit_v2.xml} | 0
tests/data/nunit/nunit_v3.xml | 0
tests/parser.nunit.spec.js | 168 +++++++++---------
3 files changed, 86 insertions(+), 82 deletions(-)
rename tests/data/nunit/{sample.xml => nunit_v2.xml} (100%)
create mode 100644 tests/data/nunit/nunit_v3.xml
diff --git a/tests/data/nunit/sample.xml b/tests/data/nunit/nunit_v2.xml
similarity index 100%
rename from tests/data/nunit/sample.xml
rename to tests/data/nunit/nunit_v2.xml
diff --git a/tests/data/nunit/nunit_v3.xml b/tests/data/nunit/nunit_v3.xml
new file mode 100644
index 0000000..e69de29
diff --git a/tests/parser.nunit.spec.js b/tests/parser.nunit.spec.js
index 955cdd9..03394b8 100644
--- a/tests/parser.nunit.spec.js
+++ b/tests/parser.nunit.spec.js
@@ -7,99 +7,103 @@ describe('Parser - NUnit', () => {
const testDataPath = "tests/data/nunit";
var result;
- before( () => {
- result = parse({ type: 'nunit', files: [`${testDataPath}/sample.xml`] });
- });
+ context('NUnit V2', () => {
- it('Should calculate totals', () => {
- // evaluate totals on the testresult
- assert.equal(result.total, 28);
- assert.equal(result.passed, 18);
- assert.equal(result.failed, 1);
- assert.equal(result.errors, 1);
- assert.equal(result.skipped, 7);
- // inconclusive is excluded from results so total is not accurate
- assert.equal(result.total - (result.passed + result.failed + result.errors + result.skipped), 1 /* not zero because inconclusive */);
-
- // compare sum of suite totals to testresult
- assert.equal( result.suites.reduce( (total, suite) => { return total + suite.total} ,0), result.total);
- assert.equal( result.suites.reduce( (total, suite) => { return total + suite.passed} ,0), result.passed);
- assert.equal( result.suites.reduce( (total, suite) => { return total + suite.failed} ,0) - /*inconclusive*/ 1, result.failed);
- assert.equal( result.suites.reduce( (total, suite) => { return total + suite.errors} ,0), result.errors);
- assert.equal( result.suites.reduce( (total, suite) => { return total + suite.skipped} ,0), result.skipped);
-
- // compare sum results of test cases to testresult totals
- assert.equal( sumCases(result, (testCase) => { return 1 /* count of testcase */ }), result.total);
- assert.equal( sumCases(result, (testCase) => { return testCase.status == "PASS" ? 1 : 0 }), result.passed);
- assert.equal( sumCases(result, (testCase) => { return testCase.status == "FAIL" ? 1 : 0 }) - 1 /*remove inconclusive */, result.failed);
- assert.equal( sumCases(result, (testCase) => { return testCase.status == "ERROR" }), result.errors);
- assert.equal( sumCases(result, (testCase) => { return testCase.status == "SKIP" ? 1 : 0 }), result.skipped);
- });
+ before( () => {
+ result = parse({ type: 'nunit', files: [`${testDataPath}/nunit_v2.xml`] });
+ });
- it('Should express durations in milliseconds', () => {
- let totalDuration = result.duration;
- let suiteDuration = result.suites[0].duration; // sample
- let testDuration = result.suites[0].cases[0].duration; // sample
- assert.equal(totalDuration > 100, true, `TestResult duration should be more than 100 millisecond (${totalDuration}`);
- assert.equal(suiteDuration > 100, true, `Suite duration should be more than 100 milliseconds (${suiteDuration})`);
- assert.equal(testDuration > 10, true, `Test duration should be more than 10 milliseconds (${testDuration})`);
- });
+ it('Should calculate totals', () => {
+ // evaluate totals on the testresult
+ assert.equal(result.total, 28);
+ assert.equal(result.passed, 18);
+ assert.equal(result.failed, 1);
+ assert.equal(result.errors, 1);
+ assert.equal(result.skipped, 7);
+ // inconclusive is excluded from results so total is not accurate
+ assert.equal(result.total - (result.passed + result.failed + result.errors + result.skipped), 1 /* not zero because inconclusive */);
+
+ // compare sum of suite totals to testresult
+ assert.equal( result.suites.reduce( (total, suite) => { return total + suite.total} ,0), result.total);
+ assert.equal( result.suites.reduce( (total, suite) => { return total + suite.passed} ,0), result.passed);
+ assert.equal( result.suites.reduce( (total, suite) => { return total + suite.failed} ,0) - /*inconclusive*/ 1, result.failed);
+ assert.equal( result.suites.reduce( (total, suite) => { return total + suite.errors} ,0), result.errors);
+ assert.equal( result.suites.reduce( (total, suite) => { return total + suite.skipped} ,0), result.skipped);
+
+ // compare sum results of test cases to testresult totals
+ assert.equal( sumCases(result, (testCase) => { return 1 /* count of testcase */ }), result.total);
+ assert.equal( sumCases(result, (testCase) => { return testCase.status == "PASS" ? 1 : 0 }), result.passed);
+ assert.equal( sumCases(result, (testCase) => { return testCase.status == "FAIL" ? 1 : 0 }) - 1 /*remove inconclusive */, result.failed);
+ assert.equal( sumCases(result, (testCase) => { return testCase.status == "ERROR" }), result.errors);
+ assert.equal( sumCases(result, (testCase) => { return testCase.status == "SKIP" ? 1 : 0 }), result.skipped);
+ });
- it('Should include names for all tests', () => {
- let count = 0;
- result.suites.forEach( s => {
- s.cases.forEach( c => {
- count++;
- assert.equal( c.name !== '' && c.name !== undefined, true);
+ it('Should express durations in milliseconds', () => {
+ let totalDuration = result.duration;
+ let suiteDuration = result.suites[0].duration; // sample
+ let testDuration = result.suites[0].cases[0].duration; // sample
+ assert.equal(totalDuration > 100, true, `TestResult duration should be more than 100 millisecond (${totalDuration}`);
+ assert.equal(suiteDuration > 100, true, `Suite duration should be more than 100 milliseconds (${suiteDuration})`);
+ assert.equal(testDuration > 10, true, `Test duration should be more than 10 milliseconds (${testDuration})`);
+ });
+
+ it('Should include names for all tests', () => {
+ let count = 0;
+ result.suites.forEach( s => {
+ s.cases.forEach( c => {
+ count++;
+ assert.equal( c.name !== '' && c.name !== undefined, true);
+ });
});
+ assert.equal( count > 0, true, "Should have evaluated multiple test cases.");
});
- assert.equal( count > 0, true, "Should have evaluated multiple test cases.");
- });
- it('Should map results correctly', () => {
- assert.equal(result.suites[0].status, "FAIL");
- assert.equal(result.suites[0].cases[0].status, "FAIL"); // Failure
- assert.equal(result.suites[0].cases[8].status, "ERROR"); // Error
- assert.equal(result.suites[1].status, "SKIP");
- assert.equal(result.suites[1].cases[0].status, "SKIP"); // NotRunnable
- assert.equal(result.suites[2].status, "PASS");
- assert.equal(result.suites[2].cases[0].status, "PASS"); // Success
- assert.equal(result.suites[6].status, "SKIP");
- assert.equal(result.suites[6].cases[0].status, "SKIP"); // Ignored
- });
+ it('Should map results correctly', () => {
+ assert.equal(result.suites[0].status, "FAIL");
+ assert.equal(result.suites[0].cases[0].status, "FAIL"); // Failure
+ assert.equal(result.suites[0].cases[8].status, "ERROR"); // Error
+ assert.equal(result.suites[1].status, "SKIP");
+ assert.equal(result.suites[1].cases[0].status, "SKIP"); // NotRunnable
+ assert.equal(result.suites[2].status, "PASS");
+ assert.equal(result.suites[2].cases[0].status, "PASS"); // Success
+ assert.equal(result.suites[6].status, "SKIP");
+ assert.equal(result.suites[6].cases[0].status, "SKIP"); // Ignored
+ });
- it('Should get reason for inconclusive', () => {
- assert.equal(result.suites[0].cases[1].status, 'FAIL');
- assert.notEqual(result.suites[0].cases[1].failure, undefined);
- });
+ it('Should get reason for inconclusive', () => {
+ assert.equal(result.suites[0].cases[1].status, 'FAIL');
+ assert.notEqual(result.suites[0].cases[1].failure, undefined);
+ });
- it('Should get failure message and stack trace for failure', () => {
- assert.equal(result.suites[0].cases[0].status, 'FAIL');
- assert.notEqual(result.suites[0].cases[0].failure, '');
- assert.notEqual(result.suites[0].cases[0].stack_trace, '');
- });
+ it('Should get failure message and stack trace for failure', () => {
+ assert.equal(result.suites[0].cases[0].status, 'FAIL');
+ assert.notEqual(result.suites[0].cases[0].failure, '');
+ assert.notEqual(result.suites[0].cases[0].stack_trace, '');
+ });
- it('Should map suite categories to testcases', () => {
- let count = 0;
- result.suites[0].cases.forEach( c => {
- count++;
- assert.equal(c.meta_data.has("FixtureCategory"), true);
- assert.equal(c.meta_data.get("Categories").includes("FixtureCategory"), true);
+ it('Should map suite categories to testcases', () => {
+ let count = 0;
+ result.suites[0].cases.forEach( c => {
+ count++;
+ assert.equal(c.meta_data.has("FixtureCategory"), true);
+ assert.equal(c.meta_data.get("Categories").includes("FixtureCategory"), true);
+ });
+ assert.equal( count > 0, true);
});
- assert.equal( count > 0, true);
- });
- it('Should map test-case categories and properties to testcases', () => {
- // case 3 has additional category + properties
- let testcase = result.suites[0].cases[3];
- // categories
- assert.equal(testcase.meta_data.has("MockCategory"), true);
- assert.equal(testcase.meta_data.get("Categories").includes("MockCategory"), true);
- assert.equal(testcase.meta_data.get("Categories"), "FixtureCategory,MockCategory"); // combined
-
- // properties
- assert.equal(testcase.meta_data.get("Severity"),"Critical");
- })
+ it('Should map test-case categories and properties to testcases', () => {
+ // case 3 has additional category + properties
+ let testcase = result.suites[0].cases[3];
+ // categories
+ assert.equal(testcase.meta_data.has("MockCategory"), true);
+ assert.equal(testcase.meta_data.get("Categories").includes("MockCategory"), true);
+ assert.equal(testcase.meta_data.get("Categories"), "FixtureCategory,MockCategory"); // combined
+
+ // properties
+ assert.equal(testcase.meta_data.get("Severity"),"Critical");
+ });
+
+ });
function sumCases(result, predicate) {
return result.suites.reduce( (total, suite) => {
From b6c799067edca510331b70c3cd20141538ebe0d8 Mon Sep 17 00:00:00 2001
From: bryan cook <3217452+bryanbcook@users.noreply.github.com>
Date: Sat, 11 Nov 2023 19:29:51 -0500
Subject: [PATCH 3/4] Refactored to support NUnit v2+v3
---
src/helpers/helper.js | 2 +-
src/parsers/nunit.js | 143 +++++++++++++++++++++--------
tests/data/nunit/nunit_v3.xml | 125 +++++++++++++++++++++++++
tests/parser.nunit.spec.js | 166 +++++++++++++++++++++++++++++++---
4 files changed, 387 insertions(+), 49 deletions(-)
diff --git a/src/helpers/helper.js b/src/helpers/helper.js
index f5c43eb..e9c80af 100644
--- a/src/helpers/helper.js
+++ b/src/helpers/helper.js
@@ -40,7 +40,7 @@ const configured_parser = new XMLParser({
return true;
}
// handle nunit deep hierarchy
- else if (jpath.startsWith("test-results")) {
+ else if (jpath.startsWith("test-results") || jpath.startsWith("test-run")) {
let parts = jpath.split(".");
switch(parts[parts.length - 1]) {
case "category":
diff --git a/src/parsers/nunit.js b/src/parsers/nunit.js
index 2aee0f9..0805480 100644
--- a/src/parsers/nunit.js
+++ b/src/parsers/nunit.js
@@ -7,19 +7,32 @@ const TestCase = require('../models/TestCase');
const SUITE_TYPES_WITH_TESTCASES = [
"TestFixture",
"ParameterizedTest",
- "GenericFixture"
+ "GenericFixture",
+ "ParameterizedMethod" // v3
]
const RESULTMAP = {
- Success: "PASS",
- Failure: "FAIL",
- Ignored: "SKIP",
- NotRunnable: "SKIP",
- Error: "ERROR",
- Inconclusive: "FAIL"
+ Success: "PASS", // v2
+ Failure: "FAIL", // v2
+ Ignored: "SKIP", // v2
+ NotRunnable: "SKIP", // v2
+ Error: "ERROR", // v2
+ Inconclusive: "FAIL", // v2
+
+ Passed: "PASS", // v3
+ Failed: "FAIL", // v3
+ Skipped: "SKIP", // v3
+}
+
+function mergeMeta(map1, map2) {
+ for(let kvp of map1) {
+ map2.set(kvp[0], kvp[1]);
+ }
}
function populateMetaData(raw, map) {
+
+ // v2 supports categories
if (raw.categories) {
let categories = raw.categories.category;
for (let i = 0; i < categories.length; i++) {
@@ -34,11 +47,56 @@ function populateMetaData(raw, map) {
}
}
}
+
+ // v2/v3 support properties
if (raw.properties) {
let properties = raw.properties.property;
for (let i = 0; i < properties.length; i++) {
let property = properties[i];
- map.set(property["@_name"], property["@_value"]);
+ let propName = property["@_name"];
+ let propValue = property["@_value"];
+
+ // v3 treats 'Categories' as property "Category"
+ if (propName == "Category") {
+
+ if (map.has("Categories")) {
+ map.set("Categories", map.get("Categories").concat(",", propValue));
+ } else {
+ map.set("Categories", propValue);
+ }
+ map.set(propValue, "");
+
+ } else {
+ map.set(propName, propValue);
+ }
+ }
+ }
+}
+
+function getNestedTestCases(rawSuite) {
+ if (rawSuite.results) {
+ return rawSuite.results["test-case"];
+ } else {
+ return rawSuite["test-case"];
+ }
+}
+
+function hasNestedSuite(rawSuite) {
+ return getNestedSuite(rawSuite) !== null;
+}
+
+function getNestedSuite(rawSuite) {
+ // nunit v2 nests test-suite inside 'results'
+ if (rawSuite.results && rawSuite.results["test-suite"]) {
+ return rawSuite.results["test-suite"];
+ } else {
+ // nunit v3 nests test-suites as immediate children
+ if (rawSuite["test-suite"]) {
+ return rawSuite["test-suite"];
+ }
+ else {
+ // not nested
+ return null;
}
}
}
@@ -46,18 +104,25 @@ function populateMetaData(raw, map) {
function getTestCases(rawSuite, parent_meta) {
var cases = [];
- let rawTestCases = rawSuite.results["test-case"];
+ let rawTestCases = getNestedTestCases(rawSuite);
if (rawTestCases) {
for (let i = 0; i < rawTestCases.length; i++) {
let rawCase = rawTestCases[i];
let testCase = new TestCase();
let result = rawCase["@_result"]
- testCase.name = rawCase["@_name"];
+ testCase.id = rawCase["@_id"] ?? "";
+ testCase.name = rawCase["@_fullname"] ?? rawCase["@_name"];
testCase.duration = rawCase["@_time"] * 1000; // in milliseconds
testCase.status = RESULTMAP[result];
+
+ // v2 : non-executed should be tests should be Ignored
if (rawCase["@_executed"] == "False") {
testCase.status = "SKIP"; // exclude failures that weren't executed.
}
+ // v3 : failed tests with error label should be Error
+ if (rawCase["@_label"] == "Error") {
+ testCase.status = "ERROR";
+ }
let errorDetails = rawCase.reason ?? rawCase.failure;
if (errorDetails !== undefined) {
testCase.setFailure(errorDetails.message);
@@ -66,9 +131,7 @@ function getTestCases(rawSuite, parent_meta) {
}
}
// copy parent_meta data to test case
- for( let kvp of parent_meta.entries()) {
- testCase.meta_data.set(kvp[0], kvp[1]);
- }
+ mergeMeta(parent_meta, testCase.meta_data);
populateMetaData(rawCase, testCase.meta_data);
cases.push( testCase );
@@ -78,22 +141,30 @@ function getTestCases(rawSuite, parent_meta) {
return cases;
}
-function getTestSuites(rawSuites) {
+function getTestSuites(rawSuites, assembly_meta) {
var suites = [];
for(let i = 0; i < rawSuites.length; i++) {
let rawSuite = rawSuites[i];
+
+ if (rawSuite["@_type"] == "Assembly") {
+ assembly_meta = new Map();
+ populateMetaData(rawSuite, assembly_meta);
+ }
- if (rawSuite.results["test-suite"]) {
+ if (hasNestedSuite(rawSuite)) {
// handle nested test-suites
- suites.push(...getTestSuites(rawSuite.results["test-suite"]));
+ suites.push(...getTestSuites(getNestedSuite(rawSuite), assembly_meta));
} else if (SUITE_TYPES_WITH_TESTCASES.indexOf(rawSuite["@_type"]) !== -1) {
let suite = new TestSuite();
+ suite.id = rawSuite["@_id"] ?? '';
+ suite.name = rawSuite["@_fullname"] ?? rawSuite["@_name"];
suite.duration = rawSuite["@_time"] * 1000; // in milliseconds
suite.status = RESULTMAP[rawSuite["@_result"]];
var meta_data = new Map();
+ mergeMeta(assembly_meta, meta_data);
populateMetaData(rawSuite, meta_data);
suite.cases.push(...getTestCases(rawSuite, meta_data));
@@ -113,29 +184,27 @@ function getTestSuites(rawSuites) {
function getTestResult(json) {
- const rawResult = json["test-results"];
- const rawSuite = rawResult["test-suite"][0];
-
+ const nunitVersion = (json["test-results"] !== undefined) ? "v2" :
+ (json["test-run"] !== undefined) ? "v3" : null;
+
+ if (nunitVersion == null) {
+ throw new Error("Unrecognized xml format");
+ }
+
const result = new TestResult();
- result.name = rawResult["@_name"];
+ const rawResult = json["test-results"] ?? json["test-run"];
+ const rawSuite = rawResult["test-suite"][0];
+
+ result.name = rawResult["@_fullname"] ?? rawResult["@_name"];
result.duration = rawSuite["@_time"] * 1000; // in milliseconds
- // test-results attributes related to totals
- // total = executed=True
- // errors = result="Error"
- // failures = result="Failure"
- // not-run = executed=False
- // inconclusive = result="Inconclusive"
- // ignored = result="Ignored"
- // skipped = sample has zero?
- // invalid = result="NotRunable"
- result.total = rawResult["@_total"] + rawResult["@_not-run"]; // total executed and not executed
- result.errors = rawResult["@_errors"];
- result.failed = rawResult["@_failures"];
- result.skipped = rawResult["@_not-run"]; // Ignored, NotRunnable
- // assume inconclusive is neither a pass or failure to prevent religious wars, total's won't match as a result.
- result.passed = rawResult["@_total"] - (result.errors + result.failed + rawResult["@_inconclusive"]);
-
- result.suites.push(getTestSuites( [ rawSuite ]) );
+
+ result.suites.push(...getTestSuites( [ rawSuite ], null));
+
+ result.total = result.suites.reduce( (total, suite) => { return total + suite.cases.length}, 0);
+ result.passed = result.suites.reduce( (total, suite) => { return total + suite.passed}, 0);
+ result.failed = result.suites.reduce( (total, suite) => { return total + suite.failed}, 0);
+ result.skipped = result.suites.reduce( (total, suite) => { return total + suite.skipped}, 0);
+ result.errors = result.suites.reduce( (total, suite) => { return total + suite.errors}, 0);
return result;
}
diff --git a/tests/data/nunit/nunit_v3.xml b/tests/data/nunit/nunit_v3.xml
index e69de29..d0d1712 100644
--- a/tests/data/nunit/nunit_v3.xml
+++ b/tests/data/nunit/nunit_v3.xml
@@ -0,0 +1,125 @@
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
\ No newline at end of file
diff --git a/tests/parser.nunit.spec.js b/tests/parser.nunit.spec.js
index 03394b8..a3d8486 100644
--- a/tests/parser.nunit.spec.js
+++ b/tests/parser.nunit.spec.js
@@ -17,23 +17,22 @@ describe('Parser - NUnit', () => {
// evaluate totals on the testresult
assert.equal(result.total, 28);
assert.equal(result.passed, 18);
- assert.equal(result.failed, 1);
+ assert.equal(result.failed, 2); // include inconclusive as a failure
assert.equal(result.errors, 1);
assert.equal(result.skipped, 7);
- // inconclusive is excluded from results so total is not accurate
- assert.equal(result.total - (result.passed + result.failed + result.errors + result.skipped), 1 /* not zero because inconclusive */);
+ assert.equal(result.total - (result.passed + result.failed + result.errors + result.skipped), 0);
// compare sum of suite totals to testresult
assert.equal( result.suites.reduce( (total, suite) => { return total + suite.total} ,0), result.total);
assert.equal( result.suites.reduce( (total, suite) => { return total + suite.passed} ,0), result.passed);
- assert.equal( result.suites.reduce( (total, suite) => { return total + suite.failed} ,0) - /*inconclusive*/ 1, result.failed);
+ assert.equal( result.suites.reduce( (total, suite) => { return total + suite.failed} ,0), result.failed);
assert.equal( result.suites.reduce( (total, suite) => { return total + suite.errors} ,0), result.errors);
assert.equal( result.suites.reduce( (total, suite) => { return total + suite.skipped} ,0), result.skipped);
// compare sum results of test cases to testresult totals
assert.equal( sumCases(result, (testCase) => { return 1 /* count of testcase */ }), result.total);
assert.equal( sumCases(result, (testCase) => { return testCase.status == "PASS" ? 1 : 0 }), result.passed);
- assert.equal( sumCases(result, (testCase) => { return testCase.status == "FAIL" ? 1 : 0 }) - 1 /*remove inconclusive */, result.failed);
+ assert.equal( sumCases(result, (testCase) => { return testCase.status == "FAIL" ? 1 : 0 }), result.failed);
assert.equal( sumCases(result, (testCase) => { return testCase.status == "ERROR" }), result.errors);
assert.equal( sumCases(result, (testCase) => { return testCase.status == "SKIP" ? 1 : 0 }), result.skipped);
});
@@ -105,13 +104,158 @@ describe('Parser - NUnit', () => {
});
+ context('NUnit V3', () => {
+ before( () => {
+ result = parse({ type: 'nunit', files: [`${testDataPath}/nunit_v3.xml`] });
+ });
+
+ it('Should calculate totals', () => {
+ // totals on the testresult
+ assert.equal(result.total, 18);
+ assert.equal(result.passed, 12);
+ assert.equal(result.failed, 2);
+ assert.equal(result.errors, 1);
+
+ // compare sum of suite totals to testresult
+ assert.equal( result.suites.reduce( (total, suite) => { return total + suite.total},0), result.total);
+ assert.equal( result.suites.reduce( (total, suite) => { return total + suite.passed},0), result.passed);
+ assert.equal( result.suites.reduce( (total, suite) => { return total + suite.failed},0), result.failed);
+ assert.equal( result.suites.reduce( (total, suite) => { return total + suite.errors},0), result.errors);
+ assert.equal( result.suites.reduce( (total, suite) => { return total + suite.skipped},0), result.skipped);
+ });
+
+ it('Should express durations in milliseconds', () => {
+ let totalDuration = result.duration;
+ let suiteDuration = result.suites[0].duration; // sample
+ let testDuration = result.suites[0].cases[0].duration; // sample
+ assert.equal(totalDuration > 100, true, `TestResult duration should be more than 100 millisecond (${totalDuration}`);
+ assert.equal(suiteDuration > 100, true, `Suite duration should be more than 100 milliseconds (${suiteDuration})`);
+ assert.equal(testDuration > 10, true, `Test duration should be more than 10 milliseconds (${testDuration})`);
+ });
+
+ it('Should include fullnames for testsuites and testcases', () => {
+ assert.equal(result.suites[0].name, "NUnit.Tests.Assemblies.MockTestFixture");
+ assert.equal(result.suites[0].cases[0].name, "NUnit.Tests.Assemblies.MockTestFixture.FailingTest")
+ });
+
+ it('Should map results correctly', () => {
+ assert.equal(result.suites.length, 8);
+
+ // assemblies.mocktestfixture
+ assert.equal(result.suites[0].status, "FAIL");
+ assert.equal(result.suites[0].cases[0].status, "FAIL");
+ assert.equal(result.suites[0].cases[1].status, "FAIL"); // inconclusive
+ assert.equal(result.suites[0].cases[2].status, "PASS");
+ assert.equal(result.suites[0].cases[3].status, "PASS");
+ assert.equal(result.suites[0].cases[4].status, "PASS");
+ assert.equal(result.suites[0].cases[5].status, "SKIP"); // ignored
+ assert.equal(result.suites[0].cases[6].status, "SKIP"); // invalid
+ assert.equal(result.suites[0].cases[7].status, "SKIP"); // invalid
+ assert.equal(result.suites[0].cases[8].status, "ERROR"); // error
+ assert.equal(result.suites[0].cases[9].status, "PASS");
+
+ // badfixture
+ assert.equal(result.suites[1].cases.length, 0); // invalid test cases
+ assert.equal(result.suites[1].status, "SKIP"); // v3 treats these as skipped
+
+ // fixturewithTestCases
+ assert.equal(result.suites[2].status, "PASS");
+ assert.equal(result.suites[4].cases[0].status, "PASS");
+ assert.equal(result.suites[4].cases[1].status, "PASS");
+
+ // ignoredfixture
+ assert.equal(result.suites[3].cases.length, 0); // invalid test cases
+ assert.equal(result.suites[3].status, "SKIP"); // v3 treats these as skipped
+
+ // parameterizedfixture(42)
+ assert.equal(result.suites[4].status, "PASS");
+ assert.equal(result.suites[4].cases[0].name, "NUnit.Tests.ParameterizedFixture(42).Test1");
+ assert.equal(result.suites[4].cases[0].status, "PASS");
+ assert.equal(result.suites[4].cases[1].status, "PASS");
+
+ // parameterizedfixture(5)
+ assert.equal(result.suites[5].status, "PASS");
+ assert.equal(result.suites[5].cases[0].name, "NUnit.Tests.ParameterizedFixture(5).Test1");
+ assert.equal(result.suites[5].cases[0].status, "PASS");
+ assert.equal(result.suites[5].cases[1].status, "PASS");
+
+ // onetestcase
+ assert.equal(result.suites[6].status, "PASS");
+ assert.equal(result.suites[6].cases[0].status, "PASS");
+
+ // testassembly.mocktestfixture
+ assert.equal(result.suites[7].status, "PASS");
+ assert.equal(result.suites[7].cases[0].status, "PASS");
+ });
+
+ it('Should include reason for invalid tests', () => {
+ // test-case 1009 has a reason for being invalid.
+ const testcase = result.suites[0].cases[7];
+ assert.equal(testcase.failure, "No arguments were provided");
+ });
+
+ it('Should include stack trace for failed tests.', () => {
+ const testcase = result.suites[0].cases[8];
+ assert.equal(testcase.failure, "System.ApplicationException : Intentional Exception");
+ assert.notEqual(testcase.stack_trace, '');
+ });
+
+ it('Should support properties defined at the Assembly level', () => {
+ const testCaseWithNoProperties = result.suites[2].cases[0].meta_data;
+ assert.equal(testCaseWithNoProperties.size, 2, "Test case without properties should inherit from assembly");
+ assert.equal(testCaseWithNoProperties.has("_PID"), true);
+ assert.equal(testCaseWithNoProperties.has("_APPDOMAIN"), true);
+ });
+
+ it('Should include both suite and assembly level properties', () => {
+ const testCaseWithSuiteProperties = result.suites[0].cases[0].meta_data
+ assert.equal(testCaseWithSuiteProperties.size, 5, "Suite with properties should inherit assembly properties");
+ assert.equal(testCaseWithSuiteProperties.has("_PID"), true);
+ assert.equal(testCaseWithSuiteProperties.has("_APPDOMAIN"), true);
+ assert.equal(testCaseWithSuiteProperties.has("Description"), true);
+ assert.equal(testCaseWithSuiteProperties.get("Description"), "Fake Test Fixture");
+ });
+
+ it('Should include properties from assembly, suite and test case', () => {
+
+ const testCaseWithProperties = result.suites[0].cases[2].meta_data;
+ assert.equal(testCaseWithProperties.size, 5, "Test case with properties should iherit assembly, suite and override");
+ assert.equal(testCaseWithProperties.has("_PID"), true);
+ assert.equal(testCaseWithProperties.has("_APPDOMAIN"), true);
+ assert.equal(testCaseWithProperties.get("Description"), "Mock Test #1");
+ });
+
+ it('Should allow multiple categories to be specified', () => {
+ const testCaseWithMultipleCategories = result.suites[0].cases[3].meta_data;
+ assert.equal(testCaseWithMultipleCategories.has("Categories"), true);
+ assert.equal(testCaseWithMultipleCategories.get("Categories"), "FixtureCategory,MockCategory");
+ assert.equal(testCaseWithMultipleCategories.has("FixtureCategory"), true);
+ assert.equal(testCaseWithMultipleCategories.has("MockCategory"), true);
+ })
+
+
+ });
+
function sumCases(result, predicate) {
- return result.suites.reduce( (total, suite) => {
- return total + suite.cases.reduce( (testcaseTotal, testcase) => {
- let output = predicate(testcase);
- return testcaseTotal + output;
- }, 0);
- }, 0);
+ return flattenTestCases(result).reduce( (total, testcase) => { return total + predicate(testcase)}, 0);
+ }
+
+ function findTestCaseById(result, id) {
+ return findTestCases(result, testcase => { return testcase.id == id})[0];
+ }
+
+ function findTestCases(result, predicate) {
+ return flattenTestCases(result).filter( testCase => {return predicate() == true});
+ }
+
+ function flattenTestCases(result) {
+ let items = [];
+ result.suites.forEach( s => {
+ s.cases.forEach( c => {
+ items.push(c);
+ });
+ });
+ return items;
}
});
\ No newline at end of file
From d3698786eab0ffaad51a48bf54d6cca2c645cd0d Mon Sep 17 00:00:00 2001
From: bryan cook <3217452+bryanbcook@users.noreply.github.com>
Date: Sat, 11 Nov 2023 19:34:14 -0500
Subject: [PATCH 4/4] updated readme
---
README.md | 1 +
1 file changed, 1 insertion(+)
diff --git a/README.md b/README.md
index 683145b..8463aa8 100644
--- a/README.md
+++ b/README.md
@@ -8,6 +8,7 @@ Parse test results from JUnit, TestNG, xUnit, Mocha(json), Cucumber(json) and ma
|-------------------------------|---------|
| TestNG | ✅ |
| JUnit | ✅ |
+| NUnit (v2 & v3) | ✅ |
| xUnit | ✅ |
| Mocha (json & mochawesome) | ✅ |
| Cucumber | ✅ |
\ No newline at end of file