From 048a34c04caae16d0d9ef9a64fa2d3bb96179c14 Mon Sep 17 00:00:00 2001 From: bryan cook <3217452+bryanbcook@users.noreply.github.com> Date: Mon, 13 Nov 2023 17:34:35 -0500 Subject: [PATCH 1/2] Add basic support for mstest #5 --- README.md | 3 +- src/helpers/helper.js | 6 +- src/parsers/index.js | 3 + src/parsers/mstest.js | 190 ++++++++++++++++++++++++++++++ tests/data/mstest/testresults.trx | 159 +++++++++++++++++++++++++ tests/parser.mstest.spec.js | 65 ++++++++++ tests/parser.nunit.spec.js | 5 - 7 files changed, 424 insertions(+), 7 deletions(-) create mode 100644 src/parsers/mstest.js create mode 100644 tests/data/mstest/testresults.trx create mode 100644 tests/parser.mstest.spec.js diff --git a/README.md b/README.md index 8463aa8..6e2f269 100644 --- a/README.md +++ b/README.md @@ -9,6 +9,7 @@ Parse test results from JUnit, TestNG, xUnit, Mocha(json), Cucumber(json) and ma | TestNG | ✅ | | JUnit | ✅ | | NUnit (v2 & v3) | ✅ | +| MSTest | ✅ | | xUnit | ✅ | | Mocha (json & mochawesome) | ✅ | -| Cucumber | ✅ | \ No newline at end of file +| Cucumber | ✅ | diff --git a/src/helpers/helper.js b/src/helpers/helper.js index e9c80af..fd95631 100644 --- a/src/helpers/helper.js +++ b/src/helpers/helper.js @@ -31,7 +31,11 @@ const FORCED_ARRAY_KEYS = [ "testng-results.suite.test", "testng-results.suite.test.class", "testng-results.suite.test.class.test-method", - "testng-results.suite.test.class.test-method.exception" + "testng-results.suite.test.class.test-method.exception", + "TestRun.Results.UnitTestResult", + "TestRun.TestDefinitions.UnitTest", + "TestRun.TestDefinitions.UnitTest.TestCategory.TestCategoryItem", + "TestRun.TestDefinitions.UnitTest.Properties.Property" ]; const configured_parser = new XMLParser({ diff --git a/src/parsers/index.js b/src/parsers/index.js index edf4d6c..2054036 100644 --- a/src/parsers/index.js +++ b/src/parsers/index.js @@ -1,6 +1,7 @@ const testng = require('./testng'); const junit = require('./junit'); const nunit = require('./nunit'); +const mstest = require('./mstest'); const xunit = require('./xunit'); const mocha = require('./mocha'); const cucumber = require('./cucumber'); @@ -40,6 +41,8 @@ function getParser(type) { return xunit; case 'nunit': return nunit; + case 'mstest': + return mstest; case 'mocha': return mocha; case 'cucumber': diff --git a/src/parsers/mstest.js b/src/parsers/mstest.js new file mode 100644 index 0000000..b38a2ab --- /dev/null +++ b/src/parsers/mstest.js @@ -0,0 +1,190 @@ +const { getJsonFromXMLFile } = require('../helpers/helper'); + +const TestResult = require('../models/TestResult'); +const TestSuite = require('../models/TestSuite'); +const TestCase = require('../models/TestCase'); + +const RESULT_MAP = { + Passed: "PASS", + Failed: "FAIL", + NotExecuted: "SKIP", +} + +function populateMetaData(rawElement, map) { + if (rawElement.TestCategory && rawElement.TestCategory.TestCategoryItem) { + let rawCategories = rawElement.TestCategory.TestCategoryItem; + for (let i = 0; i < rawCategories.length; i++) { + let categoryName = rawCategories[i]["@_TestCategory"]; + map.set(categoryName, ""); + + // create comma-delimited list of categories + if (map.has("Categories")) { + map.set("Categories", map.get("Categories").concat(",", categoryName)); + } else { + map.set("Categories", categoryName); + } + } + } + + // as per https://github.com/microsoft/vstest/issues/2480: + // - properties are supported by the XSD but are not included in TRX Visual Studio output + // - including support for properties because third-party extensions might generate this data + if (rawElement.Properties) { + let rawProperties = rawElement.Properties.Property; + for (let i = 0; i < rawProperties.length; i++) { + let key = rawProperties[i].Key ?? "not-set"; + let val = rawProperties[i].Value ?? ""; + map.set(key, val); + } + } +} + +function getTestResultDuration(rawTestResult) { + // durations are represented in a timeformat with 7 digit microsecond precision + // TODO: introduce d3-time-format after https://github.com/test-results-reporter/parser/issues/42 is fixed. + return 0; +} + +function getTestCaseName(rawDefinition) { + if (rawDefinition.TestMethod) { + let className = rawDefinition.TestMethod["@_className"]; + let name = rawDefinition.TestMethod["@_name"]; + + // attempt to produce fully-qualified name + if (className) { + className = className.split(",")[0]; // handle strong-name scenario (typeName, assembly, culture, version) + return className.concat(".", name); + } else { + return name; + } + } else { + throw new Error("Unrecognized TestDefinition"); + } +} + +function getTestSuiteName(testCase) { + // assume testCase.name is full-qualified namespace.classname.methodname + let index = testCase.name.lastIndexOf("."); + return testCase.name.substring(0, index); +} + +function getTestCase(rawTestResult, definitionMap) { + let id = rawTestResult["@_testId"]; + + if (definitionMap.has(id)) { + var rawDefinition = definitionMap.get(id); + + var testCase = new TestCase(); + testCase.id = id; + testCase.name = getTestCaseName(rawDefinition); + testCase.status = RESULT_MAP[rawTestResult["@_outcome"]]; + testCase.duration = getTestResultDuration(rawTestResult); + + // collect error messages + if (rawTestResult.Output && rawTestResult.Output.ErrorInfo) { + testCase.setFailure(rawTestResult.Output.ErrorInfo.Message); + testCase.stack_trace = rawTestResult.Output.ErrorInfo.StackTrace ?? ''; + } + // populate meta + populateMetaData(rawDefinition, testCase.meta_data); + + return testCase; + } else { + throw new Error(`Unrecognized testId ${id ?? ''}`); + } +} + +function getTestDefinitionsMap(rawTestDefinitions) { + let map = new Map(); + + // assume all definitions are 'UnitTest' elements + if (rawTestDefinitions.UnitTest) { + let rawUnitTests = rawTestDefinitions.UnitTest; + for (let i = 0; i < rawUnitTests.length; i++) { + let rawUnitTest = rawUnitTests[i]; + let id = rawUnitTest["@_id"]; + if (id) { + map.set(id, rawUnitTest); + } + } + } + + return map; +} + +function getTestResults(rawTestResults) { + let results = []; + + // assume all results are UnitTestResult elements + if (rawTestResults.UnitTestResult) { + let unitTests = rawTestResults.UnitTestResult; + for (let i = 0; i < unitTests.length; i++) { + results.push(unitTests[i]); + } + } + return results; +} + +function getTestSuites(rawTestRun) { + // outcomes + durations are stored in /TestRun/TestResults/* + const testResults = getTestResults(rawTestRun.Results); + // test names and details are stored in /TestRun/TestDefinitions/* + const testDefinitions = getTestDefinitionsMap(rawTestRun.TestDefinitions); + + // trx does not include suites, so we'll reverse engineer them by + // grouping results from the same className + let suiteMap = new Map(); + + for (let i = 0; i < testResults.length; i++) { + let rawTestResult = testResults[i]; + let testCase = getTestCase(rawTestResult, testDefinitions); + let suiteName = getTestSuiteName(testCase); + + if (!suiteMap.has(suiteName)) { + let suite = new TestSuite(); + suite.name = suiteName; + suiteMap.set(suiteName, suite); + } + suiteMap.get(suiteName).cases.push(testCase); + } + + var result = []; + for (let suite of suiteMap.values()) { + suite.total = suite.cases.length; + suite.passed = suite.cases.filter(i => i.status == "PASS").length; + suite.failed = suite.cases.filter(i => i.status == "FAIL").length; + suite.skipped = suite.cases.filter(i => i.status == "SKIP").length; + suite.errors = suite.cases.filter(i => i.status == "ERROR").length; + suite.duration = suite.cases.reduce((total, test) => { return total + test.duration }, 0); + result.push(suite); + } + + return result; +} + +function getTestResult(json) { + const rawTestRun = json.TestRun; + + let result = new TestResult(); + result.id = rawTestRun["@_id"]; + result.suites.push(...getTestSuites(rawTestRun)); + + // calculate totals + result.total = result.suites.reduce((total, suite) => { return total + suite.total }, 0); + result.passed = result.suites.reduce((total, suite) => { return total + suite.passed }, 0); + result.failed = result.suites.reduce((total, suite) => { return total + suite.failed }, 0); + result.skipped = result.suites.reduce((total, suite) => { return total + suite.skipped }, 0); + result.errors = result.suites.reduce((total, suite) => { return total + suite.errors }, 0); + result.duration = result.suites.reduce((total, suite) => { return total + suite.duration }, 0); + + return result; +} + +function parse(file) { + const json = getJsonFromXMLFile(file); + return getTestResult(json); +} + +module.exports = { + parse +} \ No newline at end of file diff --git a/tests/data/mstest/testresults.trx b/tests/data/mstest/testresults.trx new file mode 100644 index 0000000..4f5d9db --- /dev/null +++ b/tests/data/mstest/testresults.trx @@ -0,0 +1,159 @@ + + + + + + + + + + + Assert.Fail failed. + at MSTestSample.MockTestFixture.FailingTest() in C:\dev\code\_Experiments\MSTestSample\UnitTest1.cs:line 12 + + + + + + + + Assert.Inconclusive failed. + at MSTestSample.MockTestFixture.InconclusiveTest() in C:\dev\code\_Experiments\MSTestSample\UnitTest1.cs:line 19 + + + + + + + + + + + + + + Test method MSTestSample.MockTestFixture.NotRunnableTest threw exception: +Microsoft.VisualStudio.TestPlatform.MSTest.TestAdapter.ObjectModel.TestFailedException: Only data driven test methods can have parameters. Did you intend to use [DataRow] or [DynamicData]? + + + + + + + Test method MSTestSample.MockTestFixture.TestWithException threw exception: +System.NotImplementedException: The method or operation is not implemented. + at MSTestSample.MockTestFixture.TestWithException() in C:\dev\code\_Experiments\MSTestSample\UnitTest1.cs:line 70 + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + Test 'InconclusiveTest' was skipped in the test run. +Test 'MockTest4' was skipped in the test run. + + + + + [MSTest][Discovery][C:\dev\code\_Experiments\MSTestSample\bin\Debug\net6.0\MSTestSample.dll] UTA007: Method MockTest5 defined in class MSTestSample.MockTestFixture does not have correct signature. Test method marked with the [TestMethod] attribute must be non-static, public, return-type as void and should not take any parameter. Example: public void Test.Class1.Test(). Additionally, if you are using async-await in test method then return-type must be Task. Example: public async Task Test.Class1.Test2() + + + + \ No newline at end of file diff --git a/tests/parser.mstest.spec.js b/tests/parser.mstest.spec.js new file mode 100644 index 0000000..d19b15a --- /dev/null +++ b/tests/parser.mstest.spec.js @@ -0,0 +1,65 @@ +const { parse } = require('../src'); +const assert = require('assert'); + +describe('Parser - MSTest', () => { + + const testDataPath = "tests/data/mstest"; + var result; + + before(() => { + result = parse({ type: 'mstest', files: [`${testDataPath}/testresults.trx`] }); + }); + + it('Should calculate totals', () => { + assert.equal(result.total, 10); + assert.equal(result.passed, 5); + assert.equal(result.failed, 3); + assert.equal(result.skipped, 2); + + assert.equal(result.suites.length, 2); + //assert.equal(result.duration > 0, true); // TODO: Fix + }) + + it('Should express durations in milliseconds', () => { + //trx represents timestamps with microseconds + //assert.equal(result.suites[0].cases[0].duration, 259.239); // TODO: Fix + }) + + it('Should map results correctly', () => { + assert.equal(result.suites[0].cases[0].status, "FAIL"); + assert.equal(result.suites[0].cases[1].status, "SKIP"); // inconclusive + assert.equal(result.suites[0].cases[2].status, "PASS"); + assert.equal(result.suites[0].cases[3].status, "PASS"); + assert.equal(result.suites[0].cases[4].status, "PASS"); + assert.equal(result.suites[0].cases[5].status, "SKIP"); // ignore + assert.equal(result.suites[0].cases[6].status, "FAIL"); // not runnable + assert.equal(result.suites[0].cases[7].status, "FAIL"); // exception + assert.equal(result.suites[0].cases[8].status, "PASS"); + + assert.equal(result.suites[1].cases[0].status, "PASS"); // datarow + }); + + it('Should include fullnames for testsuites and testcases', () => { + assert.equal(result.suites[0].name, "MSTestSample.MockTestFixture"); + assert.equal(result.suites[0].cases[0].name, "MSTestSample.MockTestFixture.FailingTest"); + }); + + it('Should include failure and stack trace for failed test', () => { + assert.equal(result.suites[0].cases[0].failure, 'Assert.Fail failed.') + assert.equal(result.suites[0].cases[0].stack_trace, 'at MSTestSample.MockTestFixture.FailingTest() in C:\\dev\\code\\_Experiments\\MSTestSample\\UnitTest1.cs:line 12 '); + }); + + it('Should include categories from suite', () => { + const testCaseInheritedCategories = result.suites[0].cases[0]; + assert.equal(testCaseInheritedCategories.meta_data.has("FixtureCategory"), true); + assert.equal(testCaseInheritedCategories.meta_data.get("Categories"), "FixtureCategory"); + }) + + it('Should combine categories from suite and case', () => { + const testCaseWithCategories = result.suites[0].cases[3]; + assert.equal(testCaseWithCategories.meta_data.has("FixtureCategory"), true); + assert.equal(testCaseWithCategories.meta_data.has("MockCategory"), true); + assert.equal(testCaseWithCategories.meta_data.get("Categories"), "FixtureCategory,MockCategory"); + }); + +}); \ No newline at end of file diff --git a/tests/parser.nunit.spec.js b/tests/parser.nunit.spec.js index a3d8486..5cd079e 100644 --- a/tests/parser.nunit.spec.js +++ b/tests/parser.nunit.spec.js @@ -1,6 +1,5 @@ const { parse } = require('../src'); const assert = require('assert'); -const path = require('path'); describe('Parser - NUnit', () => { @@ -240,10 +239,6 @@ describe('Parser - NUnit', () => { return flattenTestCases(result).reduce( (total, testcase) => { return total + predicate(testcase)}, 0); } - function findTestCaseById(result, id) { - return findTestCases(result, testcase => { return testcase.id == id})[0]; - } - function findTestCases(result, predicate) { return flattenTestCases(result).filter( testCase => {return predicate() == true}); } From 47c0ed2fdb316c4b4bbd31d6fd379e6f156510cf Mon Sep 17 00:00:00 2001 From: bryan cook <3217452+bryanbcook@users.noreply.github.com> Date: Mon, 13 Nov 2023 17:34:55 -0500 Subject: [PATCH 2/2] Add vscode tab spacing to .vscode --- .vscode/settings.json | 3 +++ 1 file changed, 3 insertions(+) create mode 100644 .vscode/settings.json diff --git a/.vscode/settings.json b/.vscode/settings.json new file mode 100644 index 0000000..ff30c44 --- /dev/null +++ b/.vscode/settings.json @@ -0,0 +1,3 @@ +{ + "editor.tabSize": 2 +} \ No newline at end of file