Add support for testcase.attachments (#51)

* Added failing tests * add 'attachments' to model and fix impacted tests * junit implementation for attachments * mstest implementation for attachments * nunit implementation for attacments
test-results-reporter · Feb 29, 2024 · a1ba45e · a1ba45e
1 parent acf25c3
commit a1ba45e
Show file tree

Hide file tree

Showing 19 changed files with 345 additions and 28 deletions.
diff --git a/src/helpers/helper.js b/src/helpers/helper.js
@@ -33,9 +33,10 @@ const FORCED_ARRAY_KEYS = [
   "testng-results.suite.test.class.test-method",
   "testng-results.suite.test.class.test-method.exception",
   "TestRun.Results.UnitTestResult",
+  "TestRun.Results.UnitTestResult.ResultFiles.ResultFile",
   "TestRun.TestDefinitions.UnitTest",
-  "TestRun.TestDefinitions.UnitTest.TestCategory.TestCategoryItem",
-  "TestRun.TestDefinitions.UnitTest.Properties.Property"
+  "TestRun.TestDefinitions.UnitTest.Properties.Property",
+  "TestRun.TestDefinitions.UnitTest.TestCategory.TestCategoryItem"
 ];
 
 const configured_parser = new XMLParser({
@@ -51,6 +52,7 @@ const configured_parser = new XMLParser({
         case "property":
         case "test-suite":
         case "test-case":
+        case "attachment":
           return true;
         default:
           return false;

diff --git a/src/models/TestAttachment.d.ts b/src/models/TestAttachment.d.ts
@@ -0,0 +1,8 @@
+declare class TestAttachment {
+  name: string;
+  path: string;
+}
+
+declare namespace TestAttachment { }
+
+export = TestAttachment;
diff --git a/src/models/TestAttachment.js b/src/models/TestAttachment.js
@@ -0,0 +1,10 @@
+class TestAttachment {
+
+  constructor() {
+    this.name = '';
+    this.path = '';
+  }
+
+}
+
+module.exports = TestAttachment;
diff --git a/src/models/TestCase.d.ts b/src/models/TestCase.d.ts
@@ -1,4 +1,5 @@
 import * as TestStep from './TestStep';
+import * as TestAttachment from './TestAttachment';
 
 declare class TestCase {
   name: string;
@@ -12,6 +13,7 @@ declare class TestCase {
   failure: string;
   stack_trace: string;
   steps: TestStep[];
+  attachments: TestAttachment[];
   meta_data: Map<string,string>;
 
   setFailure: SetFailureFunction

diff --git a/src/models/TestCase.js b/src/models/TestCase.js
@@ -15,6 +15,7 @@ class TestCase {
       this.failure = '';
       this.stack_trace = '';
       this.steps = [];
+      this.attachments = [];
       this.meta_data = new Map();
     }
 

diff --git a/src/parsers/junit.js b/src/parsers/junit.js
@@ -3,12 +3,15 @@ const { getJsonFromXMLFile } = require('../helpers/helper');
 const TestResult = require('../models/TestResult');
 const TestSuite = require('../models/TestSuite');
 const TestCase = require('../models/TestCase');
+const TestAttachment = require('../models/TestAttachment');
+const { Test } = require('mocha');
 
 function getTestCase(rawCase) {
   const test_case = new TestCase();
   test_case.name = rawCase["@_name"];
   test_case.duration = rawCase["@_time"] * 1000;
-  setMetaData(rawCase.properties, test_case);
+  setAttachments(rawCase, test_case);
+  setMetaData(rawCase.properties, test_case);  
   if (rawCase.failure && rawCase.failure.length > 0) {
     test_case.status = 'FAIL';
     test_case.setFailure(rawCase.failure[0]["@_message"]);
@@ -59,6 +62,34 @@ function setMetaData(properties, test_element) {
   }
 }
 
+/**
+ * @param {import('./junit.result').JUnitTestCase} rawCase
+ * @param {TestCase} test_element
+ */
+function setAttachments(rawCase, test_element) {
+  if (rawCase['system.out']) {
+    const systemOut = rawCase['system.out'];
+
+    // junit attachments plug syntax is [[ATTACHMENT|/absolute/path/to/file.png]]
+    const regex = new RegExp('\\[\\[ATTACHMENT\\|([^\\]]+)\\]\\]', 'g');
+
+    while ((m = regex.exec(systemOut)) !== null) {
+      // avoid infinite loops with zero-width matches
+      if (m.index === regex.lastIndex) {
+        regex.lastIndex++;
+      }
+
+      let filePath = m[1].trim();
+
+      if (filePath.length > 0) {
+        const attachment = new TestAttachment();
+        attachment.path = filePath;        
+        test_element.attachments.push(attachment);
+      }
+    }
+  }
+}
+
 /**
  * @param {TestResult} result 
  */

diff --git a/src/parsers/junit.result.d.ts b/src/parsers/junit.result.d.ts
@@ -19,9 +19,10 @@ export type JUnitTestCase = {
   '@_id': string;
   '@_name': string;
   '@_time': number;
+  'system.out': string;
 }
 
-export type TestSuite = {
+export type JUnitTestSuite = {
   properties?: JUnitProperties;
   testcase: JUnitTestCase[];
   '@_id': string;

diff --git a/src/parsers/mstest.js b/src/parsers/mstest.js
@@ -1,8 +1,10 @@
 const { getJsonFromXMLFile } = require('../helpers/helper');
+const path = require('path');
 
 const TestResult = require('../models/TestResult');
 const TestSuite = require('../models/TestSuite');
 const TestCase = require('../models/TestCase');
+const TestAttachment = require('../models/TestAttachment');
 
 const RESULT_MAP = {
   Passed: "PASS",
@@ -39,6 +41,27 @@ function populateMetaData(rawElement, map) {
   }
 }
 
+function populateAttachments(rawResultElement, attachments, testRunName) {
+
+  // attachments are in /TestRun/Results/UnitTestResult/ResultFiles/ResultFile[@path]
+  if (rawResultElement.ResultFiles && rawResultElement.ResultFiles.ResultFile) {
+    let executionId = rawResultElement["@_executionId"];
+    let rawAttachments = rawResultElement.ResultFiles.ResultFile;
+    for (let i = 0; i < rawAttachments.length; i++) {
+      let filePath = rawAttachments[i]["@_path"];
+      if (filePath) {
+
+        // file path is relative to testresults.trx
+        // stored in ./<testrunname>/in/<executionId>/path
+
+        let attachment = new TestAttachment();
+        attachment.path = path.join(testRunName, "In", executionId, ...(filePath.split(/[\\/]/g)));
+        attachments.push(attachment);
+      }
+    }
+  }
+}
+
 function getTestResultDuration(rawTestResult) {
   // durations are represented in a timeformat with 7 digit microsecond precision
   // TODO: introduce d3-time-format after https://github.com/test-results-reporter/parser/issues/42 is fixed.
@@ -68,7 +91,16 @@ function getTestSuiteName(testCase) {
   return testCase.name.substring(0, index);
 }
 
-function getTestCase(rawTestResult, definitionMap) {
+function getTestRunName(rawTestRun) {
+  // testrun.name contains '@', spaces and ':'
+  let name = rawTestRun["@_name"];
+  if (name) {
+    return name.replace(/[ @:]/g, '_');
+  }
+  return '';
+}
+
+function getTestCase(rawTestResult, definitionMap, testRunName) {
   let id = rawTestResult["@_testId"];
 
   if (definitionMap.has(id)) {
@@ -85,6 +117,8 @@ function getTestCase(rawTestResult, definitionMap) {
       testCase.setFailure(rawTestResult.Output.ErrorInfo.Message);
       testCase.stack_trace = rawTestResult.Output.ErrorInfo.StackTrace ?? '';
     }
+    // populate attachments
+    populateAttachments(rawTestResult, testCase.attachments, testRunName);
     // populate meta
     populateMetaData(rawDefinition, testCase.meta_data);
 
@@ -126,6 +160,9 @@ function getTestResults(rawTestResults) {
 }
 
 function getTestSuites(rawTestRun) {
+
+  // test attachments are stored in a testrun specific folder <name>/in/<executionid>/<computername>
+  const testRunName = getTestRunName(rawTestRun);
   // outcomes + durations are stored in /TestRun/TestResults/*
   const testResults = getTestResults(rawTestRun.Results);
   // test names and details are stored in /TestRun/TestDefinitions/*
@@ -137,7 +174,7 @@ function getTestSuites(rawTestRun) {
 
   for (let i = 0; i < testResults.length; i++) {
     let rawTestResult = testResults[i];
-    let testCase = getTestCase(rawTestResult, testDefinitions);
+    let testCase = getTestCase(rawTestResult, testDefinitions, testRunName);
     let suiteName = getTestSuiteName(testCase);
 
     if (!suiteMap.has(suiteName)) {

diff --git a/src/parsers/nunit.js b/src/parsers/nunit.js
@@ -3,6 +3,7 @@ const { getJsonFromXMLFile } = require('../helpers/helper');
 const TestResult = require('../models/TestResult');
 const TestSuite = require('../models/TestSuite');
 const TestCase = require('../models/TestCase');
+const TestAttachment = require('../models/TestAttachment');
 
 const SUITE_TYPES_WITH_TEST_CASES = [
   "TestFixture",
@@ -24,6 +25,20 @@ const RESULT_MAP = {
   Skipped: "SKIP",        // v3
 }
 
+function populateAttachments(rawCase, attachments) {
+  if (rawCase.attachments && rawCase.attachments.attachment) {
+    let rawAttachments = rawCase.attachments.attachment;
+    for (var i = 0; i < rawAttachments.length; i++) {
+      var attachment = new TestAttachment();
+      attachment.path = rawAttachments[i].filePath;
+      if (rawAttachments[i].description) {
+        attachment.name = rawAttachments[i].description;
+      }
+      attachments.push(attachment);
+    }
+  }
+}
+
 function mergeMeta(map1, map2) {
   for (let kvp of map1) {
     map2.set(kvp[0], kvp[1]);
@@ -130,6 +145,8 @@ function getTestCases(rawSuite, parent_meta) {
           testCase.stack_trace = errorDetails["stack-trace"]
         }
       }
+      // populate attachments
+      populateAttachments(rawCase, testCase.attachments);
       // copy parent_meta data to test case
       mergeMeta(parent_meta, testCase.meta_data);
       populateMetaData(rawCase, testCase.meta_data);

diff --git a/tests/data/junit/test-case-attachments.xml b/tests/data/junit/test-case-attachments.xml
@@ -0,0 +1,29 @@
+<?xml version="1.0" encoding="UTF-8" ?>
+<testsuites id="id" name="result name" tests="1" failures="1" errors="" time="10">
+  <testsuite id="testsuite" name="suite name" tests="1" failures="1" time="10">
+    <!-- single attachment -->
+    <testcase id="testsuite.withattachment" name="include attachments" time="10">
+      <system.out>[[ATTACHMENT|/path/to/attachment.png]]
+      </system.out>
+    </testcase>
+
+    <!-- multiple attachments-->
+    <testcase id="testsuite.withattachmentmultiple" name="include multiple attachments" time="10">
+      <system.out>[[ATTACHMENT|/path/to/attachment1.png]]
+Other output
+[[ATTACHMENT|/path/to/attachment2.png]]
+      </system.out>
+    </testcase>
+
+    <!-- no attachments -->
+    <testcase id="testsuite.testcase" name="with no attachments" time="10">
+    </testcase>
+
+    <!-- zero length attachments -->
+    <testcase id="testsuite.malformedattachment" name="malformed" time="10">
+      <system.out>[[ATTACHMENT| ]]
+      </system.out>
+    </testcase>
+
+  </testsuite>
+</testsuites>
diff --git a/tests/data/mstest/testresults.trx b/tests/data/mstest/testresults.trx
@@ -6,6 +6,8 @@
         <Deployment runDeploymentRoot="bryan.b.cook_MYCOMPUTER_2023-11-12_19_21_51" />
     </TestSettings>
     <Results>
+
+        <!-- MockTestFixture -->
         <UnitTestResult executionId="e01a54d8-305c-4828-9bc0-8c935270dafe" testId="5370a586-74b0-a647-4839-100eef3a4188" testName="FailingTest" computerName="MYCOMPUTER" duration="00:00:00.0259239" startTime="2023-11-12T19:21:51.6583003-05:00" endTime="2023-11-12T19:21:51.6978162-05:00" testType="13cdc9d9-ddb5-4fa4-a97d-d965ccfc6d4b" outcome="Failed" testListId="8c84fa94-04c1-424b-9868-57a2d4851a1d" relativeResultsDirectory="e01a54d8-305c-4828-9bc0-8c935270dafe">
             <Output>
                 <ErrorInfo>
@@ -50,11 +52,25 @@ System.NotImplementedException: The method or operation is not implemented.</Mes
         </UnitTestResult>
         <UnitTestResult executionId="05c69927-e30b-4b5c-a5b8-2e44e9a88785" testId="c09170d3-a997-9a92-b6d5-9a86a4225591" testName="TestWithManyProperties" computerName="MYCOMPUTER" duration="00:00:00.0000490" startTime="2023-11-12T19:21:51.7069158-05:00" endTime="2023-11-12T19:21:51.7071317-05:00" testType="13cdc9d9-ddb5-4fa4-a97d-d965ccfc6d4b" outcome="Passed" testListId="8c84fa94-04c1-424b-9868-57a2d4851a1d" relativeResultsDirectory="05c69927-e30b-4b5c-a5b8-2e44e9a88785" />
 
+        <!-- FixtureWithTestCases -->
         <UnitTestResult executionId="0942eb3b-2633-4c82-aa3a-374236dc6dab" testId="dac0f7bf-eced-b506-706a-66ed33c7be60" testName="TestWithArg (1)" computerName="MYCOMPUTER" duration="00:00:00.0000574" startTime="2023-11-12T19:21:51.7314236-05:00" endTime="2023-11-12T19:21:51.7362844-05:00" testType="13cdc9d9-ddb5-4fa4-a97d-d965ccfc6d4b" outcome="Passed" testListId="8c84fa94-04c1-424b-9868-57a2d4851a1d" relativeResultsDirectory="0942eb3b-2633-4c82-aa3a-374236dc6dab" />
 
-
+        <!-- FixtureWithTestCaseAttachments -->
+        <UnitTestResult executionId="238c26aa-9963-4623-aeda-95ef9a6799d0" testId="40ee30b0-1c98-09a4-5020-0b5d9d182630" testName="AddOneAttachment" computerName="MYCOMPUTER" duration="00:00:00.1221116" startTime="2024-02-28T14:30:39.4675415-05:00" endTime="2024-02-28T14:30:39.6333725-05:00" testType="13cdc9d9-ddb5-4fa4-a97d-d965ccfc6d4b" outcome="Passed" testListId="8c84fa94-04c1-424b-9868-57a2d4851a1d" relativeResultsDirectory="238c26aa-9963-4623-aeda-95ef9a6799d0">
+            <ResultFiles>
+              <ResultFile path="MYCOMPUTER/dummy1.txt" />
+            </ResultFiles>
+        </UnitTestResult>
+        <UnitTestResult executionId="2b2b0f58-3d88-432c-9955-1040315f96e9" testId="b63c50f5-d54a-4566-894b-c15b37bea962" testName="AddTwoAttachment" computerName="MYCOMPUTER" duration="00:00:00.1221116" startTime="2024-02-28T14:30:39.4675415-05:00" endTime="2024-02-28T14:30:39.6333725-05:00" testType="13cdc9d9-ddb5-4fa4-a97d-d965ccfc6d4b" outcome="Passed" testListId="8c84fa94-04c1-424b-9868-57a2d4851a1d" relativeResultsDirectory="2b2b0f58-3d88-432c-9955-1040315f96e9">
+            <ResultFiles>
+              <ResultFile path="MYCOMPUTER/dummy2.txt" />
+              <ResultFile path="MYCOMPUTER/dummy3.txt" />
+            </ResultFiles>
+        </UnitTestResult>
     </Results>
     <TestDefinitions>
+
+        <!-- MockTestFixture -->
         <UnitTest name="FailingTest" storage="c:\dev\code\_experiments\nunitsample\mstestsample\bin\debug\net6.0\mstestsample.dll" id="5370a586-74b0-a647-4839-100eef3a4188">
             <TestCategory>
                 <TestCategoryItem TestCategory="FixtureCategory" />
@@ -122,10 +138,23 @@ System.NotImplementedException: The method or operation is not implemented.</Mes
             <Execution id="05c69927-e30b-4b5c-a5b8-2e44e9a88785" />
             <TestMethod codeBase="C:\dev\code\_Experiments\MSTestSample\bin\Debug\net6.0\MSTestSample.dll" adapterTypeName="executor://mstestadapter/v2" className="MSTestSample.MockTestFixture" name="TestWithManyProperties" />
         </UnitTest>
+
+        <!-- FixtureWithTestCases -->
         <UnitTest name="TestWithArg (1)" storage="c:\dev\code\_experiments\nunitsample\mstestsample\bin\debug\net6.0\mstestsample.dll" id="dac0f7bf-eced-b506-706a-66ed33c7be60">
             <Execution id="0942eb3b-2633-4c82-aa3a-374236dc6dab" />
             <TestMethod codeBase="C:\dev\code\_Experiments\MSTestSample\bin\Debug\net6.0\MSTestSample.dll" adapterTypeName="executor://mstestadapter/v2" className="MSTestSample.FixtureWithTestCases" name="TestWithArg" />
         </UnitTest>
+
+        <!-- FixtureWithTestCaseAttachments -->
+        <UnitTest name="AddOneAttachment" storage="c:\dev\code\_experiments\nunitsample\mstestsample\bin\debug\net6.0\mstestsample.dll" id="40ee30b0-1c98-09a4-5020-0b5d9d182630">
+            <Execution id="238c26aa-9963-4623-aeda-95ef9a6799d0" />
+            <TestMethod codeBase="C:\dev\code\_Experiments\NUnitSample\MSTestSample\bin\Debug\net6.0\MSTestSample.dll" adapterTypeName="executor://mstestadapter/v2" className="MSTestSample.FixtureWithTestCaseAttachments" name="AddOneAttachment" />
+        </UnitTest>
+        <UnitTest name="AddTwoAttachment" storage="c:\dev\code\_experiments\nunitsample\mstestsample\bin\debug\net6.0\mstestsample.dll" id="b63c50f5-d54a-4566-894b-c15b37bea962">
+            <Execution id="2b2b0f58-3d88-432c-9955-1040315f96e9" />
+            <TestMethod codeBase="C:\dev\code\_Experiments\NUnitSample\MSTestSample\bin\Debug\net6.0\MSTestSample.dll" adapterTypeName="executor://mstestadapter/v2" className="MSTestSample.FixtureWithTestCaseAttachments" name="AddTwoAttachment" />
+        </UnitTest>
+
     </TestDefinitions>
     <TestEntries>
         <TestEntry testId="c09170d3-a997-9a92-b6d5-9a86a4225591" executionId="05c69927-e30b-4b5c-a5b8-2e44e9a88785" testListId="8c84fa94-04c1-424b-9868-57a2d4851a1d" />
@@ -138,6 +167,7 @@ System.NotImplementedException: The method or operation is not implemented.</Mes
         <TestEntry testId="0b5c3ebb-3086-951a-1724-1821d30aef04" executionId="68e29212-0297-4843-8f1e-cad3dfe2dd06" testListId="8c84fa94-04c1-424b-9868-57a2d4851a1d" />
         <TestEntry testId="20aced79-1a63-4e66-d444-ee4575c65515" executionId="685aeaf1-023c-4eb1-8727-fc6516eb1b61" testListId="8c84fa94-04c1-424b-9868-57a2d4851a1d" />
         <TestEntry testId="6b0e8fc2-5324-dbeb-0c5f-0479cd14f351" executionId="4db5da11-faef-4e70-98d1-39ee00d6ed00" testListId="8c84fa94-04c1-424b-9868-57a2d4851a1d" />
+        <TestEntry testId="40ee30b0-1c98-09a4-5020-0b5d9d182630" executionId="238c26aa-9963-4623-aeda-95ef9a6799d0" testListId="8c84fa94-04c1-424b-9868-57a2d4851a1d" />
     </TestEntries>
     <TestLists>
         <TestList name="Results Not in a List" id="8c84fa94-04c1-424b-9868-57a2d4851a1d" />

diff --git a/tests/data/nunit/nunit_v3.xml b/tests/data/nunit/nunit_v3.xml
@@ -121,5 +121,15 @@
     <test-suite type="TestFixture" id="1014" name="MockTestFixture" fullname="NUnit.Tests.TestAssembly.MockTestFixture" testcasecount="1" result="Passed" time="0.001" total="1" passed="1" failed="0" inconclusive="0" skipped="0" asserts="0">
       <test-case id="1015" name="MyTest" fullname="NUnit.Tests.TestAssembly.MockTestFixture.MyTest" result="Passed" time="0.001" asserts="0" />
     </test-suite>
+    <test-suite type="TestFixture" id="1037" name="AttachmentsFixture" fullname="NUnit.Tests.TestAssembly.AttachmentsFixture" testcasecount="1" result="Passed" time="0.001" total="1" passed="1" failed="0" inconclusive="0" skipped="0" asserts="0">
+      <test-case id="1038" name="TestOneAttachment" fullname="NUnit.Tests.TestAssembly.AttachmentsFixture.TestOneAttachment" result="Passed" time="0.001" asserts="0">
+        <attachments>
+          <attachment>
+            <filePath>c:\absolute\filepath\dummy.txt</filePath>
+            <description><![CDATA[my description]]></description>
+          </attachment>
+        </attachments>
+      </test-case>
+    </test-suite>
   </test-suite>
 </test-run>
diff --git a/tests/parser.cucumber.spec.js b/tests/parser.cucumber.spec.js
@@ -33,6 +33,7 @@ describe('Parser - Cucumber Json', () => {
           meta_data: newMap({ tags: "blue,slow", suite: "1234", tagsRaw: "@blue,@slow" }),
           cases: [
             {
+              attachments: [],
               duration: 1.59,
               errors: 0,
               failed: 0,
@@ -97,6 +98,7 @@ describe('Parser - Cucumber Json', () => {
           meta_data: new Map(),
           cases: [
             {
+              attachments: [],
               duration: 2.56,
               errors: 0,
               failed: 0,
@@ -112,6 +114,7 @@ describe('Parser - Cucumber Json', () => {
               total: 0
             },
             {
+              attachments: [],
               duration: 0.28,
               errors: 0,
               failed: 0,
@@ -141,6 +144,7 @@ describe('Parser - Cucumber Json', () => {
           meta_data: new Map(),
           cases: [
             {
+              attachments: [],
               duration: 0.52,
               errors: 0,
               failed: 0,