Merge pull request #1925 from jgphpc/jg/xml

[feat] Add command-line option to generate a JUnit XML report

docs/config_reference.rst

@@ -1224,6 +1224,17 @@ General Configuration
       Default value has changed to avoid generating a report file per session.
 
 
+.. js:attribute:: .general[].report_junit
+
+   :required: No
+   :default: ``null``
+
+   The file where ReFrame will store its report in JUnit format.
+   The report adheres to the XSD schema `here <https://github.com/windyroad/JUnit-Schema/blob/master/JUnit.xsd>`__.
+
+   .. versionadded:: 3.6.0
+
+
 .. js:attribute:: .general[].resolve_module_conflicts
 
    :required: No

docs/manpage.rst

@@ -274,6 +274,16 @@ Options controlling ReFrame output
    .. versionadded:: 3.1
 
 
+.. option:: --report-junit=FILE
+
+   Instruct ReFrame to generate a JUnit XML report in ``FILE``.
+   The generated report adheres to the XSD schema `here <https://github.com/windyroad/JUnit-Schema/blob/master/JUnit.xsd>`__ and it takes into account only the first run, ignoring retries of failed tests.
+
+   This option can also be set using the :envvar:`RFM_REPORT_JUNIT` environment variable or the :js:attr:`report_junit` general configuration parameter.
+
+   .. versionadded:: 3.6.0
+
+
 -------------------------------------
 Options controlling ReFrame execution
 -------------------------------------
@@ -859,6 +869,21 @@ Here is an alphabetical list of the environment variables recognized by ReFrame:
       ================================== ==================
 
 
+.. envvar:: RFM_REPORT_JUNIT
+
+   The file where ReFrame will generate a JUnit XML report.
+
+   .. versionadded:: 3.6.0
+
+   .. table::
+      :align: left
+
+      ================================== ==================
+      Associated command line option     :option:`--report-junit`
+      Associated configuration parameter :js:attr:`report_junit` general configuration parameter
+      ================================== ==================
+
+
 .. envvar:: RFM_RESOLVE_MODULE_CONFLICTS
 
    Resolve module conflicts automatically.

reframe/frontend/cli.py

@@ -215,6 +215,12 @@ def main():
         envvar='RFM_REPORT_FILE',
         configvar='general/report_file'
     )
+    output_options.add_argument(
+        '--report-junit', action='store', metavar='FILE',
+        help="Store a JUnit report in FILE",
+        envvar='RFM_REPORT_JUNIT',
+        configvar='general/report_junit'
+    )
 
     # Check discovery options
     locate_options.add_argument(
@@ -1045,6 +1051,21 @@ def module_unuse(*paths):
                     f'failed to generate report in {report_file!r}: {e}'
                 )
 
+            # Generate the junit xml report for this session
+            junit_report_file = rt.get_option('general/0/report_junit')
+            if junit_report_file:
+                # Expand variables in filename
+                junit_report_file = osext.expandvars(junit_report_file)
+                junit_xml = runreport.junit_xml_report(json_report)
+                try:
+                    with open(junit_report_file, 'w') as fp:
+                        runreport.junit_dump(junit_xml, fp)
+                except OSError as e:
+                    printer.warning(
+                        f'failed to generate report in {junit_report_file!r}: '
+                        f'{e}'
+                    )
+
         if not success:
             sys.exit(1)
 

reframe/frontend/runreport.py

@@ -3,8 +3,10 @@
 #
 # SPDX-License-Identifier: BSD-3-Clause
 
+import decimal
 import json
 import jsonschema
+import lxml.etree as etree
 import os
 import re
 
@@ -13,7 +15,6 @@
 import reframe.utility.jsonext as jsonext
 import reframe.utility.versioning as versioning
 
-
 DATA_VERSION = '1.3.0'
 _SCHEMA = os.path.join(rfm.INSTALL_PREFIX, 'reframe/schemas/runreport.json')
 
@@ -152,3 +153,62 @@ def load_report(filename):
         )
 
     return _RunReport(report)
+
+
+def junit_xml_report(json_report):
+    '''Generate a JUnit report from a standard ReFrame JSON report.'''
+
+    xml_testsuites = etree.Element('testsuites')
+    xml_testsuite = etree.SubElement(
+        xml_testsuites, 'testsuite',
+        attrib={
+            'errors': '0',
+            'failures': str(json_report['session_info']['num_failures']),
+            'hostname': json_report['session_info']['hostname'],
+            'id': '0',
+            'name': 'reframe',
+            'package': 'reframe',
+            'tests': str(json_report['session_info']['num_cases']),
+            'time': str(json_report['session_info']['time_elapsed']),
+
+            # XSD schema does not like the timezone format, so we remove it
+            'timestamp': json_report['session_info']['time_start'][:-5],
+        }
+    )
+    testsuite_properties = etree.SubElement(xml_testsuite, 'properties')
+    for testid in range(len(json_report['runs'][0]['testcases'])):
+        tid = json_report['runs'][0]['testcases'][testid]
+        casename = (
+            f"{tid['name']}[{tid['system']}, {tid['environment']}]"
+        )
+        testcase = etree.SubElement(
+            xml_testsuite, 'testcase',
+            attrib={
+                'classname': tid['filename'],
+                'name': casename,
+
+                # XSD schema does not like the exponential format and since we
+                # do not want to impose a fixed width, we pass it to `Decimal`
+                # to format it automatically.
+                'time': str(decimal.Decimal(tid['time_total'])),
+            }
+        )
+        if tid['result'] == 'failure':
+            testcase_msg = etree.SubElement(
+                testcase, 'failure', attrib={'type': 'failure',
+                                             'message': tid['fail_phase']}
+            )
+            testcase_msg.text = f"{tid['fail_phase']}: {tid['fail_reason']}"
+
+    testsuite_stdout = etree.SubElement(xml_testsuite, 'system-out')
+    testsuite_stdout.text = ''
+    testsuite_stderr = etree.SubElement(xml_testsuite, 'system-err')
+    testsuite_stderr.text = ''
+    return xml_testsuites
+
+
+def junit_dump(xml, fp):
+    fp.write(
+        etree.tostring(xml, encoding='utf8', pretty_print=True,
+                       method='xml', xml_declaration=True).decode()
+    )

reframe/frontend/statistics.py

@@ -5,6 +5,7 @@
 
 import inspect
 import traceback
+
 import reframe.core.runtime as rt
 import reframe.core.exceptions as errors
 import reframe.utility as util
@@ -266,8 +267,8 @@ def print_failure_stats(self, printer):
         stats_header = row_format.format('Phase', '#', 'Failing test cases')
         num_tests = len(self.tasks(current_run))
         num_failures = 0
-        for l in failures.values():
-            num_failures += len(l)
+        for fl in failures.values():
+            num_failures += len(fl)
 
         stats_body = ['']
         stats_body.append(f'Total number of test cases: {num_tests}')

reframe/schemas/config.json

@@ -448,6 +448,7 @@
                     "non_default_craype": {"type": "boolean"},
                     "purge_environment": {"type": "boolean"},
                     "report_file": {"type": "string"},
+                    "report_junit": {"type": ["string", "null"]},
                     "resolve_module_conflicts": {"type": "boolean"},
                     "save_log_files": {"type": "boolean"},
                     "target_systems": {"$ref": "#/defs/system_ref"},
@@ -487,6 +488,7 @@
         "general/non_default_craype": false,
         "general/purge_environment": false,
         "general/report_file": "${HOME}/.reframe/reports/run-report.json",
+        "general/report_junit": null,
         "general/resolve_module_conflicts": true,
         "general/save_log_files": false,
         "general/target_systems": ["*"],