twister: coverage: Add coverage data to twister.json

New Twister option `--coverage-sections` allow to select what coverage
data to include into the `twister.json` file for its further analysis
in addition to the test meta-data and the test results.

The choices are to select either the coverage summary, or the detailed
branch coverage, ztest coverage, or all the coverage data collected.
Also the coverage run 'status' and coverage 'tool' values are added
with any non-default choice. The 'environment' top object has additional
'gcov_tool' proverty to keep the command line argument selection.

Depending on `--coverage-split` and `--disable-coverage-aggregation`
options the coverage data is attached either to its test suite object
or/and at the report's top level as the aggregated summary of the test
plan execution scope.

Currently this mode is fully supported for the default gcovr reporting
tool only, as it is based on the GCOVR json reports. For lcov only
coverage run status is reported.

Signed-off-by: Dmitrii Golovanov <[email protected]>

scripts/pylib/twister/twisterlib/coverage.py

@@ -141,7 +141,7 @@ def generate(self, outdir):
             else:
                 coverage_completed = False
         logger.debug("All coverage data processed: {}".format(coverage_completed))
-        return coverage_completed, reports
+        return coverage_completed, reports, self.gcov_tool
 
 
 class Lcov(CoverageTool):

scripts/pylib/twister/twisterlib/environment.py

@@ -304,7 +304,8 @@ def add_parse_arguments(parser = None):
         help="Only run cmake, do not build or run.")
 
     coverage_group.add_argument("--enable-coverage", action="store_true",
-                                help="Enable code coverage collection using gcov.")
+                                help="Enable code coverage data collection using gcov. "
+                                "Use --coverage to compose reports from that data.")
 
     coverage_group.add_argument("-C", "--coverage", action="store_true",
                                 help="Generate coverage reports. Implies "
@@ -348,6 +349,17 @@ def add_parse_arguments(parser = None):
                                      be active (`--coverage-split`) to set at least one of the reporting modes.
                                      Default: %(default)s""")
 
+    coverage_group.add_argument("--coverage-sections", nargs="+", default=None,
+                        choices=['all','report','ztest','summary'],
+                        help="""Selects coverage data to include into the `twister.json` report as 'coverage'
+                                object and its properties either as the sections chosen, or all of them.
+                                With `--coverage-split` each test suite will have its own 'coverage' object
+                                with these sections as properties.
+                                The aggregated coverage of all executed test suites will be a top-level object.
+                                Each 'coverage' object will also have its execution 'status' property.
+                                Currently this mode is fully supported for the default 'gcovr' tool only.
+                                Default: %(default)s""")
+
     parser.add_argument("--test-config", action="store", default=os.path.join(ZEPHYR_BASE, "tests", "test_config.yaml"),
         help="Path to file with plans and test configurations.")
 
@@ -803,14 +815,14 @@ def parse_arguments(parser, args, options = None):
 
     if options.coverage:
         options.enable_coverage = True
-
-    if not options.coverage and (options.disable_coverage_aggregation or options.coverage_split):
-        logger.error("Enable coverage reporting to set its aggregation mode.")
-        sys.exit(1)
-
-    if options.coverage and options.disable_coverage_aggregation and not options.coverage_split:
-        logger.error("At least one coverage reporting mode should be enabled: "
-                     "either aggregation, or split, or both.")
+        if options.disable_coverage_aggregation and not options.coverage_split:
+            logger.error("At least one coverage reporting mode should be enabled: "
+                         "either aggregation, or split, or both.")
+            sys.exit(1)
+    elif (options.disable_coverage_aggregation or
+          options.coverage_split or
+          options.coverage_sections):
+        logger.error("Enable coverage reporting first to set its mode.")
         sys.exit(1)
 
     if not options.coverage_platform:

scripts/pylib/twister/twisterlib/reports.py

@@ -1,7 +1,7 @@
 #!/usr/bin/env python3
 # vim: set syntax=python ts=4 :
 #
-# Copyright (c) 2018 Intel Corporation
+# Copyright (c) 2018-2023 Intel Corporation
 # SPDX-License-Identifier: Apache-2.0
 
 import os
@@ -26,6 +26,9 @@ def __init__(self, plan, env) -> None:
         self.env = env
         self.timestamp = datetime.now().isoformat()
         self.outdir = os.path.abspath(env.options.outdir)
+        self.coverage_status = None
+        self.gcov_tool = self.env.options.gcov_tool
+        self.coverage = None
 
     @staticmethod
     def process_log(log_file):
@@ -351,9 +354,40 @@ def json_report(self, filename, version="NA", platform=None):
             if instance.recording is not None:
                 suite['recording'] = instance.recording
 
+            if self.env.options.coverage_sections and instance.coverage_status is not None:
+                suite['coverage'] = { 'status': instance.coverage_status,
+                                      'tool': self.env.options.coverage_tool }
+                if instance.coverage.items:
+                    do_all = 'all' in self.env.options.coverage_sections
+                    for k,v in instance.coverage.items():
+                        if do_all or k in self.env.options.coverage_sections:
+                            logger.debug(f"Include '{instance.name}' coverage.{k} from '{v}'")
+                            with open(v, "rt") as json_file:
+                                suite['coverage'][k] = json.load(json_file)
+                else:
+                    logger.warning(f"No coverage data for '{instance.name}'")
+                #
+            #
+
             suites.append(suite)
 
         report["testsuites"] = suites
+
+        if self.env.options.coverage_sections and self.coverage_status is not None:
+            report['environment']['gcov_tool'] = self.gcov_tool
+            report['coverage'] = { 'status': self.coverage_status,
+                                   'tool': self.env.options.coverage_tool }
+            if self.coverage.items:
+                do_all = 'all' in self.env.options.coverage_sections
+                for k,v in self.coverage.items():
+                    if do_all or k in self.env.options.coverage_sections:
+                        logger.debug(f"Include aggregated coverage.{k} from '{v}'")
+                        with open(v, "rt") as json_file:
+                            report['coverage'][k] = json.load(json_file)
+            else:
+                logger.warning(f"No aggregated coverage data collected")
+        #
+
         with open(filename, "wt") as json_file:
             json.dump(report, json_file, indent=4, separators=(',',':'))
 

scripts/pylib/twister/twisterlib/runner.py

@@ -692,7 +692,7 @@ def process(self, pipeline, done, message, lock, results):
         # Run per-instance code coverage
         elif op == "coverage":
             logger.debug(f"Run coverage for '{self.instance.name}'")
-            self.instance.coverage_status, self.instance.coverage = run_coverage_instance(self.options, self.instance)
+            self.instance.coverage_status, self.instance.coverage, _ = run_coverage_instance(self.options, self.instance)
             pipeline.put({"op": "report", "test": self.instance})
 
         # Report results and output progress to screen

scripts/pylib/twister/twisterlib/twister_main.py

@@ -199,10 +199,9 @@ def main(options):
 
     report.summary(runner.results, options.disable_unrecognized_section_test, duration)
 
-    report.coverage_status = True
     if options.coverage and not options.disable_coverage_aggregation:
         if not options.build_only:
-            report.coverage_status, report.coverage = run_coverage(options, tplan)
+            report.coverage_status, report.coverage, report.gcov_tool = run_coverage(options, tplan)
         else:
             logger.info("Skipping coverage report generation due to --build-only.")