Add optional scatterplot to benchcomp output

Scatterplots should make it easier to immediately spot performance trends (and
indeed any differences) rather than having to process a (large) number
of table rows.

Uses mermaid-js to produce markdown-embedded plots that will display on
the job summary page. Scatterplots are not directly supported by
mermaid-js at this point (xycharts only do line or bar charts), so
quadrant plots are employed with various diagram items drawn in white to
make them disappear.

tools/benchcomp/benchcomp/visualizers/__init__.py

@@ -3,8 +3,10 @@
 
 
 import dataclasses
+import enum
 import json
 import logging
+import math
 import subprocess
 import sys
 import textwrap
@@ -125,11 +127,21 @@ def __call__(self, results):
 
 
 
+class Plot(enum.Enum):
+    """Scatterplot configuration options
+    """
+    OFF = 1
+    LINEAR = 2
+    LOG = 3
+
+
+
 class dump_markdown_results_table:
     """Print Markdown-formatted tables displaying benchmark results
 
     For each metric, this visualization prints out a table of benchmarks,
-    showing the value of the metric for each variant.
+    showing the value of the metric for each variant, combined with an optional
+    scatterplot.
 
     The 'out_file' key is mandatory; specify '-' to print to stdout.
 
@@ -145,12 +157,16 @@ class dump_markdown_results_table:
     particular combinations of values for different variants, such as
     regressions or performance improvements.
 
+    'scatterplot' takes the values 'off' (default), 'linear' (linearly scaled
+    axes), or 'log' (logarithmically scaled axes).
+
     Sample configuration:
 
     ```
     visualize:
     - type: dump_markdown_results_table
       out_file: "-"
+      scatterplot: linear
       extra_columns:
         runtime:
         - column_name: ratio
@@ -187,9 +203,10 @@ class dump_markdown_results_table:
     """
 
 
-    def __init__(self, out_file, extra_columns=None):
+    def __init__(self, out_file, extra_columns=None, scatterplot=None):
         self.get_out_file = benchcomp.Outfile(out_file)
         self.extra_columns = self._eval_column_text(extra_columns or {})
+        self.scatterplot = self._parse_scatterplot_config(scatterplot)
 
 
     @staticmethod
@@ -206,17 +223,50 @@ def _eval_column_text(column_spec):
         return column_spec
 
 
+    @staticmethod
+    def _parse_scatterplot_config(scatterplot_config_string):
+        if (scatterplot_config_string is None or
+                scatterplot_config_string == "off"):
+            return Plot.OFF
+        elif scatterplot_config_string == "linear":
+            return Plot.LINEAR
+        elif scatterplot_config_string == "log":
+            return Plot.LOG
+        else:
+            logging.error(
+                "Invalid scatterplot configuration '%s'",
+                scatterplot_config_string)
+            sys.exit(1)
+
+
     @staticmethod
     def _get_template():
         return textwrap.dedent("""\
             {% for metric, benchmarks in d["metrics"].items() %}
             ## {{ metric }}
 
+            {% if len(d["variants"][metric]) == 2 and scatterplot %}
+            ```mermaid
+            %%{init: { "quadrantChart": { "chartWidth": 400, "chartHeight": 400, "pointRadius": 2, "pointLabelFontSize": 3 }, "themeVariables": { "quadrant1Fill": "#FFFFFF", "quadrant2Fill": "#FFFFFF", "quadrant3Fill": "#FFFFFF", "quadrant4Fill": "#FFFFFF", "quadrant1TextFill": "#FFFFFF", "quadrant2TextFill": "#FFFFFF", "quadrant3TextFill": "#FFFFFF", "quadrant4TextFill": "#FFFFFF", "quadrantInternalBorderStrokeFill": "#FFFFFF" } }%%
+            quadrantChart
+                title {{ metric }}
+                x-axis {{ d["variants"][metric][0] }}
+                y-axis {{ d["variants"][metric][1] }}
+                quadrant-1 1
+                quadrant-2 2
+                quadrant-3 3
+                quadrant-4 4
+                {% for bench_name, bench_variants in benchmarks.items () %}
+                {{ bench_name }}: [{{ bench_variants[d["variants"][metric][0]]["scaled"] }}, {{ bench_variants[d["variants"][metric][1]]["scaled"] }}]
+                {% endfor %}
+            ```
+            {% endif %}
+
             | Benchmark | {% for variant in d["variants"][metric] %} {{ variant }} |{% endfor %}
             | --- |{% for variant in d["variants"][metric] %} --- |{% endfor -%}
             {% for bench_name, bench_variants in benchmarks.items () %}
             | {{ bench_name }} {% for variant in d["variants"][metric] -%}
-             | {{ bench_variants[variant] }} {% endfor %}|
+             | {{ bench_variants[variant]["absolute"] }} {% endfor %}|
             {%- endfor %}
             {% endfor -%}
             """)
@@ -228,7 +278,36 @@ def _get_variant_names(results):
 
 
     @staticmethod
-    def _organize_results_into_metrics(results):
+    def _add_scaled_metrics(data_for_metric, log_scaling):
+        min_value = None
+        max_value = None
+        for bench, bench_result in data_for_metric.items():
+            for variant, variant_result in bench_result.items():
+                if min_value is None or variant_result["absolute"] < min_value:
+                    min_value = variant_result["absolute"]
+                if max_value is None or variant_result["absolute"] > max_value:
+                    max_value = variant_result["absolute"]
+        if min_value is None or min_value == max_value:
+            for bench, bench_result in data_for_metric.items():
+                for variant, variant_result in bench_result.items():
+                    variant_result["scaled"] = 1.0
+        else:
+            if log_scaling:
+                min_value = math.log(min_value, 10)
+                max_value = math.log(max_value, 10)
+            value_range = max_value - min_value
+            for bench, bench_result in data_for_metric.items():
+                for variant, variant_result in bench_result.items():
+                    if log_scaling:
+                        abs_value = math.log(variant_result["absolute"], 10)
+                    else:
+                        abs_value = variant_result["absolute"]
+                    else:
+                        variant_result["scaled"] = (abs_value - min_value) / value_range
+
+
+    @staticmethod
+    def _organize_results_into_metrics(results, log_scaling):
         ret = {metric: {} for metric in results["metrics"]}
         for bench, bench_result in results["benchmarks"].items():
             for variant, variant_result in bench_result["variants"].items():
@@ -241,11 +320,15 @@ def _organize_results_into_metrics(results):
                             "the 'metrics' dict. Add '%s: {}' to the metrics "
                             "dict", bench, metric, variant, metric)
                     try:
-                        ret[metric][bench][variant] = variant_result["metrics"][metric]
+                        ret[metric][bench][variant]["absolute"] = variant_result["metrics"][metric]
                     except KeyError:
                         ret[metric][bench] = {
-                            variant: variant_result["metrics"][metric]
+                            variant: {
+                                "absolute": variant_result["metrics"][metric]
+                            }
                         }
+        for metric, bench_result in ret.items():
+            self._add_scaled_metrics(bench_result, log_scaling)
         return ret
 
 
@@ -272,7 +355,8 @@ def _get_variants(metrics):
 
 
     def __call__(self, results):
-        metrics = self._organize_results_into_metrics(results)
+        metrics = self._organize_results_into_metrics(
+                results, self.scatterplot == Plot.LOG)
         self._add_extra_columns(metrics)
 
         data = {
@@ -285,6 +369,7 @@ def __call__(self, results):
                 enabled_extensions=("html"),
                 default_for_string=True))
         template = env.from_string(self._get_template())
-        output = template.render(d=data)[:-1]
+        include_scatterplot = self.scatterplot != Plot.OFF
+        output = template.render(d=data, scatterplot=include_scatterplot)[:-1]
         with self.get_out_file() as handle:
             print(output, file=handle)

tools/benchcomp/configs/perf-regression.yaml

@@ -33,6 +33,7 @@ visualize:
 
   - type: dump_markdown_results_table
     out_file: '-'
+    scatterplot: linear
     extra_columns:
 
       # For these two metrics, display the difference between old and new and