From 2fab78eb9dbae373cf4398aed05f2945792f8b22 Mon Sep 17 00:00:00 2001 From: Michael Tautschnig Date: Wed, 13 Mar 2024 09:32:01 +0000 Subject: [PATCH] Add optional scatterplot to benchcomp output Scatterplots should make it easier to immediately spot performance trends (and indeed any differences) rather than having to process a (large) number of table rows. Uses mermaid-js to produce markdown-embedded plots that will display on the job summary page. Scatterplots are not directly supported by mermaid-js at this point (xycharts only do line or bar charts), so quadrant plots are employed with various diagram items drawn in white to make them disappear. --- .../benchcomp/visualizers/__init__.py | 101 ++++++++++++++++-- tools/benchcomp/configs/perf-regression.yaml | 1 + 2 files changed, 94 insertions(+), 8 deletions(-) diff --git a/tools/benchcomp/benchcomp/visualizers/__init__.py b/tools/benchcomp/benchcomp/visualizers/__init__.py index f9987832ad62..bf4ce2ac89db 100644 --- a/tools/benchcomp/benchcomp/visualizers/__init__.py +++ b/tools/benchcomp/benchcomp/visualizers/__init__.py @@ -3,8 +3,10 @@ import dataclasses +import enum import json import logging +import math import subprocess import sys import textwrap @@ -125,11 +127,21 @@ def __call__(self, results): +class Plot(enum.Enum): + """Scatterplot configuration options + """ + OFF = 1 + LINEAR = 2 + LOG = 3 + + + class dump_markdown_results_table: """Print Markdown-formatted tables displaying benchmark results For each metric, this visualization prints out a table of benchmarks, - showing the value of the metric for each variant. + showing the value of the metric for each variant, combined with an optional + scatterplot. The 'out_file' key is mandatory; specify '-' to print to stdout. @@ -145,12 +157,16 @@ class dump_markdown_results_table: particular combinations of values for different variants, such as regressions or performance improvements. + 'scatterplot' takes the values 'off' (default), 'linear' (linearly scaled + axes), or 'log' (logarithmically scaled axes). + Sample configuration: ``` visualize: - type: dump_markdown_results_table out_file: "-" + scatterplot: linear extra_columns: runtime: - column_name: ratio @@ -187,9 +203,10 @@ class dump_markdown_results_table: """ - def __init__(self, out_file, extra_columns=None): + def __init__(self, out_file, extra_columns=None, scatterplot=None): self.get_out_file = benchcomp.Outfile(out_file) self.extra_columns = self._eval_column_text(extra_columns or {}) + self.scatterplot = self._parse_scatterplot_config(scatterplot) @staticmethod @@ -206,17 +223,50 @@ def _eval_column_text(column_spec): return column_spec + @staticmethod + def _parse_scatterplot_config(scatterplot_config_string): + if (scatterplot_config_string is None or + scatterplot_config_string == "off"): + return Plot.OFF + elif scatterplot_config_string == "linear": + return Plot.LINEAR + elif scatterplot_config_string == "log": + return Plot.LOG + else: + logging.error( + "Invalid scatterplot configuration '%s'", + scatterplot_config_string) + sys.exit(1) + + @staticmethod def _get_template(): return textwrap.dedent("""\ {% for metric, benchmarks in d["metrics"].items() %} ## {{ metric }} + {% if len(d["variants"][metric]) == 2 and scatterplot %} + ```mermaid + %%{init: { "quadrantChart": { "chartWidth": 400, "chartHeight": 400, "pointRadius": 2, "pointLabelFontSize": 3 }, "themeVariables": { "quadrant1Fill": "#FFFFFF", "quadrant2Fill": "#FFFFFF", "quadrant3Fill": "#FFFFFF", "quadrant4Fill": "#FFFFFF", "quadrant1TextFill": "#FFFFFF", "quadrant2TextFill": "#FFFFFF", "quadrant3TextFill": "#FFFFFF", "quadrant4TextFill": "#FFFFFF", "quadrantInternalBorderStrokeFill": "#FFFFFF" } }%% + quadrantChart + title {{ metric }} + x-axis {{ d["variants"][metric][0] }} + y-axis {{ d["variants"][metric][1] }} + quadrant-1 1 + quadrant-2 2 + quadrant-3 3 + quadrant-4 4 + {% for bench_name, bench_variants in benchmarks.items () %} + {{ bench_name }}: [{{ bench_variants[d["variants"][metric][0]]["scaled"] }}, {{ bench_variants[d["variants"][metric][1]]["scaled"] }}] + {% endfor %} + ``` + {% endif %} + | Benchmark | {% for variant in d["variants"][metric] %} {{ variant }} |{% endfor %} | --- |{% for variant in d["variants"][metric] %} --- |{% endfor -%} {% for bench_name, bench_variants in benchmarks.items () %} | {{ bench_name }} {% for variant in d["variants"][metric] -%} - | {{ bench_variants[variant] }} {% endfor %}| + | {{ bench_variants[variant]["absolute"] }} {% endfor %}| {%- endfor %} {% endfor -%} """) @@ -228,7 +278,36 @@ def _get_variant_names(results): @staticmethod - def _organize_results_into_metrics(results): + def _add_scaled_metrics(data_for_metric, log_scaling): + min_value = None + max_value = None + for bench, bench_result in data_for_metric.items(): + for variant, variant_result in bench_result.items(): + if min_value is None or variant_result["absolute"] < min_value: + min_value = variant_result["absolute"] + if max_value is None or variant_result["absolute"] > max_value: + max_value = variant_result["absolute"] + if min_value is None or min_value == max_value: + for bench, bench_result in data_for_metric.items(): + for variant, variant_result in bench_result.items(): + variant_result["scaled"] = 1.0 + else: + if log_scaling: + min_value = math.log(min_value, 10) + max_value = math.log(max_value, 10) + value_range = max_value - min_value + for bench, bench_result in data_for_metric.items(): + for variant, variant_result in bench_result.items(): + if log_scaling: + abs_value = math.log(variant_result["absolute"], 10) + else: + abs_value = variant_result["absolute"] + else: + variant_result["scaled"] = (abs_value - min_value) / value_range + + + @staticmethod + def _organize_results_into_metrics(results, log_scaling): ret = {metric: {} for metric in results["metrics"]} for bench, bench_result in results["benchmarks"].items(): for variant, variant_result in bench_result["variants"].items(): @@ -241,11 +320,15 @@ def _organize_results_into_metrics(results): "the 'metrics' dict. Add '%s: {}' to the metrics " "dict", bench, metric, variant, metric) try: - ret[metric][bench][variant] = variant_result["metrics"][metric] + ret[metric][bench][variant]["absolute"] = variant_result["metrics"][metric] except KeyError: ret[metric][bench] = { - variant: variant_result["metrics"][metric] + variant: { + "absolute": variant_result["metrics"][metric] + } } + for metric, bench_result in ret.items(): + self._add_scaled_metrics(bench_result, log_scaling) return ret @@ -272,7 +355,8 @@ def _get_variants(metrics): def __call__(self, results): - metrics = self._organize_results_into_metrics(results) + metrics = self._organize_results_into_metrics( + results, self.scatterplot == Plot.LOG) self._add_extra_columns(metrics) data = { @@ -285,6 +369,7 @@ def __call__(self, results): enabled_extensions=("html"), default_for_string=True)) template = env.from_string(self._get_template()) - output = template.render(d=data)[:-1] + include_scatterplot = self.scatterplot != Plot.OFF + output = template.render(d=data, scatterplot=include_scatterplot)[:-1] with self.get_out_file() as handle: print(output, file=handle) diff --git a/tools/benchcomp/configs/perf-regression.yaml b/tools/benchcomp/configs/perf-regression.yaml index a0d88e0558db..c938b3dd861f 100644 --- a/tools/benchcomp/configs/perf-regression.yaml +++ b/tools/benchcomp/configs/perf-regression.yaml @@ -33,6 +33,7 @@ visualize: - type: dump_markdown_results_table out_file: '-' + scatterplot: linear extra_columns: # For these two metrics, display the difference between old and new and