Re-land "aggregate.py: add tab format (#6405)", reverted in #6424 (#6431

)

This reverts commit 010b6f0.

This should pass CI since CI now has the tabulate package.

.circleci/common.sh

@@ -70,6 +70,7 @@ function install_deps_pytorch_xla() {
   pip install cloud-tpu-client
   pip install absl-py
   pip install pandas
+  pip install tabulate
   pip install --upgrade "numpy>=1.18.5"
   pip install --upgrade numba
 

.circleci/docker/install_conda.sh

@@ -43,6 +43,7 @@ function install_and_setup_conda() {
   /usr/bin/yes | pip install expecttest==0.1.3
   /usr/bin/yes | pip install absl-py
   /usr/bin/yes | pip install pandas
+  /usr/bin/yes | pip install tabulate
   # Additional PyTorch requirements
   /usr/bin/yes | pip install scikit-image scipy==1.6.3
   /usr/bin/yes | pip install boto3==1.16.34

.kokoro/common.sh

@@ -60,6 +60,7 @@ function install_deps_pytorch_xla() {
   pip install cloud-tpu-client
   pip install absl-py
   pip install pandas
+  pip install tabulate
   pip install --upgrade "numpy>=1.18.5"
   pip install --upgrade numba
 

benchmarks/aggregate.py

@@ -11,6 +11,7 @@
 import sys
 import tiers
 import itertools
+from tabulate import tabulate
 from typing import Any, Dict, List, NamedTuple
 import numpy as np
 from scipy.stats.mstats import gmean
@@ -277,9 +278,45 @@ def get_pr_titles(args):
   return [titles, data_labels]
 
 
+def speedup_header(title: str, backend_name: str, args):
+  if args.format == 'tab':
+    return f'Speedup\n{title}\nover\n{args.baseline.capitalize()}\n{backend_name}'
+  return f'Speedup({title}/{args.baseline.capitalize()} {backend_name})'
+
+
+def modelname_header(model: str, args):
+  if args.format == 'tab':
+    return f'ModelName\n{model}'
+  return f'ModelName({model})'
+
+
+def percentile_header(title: str, p: str, args):
+  if args.format == 'tab':
+    return f'{title}\n{p}'
+  return f'{title} {p}'
+
+
+def pr_text(headers, rows, args):
+  if args.format == 'csv':
+    if headers:
+      headers[0] = f'# {headers[0]}'
+    print(','.join(headers))
+    for row in rows:
+      print(','.join([str(f) if f is not None else '' for f in row]))
+  elif args.format == 'tab':
+    print(
+        tabulate(rows, headers=headers, tablefmt='fancy_grid', floatfmt='.2f'))
+
+
 def pr_latest(results_map: Dict[str, Any], args, timestamps: List[str]):
   titles, data_labels = get_pr_titles(args)
+  # speedups[backend] is the list of speedups vs. the baseline for that backend.
+  # Speedups are sorted in ascending order so that speedups[backend] is
+  # monotonically increasing. That is, due to the sorting, it is unlikely that
+  # speedups["foo"][i] and speedups["bar"][i] will correspond to the same model.
   speedups = [[] for _ in titles]
+  # model_names[backend][i] contains the model name corresponding to
+  # speedups[backend][i].
   model_names = [[] for _ in titles]
   base_backend_name = _title_map[args.backends[0]]
 
@@ -297,30 +334,37 @@ def pr_latest(results_map: Dict[str, Any], args, timestamps: List[str]):
     logger.warning(f'cannot find data for accelerator {args.accelerator}')
     return
 
-  if args.format == 'csv':
-    print(','.join(['# Workload'] + [
-        f'Speedup({title}/{args.baseline.capitalize()} {base_backend_name}),StdDev,ModelName({title})'
-        for title in titles
-    ]))
-    # Note: the latest timestamp might not have results for all benchmarks.
-    max_len = max([len(l) for l in speedups])
+  if args.format == 'csv' or args.format == 'tab':
+    headers = ['Workload'] + [
+        header for title in titles for header in [
+            speedup_header(title, base_backend_name, args), 'StdDev',
+            modelname_header(title, args)
+        ]
+    ]
+
+    # Not all models run in all backends, so it is likely that
+    # len(speedups["foo"]) != len(speedups["bar"]). We therefore pad the speedup
+    # lists with "None" elements so that we have a "full table", i.e. all lists
+    # have the same length. This makes it trivial to generate correct CSV or
+    # tabular output.
+    num_rows = max([len(l) for l in speedups])
 
     def pad_array(arr, desired_len, val):
       if len(arr) >= desired_len:
         return
       arr += [val] * (desired_len - len(arr))
 
     for i in range(len(titles)):
-      pad_array(speedups[i], max_len, Datapoint('', ''))
-      pad_array(model_names[i], max_len, '')
-
-    for j in range(max_len):
-      print(','.join(
-          map(str, [j] + [
-              v for i in range(len(titles))
-              for v in (speedups[i][j].avg, speedups[i][j].std,
-                        model_names[i][j])
-          ])))
+      pad_array(speedups[i], num_rows, Datapoint(None, None))
+      pad_array(model_names[i], num_rows, None)
+
+    rows = []
+    for j in range(num_rows):
+      rows += [[j] + [
+          v for i in range(len(titles))
+          for v in (speedups[i][j].avg, speedups[i][j].std, model_names[i][j])
+      ]]
+    pr_text(headers, rows, args)
   else:
     plt.figure(figsize=(args.fig_width, args.fig_height))
     plt.axhline(y=1.0, color='lightgray')
@@ -363,24 +407,24 @@ def pr_histogram(results_map: Dict[str, Any], args, timestamps: List[str]):
   titles, data_labels = get_pr_titles(args)
   percentiles = [f'p{p}' for p in (95, 50, 5)]
   labels = [f'{pfx}:speedups:{p}' for pfx in data_labels for p in percentiles]
-  full_titles = [f'{title} {p}' for title in titles for p in percentiles]
+  full_titles = [
+      percentile_header(title, p, args) for title in titles for p in percentiles
+  ]
   base_backend_name = _title_map[args.backends[0]]
   x = []
   y = [[] for i in range(len(labels))]
   for timestamp in timestamps:
     if labels[0] in results_map[timestamp]:
-      for label in labels:
-        assert label in results_map[timestamp]
       x.append(datetime.utcfromtimestamp(float(timestamp)))
       for i, label in enumerate(labels):
-        y[i].append(
-            pr_round(results_map[timestamp][label] if label in
-                     results_map[timestamp] else Datapoint('', '')).avg)
-  if args.format == 'csv':
-    full_titles = ['# Datetime(UTC)'] + full_titles
-    print(','.join(full_titles))
+        y[i].append((pr_round(results_map[timestamp][label]) if label
+                     in results_map[timestamp] else Datapoint(None, None)).avg)
+  if args.format == 'csv' or args.format == 'tab':
+    headers = ['Datetime(UTC)'] + full_titles
+    rows = []
     for j, utc in enumerate(x):
-      print(','.join([str(utc)] + [str(y[i][j]) for i in range(len(labels))]))
+      rows += [[utc] + [y[i][j] for i in range(len(labels))]]
+    pr_text(headers, rows, args)
   else:
     fig, ax = plt.subplots(figsize=(args.fig_width, args.fig_height))
     ax.axhline(y=1.0, color='lightgray')
@@ -420,17 +464,19 @@ def pr_gmean(results_map: Dict[str, Any], args, timestamps: List[str]):
     for i, label in enumerate(labels):
       y[i].append(
           pr_round(results_map[timestamp][label]) if label in
-          results_map[timestamp] else Datapoint('', ''))
-  if args.format == 'csv':
-    print(','.join(['# Datetime(UTC)'] + [
-        f"Speedup({title}/{args.baseline.capitalize()} {base_backend_name}),StdDev"
-        for title in titles
-    ]))
+          results_map[timestamp] else Datapoint(None, None))
+  if args.format == 'csv' or args.format == 'tab':
+    headers = ['Datetime(UTC)'] + [
+        header for title in titles for header in
+        [speedup_header(title, base_backend_name, args), 'StdDev']
+    ]
+    rows = []
     for j, x in enumerate(x):
-      print(','.join(
-          map(str, [x] + [
-              v for i in range(len(labels)) for v in (y[i][j].avg, y[i][j].std)
-          ])))
+      rows += [
+          [x] +
+          [v for i in range(len(labels)) for v in (y[i][j].avg, y[i][j].std)]
+      ]
+    pr_text(headers, rows, args)
   else:
     fig, ax = plt.subplots(figsize=(args.fig_width, args.fig_height))
     ax.axhline(y=1.0, color='lightgray')
@@ -458,7 +504,7 @@ def pr_results(results_map: Dict[str, Any], args):
   timestamps = list(results_map.keys())
   timestamps.sort()
 
-  if args.format != 'csv' and not has_matplotlib:
+  if not has_matplotlib and (args.format == 'png' or args.format == 'svg'):
     sys.exit(f'Fatal: cannot find matplotlib, needed for {args.format} output.')
 
   if args.report == 'latest':
@@ -532,7 +578,7 @@ def parse_args(args=None):
   parser.add_argument(
       "--format",
       default='csv',
-      choices=['csv', 'png', 'svg'],
+      choices=['csv', 'png', 'svg', 'tab'],
       help='Output format')
   parser.add_argument('input_file', nargs='+')
   parser.add_argument(

test/benchmarks/v100.inference.histogram.lazytensor.test

@@ -0,0 +1,4 @@
+# ARGS: --backends inductor openxla+lazytensor --
+# Datetime(UTC),Inductor p95,Inductor p50,Inductor p5,XLA+LazyTensor p95,XLA+LazyTensor p50,XLA+LazyTensor p5
+2023-11-11 05:32:18.723407,1.0,1.0,1.0,,,
+2023-11-12 05:32:18,1.50833479,1.40761418,1.30689358,0.41071322,0.41071322,0.41071322

test/benchmarks/v100.inference.histogram.lazytensor_tab.test

@@ -0,0 +1,9 @@
+# ARGS: --backends inductor openxla+lazytensor --format=tab
+╒════════════════════════════╤════════════╤════════════╤════════════╤══════════════════╤══════════════════╤══════════════════╕
+│ Datetime(UTC)              │   Inductor │   Inductor │   Inductor │   XLA+LazyTensor │   XLA+LazyTensor │   XLA+LazyTensor │
+│                            │        p95 │        p50 │         p5 │              p95 │              p50 │               p5 │
+╞════════════════════════════╪════════════╪════════════╪════════════╪══════════════════╪══════════════════╪══════════════════╡
+│ 2023-11-11 05:32:18.723407 │       1.00 │       1.00 │       1.00 │                  │                  │                  │
+├────────────────────────────┼────────────┼────────────┼────────────┼──────────────────┼──────────────────┼──────────────────┤
+│ 2023-11-12 05:32:18        │       1.51 │       1.41 │       1.31 │             0.41 │             0.41 │             0.41 │
+╘════════════════════════════╧════════════╧════════════╧════════════╧══════════════════╧══════════════════╧══════════════════╛

test/benchmarks/v100.inference.histogram.tab.test

@@ -0,0 +1,9 @@
+# ARGS: --format=tab
+╒════════════════════════════╤════════════╤════════════╤════════════╤══════════════╤══════════════╤══════════════╤═══════════════════╤═══════════════════╤═══════════════════╕
+│ Datetime(UTC)              │   Inductor │   Inductor │   Inductor │   XLA+Dynamo │   XLA+Dynamo │   XLA+Dynamo │   XLA_Eval+Dynamo │   XLA_Eval+Dynamo │   XLA_Eval+Dynamo │
+│                            │        p95 │        p50 │         p5 │          p95 │          p50 │           p5 │               p95 │               p50 │                p5 │
+╞════════════════════════════╪════════════╪════════════╪════════════╪══════════════╪══════════════╪══════════════╪═══════════════════╪═══════════════════╪═══════════════════╡
+│ 2023-11-11 05:32:18.723407 │       1.00 │       1.00 │       1.00 │         0.98 │         0.86 │         0.74 │              0.94 │              0.79 │              0.65 │
+├────────────────────────────┼────────────┼────────────┼────────────┼──────────────┼──────────────┼──────────────┼───────────────────┼───────────────────┼───────────────────┤
+│ 2023-11-12 05:32:18        │       1.51 │       1.41 │       1.31 │         1.53 │         1.17 │         0.81 │              1.34 │              1.05 │              0.77 │
+╘════════════════════════════╧════════════╧════════════╧════════════╧══════════════╧══════════════╧══════════════╧═══════════════════╧═══════════════════╧═══════════════════╛

test/benchmarks/v100.inference.latest.tab.test

@@ -0,0 +1,12 @@
+# ARGS: --backends inductor openxla+dynamo openxla_eval+dynamo openxla+lazytensor --format=tab
+╒════════════╤════════════╤══════════╤════════════════════╤══════════════╤══════════╤════════════════════╤═══════════════════╤══════════╤════════════════════╤══════════════════╤══════════╤════════════════════╕
+│   Workload │    Speedup │   StdDev │ ModelName          │      Speedup │   StdDev │ ModelName          │           Speedup │   StdDev │ ModelName          │          Speedup │   StdDev │ ModelName          │
+│            │   Inductor │          │ Inductor           │   XLA+Dynamo │          │ XLA+Dynamo         │   XLA_Eval+Dynamo │          │ XLA_Eval+Dynamo    │   XLA+LazyTensor │          │ XLA+LazyTensor     │
+│            │       over │          │                    │         over │          │                    │              over │          │                    │             over │          │                    │
+│            │     Oldest │          │                    │       Oldest │          │                    │            Oldest │          │                    │           Oldest │          │                    │
+│            │   Inductor │          │                    │     Inductor │          │                    │          Inductor │          │                    │         Inductor │          │                    │
+╞════════════╪════════════╪══════════╪════════════════════╪══════════════╪══════════╪════════════════════╪═══════════════════╪══════════╪════════════════════╪══════════════════╪══════════╪════════════════════╡
+│          0 │       1.30 │     0.00 │ Background_Matting │         0.77 │     0.00 │ Background_Matting │              0.73 │     0.00 │ Background_Matting │             0.41 │     0.00 │ Background_Matting │
+├────────────┼────────────┼──────────┼────────────────────┼──────────────┼──────────┼────────────────────┼───────────────────┼──────────┼────────────────────┼──────────────────┼──────────┼────────────────────┤
+│          1 │       1.52 │     0.00 │ BERT_pytorch       │         1.57 │     0.00 │ BERT_pytorch       │              1.37 │     0.00 │ BERT_pytorch       │                  │          │                    │
+╘════════════╧════════════╧══════════╧════════════════════╧══════════════╧══════════╧════════════════════╧═══════════════════╧══════════╧════════════════════╧══════════════════╧══════════╧════════════════════╛

test/benchmarks/v100.inference.latest.test

@@ -1,3 +1,4 @@
-# Workload,Speedup(Inductor/Oldest Inductor),StdDev,ModelName(Inductor),Speedup(XLA+Dynamo/Oldest Inductor),StdDev,ModelName(XLA+Dynamo),Speedup(XLA_Eval+Dynamo/Oldest Inductor),StdDev,ModelName(XLA_Eval+Dynamo)
-0,1.2957024,0.0,Background_Matting,0.77297688,0.0,Background_Matting,0.7341254,0.0,Background_Matting
-1,1.51952596,6.914e-05,BERT_pytorch,1.56880282,7.138e-05,BERT_pytorch,1.36859903,6.227e-05,BERT_pytorch
+# ARGS: --backends inductor openxla+dynamo openxla_eval+dynamo openxla+lazytensor --
+# Workload,Speedup(Inductor/Oldest Inductor),StdDev,ModelName(Inductor),Speedup(XLA+Dynamo/Oldest Inductor),StdDev,ModelName(XLA+Dynamo),Speedup(XLA_Eval+Dynamo/Oldest Inductor),StdDev,ModelName(XLA_Eval+Dynamo),Speedup(XLA+LazyTensor/Oldest Inductor),StdDev,ModelName(XLA+LazyTensor)
+0,1.2957024,0.0,Background_Matting,0.77297688,0.0,Background_Matting,0.7341254,0.0,Background_Matting,0.41071322,0.0,Background_Matting
+1,1.51952596,6.914e-05,BERT_pytorch,1.56880282,7.138e-05,BERT_pytorch,1.36859903,6.227e-05,BERT_pytorch,,,

test/benchmarks/v100.inference.speedup.lazytensor.test

@@ -0,0 +1,4 @@
+# ARGS: --backends inductor openxla+lazytensor --
+# Datetime(UTC),Speedup(Inductor/Oldest Inductor),StdDev,Speedup(XLA+LazyTensor/Oldest Inductor),StdDev
+2023-11-11 05:32:18.723407,1.0,3.217e-05,,
+2023-11-12 05:32:18,1.40315838,3.192e-05,0.41071322,0.0