Add support for missed tests in the event handler and dashboard (#740)

Missed tests will be indicated with a capital `X` on the dashboard.

dashboard/main_heatmap.py

@@ -172,6 +172,10 @@ def _get_single_char_test_status(detailed_status):
     elif detailed_status.startswith('failure') and 'metrics' in \
         detailed_status:
       return 'M'
+    elif detailed_status.startswith('missed'):
+      # `X` is used to denote missed executions because `M` is already in use by
+      # metrics failures.
+      return 'X'
     else:
       return detailed_status[:1].upper()
   job_status_dataframe['job_status_abbrev'] = job_status_dataframe[

dashboard/main_heatmap_test.py

@@ -13,6 +13,8 @@
 
 SAMPLE_LOGS_LINK = 'https://console.cloud.google.com/logs?project=xl-ml-test&advancedFilter=resource.type%3Dk8s_container%0Aresource.labels.project_id%3Dxl-ml-test%0Aresource.labels.location=us-central1-b%0Aresource.labels.cluster_name=xl-ml-test%0Aresource.labels.namespace_name=automated%0Aresource.labels.pod_name:pt-1.5-cpp-ops-func-v2-8-1587398400&dateRangeUnbound=backwardInTime'
 
+SAMPLE_WORKLOAD_LINK = 'https://console.cloud.google.com/kubernetes/job/europe-west4/xl-ml-test-europe-west4/automated/jax-pod-latest-tpu-vm-base-func-v2-32-1vm-27799680/logs?project=xl-ml-test'
+
 def _get_values_for_failures(values, statuses):
   return [zipped[0] for zipped in zip(
       values, statuses) if zipped[1] == 'failure']
@@ -41,6 +43,11 @@ class MainHeatmapTest(parameterized.TestCase):
         'metric_statuses': ['success', 'success', 'failure'],
         'expected_overall_statuses': ['success', 'failure', 'failure'],
         'expected_job_status_abbrevs': ['', 'F', 'M']}),
+    ('some_missed_some_failed_some_oob', {
+        'job_statuses': ['missed', 'failure', 'success'],
+        'metric_statuses': ['success', 'success', 'failure'],
+        'expected_overall_statuses': ['missed', 'failure', 'failure'],
+        'expected_job_status_abbrevs': ['X', 'F', 'M']}),
   )
   def test_process_dataframes(self, args_dict):
     job_statuses = args_dict['job_statuses']
@@ -53,6 +60,7 @@ def test_process_dataframes(self, args_dict):
             len(job_statuses))]),
         'job_status': pd.Series(job_statuses),
         'logs_link': pd.Series([SAMPLE_LOGS_LINK for _ in job_statuses]),
+        'workload_link': pd.Series([SAMPLE_WORKLOAD_LINK for _ in job_statuses]),
         'logs_download_command': pd.Series(
             ['my command'] + ['' for _ in job_statuses[1:]]),
     })

metrics/handler/bigquery_client.py

@@ -31,6 +31,7 @@
   metrics_pb2.TestCompletedEvent.COMPLETED: 'success',
   metrics_pb2.TestCompletedEvent.FAILED: 'failure',
   metrics_pb2.TestCompletedEvent.TIMEOUT: 'timeout',
+  metrics_pb2.TestCompletedEvent.MISSED: 'missed',
 }
 
 @dataclasses.dataclass

metrics/handler/collectors/BUILD

@@ -22,6 +22,7 @@ py_library(
     requirement("absl-py"),
     requirement("numpy"),
     requirement("protobuf"),
+    requirement("pytz"),
   ],
   visibility = ["//visibility:public"],
 )

metrics/handler/requirements.txt

@@ -8,3 +8,4 @@ protobuf==3.15.0
 sendgrid==6.4.8
 tensorboard~=2.9.0
 tensorflow-cpu~=2.9.1
+pytz~=2022.6

metrics/metrics.proto

@@ -24,6 +24,7 @@ message TestCompletedEvent {
     COMPLETED = 0;
     FAILED = 1;
     TIMEOUT = 2;
+    MISSED = 3;
   }
 
   string benchmark_id = 1;