Fail measurements if any of the jobs returned issues

lnst/RecipeCommon/Perf/Measurements/BaseCPUMeasurement.py

@@ -1,4 +1,4 @@
-from lnst.Controller.RecipeResults import MeasurementResult
+from lnst.Controller.RecipeResults import MeasurementResult, ResultType
 from lnst.RecipeCommon.Perf.Measurements.MeasurementError import MeasurementError
 from lnst.RecipeCommon.Perf.Measurements.BaseMeasurement import BaseMeasurement
 from lnst.RecipeCommon.Perf.Measurements.Results import AggregatedCPUMeasurementResults
@@ -38,7 +38,18 @@ def _report_host_results(cls, recipe, results):
 
         desc = [result.describe() for result in results]
 
-        recipe.add_custom_result(MeasurementResult("cpu", description="\n".join(desc), data=cpu_data))
+        recipe.add_custom_result(
+            MeasurementResult(
+                "cpu",
+                result=(
+                    ResultType.PASS
+                    if all(res.measurement_success for res in results)
+                    else ResultType.FAIL
+                ),
+                description="\n".join(desc),
+                data=cpu_data,
+            )
+        )
 
     def _aggregate_hostcpu_results(self, old, new):
         if (old is not None and

lnst/RecipeCommon/Perf/Measurements/BaseFlowMeasurement.py

@@ -97,18 +97,19 @@ def _report_flow_results(cls, recipe, flow_results):
         desc = []
         desc.append(flow_results.describe())
 
-        recipe_result = ResultType.PASS
         metrics = {"Generator": generator, "Generator process": generator_cpu,
                    "Receiver": receiver, "Receiver process": receiver_cpu}
         for name, result in metrics.items():
             if cls._invalid_flow_duration(result):
-                recipe_result = ResultType.FAIL
                 desc.append("{} has invalid duration!".format(name))
 
         # TODO add flow description
         recipe_result = MeasurementResult(
             "flow",
             description="\n".join(desc),
+            result=(
+                ResultType.PASS if flow_results.measurement_success else ResultType.FAIL
+            ),
             data={
                 "generator_flow_data": generator,
                 "generator_cpu_data": generator_cpu,
@@ -169,6 +170,7 @@ def aggregate_multi_flow_results(results):
         for i in range(nr_iterations):
             parallel_result = FlowMeasurementResults(
                     measurement=sample_result.measurement,
+                    measurement_success=all(result.measurement_success for result in results),
                     flow=dummy_flow,
                     warmup_duration=dummy_flow.warmup_duration
             )
@@ -193,6 +195,7 @@ def collect_simulated_results(self):
         for test_flow in self.flows:
             flow_results = FlowMeasurementResults(
                 measurement=self,
+                measurement_success=True,
                 flow=test_flow,
                 warmup_duration=test_flow.warmup_duration,
             )

lnst/RecipeCommon/Perf/Measurements/IperfFlowMeasurement.py

@@ -118,9 +118,12 @@ def collect_results(self):
         results = []
         for test_flow in test_flows:
             flow_results = FlowMeasurementResults(
-                    measurement=self,
-                    flow=test_flow.flow,
-                    warmup_duration=test_flow.flow.warmup_duration
+                measurement=self,
+                measurement_success=(
+                    test_flow.client_job.passed and test_flow.server_job.passed
+                ),
+                flow=test_flow.flow,
+                warmup_duration=test_flow.flow.warmup_duration,
             )
             flow_results.generator_results = self._parse_job_streams(
                     test_flow.client_job)

lnst/RecipeCommon/Perf/Measurements/LinuxPerfMeasurement.py

@@ -117,11 +117,12 @@ def collect_results(self) -> list[BaseMeasurementResults]:
 
             results.append(
                 LinuxPerfMeasurementResults(
-                    self,
-                    host,
-                    dst_filepath,
-                    self._start_timestamp,
-                    self._end_timestamp,
+                    measurement=self,
+                    measurement_success=job.passed,
+                    host=host,
+                    filename=dst_filepath,
+                    start_timestamp=self._start_timestamp,
+                    end_timestamp=self._end_timestamp,
                 )
             )
         return results

lnst/RecipeCommon/Perf/Measurements/NeperFlowMeasurement.py

@@ -158,6 +158,9 @@ def collect_results(self) -> List[NeperFlowMeasurementResults]:
         for test_flow in test_flows:
             flow_results = NeperFlowMeasurementResults(
                     measurement=self,
+                    measurement_success=(
+                        test_flow.client_job.passed and test_flow.server_job.passed
+                    ),
                     flow=test_flow.flow,
                     warmup_duration=test_flow.flow.warmup_duration
             )

lnst/RecipeCommon/Perf/Measurements/RDMABandwidthMeasurement.py

@@ -4,7 +4,7 @@
 
 from lnst.Controller.Job import Job
 from lnst.Controller.Recipe import BaseRecipe
-from lnst.Controller.RecipeResults import MeasurementResult
+from lnst.Controller.RecipeResults import MeasurementResult, ResultType
 from lnst.RecipeCommon.Perf.Measurements.BaseFlowMeasurement import Flow, NetworkFlowTest
 from lnst.RecipeCommon.Perf.Results import PerfInterval
 from lnst.Tests.RDMABandwidth import RDMABandwidthServer, RDMABandwidthClient
@@ -75,6 +75,9 @@ def collect_results(self) -> list[RDMABandwidthMeasurementResults]:
             duration = endpoint_test.flow.duration
             result = RDMABandwidthMeasurementResults(
                 measurement=self,
+                measurement_success=(
+                    endpoint_test.client_job.passed and endpoint_test.server_job.passed
+                ),
                 flow=endpoint_test.flow,
             )
             result.bandwidth = PerfInterval(
@@ -156,6 +159,11 @@ def report_results(
         for aggregated_result in aggregated_results:
             measurement_result = MeasurementResult(
                 "rdma-bandwidth",
+                result=(
+                    ResultType.PASS
+                    if aggregated_result.measurement_success
+                    else ResultType.FAIL
+                ),
                 description=aggregated_result.describe(),
                 data={"bandwidth": aggregated_result.bandwidth}
             )

lnst/RecipeCommon/Perf/Measurements/Results/AggregatedCPUMeasurementResults.py

@@ -5,9 +5,16 @@
 
 class AggregatedCPUMeasurementResults(CPUMeasurementResults):
     def __init__(self, measurement, host, cpu):
-        super(AggregatedCPUMeasurementResults, self).__init__(measurement, host, cpu)
+        super(AggregatedCPUMeasurementResults, self).__init__(measurement, True, host, cpu)
         self._individual_results = []
 
+    @property
+    def measurement_success(self) -> bool:
+        if self.individual_results:
+            return all(res.measurement_success for res in self.individual_results)
+        else:
+            return False
+
     @property
     def individual_results(self):
         return self._individual_results

lnst/RecipeCommon/Perf/Measurements/Results/AggregatedFlowMeasurementResults.py

@@ -5,14 +5,21 @@
 
 class AggregatedFlowMeasurementResults(FlowMeasurementResults):
     def __init__(self, measurement, flow):
-        super(FlowMeasurementResults, self).__init__(measurement)
+        super(FlowMeasurementResults, self).__init__(measurement, True)
         self._flow = flow
         self._generator_results = SequentialPerfResult()
         self._generator_cpu_stats = SequentialPerfResult()
         self._receiver_results = SequentialPerfResult()
         self._receiver_cpu_stats = SequentialPerfResult()
         self._individual_results = []
 
+    @property
+    def measurement_success(self) -> bool:
+        if self.individual_results:
+            return all(res.measurement_success for res in self.individual_results)
+        else:
+            return False
+
     @property
     def individual_results(self):
         return self._individual_results

lnst/RecipeCommon/Perf/Measurements/Results/AggregatedLinuxPerfMeasurementResults.py

@@ -11,6 +11,13 @@ def __init__(self, result: Optional[LinuxPerfMeasurementResults] = None):
         if result:
             self._individual_results = [result]
 
+    @property
+    def measurement_success(self) -> bool:
+        if self.individual_results:
+            return all(res.measurement_success for res in self.individual_results)
+        else:
+            return False
+
     @property
     def individual_results(self) -> list[LinuxPerfMeasurementResults]:
         return self._individual_results

lnst/RecipeCommon/Perf/Measurements/Results/AggregatedRDMABandwidthMeasurementResults.py

@@ -5,9 +5,16 @@
 
 class AggregatedRDMABandwidthMeasurementResults(RDMABandwidthMeasurementResults):
     def __init__(self, measurement: BaseMeasurement, flow: "Flow"):
-        super().__init__(measurement, flow)
+        super().__init__(measurement, True, flow)
         self._individual_results = []
 
+    @property
+    def measurement_success(self) -> bool:
+        if self.individual_results:
+            return all(res.measurement_success for res in self.individual_results)
+        else:
+            return False
+
     @property
     def individual_results(self) -> list[RDMABandwidthMeasurementResults]:
         return self._individual_results

lnst/RecipeCommon/Perf/Measurements/Results/AggregatedTcRunMeasurementResults.py

@@ -15,10 +15,17 @@ def __init__(
             device: Device,
             warmup_rules=0,
     ):
-        super().__init__(measurement, device, warmup_rules=warmup_rules)
+        super().__init__(measurement, True, device, warmup_rules=warmup_rules)
         self._individual_results: list[TcRunMeasurementResults] = []
         self._rule_install_rate: SequentialPerfResult = SequentialPerfResult()
 
+    @property
+    def measurement_success(self) -> bool:
+        if self.individual_results:
+            return all(res.measurement_success for res in self.individual_results)
+        else:
+            return False
+
     @property
     def rule_install_rate(self) -> SequentialPerfResult:
         return self._rule_install_rate
@@ -27,10 +34,6 @@ def rule_install_rate(self) -> SequentialPerfResult:
     def rule_install_rate(self, result: SequentialPerfResult):
         self._rule_install_rate = result
 
-    @property
-    def run_success(self) -> bool:
-        return all((i.run_success for i in self.individual_results))
-
     @property
     def individual_results(self) -> list[TcRunMeasurementResults]:
         return self._individual_results

lnst/RecipeCommon/Perf/Measurements/Results/AggregatedXDPBenchMeasurementResults.py

@@ -6,18 +6,32 @@
 
 
 class AggregatedXDPBenchMeasurementResults(XDPBenchMeasurementResults):
-    def __init__(self, *args, **kwargs):
-        super().__init__(*args, **kwargs)
+    def __init__(self, measurement, flow):
+        super().__init__(measurement, True, flow)
+        self._individual_results: list[TcRunMeasurementResults] = []
         self._generator_results = SequentialPerfResult()
         self._receiver_results = SequentialPerfResult()
 
+    @property
+    def individual_results(self) -> list[XDPBenchMeasurementResults]:
+        return self._individual_results
+
+    @property
+    def measurement_success(self) -> bool:
+        if self.individual_results:
+            return all(res.measurement_success for res in self.individual_results)
+        else:
+            return False
+
     def add_results(self, results):
         if results is None:
             return
         elif isinstance(results, AggregatedXDPBenchMeasurementResults):
+            self._individual_results.extend(results.individual_results)
             self.generator_results.extend(results.generator_results)
             self.receiver_results.extend(results.receiver_results)
         elif isinstance(results, XDPBenchMeasurementResults):
+            self._individual_results.append(results)
             self.generator_results.append(results.generator_results)
             self.receiver_results.append(results.receiver_results)
         else:

lnst/RecipeCommon/Perf/Measurements/Results/BaseMeasurementResults.py

@@ -2,14 +2,19 @@
 
 
 class BaseMeasurementResults(object):
-    def __init__(self, measurement: BaseMeasurement, warmup=0):
+    def __init__(self, measurement: BaseMeasurement, measurement_success=True, warmup=0):
         self._measurement = measurement
+        self._measurement_success = measurement_success
         self._warmup_duration = warmup
 
     @property
     def measurement(self) -> BaseMeasurement:
         return self._measurement
 
+    @property
+    def measurement_success(self) -> bool:
+        return self._measurement_success
+
     @property
     def warmup_duration(self):
         return self._warmup_duration

lnst/RecipeCommon/Perf/Measurements/Results/CPUMeasurementResults.py

@@ -4,8 +4,8 @@
 
 
 class CPUMeasurementResults(BaseMeasurementResults):
-    def __init__(self, measurement, host, cpu):
-        super(CPUMeasurementResults, self).__init__(measurement)
+    def __init__(self, measurement, measurement_success, host, cpu):
+        super(CPUMeasurementResults, self).__init__(measurement, measurement_success)
         self._host = host
         self._cpu = cpu
         self._utilization = None

lnst/RecipeCommon/Perf/Measurements/Results/FlowMeasurementResults.py

@@ -5,9 +5,9 @@
 
 
 class FlowMeasurementResults(BaseMeasurementResults):
-    def __init__(self, measurement, flow, warmup_duration=0):
+    def __init__(self, measurement, measurement_success, flow, warmup_duration=0):
         super(FlowMeasurementResults, self).__init__(
-            measurement, warmup_duration
+            measurement, measurement_success, warmup_duration
         )
         self._flow = flow
         self._generator_results = None
@@ -92,7 +92,7 @@ def warmdown_start(self):
 
     def time_slice(self, start, end):
         result_copy = FlowMeasurementResults(
-            self.measurement, self.flow, warmup_duration=0
+            self.measurement, self.measurement_success, self.flow, warmup_duration=0
         )
 
         result_copy.generator_cpu_stats = self.generator_cpu_stats.time_slice(

lnst/RecipeCommon/Perf/Measurements/Results/LinuxPerfMeasurementResults.py

@@ -12,12 +12,13 @@ class LinuxPerfMeasurementResults(BaseMeasurementResults):
     def __init__(
         self,
         measurement,
+        measurement_success: bool,
         host: Host,
         filename: str,
         start_timestamp: float,
         end_timestamp: float,
     ):
-        super().__init__(measurement)
+        super().__init__(measurement, measurement_success)
         self._host = host
         self._filename = filename
         self._start_timestamp = start_timestamp

lnst/RecipeCommon/Perf/Measurements/Results/RDMABandwidthMeasurementResults.py

@@ -4,8 +4,8 @@
 
 
 class RDMABandwidthMeasurementResults(BaseMeasurementResults):
-    def __init__(self, measurement: BaseMeasurement, flow: "Flow"):
-        super().__init__(measurement)
+    def __init__(self, measurement: BaseMeasurement, measurement_success: bool, flow: "Flow"):
+        super().__init__(measurement, measurement_success)
 
         self._flow = flow
 
@@ -34,7 +34,7 @@ def end_timestamp(self):
         return self._bandwidth.end_timestamp
 
     def time_slice(self, start, end):
-        result_copy = RDMABandwidthMeasurementResults(self.measurement, self.flow)
+        result_copy = RDMABandwidthMeasurementResults(self.measurement, self.measurement_success, self.flow)
         result_copy.bandwidth = self.bandwidth.time_slice(start, end)
         return result_copy
 

lnst/RecipeCommon/Perf/Measurements/Results/StatCPUMeasurementResults.py

@@ -4,8 +4,8 @@
 
 
 class StatCPUMeasurementResults(CPUMeasurementResults):
-    def __init__(self, *args):
-        super(StatCPUMeasurementResults, self).__init__(*args)
+    def __init__(self, *args, **kwargs):
+        super(StatCPUMeasurementResults, self).__init__(*args, **kwargs)
         self._data = {}
 
     def update_intervals(self, intervals):
@@ -31,6 +31,7 @@ def end_timestamp(self):
     def time_slice(self, start, end):
         result_copy = StatCPUMeasurementResults(
                 self.measurement,
+                self.measurement_success,
                 self.host,
                 self.cpu
                 )