Merge pull request #23499 from vbotbuildovich/backport-pr-23400-v24.2…

….x-163

[v24.2.x] tests/failure_injector: undo the failures on exit

tests/rptest/services/failure_injector.py

@@ -71,6 +71,7 @@ def __enter__(self):
     def __exit__(self, type, value, traceback):
         self._heal_all()
         self._continue_all()
+        self._undo_all()
 
     def inject_failure(self, spec):
         if spec in self._in_flight:
@@ -97,15 +98,16 @@ def inject_failure(self, spec):
                 else:
 
                     def cleanup():
-                        if spec in self._in_flight:
-                            self._stop_func(spec.type)(spec.node)
+                        try:
                             self._in_flight.remove(spec)
-                        else:
+                        except KeyError:
                             # The stop timers may outlive the test, handle case
                             # where they run after we already had a heal_all call.
                             self.redpanda.logger.warn(
                                 f"Skipping failure stop action, already cleaned up?"
                             )
+                        else:
+                            self._stop_func(spec.type)(spec.node)
 
                     stop_timer = threading.Timer(function=cleanup,
                                                  args=[],
@@ -159,6 +161,9 @@ def _heal_all(self):
     def _continue_all(self):
         pass
 
+    def _undo_all(self):
+        pass
+
     def _suspend(self, node):
         pass
 
@@ -267,6 +272,16 @@ def _continue_all(self):
             if spec.type != FailureSpec.FAILURE_SUSPEND
         }
 
+    def _undo_all(self):
+        self.redpanda.logger.info(f"running scheduled undos earlier")
+        while self._in_flight:
+            try:
+                spec = self._in_flight.pop()
+            except KeyError:
+                pass  # timer just emptied the set: GIL off???
+            else:
+                self._stop_func(spec.type)(spec.node)
+
     def _suspend(self, node):
         self.redpanda.logger.info(
             f"suspending redpanda on {node.account.hostname}")

tests/rptest/tests/e2e_finjector.py

@@ -32,6 +32,9 @@ def const_delay(delay_seconds=10):
 
 
 class Finjector:
+    # we cannot guarantee start idempotency
+    LOG_ALLOW_LIST = ["failed to lock pidfile. already locked"]
+
     def __init__(self, redpanda, scale):
         self.redpanda = redpanda
         self.enable_manual = False
@@ -132,6 +135,7 @@ def _failure_injector_loop(self):
     def _cleanup(self):
         make_failure_injector(self.redpanda)._heal_all()
         make_failure_injector(self.redpanda)._continue_all()
+        make_failure_injector(self.redpanda)._undo_all()
 
 
 class EndToEndFinjectorTest(EndToEndTest):