Skip to content

Commit

Permalink
fix(ipv6-reboot): ignore Prometheus failures in node init
Browse files Browse the repository at this point in the history
since this issuse was declared P4, we'll gonna ignore it
for now, since after one or two retries the node does sping
up

Ref: scylladb/scylladb#20387
  • Loading branch information
fruch committed Dec 11, 2024
1 parent d199206 commit bb03234
Show file tree
Hide file tree
Showing 2 changed files with 20 additions and 0 deletions.
4 changes: 4 additions & 0 deletions sdcm/nemesis.py
Original file line number Diff line number Diff line change
Expand Up @@ -91,6 +91,7 @@
ignore_raft_transport_failing,
decorate_with_context_if_issues_open,
ignore_take_snapshot_failing,
ignore_ipv6_failure_to_assign,
)
from sdcm.sct_events.health import DataValidatorEvent
from sdcm.sct_events.loaders import CassandraStressLogEvent, ScyllaBenchEvent
Expand Down Expand Up @@ -3937,6 +3938,9 @@ def _run_commands_wait_and_cleanup( # pylint: disable=too-many-arguments
@decorate_with_context([
ignore_ycsb_connection_refused,
])
@decorate_with_context_if_issues_open(
ignore_ipv6_failure_to_assign,
issue_refs=['https://github.com/scylladb/scylladb/issues/20387'])
def reboot_node(self, target_node, hard=True, verify_ssh=True):
target_node.reboot(hard=hard, verify_ssh=verify_ssh)
if self.tester.params.get('print_kernel_callstack'):
Expand Down
16 changes: 16 additions & 0 deletions sdcm/sct_events/group_common_events.py
Original file line number Diff line number Diff line change
Expand Up @@ -474,6 +474,22 @@ def ignore_take_snapshot_failing():
yield


@contextmanager
def ignore_ipv6_failure_to_assign():
with ExitStack() as stack:
stack.enter_context(EventsSeverityChangerFilter(
new_severity=Severity.WARNING,
event_class=DatabaseLogEvent,
regex=r".*init - Startup failed:.*Cannot assign requested address",
extra_time_to_expiration=60))
stack.enter_context(EventsSeverityChangerFilter(
new_severity=Severity.WARNING,
event_class=DatabaseLogEvent,
regex=r".*init - Could not start Prometheus API server.*Cannot assign requested address",
extra_time_to_expiration=60))
yield


def decorate_with_context(context_list: list[Callable | ContextManager] | Callable | ContextManager):
"""
helper to decorate a function to run with a list of callables that return context managers
Expand Down

0 comments on commit bb03234

Please sign in to comment.