From a22d70dec66bac358930059ce127131a466b7fdd Mon Sep 17 00:00:00 2001 From: Edward Hope-Morley Date: Thu, 26 Sep 2024 14:44:54 +0100 Subject: [PATCH] Add check for oslo.messaging bug LP1934937 --- .../openstack/oslo_messaging/lp1934937.yaml | 57 +++++++++++++++++++ .../openstack/oslo_messaging/lp1934937.yaml | 20 +++++++ 2 files changed, 77 insertions(+) create mode 100644 hotsos/defs/scenarios/openstack/oslo_messaging/lp1934937.yaml create mode 100644 hotsos/defs/tests/scenarios/openstack/oslo_messaging/lp1934937.yaml diff --git a/hotsos/defs/scenarios/openstack/oslo_messaging/lp1934937.yaml b/hotsos/defs/scenarios/openstack/oslo_messaging/lp1934937.yaml new file mode 100644 index 000000000..2c26a0f49 --- /dev/null +++ b/hotsos/defs/scenarios/openstack/oslo_messaging/lp1934937.yaml @@ -0,0 +1,57 @@ +vars: + expr: 'greenlet.error: cannot switch to a different thread' + msg_part_common: >- + service(s) on this host are affected by this bug which is triggered when + a non-wsgi service that is using greenthreads and has oslo.messaging + heartbeat_in_pthread=True tries to send a heartbeat message to rabbitmq +checks: + # Include non-wsgi services that use oslo.messaging + nova_has_1934937: + input: var/log/nova/*.log + expr: $expr + neutron_has_1934937: + input: var/log/neutron/*.log + expr: $expr + cinder_has_1934937: + input: var/log/cinder/*.log + expr: $expr + gnocchi_has_1934937: + input: var/log/gnocchi/*.log + expr: $expr +conclusions: + nova_1934937: + decision: nova_has_1934937 + raises: + type: LaunchpadBug + bug-id: 1934937 + message: >- + One or more Nova {msg_part_common}. + format-dict: + msg_part_common: $msg_part_common + neutron_1934937: + decision: neutron_has_1934937 + raises: + type: LaunchpadBug + bug-id: 1934937 + message: >- + One or more Neutron {msg_part_common}. + format-dict: + msg_part_common: $msg_part_common + cinder_1934937: + decision: cinder_has_1934937 + raises: + type: LaunchpadBug + bug-id: 1934937 + message: >- + One or more Cinder {msg_part_common}. + format-dict: + msg_part_common: $msg_part_common + gnocchi_1934937: + decision: gnocchi_has_1934937 + raises: + type: LaunchpadBug + bug-id: 1934937 + message: >- + One or more Gnocchi {msg_part_common}. + format-dict: + msg_part_common: $msg_part_common diff --git a/hotsos/defs/tests/scenarios/openstack/oslo_messaging/lp1934937.yaml b/hotsos/defs/tests/scenarios/openstack/oslo_messaging/lp1934937.yaml new file mode 100644 index 000000000..03543d029 --- /dev/null +++ b/hotsos/defs/tests/scenarios/openstack/oslo_messaging/lp1934937.yaml @@ -0,0 +1,20 @@ +data-root: + files: + var/log/nova/nova-compute.log: | + 2023-06-18 08:42:58.702 1402787 ERROR oslo.messaging._drivers.impl_rabbit [-] Connection failed: [Errno 101] ENETUNREACH (retrying in 0 seconds): OSError: [Errno 101] ENETUNREACH + Traceback (most recent call last): + File "/usr/lib/python3/dist-packages/eventlet/hubs/hub.py", line 476, in fire_timers + timer() + File "/usr/lib/python3/dist-packages/eventlet/hubs/timer.py", line 59, in __call__ + cb(*args, **kw) + File "/usr/lib/python3/dist-packages/eventlet/semaphore.py", line 152, in _do_acquire + waiter.switch() + greenlet.error: cannot switch to a different thread + copy-from-original: + - sos_commands/date/date +raised-bugs: + https://bugs.launchpad.net/bugs/1934937: >- + One or more Nova service(s) on this host are affected by this bug which is + triggered when a non-wsgi service that is using greenthreads and has + oslo.messaging heartbeat_in_pthread=True tries to send a heartbeat message + to rabbitmq.