From 905204ec6fa21b554027694ebfe3eebe8f60fdbd Mon Sep 17 00:00:00 2001 From: Andreas Brenk Date: Thu, 28 Nov 2024 17:12:35 +0100 Subject: [PATCH] New liveness probe to check for deadlocked threads --- .../DeadlockedThreadsHealthIndicator.java | 132 ++++++++++++++++++ .../indicator/threads/package-info.java | 22 +++ ...adsHealthIndicatorConfigurationSpec.groovy | 20 +++ ...eadlockedThreadsHealthIndicatorSpec.groovy | 74 ++++++++++ 4 files changed, 248 insertions(+) create mode 100644 management/src/main/java/io/micronaut/management/health/indicator/threads/DeadlockedThreadsHealthIndicator.java create mode 100644 management/src/main/java/io/micronaut/management/health/indicator/threads/package-info.java create mode 100644 management/src/test/groovy/io/micronaut/management/health/indicator/threads/DeadlockedThreadsHealthIndicatorConfigurationSpec.groovy create mode 100644 management/src/test/groovy/io/micronaut/management/health/indicator/threads/DeadlockedThreadsHealthIndicatorSpec.groovy diff --git a/management/src/main/java/io/micronaut/management/health/indicator/threads/DeadlockedThreadsHealthIndicator.java b/management/src/main/java/io/micronaut/management/health/indicator/threads/DeadlockedThreadsHealthIndicator.java new file mode 100644 index 00000000000..99d11ae26ba --- /dev/null +++ b/management/src/main/java/io/micronaut/management/health/indicator/threads/DeadlockedThreadsHealthIndicator.java @@ -0,0 +1,132 @@ +/* + * Copyright 2017-2024 original authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.micronaut.management.health.indicator.threads; + +import io.micronaut.context.annotation.Requires; +import io.micronaut.core.async.publisher.Publishers; +import io.micronaut.core.util.StringUtils; +import io.micronaut.health.HealthStatus; +import io.micronaut.management.endpoint.health.HealthEndpoint; +import io.micronaut.management.health.indicator.HealthIndicator; +import io.micronaut.management.health.indicator.HealthResult; +import io.micronaut.management.health.indicator.annotation.Liveness; +import jakarta.inject.Singleton; +import org.reactivestreams.Publisher; + +import java.lang.management.ManagementFactory; +import java.lang.management.MonitorInfo; +import java.lang.management.ThreadInfo; +import java.lang.management.ThreadMXBean; +import java.util.Arrays; +import java.util.LinkedHashMap; +import java.util.Map; + +/** + *

A {@link HealthIndicator} that uses the {@link ThreadMXBean} to check for deadlocked threads. + * Returns {@link HealthStatus#DOWN} if any are found and their {@link ThreadInfo} in the details.

+ * + * @author Andreas Brenk + * @since 4.8.0 + */ +@Singleton +@Liveness +@Requires(property = HealthEndpoint.PREFIX + ".deadlocked-threads.enabled", notEquals = StringUtils.FALSE) +@Requires(beans = HealthEndpoint.class) +public class DeadlockedThreadsHealthIndicator implements HealthIndicator { + + private static final String NAME = "deadlockedThreads"; + + @Override + public Publisher getResult() { + HealthResult.Builder builder = HealthResult.builder(NAME); + + try { + ThreadMXBean threadMXBean = ManagementFactory.getThreadMXBean(); + long[] deadlockedThreads = threadMXBean.findDeadlockedThreads(); + + if (deadlockedThreads == null) { + builder.status(HealthStatus.UP); + } else { + builder.status(HealthStatus.DOWN); + builder.details( + Arrays.stream(threadMXBean.getThreadInfo(deadlockedThreads, true, true, Integer.MAX_VALUE)) + .map(DeadlockedThreadsHealthIndicator::getDetails) + .toList()); + } + } catch (Exception e) { + builder.status(HealthStatus.UNKNOWN); + builder.exception(e); + } + + return Publishers.just(builder.build()); + } + + private static Map getDetails(ThreadInfo threadInfo) { + Map details = new LinkedHashMap<>(); + details.put("threadId", String.valueOf(threadInfo.getThreadId())); + details.put("threadName", threadInfo.getThreadName()); + details.put("threadState", threadInfo.getThreadState().name()); + details.put("daemon", String.valueOf(threadInfo.isDaemon())); + details.put("priority", String.valueOf(threadInfo.getPriority())); + details.put("suspended", String.valueOf(threadInfo.isSuspended())); + details.put("inNative", String.valueOf(threadInfo.isInNative())); + details.put("lockName", threadInfo.getLockName()); + details.put("lockOwnerName", threadInfo.getLockOwnerName()); + details.put("lockOwnerId", String.valueOf(threadInfo.getLockOwnerId())); + details.put("lockedSynchronizers", Arrays.stream(threadInfo.getLockedSynchronizers()).map(String::valueOf).toList()); + details.put("stackTrace", formatStackTrace(threadInfo)); + + return details; + } + + private static String formatStackTrace(ThreadInfo threadInfo) { + StringBuilder sb = new StringBuilder(); + + int i = 0; + StackTraceElement[] stackTrace = threadInfo.getStackTrace(); + for (; i < stackTrace.length; i++) { + StackTraceElement ste = stackTrace[i]; + sb.append(ste.toString()); + sb.append('\n'); + + if (i == 0 && threadInfo.getLockInfo() != null) { + switch (threadInfo.getThreadState()) { + case BLOCKED: + sb.append("- blocked on "); + sb.append(threadInfo.getLockInfo()); + sb.append('\n'); + break; + case WAITING, TIMED_WAITING: + sb.append("- waiting on "); + sb.append(threadInfo.getLockInfo()); + sb.append('\n'); + break; + default: + } + } + + for (MonitorInfo mi : threadInfo.getLockedMonitors()) { + if (mi.getLockedStackDepth() == i) { + sb.append("- locked "); + sb.append(mi); + sb.append('\n'); + } + } + } + + return sb.toString(); + } +} diff --git a/management/src/main/java/io/micronaut/management/health/indicator/threads/package-info.java b/management/src/main/java/io/micronaut/management/health/indicator/threads/package-info.java new file mode 100644 index 00000000000..262aecb795a --- /dev/null +++ b/management/src/main/java/io/micronaut/management/health/indicator/threads/package-info.java @@ -0,0 +1,22 @@ +/* + * Copyright 2017-2024 original authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +/** + * Thread deadlock indicator. + * + * @author Andreas Brenk + * @since 4.8.0 + */ +package io.micronaut.management.health.indicator.threads; diff --git a/management/src/test/groovy/io/micronaut/management/health/indicator/threads/DeadlockedThreadsHealthIndicatorConfigurationSpec.groovy b/management/src/test/groovy/io/micronaut/management/health/indicator/threads/DeadlockedThreadsHealthIndicatorConfigurationSpec.groovy new file mode 100644 index 00000000000..ae650be0c4f --- /dev/null +++ b/management/src/test/groovy/io/micronaut/management/health/indicator/threads/DeadlockedThreadsHealthIndicatorConfigurationSpec.groovy @@ -0,0 +1,20 @@ +package io.micronaut.management.health.indicator.threads + +import io.micronaut.context.ApplicationContext +import io.micronaut.management.health.indicator.discovery.DiscoveryClientHealthIndicator +import io.micronaut.management.health.indicator.discovery.DiscoveryClientHealthIndicatorConfiguration +import spock.lang.Specification + +class DeadlockedThreadsHealthIndicatorConfigurationSpec extends Specification { + + void "bean of type DeadlockedThreadsHealthIndicator does not exist if you set endpoints.health.deadlocked-threads.enabled=false"() { + given: + ApplicationContext applicationContext = ApplicationContext.run(['endpoints.health.deadlocked-threads.enabled': 'false']) + + expect: + !applicationContext.containsBean(DeadlockedThreadsHealthIndicator) + + cleanup: + applicationContext.close() + } +} diff --git a/management/src/test/groovy/io/micronaut/management/health/indicator/threads/DeadlockedThreadsHealthIndicatorSpec.groovy b/management/src/test/groovy/io/micronaut/management/health/indicator/threads/DeadlockedThreadsHealthIndicatorSpec.groovy new file mode 100644 index 00000000000..a2bcb9d4710 --- /dev/null +++ b/management/src/test/groovy/io/micronaut/management/health/indicator/threads/DeadlockedThreadsHealthIndicatorSpec.groovy @@ -0,0 +1,74 @@ +package io.micronaut.management.health.indicator.threads + +import io.micronaut.health.HealthStatus +import org.slf4j.Logger +import org.slf4j.LoggerFactory +import reactor.core.publisher.Mono +import spock.lang.Specification + +import static java.lang.Thread.sleep + +class DeadlockedThreadsHealthIndicatorSpec extends Specification { + + Logger log = LoggerFactory.getLogger(DeadlockedThreadsHealthIndicatorSpec) + + def lock1 = new Object() + def lock2 = new Object() + def thread1 + def thread2 + + def "No deadlocked threads so status is UP"() { + given: + thread1 = new Thread() + thread2 = new Thread() + def healthIndicator = new DeadlockedThreadsHealthIndicator() + + when: + thread1.start() + thread2.start() + def result = Mono.from(healthIndicator.getResult()).block() + + then: + HealthStatus.UP == result.status + null == result.details + } + + def "Deadlocked threads found so status is DOWN"() { + given: + thread1 = new Thread(() -> { + synchronized (lock1) { + log.debug "Thread 1: Holding lock 1" + + sleep 200 + + synchronized (lock2) { + log.debug "Thread 1: Holding lock 1 and lock 2" + } + } + }) + thread2 = new Thread(() -> { + synchronized (lock2) { + log.debug "Thread 2: Holding lock 2" + + sleep 100 + + synchronized (lock1) { + log.debug "Thread 2: Holding lock 2 and lock 1" + } + } + }) + def healthIndicator = new DeadlockedThreadsHealthIndicator() + + when: + thread1.start() + thread2.start() + + Thread.sleep(300) + + def result = Mono.from(healthIndicator.getResult()).block() + + then: + HealthStatus.DOWN == result.status + null != result.details + } +}