From 02132a31e536154cb6ca836ed2be801c8bcd6695 Mon Sep 17 00:00:00 2001 From: Andreas Skoog Date: Wed, 13 Jan 2021 13:53:39 +0100 Subject: [PATCH 1/2] Adds ability to exclude deadlocked threads from exported metrics Finding deadlocked threads might be an expensive operation if there are a large number of threads in the jvm since it requires a safepoint. --- .../client/hotspot/ThreadExports.java | 35 ++++++++----- .../client/hotspot/ThreadExportsTest.java | 50 +++++++++++++++++++ 2 files changed, 73 insertions(+), 12 deletions(-) diff --git a/simpleclient_hotspot/src/main/java/io/prometheus/client/hotspot/ThreadExports.java b/simpleclient_hotspot/src/main/java/io/prometheus/client/hotspot/ThreadExports.java index 55c90a654..4a9de9458 100644 --- a/simpleclient_hotspot/src/main/java/io/prometheus/client/hotspot/ThreadExports.java +++ b/simpleclient_hotspot/src/main/java/io/prometheus/client/hotspot/ThreadExports.java @@ -32,6 +32,7 @@ */ public class ThreadExports extends Collector { private final ThreadMXBean threadBean; + private boolean includeDeadlockedThreads = true; public ThreadExports() { this(ManagementFactory.getThreadMXBean()); @@ -41,6 +42,14 @@ public ThreadExports(ThreadMXBean threadBean) { this.threadBean = threadBean; } + public final boolean isIncludeDeadlockedThreads() { + return includeDeadlockedThreads; + } + + public final void setIncludeDeadlockedThreads(boolean includeDeadlockedThreads) { + this.includeDeadlockedThreads = includeDeadlockedThreads; + } + void addThreadMetrics(List sampleFamilies) { sampleFamilies.add( new GaugeMetricFamily( @@ -66,18 +75,20 @@ void addThreadMetrics(List sampleFamilies) { "Started thread count of a JVM", threadBean.getTotalStartedThreadCount())); - sampleFamilies.add( - new GaugeMetricFamily( - "jvm_threads_deadlocked", - "Cycles of JVM-threads that are in deadlock waiting to acquire object monitors or ownable synchronizers", - nullSafeArrayLength(threadBean.findDeadlockedThreads()))); - - sampleFamilies.add( - new GaugeMetricFamily( - "jvm_threads_deadlocked_monitor", - "Cycles of JVM-threads that are in deadlock waiting to acquire object monitors", - nullSafeArrayLength(threadBean.findMonitorDeadlockedThreads()))); - + if (includeDeadlockedThreads) { + sampleFamilies.add( + new GaugeMetricFamily( + "jvm_threads_deadlocked", + "Cycles of JVM-threads that are in deadlock waiting to acquire object monitors or ownable synchronizers", + nullSafeArrayLength(threadBean.findDeadlockedThreads()))); + + sampleFamilies.add( + new GaugeMetricFamily( + "jvm_threads_deadlocked_monitor", + "Cycles of JVM-threads that are in deadlock waiting to acquire object monitors", + nullSafeArrayLength(threadBean.findMonitorDeadlockedThreads()))); + } + GaugeMetricFamily threadStateFamily = new GaugeMetricFamily( "jvm_threads_state", "Current count of threads by state", diff --git a/simpleclient_hotspot/src/test/java/io/prometheus/client/hotspot/ThreadExportsTest.java b/simpleclient_hotspot/src/test/java/io/prometheus/client/hotspot/ThreadExportsTest.java index 9bd6c7f45..994f44b1c 100644 --- a/simpleclient_hotspot/src/test/java/io/prometheus/client/hotspot/ThreadExportsTest.java +++ b/simpleclient_hotspot/src/test/java/io/prometheus/client/hotspot/ThreadExportsTest.java @@ -10,6 +10,7 @@ import java.util.Arrays; import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNull; import static org.mockito.Mockito.when; public class ThreadExportsTest { @@ -97,4 +98,53 @@ public void testThreadPools() { "jvm_threads_state", STATE_LABEL, STATE_TERMINATED_LABEL), .0000001); } + + @Test + public void testThreadPoolsNoDeadlocked() { + collectorUnderTest.setIncludeDeadlockedThreads(false); + + assertNull(registry.getSampleValue( + "jvm_threads_deadlocked", EMPTY_LABEL, EMPTY_LABEL)); + assertNull(registry.getSampleValue( + "jvm_threads_deadlocked_monitor", EMPTY_LABEL, EMPTY_LABEL)); + + assertEquals( + 300L, + registry.getSampleValue( + "jvm_threads_current", EMPTY_LABEL, EMPTY_LABEL), + .0000001); + assertEquals( + 200L, + registry.getSampleValue( + "jvm_threads_daemon", EMPTY_LABEL, EMPTY_LABEL), + .0000001); + assertEquals( + 301L, + registry.getSampleValue( + "jvm_threads_peak", EMPTY_LABEL, EMPTY_LABEL), + .0000001); + assertEquals( + 503L, + registry.getSampleValue( + "jvm_threads_started_total", EMPTY_LABEL, EMPTY_LABEL), + .0000001); + + assertEquals( + 1L, + registry.getSampleValue( + "jvm_threads_state", STATE_LABEL, STATE_BLOCKED_LABEL), + .0000001); + + assertEquals( + 2L, + registry.getSampleValue( + "jvm_threads_state", STATE_LABEL, STATE_RUNNABLE_LABEL), + .0000001); + + assertEquals( + 0L, + registry.getSampleValue( + "jvm_threads_state", STATE_LABEL, STATE_TERMINATED_LABEL), + .0000001); + } } From 07659820f88a6e4e0b2be47712ca67ea61f1ce1e Mon Sep 17 00:00:00 2001 From: Andreas Skoog Date: Wed, 13 Jan 2021 13:53:39 +0100 Subject: [PATCH 2/2] Adds ability to exclude deadlocked threads from exported metrics Finding deadlocked threads might be an expensive operation if there are a large number of threads in the jvm since it requires a safepoint. Signed-off-by: Andreas Skoog --- .../client/hotspot/ThreadExports.java | 35 ++++++++----- .../client/hotspot/ThreadExportsTest.java | 50 +++++++++++++++++++ 2 files changed, 73 insertions(+), 12 deletions(-) diff --git a/simpleclient_hotspot/src/main/java/io/prometheus/client/hotspot/ThreadExports.java b/simpleclient_hotspot/src/main/java/io/prometheus/client/hotspot/ThreadExports.java index 55c90a654..4a9de9458 100644 --- a/simpleclient_hotspot/src/main/java/io/prometheus/client/hotspot/ThreadExports.java +++ b/simpleclient_hotspot/src/main/java/io/prometheus/client/hotspot/ThreadExports.java @@ -32,6 +32,7 @@ */ public class ThreadExports extends Collector { private final ThreadMXBean threadBean; + private boolean includeDeadlockedThreads = true; public ThreadExports() { this(ManagementFactory.getThreadMXBean()); @@ -41,6 +42,14 @@ public ThreadExports(ThreadMXBean threadBean) { this.threadBean = threadBean; } + public final boolean isIncludeDeadlockedThreads() { + return includeDeadlockedThreads; + } + + public final void setIncludeDeadlockedThreads(boolean includeDeadlockedThreads) { + this.includeDeadlockedThreads = includeDeadlockedThreads; + } + void addThreadMetrics(List sampleFamilies) { sampleFamilies.add( new GaugeMetricFamily( @@ -66,18 +75,20 @@ void addThreadMetrics(List sampleFamilies) { "Started thread count of a JVM", threadBean.getTotalStartedThreadCount())); - sampleFamilies.add( - new GaugeMetricFamily( - "jvm_threads_deadlocked", - "Cycles of JVM-threads that are in deadlock waiting to acquire object monitors or ownable synchronizers", - nullSafeArrayLength(threadBean.findDeadlockedThreads()))); - - sampleFamilies.add( - new GaugeMetricFamily( - "jvm_threads_deadlocked_monitor", - "Cycles of JVM-threads that are in deadlock waiting to acquire object monitors", - nullSafeArrayLength(threadBean.findMonitorDeadlockedThreads()))); - + if (includeDeadlockedThreads) { + sampleFamilies.add( + new GaugeMetricFamily( + "jvm_threads_deadlocked", + "Cycles of JVM-threads that are in deadlock waiting to acquire object monitors or ownable synchronizers", + nullSafeArrayLength(threadBean.findDeadlockedThreads()))); + + sampleFamilies.add( + new GaugeMetricFamily( + "jvm_threads_deadlocked_monitor", + "Cycles of JVM-threads that are in deadlock waiting to acquire object monitors", + nullSafeArrayLength(threadBean.findMonitorDeadlockedThreads()))); + } + GaugeMetricFamily threadStateFamily = new GaugeMetricFamily( "jvm_threads_state", "Current count of threads by state", diff --git a/simpleclient_hotspot/src/test/java/io/prometheus/client/hotspot/ThreadExportsTest.java b/simpleclient_hotspot/src/test/java/io/prometheus/client/hotspot/ThreadExportsTest.java index 9bd6c7f45..994f44b1c 100644 --- a/simpleclient_hotspot/src/test/java/io/prometheus/client/hotspot/ThreadExportsTest.java +++ b/simpleclient_hotspot/src/test/java/io/prometheus/client/hotspot/ThreadExportsTest.java @@ -10,6 +10,7 @@ import java.util.Arrays; import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNull; import static org.mockito.Mockito.when; public class ThreadExportsTest { @@ -97,4 +98,53 @@ public void testThreadPools() { "jvm_threads_state", STATE_LABEL, STATE_TERMINATED_LABEL), .0000001); } + + @Test + public void testThreadPoolsNoDeadlocked() { + collectorUnderTest.setIncludeDeadlockedThreads(false); + + assertNull(registry.getSampleValue( + "jvm_threads_deadlocked", EMPTY_LABEL, EMPTY_LABEL)); + assertNull(registry.getSampleValue( + "jvm_threads_deadlocked_monitor", EMPTY_LABEL, EMPTY_LABEL)); + + assertEquals( + 300L, + registry.getSampleValue( + "jvm_threads_current", EMPTY_LABEL, EMPTY_LABEL), + .0000001); + assertEquals( + 200L, + registry.getSampleValue( + "jvm_threads_daemon", EMPTY_LABEL, EMPTY_LABEL), + .0000001); + assertEquals( + 301L, + registry.getSampleValue( + "jvm_threads_peak", EMPTY_LABEL, EMPTY_LABEL), + .0000001); + assertEquals( + 503L, + registry.getSampleValue( + "jvm_threads_started_total", EMPTY_LABEL, EMPTY_LABEL), + .0000001); + + assertEquals( + 1L, + registry.getSampleValue( + "jvm_threads_state", STATE_LABEL, STATE_BLOCKED_LABEL), + .0000001); + + assertEquals( + 2L, + registry.getSampleValue( + "jvm_threads_state", STATE_LABEL, STATE_RUNNABLE_LABEL), + .0000001); + + assertEquals( + 0L, + registry.getSampleValue( + "jvm_threads_state", STATE_LABEL, STATE_TERMINATED_LABEL), + .0000001); + } }