Log a warning when a health indicator takes too long to run

Update `HealthEndpointSupport` so that it logs a warning if a health
indicator takes too long to respond.

Fixes gh-31231
pull/31676/head
Phillip Webb 2 years ago
parent 2094722e5d
commit 9f8a262e6b

@ -1,5 +1,5 @@
/*
* Copyright 2012-2021 the original author or authors.
* Copyright 2012-2022 the original author or authors.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
@ -88,8 +88,9 @@ class HealthEndpointConfiguration {
@Bean
@ConditionalOnMissingBean
HealthEndpoint healthEndpoint(HealthContributorRegistry registry, HealthEndpointGroups groups) {
return new HealthEndpoint(registry, groups);
HealthEndpoint healthEndpoint(HealthContributorRegistry registry, HealthEndpointGroups groups,
HealthEndpointProperties properties) {
return new HealthEndpoint(registry, groups, properties.getLogging().getSlowIndicatorThreshold());
}
@Bean

@ -1,5 +1,5 @@
/*
* Copyright 2012-2021 the original author or authors.
* Copyright 2012-2022 the original author or authors.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
@ -16,6 +16,7 @@
package org.springframework.boot.actuate.autoconfigure.health;
import java.time.Duration;
import java.util.LinkedHashMap;
import java.util.Map;
import java.util.Set;
@ -43,6 +44,8 @@ public class HealthEndpointProperties extends HealthProperties {
*/
private Map<String, Group> group = new LinkedHashMap<>();
private Logging logging = new Logging();
@Override
public Show getShowDetails() {
return this.showDetails;
@ -56,6 +59,10 @@ public class HealthEndpointProperties extends HealthProperties {
return this.group;
}
public Logging getLogging() {
return this.logging;
}
/**
* A health endpoint group.
*/
@ -124,4 +131,24 @@ public class HealthEndpointProperties extends HealthProperties {
}
/**
* Health logging properties.
*/
public static class Logging {
/**
* Threshold after which a warning will be logged for slow health indicators.
*/
Duration slowIndicatorThreshold = Duration.ofSeconds(10);
public Duration getSlowIndicatorThreshold() {
return this.slowIndicatorThreshold;
}
public void setSlowIndicatorThreshold(Duration slowIndicatorThreshold) {
this.slowIndicatorThreshold = slowIndicatorThreshold;
}
}
}

@ -53,8 +53,10 @@ class HealthEndpointReactiveWebExtensionConfiguration {
@ConditionalOnMissingBean
@ConditionalOnBean(HealthEndpoint.class)
ReactiveHealthEndpointWebExtension reactiveHealthEndpointWebExtension(
ReactiveHealthContributorRegistry reactiveHealthContributorRegistry, HealthEndpointGroups groups) {
return new ReactiveHealthEndpointWebExtension(reactiveHealthContributorRegistry, groups);
ReactiveHealthContributorRegistry reactiveHealthContributorRegistry, HealthEndpointGroups groups,
HealthEndpointProperties properties) {
return new ReactiveHealthEndpointWebExtension(reactiveHealthContributorRegistry, groups,
properties.getLogging().getSlowIndicatorThreshold());
}
@Configuration(proxyBeanMethods = false)

@ -72,8 +72,9 @@ class HealthEndpointWebExtensionConfiguration {
@Bean
@ConditionalOnMissingBean
HealthEndpointWebExtension healthEndpointWebExtension(HealthContributorRegistry healthContributorRegistry,
HealthEndpointGroups groups) {
return new HealthEndpointWebExtension(healthContributorRegistry, groups);
HealthEndpointGroups groups, HealthEndpointProperties properties) {
return new HealthEndpointWebExtension(healthContributorRegistry, groups,
properties.getLogging().getSlowIndicatorThreshold());
}
private static ExposableWebEndpoint getHealthEndpoint(WebEndpointsSupplier webEndpointsSupplier) {

@ -1,5 +1,5 @@
/*
* Copyright 2012-2020 the original author or authors.
* Copyright 2012-2022 the original author or authors.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
@ -113,7 +113,7 @@ class CloudFoundryWebEndpointDiscovererTests {
HealthEndpoint healthEndpoint() {
HealthContributorRegistry registry = mock(HealthContributorRegistry.class);
HealthEndpointGroups groups = mock(HealthEndpointGroups.class);
return new HealthEndpoint(registry, groups);
return new HealthEndpoint(registry, groups, null);
}
@Bean

@ -1,5 +1,5 @@
/*
* Copyright 2012-2021 the original author or authors.
* Copyright 2012-2022 the original author or authors.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
@ -111,7 +111,7 @@ class HealthEndpointDocumentationTests extends MockMvcEndpointDocumentationTests
HealthContributorRegistry registry = new DefaultHealthContributorRegistry(healthContributors);
HealthEndpointGroup primary = new TestHealthEndpointGroup();
HealthEndpointGroups groups = HealthEndpointGroups.of(primary, Collections.emptyMap());
return new HealthEndpoint(registry, groups);
return new HealthEndpoint(registry, groups, null);
}
@Bean

@ -1,5 +1,5 @@
/*
* Copyright 2012-2021 the original author or authors.
* Copyright 2012-2022 the original author or authors.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
@ -16,6 +16,7 @@
package org.springframework.boot.actuate.health;
import java.time.Duration;
import java.util.Map;
import java.util.Set;
@ -51,9 +52,24 @@ public class HealthEndpoint extends HealthEndpointSupport<HealthContributor, Hea
* Create a new {@link HealthEndpoint} instance.
* @param registry the health contributor registry
* @param groups the health endpoint groups
* @deprecated since 2.6.9 for removal in 3.0.0 in favor of
* {@link #HealthEndpoint(HealthContributorRegistry, HealthEndpointGroups, Duration)}
*/
@Deprecated
public HealthEndpoint(HealthContributorRegistry registry, HealthEndpointGroups groups) {
super(registry, groups);
super(registry, groups, null);
}
/**
* Create a new {@link HealthEndpoint} instance.
* @param registry the health contributor registry
* @param groups the health endpoint groups
* @param slowIndicatorLoggingThreshold duration after which slow health indicator
* logging should occur
*/
public HealthEndpoint(HealthContributorRegistry registry, HealthEndpointGroups groups,
Duration slowIndicatorLoggingThreshold) {
super(registry, groups, slowIndicatorLoggingThreshold);
}
@ReadOperation

@ -1,5 +1,5 @@
/*
* Copyright 2012-2021 the original author or authors.
* Copyright 2012-2022 the original author or authors.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
@ -16,14 +16,21 @@
package org.springframework.boot.actuate.health;
import java.time.Duration;
import java.time.Instant;
import java.util.LinkedHashMap;
import java.util.Map;
import java.util.Set;
import java.util.stream.Collectors;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.springframework.boot.actuate.endpoint.ApiVersion;
import org.springframework.boot.actuate.endpoint.SecurityContext;
import org.springframework.boot.actuate.endpoint.web.WebServerNamespace;
import org.springframework.boot.convert.DurationStyle;
import org.springframework.core.log.LogMessage;
import org.springframework.util.Assert;
import org.springframework.util.StringUtils;
@ -37,22 +44,30 @@ import org.springframework.util.StringUtils;
*/
abstract class HealthEndpointSupport<C, T> {
private static final Log logger = LogFactory.getLog(HealthEndpointSupport.class);
static final Health DEFAULT_HEALTH = Health.up().build();
private final ContributorRegistry<C> registry;
private final HealthEndpointGroups groups;
private Duration slowIndicatorLoggingThreshold;
/**
* Create a new {@link HealthEndpointSupport} instance.
* @param registry the health contributor registry
* @param groups the health endpoint groups
* @param slowIndicatorLoggingThreshold duration after which slow health indicator
* logging should occur
*/
HealthEndpointSupport(ContributorRegistry<C> registry, HealthEndpointGroups groups) {
HealthEndpointSupport(ContributorRegistry<C> registry, HealthEndpointGroups groups,
Duration slowIndicatorLoggingThreshold) {
Assert.notNull(registry, "Registry must not be null");
Assert.notNull(groups, "Groups must not be null");
this.registry = registry;
this.groups = groups;
this.slowIndicatorLoggingThreshold = slowIndicatorLoggingThreshold;
}
HealthResult<T> getHealth(ApiVersion apiVersion, WebServerNamespace serverNamespace,
@ -127,7 +142,7 @@ abstract class HealthEndpointSupport<C, T> {
showDetails, groupNames);
}
if (contributor != null && (name.isEmpty() || group.isMember(name))) {
return getHealth((C) contributor, showDetails);
return getLoggedHealth((C) contributor, name, showDetails);
}
return null;
}
@ -151,6 +166,25 @@ abstract class HealthEndpointSupport<C, T> {
groupNames);
}
private T getLoggedHealth(C contributor, String name, boolean showDetails) {
Instant start = Instant.now();
try {
return getHealth(contributor, showDetails);
}
finally {
if (logger.isWarnEnabled() && this.slowIndicatorLoggingThreshold != null) {
Duration duration = Duration.between(start, Instant.now());
if (duration.compareTo(this.slowIndicatorLoggingThreshold) > 0) {
String contributorClassName = contributor.getClass().getName();
Object contributorIdentifier = (!StringUtils.hasLength(name)) ? contributorClassName
: contributor.getClass().getName() + " (" + name + ")";
logger.warn(LogMessage.format("Health contributor %s took %s to respond", contributorIdentifier,
DurationStyle.SIMPLE.print(duration)));
}
}
}
}
protected abstract T getHealth(C contributor, boolean includeDetails);
protected abstract T aggregateContributions(ApiVersion apiVersion, Map<String, T> contributions,

@ -1,5 +1,5 @@
/*
* Copyright 2012-2021 the original author or authors.
* Copyright 2012-2022 the original author or authors.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
@ -16,6 +16,7 @@
package org.springframework.boot.actuate.health;
import java.time.Duration;
import java.util.Arrays;
import java.util.Map;
import java.util.Set;
@ -51,9 +52,24 @@ public class HealthEndpointWebExtension extends HealthEndpointSupport<HealthCont
* Create a new {@link HealthEndpointWebExtension} instance.
* @param registry the health contributor registry
* @param groups the health endpoint groups
* @deprecated since 2.6.9 for removal in 3.0.0 in favor of
* {@link #HealthEndpointWebExtension(HealthContributorRegistry, HealthEndpointGroups, Duration)}
*/
@Deprecated
public HealthEndpointWebExtension(HealthContributorRegistry registry, HealthEndpointGroups groups) {
super(registry, groups);
super(registry, groups, null);
}
/**
* Create a new {@link HealthEndpointWebExtension} instance.
* @param registry the health contributor registry
* @param groups the health endpoint groups
* @param slowIndicatorLoggingThreshold duration after which slow health indicator
* logging should occur
*/
public HealthEndpointWebExtension(HealthContributorRegistry registry, HealthEndpointGroups groups,
Duration slowIndicatorLoggingThreshold) {
super(registry, groups, slowIndicatorLoggingThreshold);
}
@ReadOperation

@ -1,5 +1,5 @@
/*
* Copyright 2012-2021 the original author or authors.
* Copyright 2012-2022 the original author or authors.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
@ -16,6 +16,7 @@
package org.springframework.boot.actuate.health;
import java.time.Duration;
import java.util.Arrays;
import java.util.Map;
import java.util.Set;
@ -51,9 +52,24 @@ public class ReactiveHealthEndpointWebExtension
* Create a new {@link ReactiveHealthEndpointWebExtension} instance.
* @param registry the health contributor registry
* @param groups the health endpoint groups
* @deprecated since 2.6.9 for removal in 3.0.0 in favor of
* {@link #ReactiveHealthEndpointWebExtension(ReactiveHealthContributorRegistry, HealthEndpointGroups, Duration)}
*/
@Deprecated
public ReactiveHealthEndpointWebExtension(ReactiveHealthContributorRegistry registry, HealthEndpointGroups groups) {
super(registry, groups);
super(registry, groups, null);
}
/**
* Create a new {@link ReactiveHealthEndpointWebExtension} instance.
* @param registry the health contributor registry
* @param groups the health endpoint groups
* @param slowIndicatorLoggingThreshold duration after which slow health indicator
* logging should occur
*/
public ReactiveHealthEndpointWebExtension(ReactiveHealthContributorRegistry registry, HealthEndpointGroups groups,
Duration slowIndicatorLoggingThreshold) {
super(registry, groups, slowIndicatorLoggingThreshold);
}
@ReadOperation

@ -1,5 +1,5 @@
/*
* Copyright 2012-2021 the original author or authors.
* Copyright 2012-2022 the original author or authors.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
@ -16,6 +16,7 @@
package org.springframework.boot.actuate.health;
import java.time.Duration;
import java.util.Collections;
import java.util.LinkedHashMap;
import java.util.Map;
@ -34,13 +35,14 @@ import static org.assertj.core.api.Assertions.assertThatIllegalArgumentException
/**
* Base class for {@link HealthEndpointSupport} tests.
*
* @param <S> the support type
* @param <R> the registry type
* @param <C> the contributor type
* @param <T> the contributed health component type
* @author Phillip Webb
* @author Madhura Bhave
*/
abstract class HealthEndpointSupportTests<R extends ContributorRegistry<C>, C, T> {
abstract class HealthEndpointSupportTests<S extends HealthEndpointSupport<C, T>, R extends ContributorRegistry<C>, C, T> {
final R registry;
@ -352,7 +354,11 @@ abstract class HealthEndpointSupportTests<R extends ContributorRegistry<C>, C, T
assertThat(result).isEqualTo(null);
}
protected abstract HealthEndpointSupport<C, T> create(R registry, HealthEndpointGroups groups);
protected final S create(R registry, HealthEndpointGroups groups) {
return create(registry, groups, null);
}
protected abstract S create(R registry, HealthEndpointGroups groups, Duration slowIndicatorLoggingThreshold);
protected abstract R createRegistry();

@ -1,5 +1,5 @@
/*
* Copyright 2012-2020 the original author or authors.
* Copyright 2012-2022 the original author or authors.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
@ -16,12 +16,16 @@
package org.springframework.boot.actuate.health;
import java.time.Duration;
import java.util.Collections;
import java.util.Map;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.extension.ExtendWith;
import org.springframework.boot.actuate.health.HealthEndpointSupport.HealthResult;
import org.springframework.boot.test.system.CapturedOutput;
import org.springframework.boot.test.system.OutputCaptureExtension;
import static org.assertj.core.api.Assertions.assertThat;
import static org.mockito.Mockito.mock;
@ -32,8 +36,9 @@ import static org.mockito.Mockito.mock;
* @author Phillip Webb
* @author Scott Frederick
*/
class HealthEndpointTests
extends HealthEndpointSupportTests<HealthContributorRegistry, HealthContributor, HealthComponent> {
@ExtendWith(OutputCaptureExtension.class)
class HealthEndpointTests extends
HealthEndpointSupportTests<HealthEndpoint, HealthContributorRegistry, HealthContributor, HealthComponent> {
@Test
void healthReturnsSystemHealth() {
@ -66,9 +71,27 @@ class HealthEndpointTests
assertThat(health).isEqualTo(this.up);
}
@Test
void healthWhenIndicatorIsSlow(CapturedOutput output) {
HealthIndicator indicator = () -> {
try {
Thread.sleep(100);
}
catch (InterruptedException ex) {
}
return this.up;
};
this.registry.registerContributor("test", indicator);
create(this.registry, this.groups, Duration.ofMillis(10)).health();
assertThat(output).contains("Health contributor");
assertThat(output).contains("to respond");
}
@Override
protected HealthEndpoint create(HealthContributorRegistry registry, HealthEndpointGroups groups) {
return new HealthEndpoint(registry, groups);
protected HealthEndpoint create(HealthContributorRegistry registry, HealthEndpointGroups groups,
Duration slowIndicatorLoggingThreshold) {
return new HealthEndpoint(registry, groups, slowIndicatorLoggingThreshold);
}
@Override

@ -1,5 +1,5 @@
/*
* Copyright 2012-2021 the original author or authors.
* Copyright 2012-2022 the original author or authors.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
@ -16,6 +16,7 @@
package org.springframework.boot.actuate.health;
import java.time.Duration;
import java.util.Collections;
import java.util.Map;
@ -36,8 +37,8 @@ import static org.mockito.Mockito.mock;
* @author Phillip Webb
* @author Scott Frederick
*/
class HealthEndpointWebExtensionTests
extends HealthEndpointSupportTests<HealthContributorRegistry, HealthContributor, HealthComponent> {
class HealthEndpointWebExtensionTests extends
HealthEndpointSupportTests<HealthEndpointWebExtension, HealthContributorRegistry, HealthContributor, HealthComponent> {
@Test
void healthReturnsSystemHealth() {
@ -81,8 +82,9 @@ class HealthEndpointWebExtensionTests
}
@Override
protected HealthEndpointWebExtension create(HealthContributorRegistry registry, HealthEndpointGroups groups) {
return new HealthEndpointWebExtension(registry, groups);
protected HealthEndpointWebExtension create(HealthContributorRegistry registry, HealthEndpointGroups groups,
Duration slowIndicatorLoggingThreshold) {
return new HealthEndpointWebExtension(registry, groups, slowIndicatorLoggingThreshold);
}
@Override

@ -1,5 +1,5 @@
/*
* Copyright 2012-2021 the original author or authors.
* Copyright 2012-2022 the original author or authors.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
@ -188,14 +188,14 @@ class HealthEndpointWebIntegrationTests {
@Bean
HealthEndpoint healthEndpoint(HealthContributorRegistry healthContributorRegistry,
HealthEndpointGroups healthEndpointGroups) {
return new HealthEndpoint(healthContributorRegistry, healthEndpointGroups);
return new HealthEndpoint(healthContributorRegistry, healthEndpointGroups, null);
}
@Bean
@ConditionalOnWebApplication(type = Type.SERVLET)
HealthEndpointWebExtension healthWebEndpointExtension(HealthContributorRegistry healthContributorRegistry,
HealthEndpointGroups healthEndpointGroups) {
return new HealthEndpointWebExtension(healthContributorRegistry, healthEndpointGroups);
return new HealthEndpointWebExtension(healthContributorRegistry, healthEndpointGroups, null);
}
@Bean
@ -203,7 +203,8 @@ class HealthEndpointWebIntegrationTests {
ReactiveHealthEndpointWebExtension reactiveHealthWebEndpointExtension(
ReactiveHealthContributorRegistry reactiveHealthContributorRegistry,
HealthEndpointGroups healthEndpointGroups) {
return new ReactiveHealthEndpointWebExtension(reactiveHealthContributorRegistry, healthEndpointGroups);
return new ReactiveHealthEndpointWebExtension(reactiveHealthContributorRegistry, healthEndpointGroups,
null);
}
@Bean

@ -1,5 +1,5 @@
/*
* Copyright 2012-2021 the original author or authors.
* Copyright 2012-2022 the original author or authors.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
@ -16,6 +16,7 @@
package org.springframework.boot.actuate.health;
import java.time.Duration;
import java.util.Collections;
import java.util.Map;
@ -37,7 +38,7 @@ import static org.mockito.Mockito.mock;
* @author Scott Frederick
*/
class ReactiveHealthEndpointWebExtensionTests extends
HealthEndpointSupportTests<ReactiveHealthContributorRegistry, ReactiveHealthContributor, Mono<? extends HealthComponent>> {
HealthEndpointSupportTests<ReactiveHealthEndpointWebExtension, ReactiveHealthContributorRegistry, ReactiveHealthContributor, Mono<? extends HealthComponent>> {
@Test
void healthReturnsSystemHealth() {
@ -82,8 +83,8 @@ class ReactiveHealthEndpointWebExtensionTests extends
@Override
protected ReactiveHealthEndpointWebExtension create(ReactiveHealthContributorRegistry registry,
HealthEndpointGroups groups) {
return new ReactiveHealthEndpointWebExtension(registry, groups);
HealthEndpointGroups groups, Duration slowIndicatorLoggingThreshold) {
return new ReactiveHealthEndpointWebExtension(registry, groups, slowIndicatorLoggingThreshold);
}
@Override

@ -767,6 +767,10 @@ include::{docs-java}/actuator/endpoints/health/writingcustomhealthindicators/MyH
NOTE: The identifier for a given `HealthIndicator` is the name of the bean without the `HealthIndicator` suffix, if it exists.
In the preceding example, the health information is available in an entry named `my`.
TIP: Health indicators are usually called over HTTP and need to respond before any connection timeouts.
Spring Boot will log a warning message for any health indicator that takes longer than 10 seconds to respond.
If you want to configure this threshold, you can use the configprop:management.endpoint.health.logging.slow-indicator-threshold[] property
In addition to Spring Boot's predefined {spring-boot-actuator-module-code}/health/Status.java[`Status`] types, `Health` can return a custom `Status` that represents a new system state.
In such cases, you also need to provide a custom implementation of the {spring-boot-actuator-module-code}/health/StatusAggregator.java[`StatusAggregator`] interface, or you must configure the default implementation by using the configprop:management.endpoint.health.status.order[] configuration property.

Loading…
Cancel
Save