diff --git a/spring-boot/src/main/java/org/springframework/boot/web/client/RestTemplateBuilder.java b/spring-boot/src/main/java/org/springframework/boot/web/client/RestTemplateBuilder.java
index c7b7d4b84d..0b757dbc4c 100644
--- a/spring-boot/src/main/java/org/springframework/boot/web/client/RestTemplateBuilder.java
+++ b/spring-boot/src/main/java/org/springframework/boot/web/client/RestTemplateBuilder.java
@@ -463,14 +463,16 @@ public class RestTemplateBuilder {
 	private void configureRequestFactory(RestTemplate restTemplate) {
 		ClientHttpRequestFactory requestFactory = null;
 		if (this.requestFactory != null) {
-			requestFactory = unwrapRequestFactoryIfNecessary(this.requestFactory);
+			requestFactory = this.requestFactory;
 		}
 		else if (this.detectRequestFactory) {
 			requestFactory = detectRequestFactory();
 		}
 		if (requestFactory != null) {
+			ClientHttpRequestFactory unwrappedRequestFactory = unwrapRequestFactoryIfNecessary(
+					requestFactory);
 			for (RequestFactoryCustomizer customizer : this.requestFactoryCustomizers) {
-				customizer.customize(requestFactory);
+				customizer.customize(unwrappedRequestFactory);
 			}
 			restTemplate.setRequestFactory(requestFactory);
 		}
diff --git a/spring-boot/src/test/java/org/springframework/boot/web/client/RestTemplateBuilderTests.java b/spring-boot/src/test/java/org/springframework/boot/web/client/RestTemplateBuilderTests.java
index bdc81116b0..c7e7d51736 100644
--- a/spring-boot/src/test/java/org/springframework/boot/web/client/RestTemplateBuilderTests.java
+++ b/spring-boot/src/test/java/org/springframework/boot/web/client/RestTemplateBuilderTests.java
@@ -474,6 +474,16 @@ public class RestTemplateBuilderTests {
 				.isEqualTo(1234);
 	}
 
+	@Test
+	public void unwrappingDoesNotAffectRequestFactoryThatIsSetOnTheBuiltTemplate() {
+		SimpleClientHttpRequestFactory requestFactory = new SimpleClientHttpRequestFactory();
+		RestTemplate template = this.builder
+				.requestFactory(new BufferingClientHttpRequestFactory(requestFactory))
+				.build();
+		assertThat(template.getRequestFactory())
+				.isInstanceOf(BufferingClientHttpRequestFactory.class);
+	}
+
 	public static class RestTemplateSubclass extends RestTemplate {
 
 	}