Log some information about all test failures when the build completes

Closes gh-19696

buildSrc/build.gradle

@@ -81,6 +81,10 @@ gradlePlugin {
 			id = "org.springframework.boot.starter"
 			implementationClass = "org.springframework.boot.build.starters.StarterPlugin"
 		}
+		testFailuresPlugin {
+			id = "org.springframework.boot.test-failures"
+			implementationClass = "org.springframework.boot.build.testing.TestFailuresPlugin"
+		}
 	}
 }
 

buildSrc/src/main/java/org/springframework/boot/build/ConventionsPlugin.java

@@ -45,6 +45,8 @@ import org.gradle.api.tasks.compile.JavaCompile;
 import org.gradle.api.tasks.javadoc.Javadoc;
 import org.gradle.api.tasks.testing.Test;
 
+import org.springframework.boot.build.testing.TestFailuresPlugin;
+
 /**
  * Plugin to apply conventions to projects that are part of Spring Boot's build.
  * Conventions are applied in response to various plugins being applied.
@@ -55,7 +57,8 @@ import org.gradle.api.tasks.testing.Test;
  *
  * <ul>
  * <li>{@code sourceCompatibility} is set to {@code 1.8}
- * <li>Spring Java Format and Checkstyle plugins are applied
+ * <li>{@link SpringJavaFormatPlugin Spring Java Format}, {@link CheckstylePlugin
+ * Checkstyle}, and {@link TestFailuresPlugin Test Failures} plugins are applied
  * <li>{@link Test} tasks are configured to use JUnit Platform and use a max heap of 1024M
  * <li>{@link JavaCompile} tasks are configured to use UTF-8 encoding
  * <li>{@link Javadoc} tasks are configured to use UTF-8 encoding
@@ -103,6 +106,7 @@ public class ConventionsPlugin implements Plugin<Project> {
 
 	private void applyJavaConventions(Project project) {
 		project.getPlugins().withType(JavaPlugin.class, (java) -> {
+			project.getPlugins().apply(TestFailuresPlugin.class);
 			configureSpringJavaFormat(project);
 			project.setProperty("sourceCompatibility", "1.8");
 			project.getTasks().withType(JavaCompile.class, (compile) -> {

buildSrc/src/main/java/org/springframework/boot/build/testing/TestFailuresPlugin.java

@@ -0,0 +1,149 @@
+/*
+ * Copyright 2012-2020 the original author or authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.springframework.boot.build.testing;
+
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Map;
+import java.util.TreeMap;
+
+import org.gradle.BuildResult;
+import org.gradle.api.Plugin;
+import org.gradle.api.Project;
+import org.gradle.api.tasks.testing.Test;
+import org.gradle.api.tasks.testing.TestDescriptor;
+import org.gradle.api.tasks.testing.TestListener;
+import org.gradle.api.tasks.testing.TestResult;
+
+/**
+ * Plugin for recording test failures and reporting them at the end of the build.
+ *
+ * @author Andy Wilkinson
+ */
+public class TestFailuresPlugin implements Plugin<Project> {
+
+	@Override
+	public void apply(Project project) {
+		TestResultsExtension testResults = getOrCreateTestResults(project);
+		project.getTasks().withType(Test.class,
+				(test) -> test.addTestListener(new FailureRecordingTestListener(testResults, test)));
+	}
+
+	private TestResultsExtension getOrCreateTestResults(Project project) {
+		TestResultsExtension testResults = project.getRootProject().getExtensions()
+				.findByType(TestResultsExtension.class);
+		if (testResults == null) {
+			testResults = project.getRootProject().getExtensions().create("testResults", TestResultsExtension.class);
+			project.getRootProject().getGradle().buildFinished(testResults::buildFinished);
+		}
+		return testResults;
+	}
+
+	private final class FailureRecordingTestListener implements TestListener {
+
+		private List<TestFailure> failures = new ArrayList<>();
+
+		private final TestResultsExtension testResults;
+
+		private final Test test;
+
+		private FailureRecordingTestListener(TestResultsExtension testResults, Test test) {
+			this.testResults = testResults;
+			this.test = test;
+		}
+
+		@Override
+		public void afterSuite(TestDescriptor descriptor, TestResult result) {
+			if (!this.failures.isEmpty()) {
+				this.testResults.addFailures(this.test, this.failures);
+			}
+		}
+
+		@Override
+		public void afterTest(TestDescriptor descriptor, TestResult result) {
+			if (result.getFailedTestCount() > 0) {
+				this.failures.add(new TestFailure(descriptor, result.getExceptions()));
+			}
+		}
+
+		@Override
+		public void beforeSuite(TestDescriptor descriptor) {
+
+		}
+
+		@Override
+		public void beforeTest(TestDescriptor descriptor) {
+
+		}
+
+	}
+
+	private static final class TestFailure implements Comparable<TestFailure> {
+
+		private final TestDescriptor descriptor;
+
+		private final List<Throwable> exceptions;
+
+		private TestFailure(TestDescriptor descriptor, List<Throwable> exceptions) {
+			this.descriptor = descriptor;
+			this.exceptions = exceptions;
+		}
+
+		@Override
+		public int compareTo(TestFailure other) {
+			int comparison = this.descriptor.getClassName().compareTo(other.descriptor.getClassName());
+			if (comparison == 0) {
+				comparison = this.descriptor.getName().compareTo(other.descriptor.getClassName());
+			}
+			return comparison;
+		}
+
+	}
+
+	public static class TestResultsExtension {
+
+		private final Map<Test, List<TestFailure>> testFailures = new TreeMap<>(
+				(one, two) -> one.getPath().compareTo(two.getPath()));
+
+		private final Object monitor = new Object();
+
+		void addFailures(Test test, List<TestFailure> testFailures) {
+			synchronized (this.monitor) {
+				this.testFailures.put(test, testFailures);
+			}
+		}
+
+		public void buildFinished(BuildResult result) {
+			synchronized (this.monitor) {
+				if (this.testFailures.isEmpty()) {
+					return;
+				}
+				System.err.println();
+				System.err.println("Found test failures in " + this.testFailures.size() + " test task"
+						+ ((this.testFailures.size() == 1) ? ":" : "s:"));
+				this.testFailures.forEach((task, failures) -> {
+					System.err.println();
+					System.err.println(task.getPath());
+					failures.forEach((failure) -> System.err.println(
+							"    " + failure.descriptor.getClassName() + " > " + failure.descriptor.getName()));
+				});
+			}
+		}
+
+	}
+
+}

buildSrc/src/test/java/org/springframework/boot/build/testing/TestFailuresPluginIntegrationTests.java

@@ -0,0 +1,185 @@
+/*
+ * Copyright 2012-2020 the original author or authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.springframework.boot.build.testing;
+
+import java.io.BufferedReader;
+import java.io.File;
+import java.io.FileWriter;
+import java.io.IOException;
+import java.io.PrintWriter;
+import java.io.StringReader;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.function.Consumer;
+
+import org.gradle.testkit.runner.BuildResult;
+import org.gradle.testkit.runner.GradleRunner;
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.io.TempDir;
+
+import static org.assertj.core.api.Assertions.assertThat;
+
+/**
+ * Integrations tests for {@link TestFailuresPlugin}.
+ *
+ * @author Andy Wilkinson
+ */
+class TestFailuresPluginIntegrationTests {
+
+	private File projectDir;
+
+	@BeforeEach
+	void setup(@TempDir File projectDir) throws IOException {
+		this.projectDir = projectDir;
+	}
+
+	@Test
+	void singleProject() throws IOException {
+		createProject(this.projectDir);
+		BuildResult result = GradleRunner.create().withDebug(true).withProjectDir(this.projectDir)
+				.withArguments("build").withPluginClasspath().buildAndFail();
+		assertThat(readLines(result.getOutput())).containsSequence("Found test failures in 1 test task:", "", ":test",
+				"    example.ExampleTests > bad()", "    example.ExampleTests > fail()",
+				"    example.MoreTests > bad()", "    example.MoreTests > fail()", "");
+	}
+
+	@Test
+	void multiProject() throws IOException {
+		createMultiProjectBuild();
+		BuildResult result = GradleRunner.create().withDebug(true).withProjectDir(this.projectDir)
+				.withArguments("build").withPluginClasspath().buildAndFail();
+		assertThat(readLines(result.getOutput())).containsSequence("Found test failures in 1 test task:", "",
+				":project-one:test", "    example.ExampleTests > bad()", "    example.ExampleTests > fail()",
+				"    example.MoreTests > bad()", "    example.MoreTests > fail()", "");
+	}
+
+	@Test
+	void multiProjectContinue() throws IOException {
+		createMultiProjectBuild();
+		BuildResult result = GradleRunner.create().withDebug(true).withProjectDir(this.projectDir)
+				.withArguments("build", "--continue").withPluginClasspath().buildAndFail();
+		assertThat(readLines(result.getOutput())).containsSequence("Found test failures in 2 test tasks:", "",
+				":project-one:test", "    example.ExampleTests > bad()", "    example.ExampleTests > fail()",
+				"    example.MoreTests > bad()", "    example.MoreTests > fail()", "", ":project-two:test",
+				"    example.ExampleTests > bad()", "    example.ExampleTests > fail()",
+				"    example.MoreTests > bad()", "    example.MoreTests > fail()", "");
+	}
+
+	@Test
+	void multiProjectParallel() throws IOException {
+		createMultiProjectBuild();
+		BuildResult result = GradleRunner.create().withDebug(true).withProjectDir(this.projectDir)
+				.withArguments("build", "--parallel").withPluginClasspath().buildAndFail();
+		assertThat(readLines(result.getOutput())).containsSequence("Found test failures in 2 test tasks:", "",
+				":project-one:test", "    example.ExampleTests > bad()", "    example.ExampleTests > fail()",
+				"    example.MoreTests > bad()", "    example.MoreTests > fail()", "", ":project-two:test",
+				"    example.ExampleTests > bad()", "    example.ExampleTests > fail()",
+				"    example.MoreTests > bad()", "    example.MoreTests > fail()", "");
+	}
+
+	private void createProject(File dir) {
+		File examplePackage = new File(dir, "src/test/java/example");
+		examplePackage.mkdirs();
+		createTestSource("ExampleTests", examplePackage);
+		createTestSource("MoreTests", examplePackage);
+		createBuildScript(dir);
+	}
+
+	private void createMultiProjectBuild() {
+		createProject(new File(this.projectDir, "project-one"));
+		createProject(new File(this.projectDir, "project-two"));
+		withPrintWriter(new File(this.projectDir, "settings.gradle"), (writer) -> {
+			writer.println("include 'project-one'");
+			writer.println("include 'project-two'");
+		});
+	}
+
+	private void createTestSource(String name, File dir) {
+		withPrintWriter(new File(dir, name + ".java"), (writer) -> {
+			writer.println("package example;");
+			writer.println();
+			writer.println("import org.junit.jupiter.api.Test;");
+			writer.println();
+			writer.println("import static org.assertj.core.api.Assertions.assertThat;");
+			writer.println();
+			writer.println("class " + name + "{");
+			writer.println();
+			writer.println("	@Test");
+			writer.println("	void fail() {");
+			writer.println("		assertThat(true).isFalse();");
+			writer.println("	}");
+			writer.println();
+			writer.println("	@Test");
+			writer.println("	void bad() {");
+			writer.println("		assertThat(5).isLessThan(4);");
+			writer.println("	}");
+			writer.println();
+			writer.println("	@Test");
+			writer.println("	void ok() {");
+			writer.println("	}");
+			writer.println();
+			writer.println("}");
+		});
+	}
+
+	private void createBuildScript(File dir) {
+		withPrintWriter(new File(dir, "build.gradle"), (writer) -> {
+			writer.println("plugins {");
+			writer.println("	id 'java'");
+			writer.println("	id 'org.springframework.boot.test-failures'");
+			writer.println("}");
+			writer.println();
+			writer.println("repositories {");
+			writer.println("	mavenCentral()");
+			writer.println("}");
+			writer.println();
+			writer.println("dependencies {");
+			writer.println("	testImplementation 'org.junit.jupiter:junit-jupiter:5.5.2'");
+			writer.println("	testImplementation 'org.assertj:assertj-core:3.11.1'");
+			writer.println("}");
+			writer.println();
+			writer.println("test {");
+			writer.println("	useJUnitPlatform()");
+			writer.println("}");
+		});
+	}
+
+	private void withPrintWriter(File file, Consumer<PrintWriter> consumer) {
+		try (PrintWriter writer = new PrintWriter(new FileWriter(file))) {
+			consumer.accept(writer);
+		}
+		catch (IOException ex) {
+			throw new RuntimeException(ex);
+		}
+	}
+
+	private List<String> readLines(String output) {
+		List<String> lines = new ArrayList<>();
+		try (BufferedReader reader = new BufferedReader(new StringReader(output))) {
+			String line;
+			while ((line = reader.readLine()) != null) {
+				lines.add(line);
+			}
+		}
+		catch (IOException ex) {
+			throw new RuntimeException(ex);
+		}
+		return lines;
+	}
+
+}