Synthesize start/finished events for reported failures

marcphilipp · marcphilipp · commit f9c509d3616c · 2019-04-07T16:44:00.000+02:00
When Spock data-provider preparation fails early, it only calls `RunListener.testFailure` but not `testFinished` which caused the Vintage engine to never report the failure. Now, when `testFailure` is called a `testStarted` event is synthesized unless it was already reported. The existing handling of synthetic start events then reports it as finished. Fixes #1845.
diff --git a/junit-vintage-engine/src/main/java/org/junit/vintage/engine/execution/RunListenerAdapter.java b/junit-vintage-engine/src/main/java/org/junit/vintage/engine/execution/RunListenerAdapter.java
@@ -124,16 +124,11 @@ private void handleFailure(Failure failure, Function<Throwable, TestExecutionRes
 			TestDescriptor testDescriptor) {
 		TestExecutionResult result = resultCreator.apply(failure.getException());
 		testRun.storeResult(testDescriptor, result);
-		if (testDescriptor.isContainer() && testRun.isDescendantOfRunnerTestDescriptor(testDescriptor)) {
-			fireMissingContainerEvents(testDescriptor);
-		}
-	}
-
-	private void fireMissingContainerEvents(TestDescriptor testDescriptor) {
 		if (testRun.isNotStarted(testDescriptor)) {
 			testStarted(testDescriptor, EventType.SYNTHETIC);
 		}
-		if (testRun.isNotFinished(testDescriptor)) {
+		if (testRun.isNotFinished(testDescriptor) && testDescriptor.isContainer()
+				&& testRun.isDescendantOfRunnerTestDescriptor(testDescriptor)) {
 			testFinished(testDescriptor);
 		}
 	}
diff --git a/junit-vintage-engine/src/test/java/org/junit/vintage/engine/VintageTestEngineExecutionTests.java b/junit-vintage-engine/src/test/java/org/junit/vintage/engine/VintageTestEngineExecutionTests.java
@@ -61,12 +61,14 @@
 import org.junit.vintage.engine.samples.junit4.JUnit4SuiteWithJUnit3SuiteWithSingleTestCase;
 import org.junit.vintage.engine.samples.junit4.JUnit4SuiteWithJUnit4TestCaseWithAssumptionFailureInBeforeClass;
 import org.junit.vintage.engine.samples.junit4.JUnit4SuiteWithJUnit4TestCaseWithErrorInBeforeClass;
+import org.junit.vintage.engine.samples.junit4.JUnit4SuiteWithJUnit4TestCaseWithFailingDescriptionThatIsNotReportedAsFinished;
 import org.junit.vintage.engine.samples.junit4.JUnit4SuiteWithPlainJUnit4TestCaseWithSingleTestWhichIsIgnored;
 import org.junit.vintage.engine.samples.junit4.JUnit4TestCaseWithAssumptionFailureInBeforeClass;
 import org.junit.vintage.engine.samples.junit4.JUnit4TestCaseWithErrorCollectorStoringMultipleFailures;
 import org.junit.vintage.engine.samples.junit4.JUnit4TestCaseWithErrorInAfterClass;
 import org.junit.vintage.engine.samples.junit4.JUnit4TestCaseWithErrorInBeforeClass;
 import org.junit.vintage.engine.samples.junit4.JUnit4TestCaseWithExceptionThrowingRunner;
+import org.junit.vintage.engine.samples.junit4.JUnit4TestCaseWithFailingDescriptionThatIsNotReportedAsFinished;
 import org.junit.vintage.engine.samples.junit4.JUnit4TestCaseWithOverloadedMethod;
 import org.junit.vintage.engine.samples.junit4.JUnit4TestCaseWithRunnerWithCustomUniqueIds;
 import org.junit.vintage.engine.samples.junit4.MalformedJUnit4TestCase;
@@ -624,6 +626,40 @@ void executesJUnit4TestCaseWithErrorCollectorStoringMultipleFailures() {
 			event(engine(), finishedSuccessfully()));
 	}
 
+	@Test
+	void executesJUnit4TestCaseWithFailingDescriptionThatIsNotReportedAsFinished() {
+		Class<?> testClass = JUnit4TestCaseWithFailingDescriptionThatIsNotReportedAsFinished.class;
+
+		execute(testClass).assertEventsMatchExactly( //
+			event(engine(), started()), //
+			event(container(testClass), started()), //
+			event(test("testWithMissingEvents"), started()), //
+			event(test("testWithMissingEvents"), finishedWithFailure()), //
+			event(container(testClass), finishedSuccessfully()), //
+			event(engine(), finishedSuccessfully()));
+	}
+
+	@Test
+	void executesJUnit4SuiteWithJUnit4TestCaseWithFailingDescriptionThatIsNotReportedAsFinished() {
+		Class<?> suiteClass = JUnit4SuiteWithJUnit4TestCaseWithFailingDescriptionThatIsNotReportedAsFinished.class;
+		Class<?> firstTestClass = JUnit4TestCaseWithFailingDescriptionThatIsNotReportedAsFinished.class;
+		Class<?> secondTestClass = PlainJUnit4TestCaseWithSingleTestWhichFails.class;
+
+		execute(suiteClass).assertEventsMatchExactly( //
+			event(engine(), started()), //
+			event(container(suiteClass), started()), //
+			event(container(firstTestClass), started()), //
+			event(test("testWithMissingEvents"), started()), //
+			event(test("testWithMissingEvents"), finishedWithFailure()), //
+			event(container(firstTestClass), finishedSuccessfully()), //
+			event(container(secondTestClass), started()), //
+			event(test("failingTest"), started()), //
+			event(test("failingTest"), finishedWithFailure()), //
+			event(container(secondTestClass), finishedSuccessfully()), //
+			event(container(suiteClass), finishedSuccessfully()), //
+			event(engine(), finishedSuccessfully()));
+	}
+
 	private static Events execute(Class<?> testClass) {
 		return EngineTestKit.execute(new VintageTestEngine(), request(testClass)).all();
 	}
diff --git a/junit-vintage-engine/src/test/java/org/junit/vintage/engine/samples/junit4/JUnit4SuiteWithJUnit4TestCaseWithFailingDescriptionThatIsNotReportedAsFinished.java b/junit-vintage-engine/src/test/java/org/junit/vintage/engine/samples/junit4/JUnit4SuiteWithJUnit4TestCaseWithFailingDescriptionThatIsNotReportedAsFinished.java
@@ -0,0 +1,21 @@
+/*
+ * Copyright 2015-2019 the original author or authors.
+ *
+ * All rights reserved. This program and the accompanying materials are
+ * made available under the terms of the Eclipse Public License v2.0 which
+ * accompanies this distribution and is available at
+ *
+ * http://www.eclipse.org/legal/epl-v20.html
+ */
+
+package org.junit.vintage.engine.samples.junit4;
+
+import org.junit.runner.RunWith;
+import org.junit.runners.Suite;
+import org.junit.runners.Suite.SuiteClasses;
+
+@RunWith(Suite.class)
+@SuiteClasses({ JUnit4TestCaseWithFailingDescriptionThatIsNotReportedAsFinished.class,
+		PlainJUnit4TestCaseWithSingleTestWhichFails.class })
+public class JUnit4SuiteWithJUnit4TestCaseWithFailingDescriptionThatIsNotReportedAsFinished {
+}
diff --git a/junit-vintage-engine/src/test/java/org/junit/vintage/engine/samples/junit4/JUnit4TestCaseWithFailingDescriptionThatIsNotReportedAsFinished.java b/junit-vintage-engine/src/test/java/org/junit/vintage/engine/samples/junit4/JUnit4TestCaseWithFailingDescriptionThatIsNotReportedAsFinished.java
@@ -0,0 +1,24 @@
+/*
+ * Copyright 2015-2019 the original author or authors.
+ *
+ * All rights reserved. This program and the accompanying materials are
+ * made available under the terms of the Eclipse Public License v2.0 which
+ * accompanies this distribution and is available at
+ *
+ * http://www.eclipse.org/legal/epl-v20.html
+ */
+
+package org.junit.vintage.engine.samples.junit4;
+
+import static org.junit.Assert.fail;
+
+import org.junit.Test;
+import org.junit.runner.RunWith;
+
+@RunWith(RunnerThatOnlyReportsFailures.class)
+public class JUnit4TestCaseWithFailingDescriptionThatIsNotReportedAsFinished {
+	@Test
+	public void testWithMissingEvents() {
+		fail("boom");
+	}
+}
diff --git a/junit-vintage-engine/src/test/java/org/junit/vintage/engine/samples/junit4/RunnerThatOnlyReportsFailures.java b/junit-vintage-engine/src/test/java/org/junit/vintage/engine/samples/junit4/RunnerThatOnlyReportsFailures.java
@@ -0,0 +1,35 @@
+/*
+ * Copyright 2015-2019 the original author or authors.
+ *
+ * All rights reserved. This program and the accompanying materials are
+ * made available under the terms of the Eclipse Public License v2.0 which
+ * accompanies this distribution and is available at
+ *
+ * http://www.eclipse.org/legal/epl-v20.html
+ */
+
+package org.junit.vintage.engine.samples.junit4;
+
+import org.junit.runner.notification.Failure;
+import org.junit.runner.notification.RunNotifier;
+import org.junit.runners.BlockJUnit4ClassRunner;
+import org.junit.runners.model.FrameworkMethod;
+import org.junit.runners.model.InitializationError;
+import org.junit.runners.model.Statement;
+
+public class RunnerThatOnlyReportsFailures extends BlockJUnit4ClassRunner {
+	public RunnerThatOnlyReportsFailures(Class<?> klass) throws InitializationError {
+		super(klass);
+	}
+
+	@Override
+	protected void runChild(FrameworkMethod method, RunNotifier notifier) {
+		Statement statement = methodBlock(method);
+		try {
+			statement.evaluate();
+		}
+		catch (Throwable e) {
+			notifier.fireTestFailure(new Failure(describeChild(method), e));
+		}
+	}
+}

Original file line number	Diff line number	Diff line change
`@@ -124,16 +124,11 @@ private void handleFailure(Failure failure, Function<Throwable, TestExecutionRes`
`124`	`124`	`TestDescriptor testDescriptor) {`
`125`	`125`	`TestExecutionResult result = resultCreator.apply(failure.getException());`
`126`	`126`	`testRun.storeResult(testDescriptor, result);`
`127`		`- if (testDescriptor.isContainer() && testRun.isDescendantOfRunnerTestDescriptor(testDescriptor)) {`
`128`		`- fireMissingContainerEvents(testDescriptor);`
`129`		`- }`
`130`		`- }`
`131`		`-`
`132`		`- private void fireMissingContainerEvents(TestDescriptor testDescriptor) {`
`133`	`127`	`if (testRun.isNotStarted(testDescriptor)) {`
`134`	`128`	`testStarted(testDescriptor, EventType.SYNTHETIC);`
`135`	`129`	`}`
`136`		`- if (testRun.isNotFinished(testDescriptor)) {`
	`130`	`+ if (testRun.isNotFinished(testDescriptor) && testDescriptor.isContainer()`
	`131`	`+ && testRun.isDescendantOfRunnerTestDescriptor(testDescriptor)) {`
`137`	`132`	`testFinished(testDescriptor);`
`138`	`133`	`}`
`139`	`134`	`}`