diff --git a/surefire-its/src/test/java/org/apache/maven/surefire/its/JUnitPlatformEnginesIT.java b/surefire-its/src/test/java/org/apache/maven/surefire/its/JUnitPlatformEnginesIT.java index ba79ccb9ff..255aea2918 100644 --- a/surefire-its/src/test/java/org/apache/maven/surefire/its/JUnitPlatformEnginesIT.java +++ b/surefire-its/src/test/java/org/apache/maven/surefire/its/JUnitPlatformEnginesIT.java @@ -296,6 +296,45 @@ public void testJupiterEngineWithFailureInTestTemplateProvider() { .assertContainsText("Encountered failure in TestTemplate provideTestTemplateInvocationContexts()"); } + @Test + public void testJupiterEngineWithTestTemplateNotClassifiedAsFlake() { + unpack("junit5-testtemplate-bug", "-" + jupiter) + .setTestToRun("FieldSettingTest") + .sysProp("junit5.version", jupiter) + .maven() + .withFailure() + .executeTest() + .verifyTextInLog("AssertionFailedError") + .assertTestSuiteResults(2, 0, 1, 0, 0); + + unpack("junit5-testtemplate-bug", "-" + jupiter) + .debugLogging() + .setTestToRun("FieldSettingTest") + .sysProp("junit5.version", jupiter) + // The tests are failing deterministically, so rerunning them should not change the result + .sysProp("surefire.rerunFailingTestsCount", "1") + .maven() + .withFailure() + .executeTest() + .verifyTextInLog("AssertionFailedError") + .assertTestSuiteResults(2, 0, 1, 0, 0); + } + + @Test + public void testJupiterEngineWithParameterizedTestsNotClassifiedAsFlake() { + unpack("junit5-testtemplate-bug", "-" + jupiter) + .debugLogging() + .setTestToRun("ParamsContextTest") + .sysProp("junit5.version", jupiter) + // The tests are failing deterministically, so rerunning them should not change the result + .sysProp("surefire.rerunFailingTestsCount", "1") + .maven() + .withFailure() + .executeTest() + .verifyTextInLog("AssertionFailedError") + .assertTestSuiteResults(2, 0, 1, 0, 0); + } + @Test public void testJupiterEngineWithAssertionsFailNoParameters() { // `Assertions.fail()` not supported until 5.2.0 diff --git a/surefire-its/src/test/resources/junit5-testtemplate-bug/pom.xml b/surefire-its/src/test/resources/junit5-testtemplate-bug/pom.xml new file mode 100644 index 0000000000..335ba72b53 --- /dev/null +++ b/surefire-its/src/test/resources/junit5-testtemplate-bug/pom.xml @@ -0,0 +1,55 @@ + + + 4.0.0 + + org.apache.maven.plugins.surefire + surefire-junit-testtemplate-retry-bug + 1.0-SNAPSHOT + Test for JUnit 5 TestTemplate with retries + + + ${java.specification.version} + ${java.specification.version} + UTF-8 + + + + + org.junit.jupiter + junit-jupiter-engine + ${junit5.version} + test + + + org.junit.jupiter + junit-jupiter-api + ${junit5.version} + test + + + org.junit.jupiter + junit-jupiter-params + ${junit5.version} + test + + + + + + maven-compiler-plugin + 3.8.1 + + UTF-8 + + + + org.apache.maven.plugins + maven-surefire-plugin + ${surefire.version} + + + + + diff --git a/surefire-its/src/test/resources/junit5-testtemplate-bug/src/test/java/pkg/FieldSettingTest.java b/surefire-its/src/test/resources/junit5-testtemplate-bug/src/test/java/pkg/FieldSettingTest.java new file mode 100644 index 0000000000..61f5fa5c52 --- /dev/null +++ b/surefire-its/src/test/resources/junit5-testtemplate-bug/src/test/java/pkg/FieldSettingTest.java @@ -0,0 +1,63 @@ +package pkg; + +import org.junit.jupiter.api.TestTemplate; +import org.junit.jupiter.api.extension.BeforeEachCallback; +import org.junit.jupiter.api.extension.ExtendWith; +import org.junit.jupiter.api.extension.Extension; +import org.junit.jupiter.api.extension.ExtensionContext; +import org.junit.jupiter.api.extension.TestTemplateInvocationContext; +import org.junit.jupiter.api.extension.TestTemplateInvocationContextProvider; + +import java.util.Arrays; +import java.util.List; +import java.util.stream.Stream; + +import static org.junit.jupiter.api.Assertions.assertEquals; + +public class FieldSettingTest { + private int testValue = 42; + + // We're calling this in the provider underneath + public void setTestValue(int testValue) { + this.testValue = testValue; + } + + @TestTemplate + @ExtendWith(FieldSettingContextProvider.class) + public void testTemplatePartiallyFails() { + assertEquals(42, testValue); + } +} + + +class FieldSettingContextProvider implements TestTemplateInvocationContextProvider { + @Override + public boolean supportsTestTemplate(ExtensionContext extensionContext) { + return true; + } + + @Override + public Stream provideTestTemplateInvocationContexts(ExtensionContext extensionContext) { + return Stream.of(context(0), context(42)); + } + + private TestTemplateInvocationContext context(int parameter) { + return new TestTemplateInvocationContext() { + @Override + public String getDisplayName(int invocationIndex) { + return String.format("[%d] %s", invocationIndex, parameter); + } + + @Override + public List getAdditionalExtensions() { + return getBeforeEachCallbacks(parameter); + } + }; + } + + private List getBeforeEachCallbacks(int value) { + return Arrays.asList((BeforeEachCallback) ctx -> + ((FieldSettingTest) ctx.getRequiredTestInstance()).setTestValue(value) + ); + } +} diff --git a/surefire-its/src/test/resources/junit5-testtemplate-bug/src/test/java/pkg/ParamsContextTest.java b/surefire-its/src/test/resources/junit5-testtemplate-bug/src/test/java/pkg/ParamsContextTest.java new file mode 100644 index 0000000000..4022e2cbb1 --- /dev/null +++ b/surefire-its/src/test/resources/junit5-testtemplate-bug/src/test/java/pkg/ParamsContextTest.java @@ -0,0 +1,62 @@ +package pkg; + +import org.junit.jupiter.api.TestTemplate; +import org.junit.jupiter.api.extension.ExtendWith; +import org.junit.jupiter.api.extension.Extension; +import org.junit.jupiter.api.extension.ExtensionContext; +import org.junit.jupiter.api.extension.ParameterContext; +import org.junit.jupiter.api.extension.ParameterResolver; +import org.junit.jupiter.api.extension.TestTemplateInvocationContext; +import org.junit.jupiter.api.extension.TestTemplateInvocationContextProvider; + +import java.util.Collections; +import java.util.List; +import java.util.stream.Stream; + +import static org.junit.jupiter.api.Assertions.assertEquals; + +public class ParamsContextTest { + + @TestTemplate + @ExtendWith(ParamsContextProvider.class) + void testTemplatePartiallyFails(int value) { + assertEquals(42, value); + } +} + +class ParamsContextProvider implements TestTemplateInvocationContextProvider { + + @Override + public boolean supportsTestTemplate(ExtensionContext context) { + return true; + } + + @Override + public Stream provideTestTemplateInvocationContexts(ExtensionContext context) { + return Stream.of(invocationContext(0), invocationContext(42)); + } + + private TestTemplateInvocationContext invocationContext(int parameter) { + return new TestTemplateInvocationContext() { + @Override + public String getDisplayName(int invocationIndex) { + return String.format("[%d] %s", invocationIndex, parameter); + } + + @Override + public List getAdditionalExtensions() { + return Collections.singletonList(new ParameterResolver() { + @Override + public boolean supportsParameter(ParameterContext parameterContext, ExtensionContext extensionContext) { + return true; + } + + @Override + public Object resolveParameter(ParameterContext parameterContext, ExtensionContext extensionContext) { + return parameter; + } + }); + } + }; + } +} diff --git a/surefire-providers/surefire-junit-platform/src/main/java/org/apache/maven/surefire/junitplatform/RunListenerAdapter.java b/surefire-providers/surefire-junit-platform/src/main/java/org/apache/maven/surefire/junitplatform/RunListenerAdapter.java index 8c274fd1c9..7ae8cc79d3 100644 --- a/surefire-providers/surefire-junit-platform/src/main/java/org/apache/maven/surefire/junitplatform/RunListenerAdapter.java +++ b/surefire-providers/surefire-junit-platform/src/main/java/org/apache/maven/surefire/junitplatform/RunListenerAdapter.java @@ -317,12 +317,14 @@ private String[] toClassMethodName(TestIdentifier testIdentifier) { boolean needsSpaceSeparator = isNotBlank(parentDisplay) && !display.startsWith("["); String methodDisplay = parentDisplay + (needsSpaceSeparator ? " " : "") + display; + boolean isParameterized = isNotBlank(methodSource.getMethodParameterTypes()); boolean hasParameterizedParent = collectAllTestIdentifiersInHierarchy(testIdentifier) .filter(identifier -> !identifier.getSource().isPresent()) .map(TestIdentifier::getLegacyReportingName) .anyMatch(legacyReportingName -> legacyReportingName.matches("^\\[.+]$")); + boolean isTestTemplate = testIdentifier.getLegacyReportingName().matches("^.*\\[\\d+]$"); - boolean parameterized = isNotBlank(methodSource.getMethodParameterTypes()) || hasParameterizedParent; + boolean parameterized = isParameterized || hasParameterizedParent || isTestTemplate; String methodName = methodSource.getMethodName(); String description = testIdentifier.getLegacyReportingName(); boolean equalDescriptions = methodDisplay.equals(description);