diff --git a/java-reporter-core/pom.xml b/java-reporter-core/pom.xml index 4ac28c8..6c910c6 100644 --- a/java-reporter-core/pom.xml +++ b/java-reporter-core/pom.xml @@ -7,7 +7,7 @@ io.testomat java-reporter-core - 0.18.1 + 0.18.2 jar Testomat.io Reporter Core diff --git a/java-reporter-core/src/main/java/io/testomat/core/constants/CommonConstants.java b/java-reporter-core/src/main/java/io/testomat/core/constants/CommonConstants.java index 2d4c9f2..773c3c6 100644 --- a/java-reporter-core/src/main/java/io/testomat/core/constants/CommonConstants.java +++ b/java-reporter-core/src/main/java/io/testomat/core/constants/CommonConstants.java @@ -1,7 +1,7 @@ package io.testomat.core.constants; public class CommonConstants { - public static final String REPORTER_VERSION = "0.18.1"; + public static final String REPORTER_VERSION = "0.18.2"; public static final String TESTS_STRING = "tests"; public static final String API_KEY_STRING = "api_key"; diff --git a/java-reporter-cucumber/pom.xml b/java-reporter-cucumber/pom.xml index 0ed0d30..9546d87 100644 --- a/java-reporter-cucumber/pom.xml +++ b/java-reporter-cucumber/pom.xml @@ -6,7 +6,7 @@ io.testomat java-reporter-cucumber - 0.9.1 + 0.9.2 jar Testomat.io Java Reporter Cucumber @@ -51,7 +51,7 @@ io.testomat java-reporter-core - 0.18.1 + 0.18.2 org.slf4j diff --git a/java-reporter-junit/pom.xml b/java-reporter-junit/pom.xml index 9374d2b..d3e7bfa 100644 --- a/java-reporter-junit/pom.xml +++ b/java-reporter-junit/pom.xml @@ -6,7 +6,7 @@ io.testomat java-reporter-junit - 0.9.5 + 0.9.6 jar Testomat.io Java Reporter JUnit @@ -51,7 +51,7 @@ io.testomat java-reporter-core - 0.18.1 + 0.18.2 org.slf4j diff --git a/java-reporter-karate/pom.xml b/java-reporter-karate/pom.xml index 747108d..c11faeb 100644 --- a/java-reporter-karate/pom.xml +++ b/java-reporter-karate/pom.xml @@ -6,7 +6,7 @@ io.testomat java-reporter-karate - 0.3.5 + 0.3.6 jar Testomat.io Java Reporter Karate @@ -52,7 +52,7 @@ io.testomat java-reporter-core - 0.18.1 + 0.18.2 io.karatelabs diff --git a/java-reporter-testng/pom.xml b/java-reporter-testng/pom.xml index 309a2aa..f00c94a 100644 --- a/java-reporter-testng/pom.xml +++ b/java-reporter-testng/pom.xml @@ -6,7 +6,7 @@ io.testomat java-reporter-testng - 0.8.5 + 0.8.6 jar Testomat.io Java Reporter TestNG @@ -47,7 +47,7 @@ io.testomat java-reporter-core - 0.18.1 + 0.18.2 org.slf4j diff --git a/java-reporter-testng/src/main/java/io/testomat/testng/listener/AbstractHooksContainer.java b/java-reporter-testng/src/main/java/io/testomat/testng/listener/AbstractHooksContainer.java index c065273..bb6a865 100644 --- a/java-reporter-testng/src/main/java/io/testomat/testng/listener/AbstractHooksContainer.java +++ b/java-reporter-testng/src/main/java/io/testomat/testng/listener/AbstractHooksContainer.java @@ -4,12 +4,17 @@ import java.util.Collections; import java.util.List; import java.util.ServiceLoader; +import java.util.function.Consumer; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; import org.testng.IInvokedMethod; import org.testng.ISuite; import org.testng.ITestResult; public abstract class AbstractHooksContainer { + private static final Logger log = LoggerFactory.getLogger(AbstractHooksContainer.class); + private static final List HOOKS; static { @@ -20,28 +25,24 @@ public abstract class AbstractHooksContainer { HOOKS = Collections.unmodifiableList(hooks); } - protected void onSuiteStartHookAfterExecution(ISuite suite) { - for (TestomatHook hook : HOOKS) { - hook.onSuiteStartHookAfterExecution(suite); - } + protected boolean onSuiteStartHookAfterExecution(ISuite suite) { + return runHooks("onSuiteStartHookAfterExecution", + hook -> hook.onSuiteStartHookAfterExecution(suite)); } - protected void onSuiteStartHookBeforeExecution(ISuite suite) { - for (TestomatHook hook : HOOKS) { - hook.onSuiteStartHookBeforeExecution(suite); - } + protected boolean onSuiteStartHookBeforeExecution(ISuite suite) { + return runHooks("onSuiteStartHookBeforeExecution", + hook -> hook.onSuiteStartHookBeforeExecution(suite)); } protected void onSuiteFinishHookAfterExecution(ISuite suite) { - for (TestomatHook hook : HOOKS) { - hook.onSuiteFinishHookAfterExecution(suite); - } + runHooks("onSuiteFinishHookAfterExecution", + hook -> hook.onSuiteFinishHookAfterExecution(suite)); } protected void onSuiteFinishHookBeforeExecution(ISuite suite) { - for (TestomatHook hook : HOOKS) { - hook.onSuiteFinishHookBeforeExecution(suite); - } + runHooks("onSuiteFinishHookBeforeExecution", + hook -> hook.onSuiteFinishHookBeforeExecution(suite)); } protected void onTestSuccessHookAfterExecution(ITestResult result) { @@ -143,4 +144,17 @@ protected void onExecutionFinishHookBeforeExecution() { hook.onExecutionFinishHookBeforeExecution(); } } + + private boolean runHooks(String hookName, Consumer action) { + boolean failed = false; + for (TestomatHook hook : HOOKS) { + try { + action.accept(hook); + } catch (Exception e) { + log.error("Hook '{}' failed: {}", hookName, e.getMessage(), e); + failed = true; + } + } + return failed; + } } diff --git a/java-reporter-testng/src/main/java/io/testomat/testng/listener/TestNgListener.java b/java-reporter-testng/src/main/java/io/testomat/testng/listener/TestNgListener.java index 123b8e7..127e152 100644 --- a/java-reporter-testng/src/main/java/io/testomat/testng/listener/TestNgListener.java +++ b/java-reporter-testng/src/main/java/io/testomat/testng/listener/TestNgListener.java @@ -88,11 +88,15 @@ public final void onStart(ISuite suite) { if (!isListeningRequired()) { return; } - onSuiteStartHookBeforeExecution(suite); - log.debug("Suite started: {}", suite.getName()); + runManager.incrementSuiteCounter(); + boolean failed = onSuiteStartHookBeforeExecution(suite); + log.debug("Suite started: {}", suite.getName()); reporter.reportTestResult(suite); - onSuiteStartHookAfterExecution(suite); + failed |= onSuiteStartHookAfterExecution(suite); + if (failed) { + suite.getSuiteState().failed(); + } } @Override @@ -100,9 +104,9 @@ public final void onFinish(ISuite suite) { if (!isListeningRequired()) { return; } + runManager.decrementSuiteCounter(); onSuiteFinishHookBeforeExecution(suite); log.debug("Suite finished: {}", suite.getName()); - runManager.decrementSuiteCounter(); onSuiteFinishHookAfterExecution(suite); } diff --git a/java-reporter-testng/src/main/java/io/testomat/testng/reporter/TestNgTestResultReporter.java b/java-reporter-testng/src/main/java/io/testomat/testng/reporter/TestNgTestResultReporter.java index 920c177..a2c7a8f 100644 --- a/java-reporter-testng/src/main/java/io/testomat/testng/reporter/TestNgTestResultReporter.java +++ b/java-reporter-testng/src/main/java/io/testomat/testng/reporter/TestNgTestResultReporter.java @@ -15,10 +15,11 @@ import io.testomat.testng.extractor.TestNgParameterExtractor; import io.testomat.testng.extractor.TestNgTestWrapper; import java.lang.reflect.Method; -import java.util.ArrayList; import java.util.HashSet; -import java.util.List; +import java.util.Map; import java.util.Set; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.atomic.AtomicInteger; import org.testng.ISuite; import org.testng.ITestResult; import org.testng.annotations.Test; @@ -35,16 +36,16 @@ public class TestNgTestResultReporter { private final TestNgMetaDataExtractor metaDataExtractor; private final TestNgParameterExtractor parameterExtractor; private final GlobalRunManager runManager; - private final List processedTests; - private final Set testIds; + private final Set processedTests; + private final Map testStates; public TestNgTestResultReporter() { this.resultConstructor = new TestNgTestResultConstructor(); this.metaDataExtractor = new TestNgMetaDataExtractor(); this.parameterExtractor = new TestNgParameterExtractor(); this.runManager = GlobalRunManager.getInstance(); - this.processedTests = new ArrayList<>(); - this.testIds = new HashSet<>(); + this.processedTests = new HashSet<>(); + this.testStates = new ConcurrentHashMap<>(); } /** @@ -58,8 +59,8 @@ public TestNgTestResultReporter(TestNgTestResultConstructor resultConstructor, this.metaDataExtractor = metaDataExtractor; this.parameterExtractor = parameterExtractor; this.runManager = runManager; - this.testIds = new HashSet<>(); - this.processedTests = new ArrayList<>(); + this.processedTests = new HashSet<>(); + this.testStates = new ConcurrentHashMap<>(); } /** @@ -71,26 +72,16 @@ public void reportTestResult(ITestResult result, String status) { return; } - String baseKey = result.getTestClass().getName() - + "." - + result.getMethod().getMethodName(); - String rid = parameterExtractor.generateRid(result); - String methodKey = rid != null ? baseKey + "-" + rid : baseKey; + + boolean isRetry = isTestRetried(result, rid); TestNgTestWrapper wrapper = TestNgTestWrapper.forRegularTest(result); TestMetadata metadata = metaDataExtractor.extractTestMetadata(wrapper); - if (processedTests.contains(methodKey) && !testIds.contains(metadata.getTestId())) { - return; - } - - processedTests.add(methodKey); - testIds.add(metadata.getTestId()); - Object example = parameterExtractor.extractExample(result); - reportTestResultWithParameters(metadata, status, null, result, example, rid); + reportTestResultWithParameters(metadata, status, null, result, example, rid, isRetry); } /** @@ -120,7 +111,7 @@ public void reportTestResult(ISuite suite) { } } catch (ClassNotFoundException e) { throw new TestClassNotFoundException("Failed to load test class: " - + xmlClass.getName(), e); + + xmlClass.getName(), e); } }); }); @@ -132,7 +123,7 @@ public void reportTestResult(ISuite suite) { private void reportTestResult(TestMetadata metadata, String status, String message, Object frameworkSpecificData) { reportTestResultWithParameters(metadata, status, message, - frameworkSpecificData, null, null); + frameworkSpecificData, null, null, false); } /** @@ -140,7 +131,7 @@ private void reportTestResult(TestMetadata metadata, String status, */ private void reportTestResultWithParameters(TestMetadata metadata, String status, String message, Object frameworkSpecificData, - Object example, String rid) { + Object example, String rid, boolean isRetry) { if (!runManager.isActive()) { return; } @@ -163,7 +154,7 @@ private void reportTestResultWithParameters(TestMetadata metadata, String status TestResultWrapper wrapper = builder.build(); TestResult result = resultConstructor.constructTestRunResult(wrapper); - if (testIds.contains(result.getTestId())) { + if (isRetry) { result.setOverwrite(false); if (!PASSED.equals(result.getStatus())) { @@ -179,6 +170,30 @@ private void reportTestResultWithParameters(TestMetadata metadata, String status } } + /** + * Checks whether the test is being executed as a retry. + * + * @param result TestNG test result + * @param rid report identifier + * @return true if the test was retried, false otherwise + */ + private boolean isTestRetried(ITestResult result, String rid) { + String methodKey = result.getTestClass().getName() + + "." + + result.getMethod().getMethodName() + + "-" + rid; + TestState state = testStates.computeIfAbsent(methodKey, k -> new TestState()); + return state.markRetried() || result.wasRetried(); + } + + private static final class TestState { + private final AtomicInteger reportCount = new AtomicInteger(); + + boolean markRetried() { + return reportCount.getAndIncrement() > 0; + } + } + private void reportDisabledTest(Method method, Class testClass) { TestNgTestWrapper wrapper = TestNgTestWrapper.forDisabledTest(method, testClass); TestMetadata metadata = metaDataExtractor.extractTestMetadata(wrapper); diff --git a/testomat-allure-adapter/pom.xml b/testomat-allure-adapter/pom.xml index 2ac3f65..3767577 100644 --- a/testomat-allure-adapter/pom.xml +++ b/testomat-allure-adapter/pom.xml @@ -6,7 +6,7 @@ io.testomat testomat-allure-adapter - 0.1.6 + 0.1.7 jar Testomat.io Testomat Allure adapter @@ -67,7 +67,7 @@ io.testomat java-reporter-core - 0.18.1 + 0.18.2 io.qameta.allure