diff --git a/java-reporter-core/pom.xml b/java-reporter-core/pom.xml
index 4ac28c8..6c910c6 100644
--- a/java-reporter-core/pom.xml
+++ b/java-reporter-core/pom.xml
@@ -7,7 +7,7 @@
io.testomat
java-reporter-core
- 0.18.1
+ 0.18.2
jar
Testomat.io Reporter Core
diff --git a/java-reporter-core/src/main/java/io/testomat/core/constants/CommonConstants.java b/java-reporter-core/src/main/java/io/testomat/core/constants/CommonConstants.java
index 2d4c9f2..773c3c6 100644
--- a/java-reporter-core/src/main/java/io/testomat/core/constants/CommonConstants.java
+++ b/java-reporter-core/src/main/java/io/testomat/core/constants/CommonConstants.java
@@ -1,7 +1,7 @@
package io.testomat.core.constants;
public class CommonConstants {
- public static final String REPORTER_VERSION = "0.18.1";
+ public static final String REPORTER_VERSION = "0.18.2";
public static final String TESTS_STRING = "tests";
public static final String API_KEY_STRING = "api_key";
diff --git a/java-reporter-cucumber/pom.xml b/java-reporter-cucumber/pom.xml
index 0ed0d30..9546d87 100644
--- a/java-reporter-cucumber/pom.xml
+++ b/java-reporter-cucumber/pom.xml
@@ -6,7 +6,7 @@
io.testomat
java-reporter-cucumber
- 0.9.1
+ 0.9.2
jar
Testomat.io Java Reporter Cucumber
@@ -51,7 +51,7 @@
io.testomat
java-reporter-core
- 0.18.1
+ 0.18.2
org.slf4j
diff --git a/java-reporter-junit/pom.xml b/java-reporter-junit/pom.xml
index 9374d2b..d3e7bfa 100644
--- a/java-reporter-junit/pom.xml
+++ b/java-reporter-junit/pom.xml
@@ -6,7 +6,7 @@
io.testomat
java-reporter-junit
- 0.9.5
+ 0.9.6
jar
Testomat.io Java Reporter JUnit
@@ -51,7 +51,7 @@
io.testomat
java-reporter-core
- 0.18.1
+ 0.18.2
org.slf4j
diff --git a/java-reporter-karate/pom.xml b/java-reporter-karate/pom.xml
index 747108d..c11faeb 100644
--- a/java-reporter-karate/pom.xml
+++ b/java-reporter-karate/pom.xml
@@ -6,7 +6,7 @@
io.testomat
java-reporter-karate
- 0.3.5
+ 0.3.6
jar
Testomat.io Java Reporter Karate
@@ -52,7 +52,7 @@
io.testomat
java-reporter-core
- 0.18.1
+ 0.18.2
io.karatelabs
diff --git a/java-reporter-testng/pom.xml b/java-reporter-testng/pom.xml
index 309a2aa..f00c94a 100644
--- a/java-reporter-testng/pom.xml
+++ b/java-reporter-testng/pom.xml
@@ -6,7 +6,7 @@
io.testomat
java-reporter-testng
- 0.8.5
+ 0.8.6
jar
Testomat.io Java Reporter TestNG
@@ -47,7 +47,7 @@
io.testomat
java-reporter-core
- 0.18.1
+ 0.18.2
org.slf4j
diff --git a/java-reporter-testng/src/main/java/io/testomat/testng/listener/AbstractHooksContainer.java b/java-reporter-testng/src/main/java/io/testomat/testng/listener/AbstractHooksContainer.java
index c065273..bb6a865 100644
--- a/java-reporter-testng/src/main/java/io/testomat/testng/listener/AbstractHooksContainer.java
+++ b/java-reporter-testng/src/main/java/io/testomat/testng/listener/AbstractHooksContainer.java
@@ -4,12 +4,17 @@
import java.util.Collections;
import java.util.List;
import java.util.ServiceLoader;
+import java.util.function.Consumer;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
import org.testng.IInvokedMethod;
import org.testng.ISuite;
import org.testng.ITestResult;
public abstract class AbstractHooksContainer {
+ private static final Logger log = LoggerFactory.getLogger(AbstractHooksContainer.class);
+
private static final List HOOKS;
static {
@@ -20,28 +25,24 @@ public abstract class AbstractHooksContainer {
HOOKS = Collections.unmodifiableList(hooks);
}
- protected void onSuiteStartHookAfterExecution(ISuite suite) {
- for (TestomatHook hook : HOOKS) {
- hook.onSuiteStartHookAfterExecution(suite);
- }
+ protected boolean onSuiteStartHookAfterExecution(ISuite suite) {
+ return runHooks("onSuiteStartHookAfterExecution",
+ hook -> hook.onSuiteStartHookAfterExecution(suite));
}
- protected void onSuiteStartHookBeforeExecution(ISuite suite) {
- for (TestomatHook hook : HOOKS) {
- hook.onSuiteStartHookBeforeExecution(suite);
- }
+ protected boolean onSuiteStartHookBeforeExecution(ISuite suite) {
+ return runHooks("onSuiteStartHookBeforeExecution",
+ hook -> hook.onSuiteStartHookBeforeExecution(suite));
}
protected void onSuiteFinishHookAfterExecution(ISuite suite) {
- for (TestomatHook hook : HOOKS) {
- hook.onSuiteFinishHookAfterExecution(suite);
- }
+ runHooks("onSuiteFinishHookAfterExecution",
+ hook -> hook.onSuiteFinishHookAfterExecution(suite));
}
protected void onSuiteFinishHookBeforeExecution(ISuite suite) {
- for (TestomatHook hook : HOOKS) {
- hook.onSuiteFinishHookBeforeExecution(suite);
- }
+ runHooks("onSuiteFinishHookBeforeExecution",
+ hook -> hook.onSuiteFinishHookBeforeExecution(suite));
}
protected void onTestSuccessHookAfterExecution(ITestResult result) {
@@ -143,4 +144,17 @@ protected void onExecutionFinishHookBeforeExecution() {
hook.onExecutionFinishHookBeforeExecution();
}
}
+
+ private boolean runHooks(String hookName, Consumer action) {
+ boolean failed = false;
+ for (TestomatHook hook : HOOKS) {
+ try {
+ action.accept(hook);
+ } catch (Exception e) {
+ log.error("Hook '{}' failed: {}", hookName, e.getMessage(), e);
+ failed = true;
+ }
+ }
+ return failed;
+ }
}
diff --git a/java-reporter-testng/src/main/java/io/testomat/testng/listener/TestNgListener.java b/java-reporter-testng/src/main/java/io/testomat/testng/listener/TestNgListener.java
index 123b8e7..127e152 100644
--- a/java-reporter-testng/src/main/java/io/testomat/testng/listener/TestNgListener.java
+++ b/java-reporter-testng/src/main/java/io/testomat/testng/listener/TestNgListener.java
@@ -88,11 +88,15 @@ public final void onStart(ISuite suite) {
if (!isListeningRequired()) {
return;
}
- onSuiteStartHookBeforeExecution(suite);
- log.debug("Suite started: {}", suite.getName());
+
runManager.incrementSuiteCounter();
+ boolean failed = onSuiteStartHookBeforeExecution(suite);
+ log.debug("Suite started: {}", suite.getName());
reporter.reportTestResult(suite);
- onSuiteStartHookAfterExecution(suite);
+ failed |= onSuiteStartHookAfterExecution(suite);
+ if (failed) {
+ suite.getSuiteState().failed();
+ }
}
@Override
@@ -100,9 +104,9 @@ public final void onFinish(ISuite suite) {
if (!isListeningRequired()) {
return;
}
+ runManager.decrementSuiteCounter();
onSuiteFinishHookBeforeExecution(suite);
log.debug("Suite finished: {}", suite.getName());
- runManager.decrementSuiteCounter();
onSuiteFinishHookAfterExecution(suite);
}
diff --git a/java-reporter-testng/src/main/java/io/testomat/testng/reporter/TestNgTestResultReporter.java b/java-reporter-testng/src/main/java/io/testomat/testng/reporter/TestNgTestResultReporter.java
index 920c177..a2c7a8f 100644
--- a/java-reporter-testng/src/main/java/io/testomat/testng/reporter/TestNgTestResultReporter.java
+++ b/java-reporter-testng/src/main/java/io/testomat/testng/reporter/TestNgTestResultReporter.java
@@ -15,10 +15,11 @@
import io.testomat.testng.extractor.TestNgParameterExtractor;
import io.testomat.testng.extractor.TestNgTestWrapper;
import java.lang.reflect.Method;
-import java.util.ArrayList;
import java.util.HashSet;
-import java.util.List;
+import java.util.Map;
import java.util.Set;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.atomic.AtomicInteger;
import org.testng.ISuite;
import org.testng.ITestResult;
import org.testng.annotations.Test;
@@ -35,16 +36,16 @@ public class TestNgTestResultReporter {
private final TestNgMetaDataExtractor metaDataExtractor;
private final TestNgParameterExtractor parameterExtractor;
private final GlobalRunManager runManager;
- private final List processedTests;
- private final Set testIds;
+ private final Set processedTests;
+ private final Map testStates;
public TestNgTestResultReporter() {
this.resultConstructor = new TestNgTestResultConstructor();
this.metaDataExtractor = new TestNgMetaDataExtractor();
this.parameterExtractor = new TestNgParameterExtractor();
this.runManager = GlobalRunManager.getInstance();
- this.processedTests = new ArrayList<>();
- this.testIds = new HashSet<>();
+ this.processedTests = new HashSet<>();
+ this.testStates = new ConcurrentHashMap<>();
}
/**
@@ -58,8 +59,8 @@ public TestNgTestResultReporter(TestNgTestResultConstructor resultConstructor,
this.metaDataExtractor = metaDataExtractor;
this.parameterExtractor = parameterExtractor;
this.runManager = runManager;
- this.testIds = new HashSet<>();
- this.processedTests = new ArrayList<>();
+ this.processedTests = new HashSet<>();
+ this.testStates = new ConcurrentHashMap<>();
}
/**
@@ -71,26 +72,16 @@ public void reportTestResult(ITestResult result, String status) {
return;
}
- String baseKey = result.getTestClass().getName()
- + "."
- + result.getMethod().getMethodName();
-
String rid = parameterExtractor.generateRid(result);
- String methodKey = rid != null ? baseKey + "-" + rid : baseKey;
+
+ boolean isRetry = isTestRetried(result, rid);
TestNgTestWrapper wrapper = TestNgTestWrapper.forRegularTest(result);
TestMetadata metadata = metaDataExtractor.extractTestMetadata(wrapper);
- if (processedTests.contains(methodKey) && !testIds.contains(metadata.getTestId())) {
- return;
- }
-
- processedTests.add(methodKey);
- testIds.add(metadata.getTestId());
-
Object example = parameterExtractor.extractExample(result);
- reportTestResultWithParameters(metadata, status, null, result, example, rid);
+ reportTestResultWithParameters(metadata, status, null, result, example, rid, isRetry);
}
/**
@@ -120,7 +111,7 @@ public void reportTestResult(ISuite suite) {
}
} catch (ClassNotFoundException e) {
throw new TestClassNotFoundException("Failed to load test class: "
- + xmlClass.getName(), e);
+ + xmlClass.getName(), e);
}
});
});
@@ -132,7 +123,7 @@ public void reportTestResult(ISuite suite) {
private void reportTestResult(TestMetadata metadata, String status,
String message, Object frameworkSpecificData) {
reportTestResultWithParameters(metadata, status, message,
- frameworkSpecificData, null, null);
+ frameworkSpecificData, null, null, false);
}
/**
@@ -140,7 +131,7 @@ private void reportTestResult(TestMetadata metadata, String status,
*/
private void reportTestResultWithParameters(TestMetadata metadata, String status,
String message, Object frameworkSpecificData,
- Object example, String rid) {
+ Object example, String rid, boolean isRetry) {
if (!runManager.isActive()) {
return;
}
@@ -163,7 +154,7 @@ private void reportTestResultWithParameters(TestMetadata metadata, String status
TestResultWrapper wrapper = builder.build();
TestResult result = resultConstructor.constructTestRunResult(wrapper);
- if (testIds.contains(result.getTestId())) {
+ if (isRetry) {
result.setOverwrite(false);
if (!PASSED.equals(result.getStatus())) {
@@ -179,6 +170,30 @@ private void reportTestResultWithParameters(TestMetadata metadata, String status
}
}
+ /**
+ * Checks whether the test is being executed as a retry.
+ *
+ * @param result TestNG test result
+ * @param rid report identifier
+ * @return true if the test was retried, false otherwise
+ */
+ private boolean isTestRetried(ITestResult result, String rid) {
+ String methodKey = result.getTestClass().getName()
+ + "."
+ + result.getMethod().getMethodName()
+ + "-" + rid;
+ TestState state = testStates.computeIfAbsent(methodKey, k -> new TestState());
+ return state.markRetried() || result.wasRetried();
+ }
+
+ private static final class TestState {
+ private final AtomicInteger reportCount = new AtomicInteger();
+
+ boolean markRetried() {
+ return reportCount.getAndIncrement() > 0;
+ }
+ }
+
private void reportDisabledTest(Method method, Class> testClass) {
TestNgTestWrapper wrapper = TestNgTestWrapper.forDisabledTest(method, testClass);
TestMetadata metadata = metaDataExtractor.extractTestMetadata(wrapper);
diff --git a/testomat-allure-adapter/pom.xml b/testomat-allure-adapter/pom.xml
index 2ac3f65..3767577 100644
--- a/testomat-allure-adapter/pom.xml
+++ b/testomat-allure-adapter/pom.xml
@@ -6,7 +6,7 @@
io.testomat
testomat-allure-adapter
- 0.1.6
+ 0.1.7
jar
Testomat.io Testomat Allure adapter
@@ -67,7 +67,7 @@
io.testomat
java-reporter-core
- 0.18.1
+ 0.18.2
io.qameta.allure