Improve TestNG retry reporting to Testomat

java-reporter-core/pom.xml

@@ -7,7 +7,7 @@
 
     <groupId>io.testomat</groupId>
     <artifactId>java-reporter-core</artifactId>
-    <version>0.18.1</version>
+    <version>0.18.2</version>
     <packaging>jar</packaging>
 
     <name>Testomat.io Reporter Core</name>

java-reporter-core/src/main/java/io/testomat/core/constants/CommonConstants.java

@@ -1,7 +1,7 @@
 package io.testomat.core.constants;
 
 public class CommonConstants {
-    public static final String REPORTER_VERSION = "0.18.1";
+    public static final String REPORTER_VERSION = "0.18.2";
 
     public static final String TESTS_STRING = "tests";
     public static final String API_KEY_STRING = "api_key";

java-reporter-cucumber/pom.xml

@@ -6,7 +6,7 @@
 
     <groupId>io.testomat</groupId>
     <artifactId>java-reporter-cucumber</artifactId>
-    <version>0.9.1</version>
+    <version>0.9.2</version>
     <packaging>jar</packaging>
 
     <name>Testomat.io Java Reporter Cucumber</name>
@@ -51,7 +51,7 @@
         <dependency>
             <groupId>io.testomat</groupId>
             <artifactId>java-reporter-core</artifactId>
-            <version>0.18.1</version>
+            <version>0.18.2</version>
         </dependency>
         <dependency>
             <groupId>org.slf4j</groupId>

java-reporter-junit/pom.xml

@@ -6,7 +6,7 @@
 
     <groupId>io.testomat</groupId>
     <artifactId>java-reporter-junit</artifactId>
-    <version>0.9.5</version>
+    <version>0.9.6</version>
     <packaging>jar</packaging>
 
     <name>Testomat.io Java Reporter JUnit</name>
@@ -51,7 +51,7 @@
         <dependency>
             <groupId>io.testomat</groupId>
             <artifactId>java-reporter-core</artifactId>
-            <version>0.18.1</version>
+            <version>0.18.2</version>
         </dependency>
         <dependency>
             <groupId>org.slf4j</groupId>

java-reporter-karate/pom.xml

@@ -6,7 +6,7 @@
 
     <groupId>io.testomat</groupId>
     <artifactId>java-reporter-karate</artifactId>
-    <version>0.3.5</version>
+    <version>0.3.6</version>
     <packaging>jar</packaging>
 
     <name>Testomat.io Java Reporter Karate</name>
@@ -52,7 +52,7 @@
         <dependency>
             <groupId>io.testomat</groupId>
             <artifactId>java-reporter-core</artifactId>
-            <version>0.18.1</version>
+            <version>0.18.2</version>
         </dependency>
         <dependency>
             <groupId>io.karatelabs</groupId>

java-reporter-testng/pom.xml

@@ -6,7 +6,7 @@
 
     <groupId>io.testomat</groupId>
     <artifactId>java-reporter-testng</artifactId>
-    <version>0.8.5</version>
+    <version>0.8.6</version>
     <packaging>jar</packaging>
 
     <name>Testomat.io Java Reporter TestNG</name>
@@ -47,7 +47,7 @@
         <dependency>
             <groupId>io.testomat</groupId>
             <artifactId>java-reporter-core</artifactId>
-            <version>0.18.1</version>
+            <version>0.18.2</version>
         </dependency>
         <dependency>
             <groupId>org.slf4j</groupId>

java-reporter-testng/src/main/java/io/testomat/testng/listener/AbstractHooksContainer.java

@@ -4,12 +4,17 @@
 import java.util.Collections;
 import java.util.List;
 import java.util.ServiceLoader;
+import java.util.function.Consumer;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
 import org.testng.IInvokedMethod;
 import org.testng.ISuite;
 import org.testng.ITestResult;
 
 public abstract class AbstractHooksContainer {
 
+    private static final Logger log = LoggerFactory.getLogger(AbstractHooksContainer.class);
+
     private static final List<TestomatHook> HOOKS;
 
     static {
@@ -20,28 +25,24 @@ public abstract class AbstractHooksContainer {
         HOOKS = Collections.unmodifiableList(hooks);
     }
 
-    protected void onSuiteStartHookAfterExecution(ISuite suite) {
-        for (TestomatHook hook : HOOKS) {
-            hook.onSuiteStartHookAfterExecution(suite);
-        }
+    protected boolean onSuiteStartHookAfterExecution(ISuite suite) {
+        return runHooks("onSuiteStartHookAfterExecution",
+                hook -> hook.onSuiteStartHookAfterExecution(suite));
     }
 
-    protected void onSuiteStartHookBeforeExecution(ISuite suite) {
-        for (TestomatHook hook : HOOKS) {
-            hook.onSuiteStartHookBeforeExecution(suite);
-        }
+    protected boolean onSuiteStartHookBeforeExecution(ISuite suite) {
+        return runHooks("onSuiteStartHookBeforeExecution",
+                hook -> hook.onSuiteStartHookBeforeExecution(suite));
     }
 
     protected void onSuiteFinishHookAfterExecution(ISuite suite) {
-        for (TestomatHook hook : HOOKS) {
-            hook.onSuiteFinishHookAfterExecution(suite);
-        }
+        runHooks("onSuiteFinishHookAfterExecution",
+                hook -> hook.onSuiteFinishHookAfterExecution(suite));
     }
 
     protected void onSuiteFinishHookBeforeExecution(ISuite suite) {
-        for (TestomatHook hook : HOOKS) {
-            hook.onSuiteFinishHookBeforeExecution(suite);
-        }
+        runHooks("onSuiteFinishHookBeforeExecution",
+                hook -> hook.onSuiteFinishHookBeforeExecution(suite));
     }
 
     protected void onTestSuccessHookAfterExecution(ITestResult result) {
@@ -143,4 +144,17 @@ protected void onExecutionFinishHookBeforeExecution() {
             hook.onExecutionFinishHookBeforeExecution();
         }
     }
+
+    private boolean runHooks(String hookName, Consumer<TestomatHook> action) {
+        boolean failed = false;
+        for (TestomatHook hook : HOOKS) {
+            try {
+                action.accept(hook);
+            } catch (Exception e) {
+                log.error("Hook '{}' failed: {}", hookName, e.getMessage(), e);
+                failed = true;
+            }
+        }
+        return failed;
+    }
 }

java-reporter-testng/src/main/java/io/testomat/testng/listener/TestNgListener.java

@@ -88,21 +88,25 @@ public final void onStart(ISuite suite) {
         if (!isListeningRequired()) {
             return;
         }
-        onSuiteStartHookBeforeExecution(suite);
-        log.debug("Suite started: {}", suite.getName());
+
         runManager.incrementSuiteCounter();
+        boolean failed = onSuiteStartHookBeforeExecution(suite);
+        log.debug("Suite started: {}", suite.getName());
         reporter.reportTestResult(suite);
-        onSuiteStartHookAfterExecution(suite);
+        failed |= onSuiteStartHookAfterExecution(suite);
+        if (failed) {
+            suite.getSuiteState().failed();
+        }
     }
 
     @Override
     public final void onFinish(ISuite suite) {
         if (!isListeningRequired()) {
             return;
         }
+        runManager.decrementSuiteCounter();
         onSuiteFinishHookBeforeExecution(suite);
         log.debug("Suite finished: {}", suite.getName());
-        runManager.decrementSuiteCounter();
         onSuiteFinishHookAfterExecution(suite);
     }
 

java-reporter-testng/src/main/java/io/testomat/testng/reporter/TestNgTestResultReporter.java

@@ -15,10 +15,11 @@
 import io.testomat.testng.extractor.TestNgParameterExtractor;
 import io.testomat.testng.extractor.TestNgTestWrapper;
 import java.lang.reflect.Method;
-import java.util.ArrayList;
 import java.util.HashSet;
-import java.util.List;
+import java.util.Map;
 import java.util.Set;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.atomic.AtomicInteger;
 import org.testng.ISuite;
 import org.testng.ITestResult;
 import org.testng.annotations.Test;
@@ -35,16 +36,16 @@ public class TestNgTestResultReporter {
     private final TestNgMetaDataExtractor metaDataExtractor;
     private final TestNgParameterExtractor parameterExtractor;
     private final GlobalRunManager runManager;
-    private final List<String> processedTests;
-    private final Set<String> testIds;
+    private final Set<String> processedTests;
+    private final Map<String, TestState> testStates;
 
     public TestNgTestResultReporter() {
         this.resultConstructor = new TestNgTestResultConstructor();
         this.metaDataExtractor = new TestNgMetaDataExtractor();
         this.parameterExtractor = new TestNgParameterExtractor();
         this.runManager = GlobalRunManager.getInstance();
-        this.processedTests = new ArrayList<>();
-        this.testIds = new HashSet<>();
+        this.processedTests = new HashSet<>();
+        this.testStates = new ConcurrentHashMap<>();
     }
 
     /**
@@ -58,8 +59,8 @@ public TestNgTestResultReporter(TestNgTestResultConstructor resultConstructor,
         this.metaDataExtractor = metaDataExtractor;
         this.parameterExtractor = parameterExtractor;
         this.runManager = runManager;
-        this.testIds = new HashSet<>();
-        this.processedTests = new ArrayList<>();
+        this.processedTests = new HashSet<>();
+        this.testStates = new ConcurrentHashMap<>();
     }
 
     /**
@@ -71,26 +72,16 @@ public void reportTestResult(ITestResult result, String status) {
             return;
         }
 
-        String baseKey = result.getTestClass().getName()
-                + "."
-                + result.getMethod().getMethodName();
-
         String rid = parameterExtractor.generateRid(result);
-        String methodKey = rid != null ? baseKey + "-" + rid : baseKey;
+
+        boolean isRetry = isTestRetried(result, rid);
 
         TestNgTestWrapper wrapper = TestNgTestWrapper.forRegularTest(result);
         TestMetadata metadata = metaDataExtractor.extractTestMetadata(wrapper);
 
-        if (processedTests.contains(methodKey) && !testIds.contains(metadata.getTestId())) {
-            return;
-        }
-
-        processedTests.add(methodKey);
-        testIds.add(metadata.getTestId());
-
         Object example = parameterExtractor.extractExample(result);
 
-        reportTestResultWithParameters(metadata, status, null, result, example, rid);
+        reportTestResultWithParameters(metadata, status, null, result, example, rid, isRetry);
     }
 
     /**
@@ -120,7 +111,7 @@ public void reportTestResult(ISuite suite) {
                     }
                 } catch (ClassNotFoundException e) {
                     throw new TestClassNotFoundException("Failed to load test class: "
-                            + xmlClass.getName(), e);
+                        + xmlClass.getName(), e);
                 }
             });
         });
@@ -132,15 +123,15 @@ public void reportTestResult(ISuite suite) {
     private void reportTestResult(TestMetadata metadata, String status,
                                   String message, Object frameworkSpecificData) {
         reportTestResultWithParameters(metadata, status, message,
-                frameworkSpecificData, null, null);
+                frameworkSpecificData, null, null, false);
     }
 
     /**
      * Enhanced method to report test results with parameterized test support.
      */
     private void reportTestResultWithParameters(TestMetadata metadata, String status,
                                                 String message, Object frameworkSpecificData,
-                                                Object example, String rid) {
+                                                Object example, String rid, boolean isRetry) {
         if (!runManager.isActive()) {
             return;
         }
@@ -163,7 +154,7 @@ private void reportTestResultWithParameters(TestMetadata metadata, String status
             TestResultWrapper wrapper = builder.build();
             TestResult result = resultConstructor.constructTestRunResult(wrapper);
 
-            if (testIds.contains(result.getTestId())) {
+            if (isRetry) {
                 result.setOverwrite(false);
 
                 if (!PASSED.equals(result.getStatus())) {
@@ -179,6 +170,30 @@ private void reportTestResultWithParameters(TestMetadata metadata, String status
         }
     }
 
+    /**
+     * Checks whether the test is being executed as a retry.
+     *
+     * @param result TestNG test result
+     * @param rid report identifier
+     * @return true if the test was retried, false otherwise
+     */
+    private boolean isTestRetried(ITestResult result, String rid) {
+        String methodKey = result.getTestClass().getName()
+                + "."
+                + result.getMethod().getMethodName()
+                + "-" + rid;
+        TestState state = testStates.computeIfAbsent(methodKey, k -> new TestState());
+        return state.markRetried() || result.wasRetried();
+    }
+
+    private static final class TestState {
+        private final AtomicInteger reportCount = new AtomicInteger();
+
+        boolean markRetried() {
+            return reportCount.getAndIncrement() > 0;
+        }
+    }
+
     private void reportDisabledTest(Method method, Class<?> testClass) {
         TestNgTestWrapper wrapper = TestNgTestWrapper.forDisabledTest(method, testClass);
         TestMetadata metadata = metaDataExtractor.extractTestMetadata(wrapper);

testomat-allure-adapter/pom.xml

@@ -6,7 +6,7 @@
 
     <groupId>io.testomat</groupId>
     <artifactId>testomat-allure-adapter</artifactId>
-    <version>0.1.6</version>
+    <version>0.1.7</version>
     <packaging>jar</packaging>
 
     <name>Testomat.io Testomat Allure adapter</name>
@@ -67,7 +67,7 @@
         <dependency>
             <groupId>io.testomat</groupId>
             <artifactId>java-reporter-core</artifactId>
-            <version>0.18.1</version>
+            <version>0.18.2</version>
         </dependency>
         <dependency>
             <groupId>io.qameta.allure</groupId>