Read the Catch2 dialect, so the guard covers a DuckDB consumer

.github/actions/check-test-outcome/action.yml

@@ -9,8 +9,8 @@ description: >-
 
   The rules themselves are tools/check-test-outcome.py in this repository, the
   one runnable definition this action and a developer both call, so the CI
-  answer and the by-hand answer cannot differ. It reads surefire and pytest
-  summaries and sums every module's line.
+  answer and the by-hand answer cannot differ. It reads the surefire, pytest,
+  Go, VSTest and Catch2 summaries, and sums every module's line.
 
 inputs:
   log:

.github/workflows/pytest.yml

@@ -96,7 +96,7 @@ jobs:
       # carries, or a change to them is not exercised until after it merges.
       # Consumers use the action; this repository owns the rules.
       - name: Refuse a skip, and a suite that shrank
-        run: tools/check-test-outcome.py "$RUNNER_TEMP/pytest.log" --min-tests 293
+        run: tools/check-test-outcome.py "$RUNNER_TEMP/pytest.log" --min-tests 300
 
       # The rules earn their place by refusing a log that carries what they
       # name. Both fixtures are written here rather than tracked, and the
@@ -178,6 +178,34 @@ jobs:
             echo "::error::the check passed a VSTest run over an assembly holding no test"
             exit 1
           fi
+          # The Catch2 dialect, which the DuckDB test runner writes and a
+          # MobilityDuck suite tees. The passing form's test-case count
+          # EXCLUDES the skipped cases, so the floor is read off the sum.
+          printf 'All tests passed (2695 assertions in 102 test cases)\n' \
+            > "$RUNNER_TEMP/fx-catch-ok.log"
+          if ! tools/check-test-outcome.py "$RUNNER_TEMP/fx-catch-ok.log" --min-tests 102; then
+            echo "::error::the check refused a Catch2 log that satisfies both rules"
+            exit 1
+          fi
+          if tools/check-test-outcome.py "$RUNNER_TEMP/fx-catch-ok.log" --min-tests 103; then
+            echo "::error::the check passed a Catch2 suite below its floor"
+            exit 1
+          fi
+          printf 'All tests passed (3 skip''ped tests, 2695 assertions in 99 test cases)\n' \
+            > "$RUNNER_TEMP/fx-catch-skip.log"
+          if tools/check-test-outcome.py "$RUNNER_TEMP/fx-catch-skip.log" --min-tests 102; then
+            echo "::error::the check passed a Catch2 log reporting skipped tests"
+            exit 1
+          fi
+          # The failure table carries `101 passed`, which the pytest pattern
+          # matches too; reading it as pytest would report 101 as the total.
+          printf 'test cases: 102 | 101 passed | 1 failed\n' \
+            > "$RUNNER_TEMP/fx-catch-table.log"
+          if tools/check-test-outcome.py "$RUNNER_TEMP/fx-catch-table.log" --min-tests 102 \
+               | grep -q 'pytest'; then
+            echo "::error::the check read a Catch2 failure table as pytest"
+            exit 1
+          fi
           echo "the check refuses a skip, a shrunken suite, and a log with no summary, in every dialect it reads"
 
       - name: Upload meos-idl.json as artefact

tests/test_test_outcome.py

@@ -76,6 +76,57 @@ def test_vstest_sums_every_assembly(self):
         self.assertEqual(2, len(lines))
 
 
+class Catch2Tests(unittest.TestCase):
+    """The Catch2 console reporter, which DuckDB's test runner writes."""
+
+    def test_catch2_reads_the_line_a_mobilityduck_run_writes(self):
+        # Verbatim from a MobilityDuck CI run of its 102 sqllogictest files.
+        total, skipped, dialect, _ = outcome.read_summaries(
+            "All tests passed (2695 assertions in 102 test cases)\n")
+        self.assertEqual(("catch2", 102, 0), (dialect, total, skipped))
+
+    def test_catch2_adds_the_skipped_cases_the_passing_count_excludes(self):
+        # The reporter prints `testCases.passed - skippedTests`, so 99 printed
+        # beside 3 skipped is a suite of 102.
+        total, skipped, dialect, _ = outcome.read_summaries(
+            "All tests passed (3 skipped tests, 2695 assertions in "
+            "99 test cases)\n")
+        self.assertEqual(("catch2", 102, 3), (dialect, total, skipped))
+
+    def test_catch2_reads_the_singular_forms(self):
+        total, skipped, dialect, _ = outcome.read_summaries(
+            "All tests passed (1 skipped test, 1 assertion in 1 test case)\n")
+        self.assertEqual(("catch2", 2, 1), (dialect, total, skipped))
+
+    def test_catch2_reads_the_failure_table(self):
+        log = (
+            "test cases: 102 | 101 passed | 1 failed\n"
+            "assertions: 2695 | 2694 passed | 1 failed\n"
+        )
+        total, skipped, dialect, _ = outcome.read_summaries(log)
+        self.assertEqual(("catch2", 102, 0), (dialect, total, skipped))
+
+    def test_catch2_reads_the_skipped_column_when_the_table_carries_one(self):
+        # A column whose count is zero is omitted, so the skipped column is
+        # present only when it is not zero, and the leading total includes it.
+        log = "test cases: 102 | 99 passed | 1 failed | 2 skipped\n"
+        total, skipped, dialect, _ = outcome.read_summaries(log)
+        self.assertEqual(("catch2", 102, 2), (dialect, total, skipped))
+
+    def test_catch2_reads_a_wholly_skipped_run(self):
+        total, skipped, dialect, _ = outcome.read_summaries(
+            "All tests were skipped (total skipped 7)\n")
+        self.assertEqual(("catch2", 7, 7), (dialect, total, skipped))
+
+    def test_catch2_is_read_before_pytest(self):
+        # THE ORDER IS THE TEST: `101 passed` in the failure table satisfies the
+        # pytest pattern too, and pytest-first would report 101 as the total,
+        # missing the failure and any skip beside it.
+        log = "test cases: 102 | 101 passed | 1 failed\n"
+        total, _, dialect, _ = outcome.read_summaries(log)
+        self.assertEqual(("catch2", 102), (dialect, total))
+
+
 class NoSummaryTests(unittest.TestCase):
     """A log carrying no summary at all names no dialect, which is a failure."""
 

tools/check-test-outcome.py

@@ -28,6 +28,15 @@
 # reading only the last understates the total (measured: MobilityKafka prints 7
 # and 4, so its total is 11 rather than the 4 a tail would report).
 #
+# The Catch2 dialect reads the runner DuckDB builds, under which a
+# MobilityDuck sqllogictest FILE is one test case. ITS SKIP COUNT CANNOT SEE A
+# `mode skip`: that directive silences the statements after it while the file
+# still reports as a passing test case, so a suite can retire assertions
+# wholesale and this dialect reads `0 skipped` over it. What the dialect does
+# hold for such a consumer is the FLOOR — the total moves when a whole file
+# leaves the suite. A sqllogictest consumer needs a census of `mode skip` in the
+# test SOURCES beside this, and a green answer here is not evidence about them.
+#
 # The Go dialect prints no counts at all: `go test -v` writes one result line
 # per test and per subtest, and the totals are their tally. A run without -v
 # writes only `ok <pkg> 0.42s`, which carries neither a total nor a skip count,
@@ -69,6 +78,28 @@
     r"(?:Passed|Failed)!\s+-\s+Failed:\s*(\d+),\s*Passed:\s*(\d+),"
     r"\s*Skipped:\s*(\d+),\s*Total:\s*(\d+)")
 
+# `All tests passed (2695 assertions in 102 test cases)` — what the Catch2
+# console reporter writes when nothing failed, carrying a leading
+# `3 skipped tests, ` when any were. The test-case count it prints EXCLUDES the
+# skipped ones, so the total is the two added: the reporter writes
+# `testCases.passed - skippedTests` there.
+CATCH_PASSED = re.compile(
+    r"All tests passed\s*\(\s*(?:(\d+)\s+skipped tests?,\s*)?"
+    r"(\d+)\s+assertions?\s+in\s+(\d+)\s+test cases?\s*\)")
+
+# `test cases: 102 | 101 passed | 1 failed` — the table the same reporter writes
+# when anything failed. Here the leading number IS the whole total, skips
+# included, and a column whose count is zero is omitted rather than printed, so
+# the skipped column is read out of the tail when it is there at all. That
+# column sums expected failures with skips; both are tests that did not assert,
+# and the guard treats them alike.
+CATCH_TOTALS = re.compile(r"\btest cases:\s*(\d+)((?:\s*\|\s*\d+\s+\w+)*)")
+
+# `All tests were skipped (total skipped 7)` — the one form that reports no
+# passing count at all, written when every test skipped.
+CATCH_ALL_SKIPPED = re.compile(
+    r"All tests were skipped\s*\(\s*total skipped\s*(\d+)\s*\)")
+
 
 def read_summaries(text: str):
     """Return (total, skipped, dialect, lines) summed over every summary found."""
@@ -104,6 +135,35 @@ def read_summaries(text: str):
     if lines:
         return total, skipped, "vstest", lines
 
+    # Catch2 is read BEFORE pytest, and the order is load-bearing: the failure
+    # table's `101 passed` satisfies the pytest pattern, so a failing Catch2 run
+    # read pytest-first reports the passing count as the total and misses both
+    # the failures and the skips.
+    for raw in text.splitlines():
+        line = raw.rstrip()
+        m = CATCH_ALL_SKIPPED.search(line)
+        if m:
+            every = int(m.group(1))
+            total += every
+            skipped += every
+            lines.append(line.strip())
+            continue
+        m = CATCH_PASSED.search(line)
+        if m:
+            some = int(m.group(1) or 0)
+            total += int(m.group(3)) + some
+            skipped += some
+            lines.append(line.strip())
+            continue
+        m = CATCH_TOTALS.search(line)
+        if m:
+            total += int(m.group(1))
+            column = re.search(r"(\d+)\s+skipped", m.group(2))
+            skipped += int(column.group(1)) if column else 0
+            lines.append(line.strip())
+    if lines:
+        return total, skipped, "catch2", lines
+
     for raw in text.splitlines():
         m = PYTEST.search(raw)
         if m: