test() errors on unexpected output (#7847)

NEWS.md

@@ -112,6 +112,8 @@
 
 10. `fwrite()` returns a clearer error message when `na = data.frame()` is used, [#7866](https://github.com/Rdatatable/data.table/issues/7866). Thanks @mcol for the report and the fix.
 
+11. `test()` now fails on unexpected console output, with the message `Test <num> produced unexpected output`, consistent with how unexpected warnings and messages are already handled, [#7847](https://github.com/Rdatatable/data.table/issues/7847). Thanks @MichaelChirico for the suggestion and @0xtch for the implementation.
+
 ## data.table [v1.18.4](https://github.com/Rdatatable/data.table/milestone/45) (6 May 2026)
 
 ### BUG FIXES

R/test.data.table.R

@@ -538,16 +538,11 @@ test = function(num, x, y=TRUE, ...,
     old_options = do.call(base::options, as.list(options)) # as.list(): allow passing named character vector for convenience
     on.exit(base::options(old_options), add=TRUE)
   }
-  if (is.null(output) && is.null(notOutput)) {
-    x = suppressMessages(withCallingHandlers(tryCatch(x, error=eHandler), warning=wHandler, message=mHandler))
-    # save the overhead of capture.output() since there are a lot of tests, often called in loops
+  out = if (is.null(output) && is.null(notOutput) || xsub %iscall% "print") {
+    capture.output(x <- suppressMessages(withCallingHandlers(tryCatch(x, error=eHandler), warning=wHandler, message=mHandler)))
     # Thanks to tryCatch2 by Jan here : https://github.com/jangorecki/logR/blob/master/R/logR.R#L21
   } else {
-    out = if (xsub %iscall% "print") {
-      capture.output(x <- suppressMessages(withCallingHandlers(tryCatch(x, error=eHandler), warning=wHandler, message=mHandler)))
-    } else {
-      capture.output(print(x <- suppressMessages(withCallingHandlers(tryCatch(x, error=eHandler), warning=wHandler, message=mHandler))))
-    }
+    capture.output(print(x <- suppressMessages(withCallingHandlers(tryCatch(x, error=eHandler), warning=wHandler, message=mHandler))))
   }
   if (!is.null(options)) {
     # some of the options passed to test() may break internal data.table use below (e.g. invalid datatable.alloccol), so undo them ASAP
@@ -600,12 +595,19 @@ test = function(num, x, y=TRUE, ...,
       }
     }
   }
-  if (fail && exists("out",inherits=FALSE)) {
+  if (fail && length(out)) {
     # nocov start
     catf("Output captured before unexpected warning/error/message:\n")
     writeLines(out)
     # nocov end
   }
+  if (!fail && is.null(output) && is.null(notOutput) && length(out) && !isTRUE(getOption("datatable.verbose"))) {
+    # nocov start
+    catf("Test %s produced unexpected output:\n", numStr)
+    writeLines(out)
+    fail = TRUE
+    # nocov end
+  }
   if (!fail && !length(error) && (length(output) || length(notOutput))) {
     if (out[length(out)] == "NULL") out = out[-length(out)]
     out = paste(out, collapse="\n")

inst/tests/self.Rraw

@@ -12,3 +12,11 @@ test(1.1, test(0, warning("a"), warning=c("a", "b"), check_value=FALSE), FALSE,
           output="Test 0 produced 1 warnings but expected 2\nExpected: a\n          b\nObserved: a")
 test(1.2, test(0, {warning("a"); warning("b")}, warning="a", check_value=FALSE), FALSE,
           output="Test 0 produced 2 warnings but expected 1\nExpected: a\nObserved: a\n          b")
+
+# test() catches unexpected console output, #7847
+test(2.1, test(0, {cat("123\n"); TRUE}), FALSE,
+          output="Test 0 produced unexpected output:\n123")
+# output printed by verbose mode itself doesn't count as unexpected, #7847
+test(2.2, test(0, data.table(a = 1:3)[, sum(a)], 6L), 
+          options=c(datatable.verbose=TRUE),
+          notOutput="Detected that j")

man/test.Rd

@@ -37,6 +37,8 @@ test(num, x, y = TRUE, ...,
 
    Multiple warnings are supported; supply a vector of strings to \code{warning=}. If \code{x} does not produce the correct number of warnings in the correct order, the test will fail.
 
+   Unless \code{output=} or \code{notOutput=} is supplied, you are automatically asserting that \code{x} prints nothing to the console; the test will fail if it does. This does not apply when \code{datatable.verbose} is \code{TRUE}.
+
    Strings passed to \code{notOutput=} should be minimal; e.g. pick out single words from the output that you desire to check does not occur. The reason being so that the test does not incorrectly pass just because the output has slightly changed. For example \code{notOutput="revised"} is better than \code{notOutput="revised flag to true"}. \code{notOutput=} is automatically case insensitive for this reason.
 }
 \value{