Add SingleLineTestRunner/Result. NFC

This test runner does a few things differ the base TextTestRunner:

1. It improves the behviour of `--buffer` by also buffering/redirecting
   logging output that occurs during the test run.
2. It displays all results on a single line, each result erasing the
   contents of the line before re-drawing it.
3. It uses ANSI colors to the show the results.
4. It should the progress as each results is displayed so its easy to
   see how far you are through the test suite "[XX/YY]"

I also updated parallel_testsuite.py use the same "XX/YY" progress
rather than a percent.

See emscripten-core#25752, which implements similar thing in the parallel_runner.

Author: sbc100

test/parallel_testsuite.py

@@ -125,9 +125,8 @@ def addTest(self, test):
     test.is_parallel = True
 
   def printOneResult(self, res):
-    percent = int(self.progress_counter * 100 / self.num_tests)
-    progress = f'[{percent:2d}%] '
     self.progress_counter += 1
+    progress = f'[{self.progress_counter}/{self.num_tests}] '
 
     if res.test_result == 'success':
       msg = 'ok'
@@ -165,7 +164,7 @@ def run(self, result):
     # multiprocessing.set_start_method('spawn')
 
     tests = self.get_sorted_tests()
-    self.num_tests = len(tests)
+    self.num_tests = self.countTestCases()
     contains_browser_test = any(test.is_browser_test() for test in tests)
     use_cores = cap_max_workers_in_pool(min(self.max_cores, len(tests), num_cores()), contains_browser_test)
     errlog(f'Using {use_cores} parallel test processes')

test/runner.py

@@ -41,8 +41,10 @@
 import jsrun
 import parallel_testsuite
 from common import errlog
+from single_line_runner import SingleLineTestRunner
 
 from tools import config, shared, utils
+from tools.colored_logger import ansi_color_available
 
 logger = logging.getLogger("runner")
 
@@ -427,8 +429,12 @@ def run_tests(options, suites):
     testRunner = xmlrunner.XMLTestRunner(output=output, verbosity=2,
                                          failfast=options.failfast)
     print('Writing XML test output to ' + os.path.abspath(output.name))
+  elif options.buffer and options.ansi and not options.verbose:
+    # And buffering is enabled and ansi color output is available use our nice single-line
+    # result display.
+    testRunner = SingleLineTestRunner(verbosity=2, failfast=options.failfast)
   else:
-    testRunner = unittest.TextTestRunner(verbosity=2, buffer=options.buffer, failfast=options.failfast)
+    testRunner = unittest.TextTestRunner(verbosity=2, failfast=options.failfast)
 
   total_core_time = 0
   run_start_time = time.perf_counter()
@@ -467,6 +473,9 @@ def parse_args():
   parser.add_argument('--no-clean', action='store_true',
                       help='Do not clean the temporary directory before each test run')
   parser.add_argument('--verbose', '-v', action='store_true')
+  # TODO: Replace with BooleanOptionalAction once we can depend on python3.9
+  parser.add_argument('--ansi', action='store_true', default=None)
+  parser.add_argument('--no-ansi', action='store_false', dest='ansi', default=None)
   parser.add_argument('--all-engines', action='store_true')
   parser.add_argument('--detect-leaks', action='store_true')
   parser.add_argument('--skip-slow', action='store_true', help='Skip tests marked as slow')
@@ -499,6 +508,9 @@ def parse_args():
 
   options = parser.parse_args()
 
+  if options.ansi is None:
+    options.ansi = ansi_color_available()
+
   if options.failfast:
     if options.max_failures != 2**31 - 1:
       utils.exit_with_error('--failfast and --max-failures are mutually exclusive!')

test/single_line_runner.py

@@ -0,0 +1,114 @@
+# Copyright 2025 The Emscripten Authors.  All rights reserved.
+# Emscripten is available under two separate licenses, the MIT license and the
+# University of Illinois/NCSA Open Source License.  Both these licenses can be
+# found in the LICENSE file.
+
+import logging
+import shutil
+import unittest
+
+from tools.colored_logger import CYAN, GREEN, RED, with_color
+
+
+def clearline(stream):
+  stream.write('\r\033[K')
+  stream.flush()
+
+
+def term_width():
+  return shutil.get_terminal_size()[0]
+
+
+class SingleLineTestResult(unittest.TextTestResult):
+  """Similar to the standard TextTestResult but uses ANSI escape codes
+  for color output and reusing a single line on the terminal.
+
+  This class also takes care of redirectting `logging` output when
+  in `buffer=True` mode.
+  """
+  test_count = 0
+
+  def __init__(self, *args, **kwargs):
+    super().__init__(*args, **kwargs)
+    self.progress_counter = 0
+
+  def writeStatusLine(self, line):
+    clearline(self._original_stderr)
+    self._original_stderr.write(line)
+    self._original_stderr.flush()
+
+  def updateStatus(self, test, msg, color):
+    self.progress_counter += 1
+    progress = f'[{self.progress_counter}/{self.test_count}] '
+    # Format the line so that it fix within the terminal width, unless its less then min_len
+    # in which case there is not much we can do, and we just overflow the line.
+    min_len = len(progress) + len(msg) + 5
+    test_name = str(test)
+    if term_width() > min_len:
+      max_name = term_width() - min_len
+      test_name = test_name[:max_name]
+    line = f'{with_color(CYAN, progress)}{test_name} ... {with_color(color, msg)}'
+    self.writeStatusLine(line)
+
+  def startTest(self, test):
+    assert self.test_count > 0
+    # Note: We explicitly do not use `super()` here but instead call `unittest.TestResult`. i.e.
+    # we skip the superclass (since we don't want its specific behaviour) and instead call its
+    # superclass.
+    unittest.TestResult.startTest(self, test)
+    # When we start a test in buffering mode, also update the python
+    # loggers to redirect to the _stderr_buffer
+    if self.buffer:
+      for handler in logging.root.handlers:
+        if handler.stream == self._original_stderr:
+          handler.stream = self._stderr_buffer
+    if self.progress_counter == 0:
+      self.writeStatusLine('.. awaiting first test result')
+
+  def stopTest(self, test):
+    unittest.TestResult.stopTest(self, test)
+    if self.buffer:
+      for handler in logging.root.handlers:
+        if handler.stream == self._stderr_buffer:
+          handler.stream = self._original_stderr
+
+  def addSuccess(self, test):
+    unittest.TestResult.addSuccess(self, test)
+    self.updateStatus(test, 'ok', GREEN)
+
+  def addFailure(self, test, err):
+    unittest.TestResult.addFailure(self, test, err)
+    self.updateStatus(test, 'FAIL', RED)
+
+  def addError(self, test, err):
+    unittest.TestResult.addError(self, test, err)
+    self.updateStatus(test, 'ERROR', RED)
+
+  def addExpectedFailure(self, test, err):
+    unittest.TestResult.addExpectedFailure(self, test, err)
+    self.updateStatus(test, 'expected failure', RED)
+
+  def addUnexpectedSuccess(self, test, err):
+    unittest.TestResult.addUnexpectedSuccess(self, test, err)
+    self.updateStatus(test, 'UNEXPECTED SUCCESS', RED)
+
+  def addSkip(self, test, reason):
+    unittest.TestResult.addSkip(self, test, reason)
+    self.updateStatus(test, f"skipped '{reason}'", CYAN)
+
+
+class SingleLineTestRunner(unittest.TextTestRunner):
+  """Subclass of TextTestResult that uses SingleLineTestResult"""
+  resultclass = SingleLineTestResult
+
+  def __init__(self, *args, **kwargs):
+    super().__init__(*args, buffer=True, **kwargs)
+
+  def _makeResult(self):
+    result = super()._makeResult()
+    result.test_count = self.test_count
+    return result
+
+  def run(self, test):
+    self.test_count = test.countTestCases()
+    return super().run(test)