[CI] Add Python code coverage to test-other and modernize tooling

Revive and modernize Python code coverage collection, integrating it into
the CircleCI `test-other` suite.

1. In `tools/emcoverage.py`, strip incoming Python interpreter flags
   (such as `-E` passed by compiler entry point wrappers) and resolve
   bare command names to their `.py` equivalents in the root directory.
2. In `tools/config.py`, normalize `EMSDK_PYTHON` to an absolute path
   rather than clobbering it with `sys.executable`, which prevented
   child processes from executing under custom wrappers.
3. In `pyproject.toml`, enable parallel coverage measurement and
   multiprocessing concurrency, and configure omit patterns and report
   settings.
4. Add `diff-cover` to `requirements-dev.txt`.
5. In `.circleci/config.yml`, add an optional `coverage` parameter to
   `run-tests` and `run-tests-linux`. Enable coverage in `test-other`,
   upload HTML and XML reports as artifacts, and run `diff-cover` on PRs.

Author: sbc100

.circleci/config.yml

@@ -313,15 +313,45 @@ commands:
         description: "Extra EMCC_CFLAGS"
         type: string
         default: ""
+      coverage:
+        description: "Whether to collect python code coverage"
+        type: boolean
+        default: false
     steps:
+      - when:
+          condition: << parameters.coverage >>
+          steps:
+            - run:
+                name: reset coverage
+                command: ./tools/emcoverage.py reset
       - run:
           name: run tests (<< parameters.title >>)
           environment:
             EMCC_CFLAGS: << parameters.extra-cflags >>
           command: |
             env
-            ./test/runner << parameters.test_targets >>
+            if [ "<< parameters.coverage >>" = "true" ]; then
+              ./tools/emcoverage.py ./test/runner << parameters.test_targets >>
+            else
+              ./test/runner << parameters.test_targets >>
+            fi
             $EMSDK_PYTHON ./test/check_clean.py
+      - when:
+          condition: << parameters.coverage >>
+          steps:
+            - run:
+                name: generate coverage report
+                command: |
+                  ./tools/emcoverage.py report
+                  ./tools/emcoverage.py xml
+                  ./tools/emcoverage.py html
+                  if [ -n "$CIRCLE_PULL_REQUEST" ]; then
+                    diff-cover coverage.xml --compare-branch=origin/main || true
+                  fi
+            - store_artifacts:
+                path: coverage.xml
+            - store_artifacts:
+                path: htmlcov
   freeze-cache:
     description: "Freeze emscripten cache"
     steps:
@@ -348,6 +378,10 @@ commands:
         description: "Name of given test suite"
         type: string
         default: ""
+      coverage:
+        description: "Whether to collect python code coverage"
+        type: boolean
+        default: false
     steps:
       - prepare-for-tests
       - when:
@@ -357,6 +391,7 @@ commands:
       - run-tests:
           test_targets: << parameters.test_targets >>
           title: << parameters.title >>
+          coverage: << parameters.coverage >>
       - upload-test-results
   run-tests-chrome:
     description: "Runs browser tests under chrome"
@@ -1043,6 +1078,7 @@ jobs:
           # some native-dependent tests fail because of the lack of native
           # headers on emsdk-bundled clang
           test_targets: "other jslib skip:other.test_native_link_error_message"
+          coverage: true
       - run-tests-linux:
           # Include a single test from test_benchmark.py to ensure it's always
           # kept functional. Must be run separately from above because we cannot

pyproject.toml

@@ -91,13 +91,26 @@ lint.pylint.max-statements = 142 # Recommended: 50
 
 [tool.coverage.run]
 source = [ "." ]
+parallel = true
+concurrency = [ "multiprocessing" ]
 omit = [
+  "test/*",
   "./test/*",
+  "third_party/*",
   "./third_party/*",
+  "tools/emcoverage.py",
   "./tools/emcoverage.py",
   "test.py",
 ]
 
+[tool.coverage.report]
+show_missing = true
+exclude_lines = [
+  "# pragma: no cover",
+  "if __name__ == '__main__':",
+  "raise NotImplementedError",
+]
+
 [tool.mypy]
 mypy_path = "third_party/,third_party/ply,third_party/websockify"
 files = [ "." ]

requirements-dev.txt

@@ -5,6 +5,7 @@
 # Install with `pip3 install -r requirements-dev.txt`
 
 coverage[toml]==7.15.0
+diff-cover==10.6.0
 mypy==2.3.0
 psutil==7.0.0
 ruff==0.16.6

tools/config.py

@@ -70,8 +70,9 @@ def normalize_relative_python_path():
   # libraries to cache. These sub-emcc tasks will run in a different CWD, so
   # reinitialize EMSDK_PYTHON here so that sub-tool spawns will use the same
   # Python interpreter as the parent.
-  if os.environ.get('EMSDK_PYTHON'):
-    os.environ['EMSDK_PYTHON'] = sys.executable
+  python = os.environ.get('EMSDK_PYTHON')
+  if python:
+    os.environ['EMSDK_PYTHON'] = os.path.abspath(python)
 
 
 def set_config_from_tool_location(config_key, tool_binary, f):

tools/emcoverage.py

@@ -29,7 +29,6 @@
 """
 
 import contextlib
-import errno
 import os
 import shutil
 import sys
@@ -39,44 +38,67 @@
 import coverage.cmdline  # type: ignore
 
 SCRIPT_DIR = os.path.dirname(os.path.abspath(__file__))
+ROOT_DIR = os.path.dirname(SCRIPT_DIR)
 
 
 def main():
   # We set EMSDK_PYTHON to point to this file, which is executable via #! line.
   # Emscripten uses EMSDK_PYTHON to invoke all python subprocesses. By making this
   # script run all python subprocesses, all of them will execute under the
-  # watchful eye of emcoverage.py, and resulting in their code coverage being
+  # watchful eye of emcoverage.py, resulting in their code coverage being
   # tracked.
   os.environ['EMSDK_PYTHON'] = os.path.abspath(__file__)
 
   store = os.path.join(SCRIPT_DIR, 'coverage')
 
-  if len(sys.argv) < 2 or sys.argv[1] == 'help':
+  if len(sys.argv) < 2 or sys.argv[1] in {'help', '-h', '--help'}:
     print(__doc__.replace('emcoverage.py', sys.argv[0]).strip())
-    return
+    return 0
 
   if sys.argv[1] == 'reset':
-    shutil.rmtree(store)
-    return
-
-  if sys.argv[1] in {'html', 'report', 'xml'}:
+    shutil.rmtree(store, ignore_errors=True)
+    with contextlib.suppress(FileNotFoundError):
+      os.remove('.coverage')
+    for f in glob('.coverage.*'):
+      with contextlib.suppress(FileNotFoundError):
+        os.remove(f)
+    return 0
+
+  if sys.argv[1] in {'html', 'report', 'xml', 'annotate', 'json', 'lcov'}:
     old_argv = sys.argv
-    sys.argv = ['coverage', 'combine', *glob(os.path.join(store, '*'))]
-    with contextlib.suppress(SystemExit):
-      coverage.cmdline.main()
-    sys.argv = [*old_argv, '-i']
+    files = glob(os.path.join(store, '*'))
+    if files:
+      sys.argv = ['coverage', 'combine', *files]
+      with contextlib.suppress(SystemExit):
+        coverage.cmdline.main()
+    extra_args = []
+    if '-i' not in old_argv and '--ignore-errors' not in old_argv:
+      extra_args.append('-i')
+    sys.argv = ['coverage', *old_argv[1:], *extra_args]
     return coverage.cmdline.main()
 
-  if not os.path.exists(sys.argv[1]):
-    # If argv[1] is not a file path, instead try to interpret it as an emscripten command.
-    # This allows `emcoverage.py emcc` or `emcoverage.py embuilder` to work.
-    sys.argv[1] = os.path.join(os.path.dirname(sys.executable), '..', sys.argv[1] + '.py')
+  # Strip python interpreter flags passed by launcher scripts (e.g. -E in emcc wrapper)
+  while len(sys.argv) > 1 and sys.argv[1] in {'-E', '-B', '-O', '-s', '-S', '-u'}:
+    sys.argv.pop(1)
+
+  if len(sys.argv) > 1 and sys.argv[1] in {'python', 'python3'}:
+    sys.argv.pop(1)
+
+  if len(sys.argv) < 2:
+    print('emcoverage.py: no command specified', file=sys.stderr)
+    return 1
+
+  # If argv[1] is an emscripten command without .py extension, resolve it to root script.
+  # This allows `emcoverage.py emcc` or `emcoverage.py embuilder` to work even though
+  # executable shell wrappers exist in the repository root.
+  if not sys.argv[1].endswith('.py'):
+    candidate = os.path.join(ROOT_DIR, sys.argv[1] + '.py')
+    if os.path.isfile(candidate):
+      sys.argv[1] = candidate
+    elif not os.path.exists(sys.argv[1]):
+      sys.argv[1] = candidate
 
-  try:
-    os.mkdir(store)
-  except OSError as e:
-    if e.errno != errno.EEXIST:
-      raise
+  os.makedirs(store, exist_ok=True)
   os.environ['COVERAGE_FILE'] = os.path.join(store, str(uuid.uuid4()))
   sys.argv[0:1] = ['coverage', 'run', '--parallel-mode', '--']