Implement native clang launcher

Implement Phase 1 of the native launcher design document. Provides
high-performance native C++ compiler drivers (emcc, em++) using C++20
that directly invoke Clang for compile steps (-c, -S, -E) and seamlessly
fall back to python (emcc.py / em++.py) for link steps or complex
post-processing options.

Output executables are placed in ./bin using CMake and Ninja. Adds CI
matrix testing for Linux, macOS, and Windows.

These new binaries are currently 100% optional and the python versions
can still be used without building them.

Tested using `embuilder build libc --force` on Linux CI (compiling 1,075 files sequentially with `EMCC_CORES=1`, `EMCC_USE_NINJA=0`, and `EMCC_BATCH_BUILD=0`):

| Metric | Before (Python Baseline) | After (Native Launcher) | Improvement |
| :--- | :---: | :---: | :---: |
| **Total Time** | **181.96 s** (3m 2s) | **64.82 s** (1m 5s) | **-117.14 s (64.4% faster)** |
| **Time per File** | 169.3 ms | 60.3 ms | **-109.0 ms / invocation** |
| **Speedup Factor** | 1.0x | **2.81x** | — |

**Key Takeaways:**

* **64.4% Reduction in Build Time:** Bypassing Python interpreter startup
  shaves off **~109.0 ms of overhead per compiler invocation** on Linux CI
  (a **2.81x speedup**).
* **Windows Impact:** Because process spawning and `python.exe` startup
  carry significantly higher overhead on Windows than on Linux, the
  percentage speedup on Windows CI is expected to be even larger.

See: #26453

Author: sbc100

.circleci/config.yml

@@ -53,6 +53,12 @@ commands:
   bootstrap:
     description: "bootstrap"
     steps:
+      - run:
+          name: Install dependencies (Linux)
+          command: |
+            if command -v apt-get >/dev/null 2>&1; then
+              apt-get install -q -y cmake ninja-build
+            fi
       - run: "$EMSDK_PYTHON ./bootstrap.py"
   pip-install:
     description: "pip install"
@@ -1182,30 +1188,6 @@ jobs:
     steps:
       - test-sockets-chrome
 
-  build-windows-launcher:
-    executor:
-      name: win/server-2022
-      shell: bash.exe -eo pipefail
-    steps:
-      - checkout
-      - run:
-          name: "build pylauncher"
-          shell: cmd.exe
-          command: .circleci\setup_vs2022.bat && cd tools\pylauncher && call build.bat
-      - store_artifacts:
-          path: tools/pylauncher/pylauncher.exe
-          destination: pylauncher.exe
-      - install-emsdk
-      - pip-install:
-          python: "$EMSDK_PYTHON"
-      - run:
-          name: "create_entry_points"
-          command: $EMSDK_PYTHON tools/maint/create_entry_points.py --exe-files
-      - run:
-          name: "crossplatform tests"
-          command: test/runner.exe core0.test_hello_world
-
-
   # windows and mac do not have separate build and test jobs, as they only run
   # a limited set of tests; it is simpler and faster to do it all in one job.
   test-windows:
@@ -1233,10 +1215,6 @@ jobs:
       EMTEST_BROWSER: "0"
     steps:
       - checkout
-      - run:
-          name: Build launcher
-          command: call .circleci\setup_vs2019.bat && cd tools\pylauncher && call build.bat
-          shell: cmd.exe
       - run:
           name: Install packages
           command: |
@@ -1257,9 +1235,7 @@ jobs:
       - upload-test-results
       - run:
           name: "check clean"
-          command: |
-            git checkout tools/pylauncher
-            $EMSDK_PYTHON test/check_clean.py
+          command: $EMSDK_PYTHON test/check_clean.py
 
   test-mac-arm64:
     executor: mac-arm64
@@ -1357,7 +1333,6 @@ workflows:
       - test-node-compat
       - test-windows
       - test-windows-browser-firefox
-      - build-windows-launcher
       - test-mac-arm64:
          requires:
            - build-linux

.github/workflows/ci.yml

@@ -66,6 +66,9 @@ jobs:
             echo "Be sure that you have installed the current emsdk version.  See test/emsdk_version.txt ($(cat test/emsdk_version.txt))."
             exit 1
           fi
+      - name: Check emcc_native generated settings
+        run: |
+          ./tools/emcc_native/gen_settings.py --check
 
   clang-format-diff:
     # This job is disabled until we can make it more precise
@@ -83,22 +86,3 @@ jobs:
         sudo apt-get install clang-format-19
         sudo update-alternatives --install /usr/bin/git-clang-format git-clang-format /usr/bin/git-clang-format-19 100
     - run: tools/maint/clang-format-diff.sh origin/$GITHUB_BASE_REF
-
-  build-pylauncher-arm64:
-    name: Build pylauncher (ARM64)
-    runs-on: windows-11-arm
-    steps:
-    - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
-    - name: Setup MSVC
-      uses: ilammy/msvc-dev-cmd@0b201ec74fa43914dc39ae48a89fd1d8cb592756 # v1.13.0
-      with:
-        arch: arm64
-    - name: Build pylauncher
-      run: |
-        cd tools\pylauncher
-        call build.bat arm64
-      shell: cmd
-    - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
-      with:
-        name: pylauncher-arm64.exe
-        path: tools/pylauncher/pylauncher-arm64.exe

Makefile

@@ -13,10 +13,18 @@ install:
 	./tools/install.py $(DESTDIR)
 	npm install --omit=dev --prefix $(DESTDIR)
 
+out/build_emcc_native/native_tests emcc_native:
+	cmake -B out/build_emcc_native -S tools/emcc_native -G Ninja -DCMAKE_BUILD_TYPE=Release
+	cmake --build out/build_emcc_native --config Release
+	cmake --install out/build_emcc_native --config Release
+
+test: out/build_emcc_native/native_tests
+	ctest --test-dir out/build_emcc_native --output-on-failure
+
 # Create an distributable archive of emscripten suitable for use
 # by end users. This archive excludes node_modules as it can include native
 # modules which can't be safely pre-packaged.
 $(DISTFILE): install
 	tar cf $@ --exclude=node_modules -C `dirname $(DESTDIR)` `basename $(DESTDIR)`
 
-.PHONY: dist install
+.PHONY: dist install emcc_native test

README.md

@@ -39,11 +39,11 @@ There are two primary ways to install Emscripten:
     The easiest way to get started is by using the Emscripten SDK. Follow the instructions on the [downloads page](https://emscripten.org/docs/getting_started/downloads.html) to install it.
 
 2.  **From a Git Checkout (Manual Installation)**
-    If you have cloned the repository from Git, you can install the dependencies manually and then run the bootstrap script:
+    If you have cloned the repository from Git, run `bootstrap.py` to build the native launcher (`emcc_native`) and set up dependencies:
     ```bash
     ./bootstrap.py
     ```
-    For more details, see the [developer guide](https://emscripten.org/docs/contributing/developers_guide.html).
+    Building `emcc_native` requires CMake 3.20+ and a C++20 host compiler toolchain. Alternatively, setting `EMCC_NATIVE=0` in your environment before running `./bootstrap.py` will generate legacy Python launcher scripts (e.g. `.bat` / `.ps1` files on Windows) via `./tools/maint/create_entry_points.py`. For more details, see the [developer guide](https://emscripten.org/docs/contributing/developers_guide.html).
 
 ## Using the compiler
 

bootstrap.py

@@ -95,18 +95,22 @@ def run_cmd(cmd):
   subprocess.run(cmd, check=True, text=True, encoding='utf-8', cwd=utils.path_from_root())
 
 
+def build_emcc_native():
+  build_dir = utils.path_from_root('out/build_emcc_native')
+  source_dir = utils.path_from_root('tools/emcc_native')
+  cmd = ['cmake', '-B', build_dir, source_dir, '-DCMAKE_BUILD_TYPE=Release']
+  if not utils.WINDOWS and shutil.which('ninja'):
+    cmd.extend(['-G', 'Ninja'])
+  run_cmd(cmd)
+  run_cmd(['cmake', '--build', build_dir, '--config', 'Release'])
+  run_cmd(['cmake', '--install', build_dir, '--config', 'Release'])
+
+
 actions = [
   ('npm packages', [
      'package.json',
      'package-lock.json',
    ], ['npm', 'ci']),
-  ('create entry points', [
-     'tools/maint/create_entry_points.py',
-     'tools/pylauncher/pylauncher.exe',
-     'tools/maint/run_python.bat',
-     'tools/maint/run_python.sh',
-     'tools/maint/run_python.ps1',
-   ], [sys.executable, 'tools/maint/create_entry_points.py']),
   ('git submodules', [
      'test/third_party/posixtestsuite/',
      'test/third_party/googletest',
@@ -118,6 +122,28 @@ def run_cmd(cmd):
    ], maybe_install_hooks),
 ]
 
+if os.environ.get('EMCC_NATIVE') == '0':
+  actions.append(('legacy entry points', [
+     'tools/maint/create_entry_points.py',
+     'tools/maint/run_python.sh',
+     'tools/maint/run_python.bat',
+     'tools/maint/run_python.ps1',
+     'tools/maint/run_python_compiler.sh',
+     'tools/maint/run_python_compiler.bat',
+     'tools/maint/run_python_compiler.ps1',
+   ], [sys.executable, utils.path_from_root('tools/maint/create_entry_points.py')]))
+else:
+  actions.append(('build emcc_native', [
+     'tools/emcc_native/CMakeLists.txt',
+     'tools/emcc_native/main.cpp',
+     'tools/emcc_native/driver.cpp',
+     'tools/emcc_native/driver.h',
+     'tools/emcc_native/exec.cpp',
+     'tools/emcc_native/exec.h',
+     'tools/emcc_native/config.cpp',
+     'tools/emcc_native/config.h',
+   ], build_emcc_native))
+
 
 def main(args):
   parser = argparse.ArgumentParser(description=__doc__)

docs/design/03-native-clang-frontend.md

@@ -1,6 +1,6 @@
 # Design Doc: Native Launcher / Clang Frontend
 
-- **Status**: Draft
+- **Status**: Phase 1 Completed
 - **Bug**: https://github.com/emscripten-core/emscripten/issues/26453
 
 ## Context

site/source/docs/building_from_source/index.rst

@@ -7,11 +7,18 @@ Building Emscripten from Source
 Building Emscripten yourself is an alternative to getting binaries using the
 emsdk.
 
-Emscripten itself is written in Python and JavaScript so it does not need to be
-compiled.  However, after checkout you will need to run the top level
-``bootstrap.py`` script before the toolchain is usable.  This performs
-various steps including ``npm install`` and the creation of compiler entry
-points (e.g. `.bat` files on windows).
+Emscripten itself is primarily written in Python and JavaScript. However,
+after checkout you will need to run the top-level ``bootstrap.py`` script
+before the toolchain is usable. This performs various steps including ``npm
+install`` and building the native compiler frontend launcher (``emcc_native``),
+which provides high-performance ``emcc`` and ``em++`` binaries.
+
+Building ``emcc_native`` requires CMake 3.20+ and a C++20 host compiler
+toolchain (which you already need for building LLVM and Binaryen). If you prefer
+not to perform a native build, setting ``EMCC_NATIVE=0`` in your environment
+before running ``./bootstrap.py`` instructs it to generate legacy Python launcher
+scripts (e.g., ``.bat`` / ``.ps1`` files on Windows) via
+``./tools/maint/create_entry_points.py``.
 
 Emscripten comes with its own versions of some C/C++ system libraries which
 ``emcc`` builds automatically as and when needed (in the emsdk builds, these are

site/source/docs/building_from_source/toolchain_what_is_needed.rst

@@ -39,7 +39,9 @@ In general a complete Emscripten environment requires the following tools. First
 Compiler toolchain
 ------------------
 
-When building LLVM and Binaryen from source code, whether "manually" or using the SDK, you will need a *compiler toolchain*:
+When building Emscripten from source (via ``bootstrap.py`` to build
+``emcc_native``), or when building LLVM and Binaryen from source, you will need
+a C++20 host *compiler toolchain* and CMake 3.20+:
 
 - Windows: You will need `Visual Studio <https://visualstudio.microsoft.com/>`_ (2019 or above) and `cmake <http://www.cmake.org/cmake/resources/software.html>`_ (3.20 or above).
 

site/source/docs/contributing/developers_guide.rst

@@ -16,10 +16,22 @@ interested in helping out!
 Setting up
 ==========
 
-For contributing to core Emscripten code, such as ``emcc.py``, you don't need to
-build any binaries as ``emcc.py`` is in Python, and the core JS generation is
-in JavaScript. You do still need binaries for LLVM and Binaryen, which you can
-get using the emsdk.
+When setting up a Git checkout of Emscripten, run the top-level ``bootstrap.py``
+script to set up dependencies (such as ``npm install``) and build the native
+compiler frontend launcher (``emcc_native``):
+
+::
+
+    ./bootstrap.py
+
+Building ``emcc_native`` requires CMake 3.20+ and a C++20 host compiler
+toolchain. If you prefer not to build the native launcher, setting ``EMCC_NATIVE=0``
+in your environment before running ``./bootstrap.py`` will generate legacy Python
+launcher scripts (e.g., ``.bat`` / ``.ps1`` files on Windows) via
+``./tools/maint/create_entry_points.py``.
+
+For LLVM and Binaryen binaries, you don't need to build them from source if you
+are only contributing to Emscripten; you can get them using the emsdk.
 
 If you want to contribute back to Emscripten, it is recommended that you install
 the precise version of the emsdk binaries that are used by Emscripten CI when

test/test_other.py

@@ -12278,9 +12278,9 @@ def test_xclang_flag(self):
     self.run_process([EMCC, '-c', '-o', 'out.o', '-Xclang', '-include', '-Xclang', 'foo.h', test_file('hello_world.c')])
 
   def test_emcc_size_parsing(self):
-    create_file('foo.h', ' ')
-    self.assert_fail([EMCC, '-sTOTAL_MEMORY=X', 'foo.h'], 'error: invalid byte size `X`.  Valid suffixes are: kb, mb, gb, tb')
-    self.assert_fail([EMCC, '-sTOTAL_MEMORY=11PB', 'foo.h'], 'error: invalid byte size `11PB`.  Valid suffixes are: kb, mb, gb, tb')
+    create_file('foo.c', ' ')
+    self.assert_fail([EMCC, '-sTOTAL_MEMORY=X', 'foo.c'], 'error: invalid byte size `X`.  Valid suffixes are: kb, mb, gb, tb')
+    self.assert_fail([EMCC, '-sTOTAL_MEMORY=11PB', 'foo.c'], 'error: invalid byte size `11PB`.  Valid suffixes are: kb, mb, gb, tb')
 
   def test_native_call_before_init(self):
     self.set_setting('ASSERTIONS')