gh-145219: Cache Emscripten build dependencies, add install-emscripten (#145664)

Modifies the Emscripten build script to allow for caching of dependencies, and
for automated installation of new EMSDK versions.

Co-authored-by: Russell Keith-Magee <russell@keith-magee.com>

Author: hoodmane

Tools/wasm/emscripten/__main__.py

@@ -4,6 +4,7 @@
 import contextlib
 import functools
 import hashlib
+import json
 import os
 import shutil
 import subprocess
@@ -14,6 +15,8 @@
 from textwrap import dedent
 from urllib.request import urlopen
 
+import tomllib
+
 try:
     from os import process_cpu_count as cpu_count
 except ImportError:
@@ -22,48 +25,62 @@
 
 EMSCRIPTEN_DIR = Path(__file__).parent
 CHECKOUT = EMSCRIPTEN_DIR.parent.parent.parent
-EMSCRIPTEN_VERSION_FILE = EMSCRIPTEN_DIR / "emscripten_version.txt"
+CONFIG_FILE = EMSCRIPTEN_DIR / "config.toml"
 
 DEFAULT_CROSS_BUILD_DIR = CHECKOUT / "cross-build"
 HOST_TRIPLE = "wasm32-emscripten"
 
 
-def get_build_paths(cross_build_dir=None):
+@functools.cache
+def load_config_toml():
+    with CONFIG_FILE.open("rb") as file:
+        return tomllib.load(file)
+
+
+@functools.cache
+def required_emscripten_version():
+    return load_config_toml()["emscripten-version"]
+
+
+@functools.cache
+def emsdk_cache_root(emsdk_cache):
+    required_version = required_emscripten_version()
+    return Path(emsdk_cache).absolute() / required_version
+
+
+@functools.cache
+def emsdk_activate_path(emsdk_cache):
+    return emsdk_cache_root(emsdk_cache) / "emsdk/emsdk_env.sh"
+
+
+def get_build_paths(cross_build_dir=None, emsdk_cache=None):
     """Compute all build paths from the given cross-build directory."""
     if cross_build_dir is None:
         cross_build_dir = DEFAULT_CROSS_BUILD_DIR
     cross_build_dir = Path(cross_build_dir).absolute()
     host_triple_dir = cross_build_dir / HOST_TRIPLE
+    prefix_dir = host_triple_dir / "prefix"
+    if emsdk_cache:
+        prefix_dir = emsdk_cache_root(emsdk_cache) / "prefix"
+
     return {
         "cross_build_dir": cross_build_dir,
         "native_build_dir": cross_build_dir / "build",
         "host_triple_dir": host_triple_dir,
         "host_build_dir": host_triple_dir / "build",
         "host_dir": host_triple_dir / "build" / "python",
-        "prefix_dir": host_triple_dir / "prefix",
+        "prefix_dir": prefix_dir,
     }
 
 
 LOCAL_SETUP = CHECKOUT / "Modules" / "Setup.local"
 LOCAL_SETUP_MARKER = b"# Generated by Tools/wasm/emscripten.py\n"
 
 
-@functools.cache
-def get_required_emscripten_version():
-    """Read the required emscripten version from emscripten_version.txt."""
-    return EMSCRIPTEN_VERSION_FILE.read_text().strip()
-
-
-@functools.cache
-def get_emsdk_activate_path(emsdk_cache):
-    required_version = get_required_emscripten_version()
-    return Path(emsdk_cache) / required_version / "emsdk_env.sh"
-
-
 def validate_emsdk_version(emsdk_cache):
     """Validate that the emsdk cache contains the required emscripten version."""
-    required_version = get_required_emscripten_version()
-    emsdk_env = get_emsdk_activate_path(emsdk_cache)
+    required_version = required_emscripten_version()
+    emsdk_env = emsdk_activate_path(emsdk_cache)
     if not emsdk_env.is_file():
         print(
             f"Required emscripten version {required_version} not found in {emsdk_cache}",
@@ -90,7 +107,7 @@ def get_emsdk_environ(emsdk_cache):
         [
             "bash",
             "-c",
-            f"EMSDK_QUIET=1 source {get_emsdk_activate_path(emsdk_cache)} && env",
+            f"EMSDK_QUIET=1 source {emsdk_activate_path(emsdk_cache)} && env",
         ],
         text=True,
     )
@@ -207,6 +224,35 @@ def build_python_path(context):
     return binary
 
 
+def install_emscripten(context):
+    emsdk_cache = context.emsdk_cache
+    if emsdk_cache is None:
+        print("install-emscripten requires --emsdk-cache", file=sys.stderr)
+        sys.exit(1)
+    version = required_emscripten_version()
+    emsdk_target = emsdk_cache_root(emsdk_cache) / "emsdk"
+    if emsdk_target.exists():
+        if not context.quiet:
+            print(f"Emscripten version {version} already installed")
+        return
+    if not context.quiet:
+        print(f"Installing emscripten version {version}")
+    emsdk_target.mkdir(parents=True)
+    call(
+        [
+            "git",
+            "clone",
+            "https://github.com/emscripten-core/emsdk.git",
+            emsdk_target,
+        ],
+        quiet=context.quiet,
+    )
+    call([emsdk_target / "emsdk", "install", version], quiet=context.quiet)
+    call([emsdk_target / "emsdk", "activate", version], quiet=context.quiet)
+    if not context.quiet:
+        print(f"Installed emscripten version {version}")
+
+
 @subdir("native_build_dir", clean_ok=True)
 def configure_build_python(context, working_dir):
     """Configure the build/host Python."""
@@ -258,43 +304,95 @@ def download_and_unpack(working_dir: Path, url: str, expected_shasum: str):
         shutil.unpack_archive(tmp_file.name, working_dir)
 
 
+def should_build_library(prefix, name, config, quiet):
+    cached_config = prefix / (name + ".json")
+    if not cached_config.exists():
+        if not quiet:
+            print(
+                f"No cached build of {name} version {config['version']} found, building"
+            )
+        return True
+
+    try:
+        with cached_config.open("rb") as f:
+            cached_config = json.load(f)
+    except json.JSONDecodeError:
+        if not quiet:
+            print(f"Cached data for {name} invalid, rebuilding")
+        return True
+    if config == cached_config:
+        if not quiet:
+            print(
+                f"Found cached build of {name} version {config['version']}, not rebuilding"
+            )
+        return False
+
+    if not quiet:
+        print(
+            f"Found cached build of {name} version {config['version']} but it's out of date, rebuilding"
+        )
+    return True
+
+
+def write_library_config(prefix, name, config, quiet):
+    cached_config = prefix / (name + ".json")
+    with cached_config.open("w") as f:
+        json.dump(config, f)
+    if not quiet:
+        print(f"Succeded building {name}, wrote config to {cached_config}")
+
+
 @subdir("host_build_dir", clean_ok=True)
 def make_emscripten_libffi(context, working_dir):
-    ver = "3.4.6"
-    libffi_dir = working_dir / f"libffi-{ver}"
+    prefix = context.build_paths["prefix_dir"]
+    libffi_config = load_config_toml()["libffi"]
+    if not should_build_library(
+        prefix, "libffi", libffi_config, context.quiet
+    ):
+        return
+    url = libffi_config["url"]
+    version = libffi_config["version"]
+    shasum = libffi_config["shasum"]
+    libffi_dir = working_dir / f"libffi-{version}"
     shutil.rmtree(libffi_dir, ignore_errors=True)
     download_and_unpack(
         working_dir,
-        f"https://github.com/libffi/libffi/releases/download/v{ver}/libffi-{ver}.tar.gz",
-        "b0dea9df23c863a7a50e825440f3ebffabd65df1497108e5d437747843895a4e",
+        url.format(version=version),
+        shasum,
     )
     call(
         [EMSCRIPTEN_DIR / "make_libffi.sh"],
-        env=updated_env(
-            {"PREFIX": context.build_paths["prefix_dir"]}, context.emsdk_cache
-        ),
+        env=updated_env({"PREFIX": prefix}, context.emsdk_cache),
         cwd=libffi_dir,
         quiet=context.quiet,
     )
+    write_library_config(prefix, "libffi", libffi_config, context.quiet)
 
 
 @subdir("host_build_dir", clean_ok=True)
 def make_mpdec(context, working_dir):
-    ver = "4.0.1"
-    mpdec_dir = working_dir / f"mpdecimal-{ver}"
+    prefix = context.build_paths["prefix_dir"]
+    mpdec_config = load_config_toml()["mpdec"]
+    if not should_build_library(prefix, "mpdec", mpdec_config, context.quiet):
+        return
+
+    url = mpdec_config["url"]
+    version = mpdec_config["version"]
+    shasum = mpdec_config["shasum"]
+    mpdec_dir = working_dir / f"mpdecimal-{version}"
     shutil.rmtree(mpdec_dir, ignore_errors=True)
     download_and_unpack(
         working_dir,
-        f"https://www.bytereef.org/software/mpdecimal/releases/mpdecimal-{ver}.tar.gz",
-        "96d33abb4bb0070c7be0fed4246cd38416188325f820468214471938545b1ac8",
+        url.format(version=version),
+        shasum,
     )
     call(
         [
             "emconfigure",
             mpdec_dir / "configure",
             "CFLAGS=-fPIC",
             "--prefix",
-            context.build_paths["prefix_dir"],
+            prefix,
             "--disable-shared",
         ],
         cwd=mpdec_dir,
@@ -306,6 +404,7 @@ def make_mpdec(context, working_dir):
         cwd=mpdec_dir,
         quiet=context.quiet,
     )
+    write_library_config(prefix, "mpdec", mpdec_config, context.quiet)
 
 
 @subdir("host_dir", clean_ok=True)
@@ -436,16 +535,24 @@ def make_emscripten_python(context, working_dir):
     subprocess.check_call([exec_script, "--version"])
 
 
-def build_all(context):
-    """Build everything."""
-    steps = [
-        configure_build_python,
-        make_build_python,
-        make_emscripten_libffi,
-        make_mpdec,
-        configure_emscripten_python,
-        make_emscripten_python,
-    ]
+def build_target(context):
+    """Build one or more targets."""
+    steps = []
+    if context.target in {"all"}:
+        steps.append(install_emscripten)
+    if context.target in {"build", "all"}:
+        steps.extend([
+            configure_build_python,
+            make_build_python,
+        ])
+    if context.target in {"host", "all"}:
+        steps.extend([
+            make_emscripten_libffi,
+            make_mpdec,
+            configure_emscripten_python,
+            make_emscripten_python,
+        ])
+
     for step in steps:
         step(context)
 
@@ -475,7 +582,22 @@ def main():
 
     parser = argparse.ArgumentParser()
     subcommands = parser.add_subparsers(dest="subcommand")
+    install_emscripten_cmd = subcommands.add_parser(
+        "install-emscripten",
+        help="Install the appropriate version of Emscripten",
+    )
     build = subcommands.add_parser("build", help="Build everything")
+    build.add_argument(
+        "target",
+        nargs="?",
+        default="all",
+        choices=["all", "host", "build"],
+        help=(
+            "What should be built. 'build' for just the build platform, or "
+            "'host' for the host platform, or 'all' for both. Defaults to 'all'."
+        ),
+    )
+
     configure_build = subcommands.add_parser(
         "configure-build-python", help="Run `configure` for the build Python"
     )
@@ -512,6 +634,7 @@ def main():
     )
 
     for subcommand in (
+        install_emscripten_cmd,
         build,
         configure_build,
         make_libffi_cmd,
@@ -568,22 +691,25 @@ def main():
 
     context = parser.parse_args()
 
-    context.build_paths = get_build_paths(context.cross_build_dir)
-
-    if context.emsdk_cache:
+    if context.emsdk_cache and context.subcommand != "install-emscripten":
         validate_emsdk_version(context.emsdk_cache)
         context.emsdk_cache = Path(context.emsdk_cache).absolute()
     else:
         print("Build will use EMSDK from current environment.")
 
+    context.build_paths = get_build_paths(
+        context.cross_build_dir, context.emsdk_cache
+    )
+
     dispatch = {
+        "install-emscripten": install_emscripten,
         "make-libffi": make_emscripten_libffi,
         "make-mpdec": make_mpdec,
         "configure-build-python": configure_build_python,
         "make-build-python": make_build_python,
         "configure-host": configure_emscripten_python,
         "make-host": make_emscripten_python,
-        "build": build_all,
+        "build": build_target,
         "clean": clean_contents,
     }
 

Tools/wasm/emscripten/config.toml

@@ -0,0 +1,14 @@
+# Any data that can vary between Python versions is to be kept in this file.
+# This allows for blanket copying of the Emscripten build code between supported
+# Python versions.
+emscripten-version = "4.0.12"
+
+[libffi]
+url = "https://github.com/libffi/libffi/releases/download/v{version}/libffi-{version}.tar.gz"
+version = "3.4.6"
+shasum = "b0dea9df23c863a7a50e825440f3ebffabd65df1497108e5d437747843895a4e"
+
+[mpdec]
+url = "https://www.bytereef.org/software/mpdecimal/releases/mpdecimal-{version}.tar.gz"
+version = "4.0.1"
+shasum = "96d33abb4bb0070c7be0fed4246cd38416188325f820468214471938545b1ac8"