Bake SciCode sandbox image once; pre-build from launch host

src/olmo_eval/cli/beaker/job_assembler.py

@@ -2,6 +2,7 @@
 
 from __future__ import annotations
 
+import json
 from typing import TYPE_CHECKING, Any
 
 from olmo_eval.common.constants.infrastructure import (
@@ -204,12 +205,12 @@ def assemble_external_eval_job(
     # Add eval_args
     if eval_args:
         for key, value in eval_args.items():
-            command.extend(["-a", f"{key}={value}"])
+            command.extend(["-a", f"{key}={json.dumps(value)}"])
 
     # Add provider_kwargs
     if provider_kwargs:
         for key, value in provider_kwargs.items():
-            command.extend(["-K", f"{key}={value}"])
+            command.extend(["-K", f"{key}={json.dumps(value)}"])
 
     # Add storage options (only when --store is enabled)
     if store:

src/olmo_eval/cli/beaker/launch.py

@@ -985,6 +985,56 @@ def _apply_harness_overrides(harness_config, overrides: list[str]):
     return HarnessConfig.from_dict(harness_dict)
 
 
+def _prebuild_external_eval_sandbox_images(external_evals: list[str]) -> None:
+    """Build & push sandbox images on the launch host before submitting the GPU job.
+
+    Each external eval may declare swerex-derived images (base + dockerfile_extra)
+    via ExternalEval.prebuild_sandbox_specs. Building them here, on the user's
+    machine, means the GPU node only has to pull a ready image instead of running
+    apt/uv at startup — keeping the GPU from idling while deps install.
+    """
+    import os
+    import shutil
+
+    from olmo_eval.common.config import infrastructure as infra_config_module
+    from olmo_eval.evals.external.registry import get_external_eval
+    from olmo_eval.harness.sandbox.image import get_swerex_image
+    from olmo_eval.launch.beaker.constants import BEAKER_INFRA_ENV_VARS
+
+    # The launch host must push to the same registry the Beaker job pulls from.
+    os.environ["SWEREX_REGISTRY"] = BEAKER_INFRA_ENV_VARS["SWEREX_REGISTRY"]
+    infra_config_module.reset_infra_config()
+
+    runtime = "docker" if shutil.which("docker") else "podman"
+    if not shutil.which(runtime):
+        console.print(
+            "[yellow]Skipping sandbox image pre-bake: no docker or podman on PATH[/yellow]"
+        )
+        return
+
+    seen: set[tuple[str, tuple[str, ...]]] = set()
+    specs: list[tuple[str, tuple[str, ...]]] = []
+    for eval_name in external_evals:
+        for spec in get_external_eval(eval_name).prebuild_sandbox_specs():
+            if spec not in seen:
+                seen.add(spec)
+                specs.append(spec)
+
+    if not specs:
+        return
+
+    console.print(f"[bold]Pre-building {len(specs)} sandbox image(s) for external evals[/bold]")
+    for base_image, dockerfile_extra in specs:
+        console.print(f"  building from [blue]{base_image}[/blue]…")
+        image = get_swerex_image(
+            base_image,
+            container_runtime=runtime,
+            dockerfile_extra=dockerfile_extra,
+            require_registry=True,
+        )
+        console.print(f"  [green]ready:[/green] {image}")
+
+
 def _launch_external_evals(
     external_evals: list[str],
     model: tuple[str, ...],
@@ -1233,6 +1283,9 @@ def _launch_external_evals(
         console.print("[yellow]Launch cancelled[/yellow]")
         raise SystemExit(0)
 
+    if not dry_run:
+        _prebuild_external_eval_sandbox_images(external_evals)
+
     launched_experiments = _launch_jobs(launcher, job_configs, dry_run)
 
     # Follow launched experiments

src/olmo_eval/cli/utils.py

@@ -6,7 +6,8 @@
 from typing import TYPE_CHECKING, Any
 
 import click
-from rich.console import Console
+
+from olmo_eval.common.console import console
 
 if TYPE_CHECKING:
     from olmo_eval.evals.external.base import ExternalEval
@@ -16,9 +17,6 @@
     from olmo_eval.launch.beaker.launcher import BeakerJobConfig
 
 
-console = Console(force_terminal=True, width=120)
-
-
 @dataclass
 class FlaggedArg:
     """Argument with its flag type for order tracking."""
@@ -233,14 +231,16 @@ def _coerce_value(value: str) -> Any:
 
     Handles booleans, integers, floats, JSON objects/arrays, and plain strings.
     """
-    # Booleans
+    # JSON scalar literals
     if value.lower() == "true":
         return True
     if value.lower() == "false":
         return False
+    if value == "null":
+        return None
 
-    # JSON objects and arrays
-    if value.startswith("{") or value.startswith("["):
+    # JSON objects, arrays, and quoted strings (so launcher round-trips survive)
+    if value.startswith(("{", "[", '"')):
         try:
             return json.loads(value)
         except json.JSONDecodeError:

src/olmo_eval/common/console.py

@@ -0,0 +1,9 @@
+"""Shared Rich console singleton.
+
+Lives in `common` (not `cli`) so non-CLI modules can import it without pulling in
+the CLI package and triggering circular imports through `harness.presets`.
+"""
+
+from rich.console import Console
+
+console = Console(force_terminal=True, width=120)

src/olmo_eval/common/inspection.py

@@ -17,6 +17,8 @@
 from rich.table import Table
 from rich.text import Text
 
+from olmo_eval.common.console import console as shared_console
+
 if TYPE_CHECKING:
     from olmo_eval.common.types import Instance, LMRequest, Response
 
@@ -161,8 +163,6 @@ def inspect_object(
         show_none: Whether to show fields with None values.
     """
     if console is None:
-        from olmo_eval.cli.utils import console as shared_console
-
         console = shared_console
 
     if not is_dataclass(obj) or isinstance(obj, type):
@@ -208,8 +208,6 @@ def inspect_instance(
     import json
 
     if console is None:
-        from olmo_eval.cli.utils import console as shared_console
-
         console = shared_console
 
     renderables: list[Any] = []
@@ -308,8 +306,6 @@ def inspect_request(
         max_string_length: Maximum length for string values.
     """
     if console is None:
-        from olmo_eval.cli.utils import console as shared_console
-
         console = shared_console
 
     renderables: list[Any] = []
@@ -832,8 +828,6 @@ def inspect_formatted_request(
             When truncating, shows first half and last half.
     """
     if console is None:
-        from olmo_eval.cli.utils import console as shared_console
-
         console = shared_console
 
     # Pattern to match common special tokens
@@ -909,8 +903,6 @@ def inspect_tokens(
         show_decoded: Whether to show decoded token values.
     """
     if console is None:
-        from olmo_eval.cli.utils import console as shared_console
-
         console = shared_console
 
     # Get special token IDs
@@ -1051,8 +1043,6 @@ def inspect_response(
     import json
 
     if console is None:
-        from olmo_eval.cli.utils import console as shared_console
-
         console = shared_console
 
     renderables: list[Any] = []
@@ -1153,8 +1143,6 @@ def inspect_task_instances(
     logger = get_logger(__name__)
 
     if console is None:
-        from olmo_eval.cli.utils import console as shared_console
-
         console = shared_console
 
     tokenizer = None

src/olmo_eval/evals/external/base.py

@@ -65,6 +65,16 @@ def backend(self) -> str | None:
         """Backend name used by this evaluation, if any."""
         return None
 
+    def prebuild_sandbox_specs(self) -> tuple[tuple[str, tuple[str, ...]], ...]:
+        """Sandbox images this eval would build at runtime, for CLI pre-baking.
+
+        Returns a tuple of (base_image, dockerfile_extra) pairs. The launcher uses
+        these to build and push images to the registry on the launch host before
+        submitting the GPU job, so the GPU node only needs to pull a ready image.
+        Default: empty (no sandbox images to pre-build).
+        """
+        return ()
+
     @abstractmethod
     async def execute(
         self,

src/olmo_eval/evals/external/benchmarks/scicode/eval.py

@@ -11,6 +11,7 @@
 from __future__ import annotations
 
 import asyncio
+import base64
 import logging
 import time
 from dataclasses import dataclass
@@ -20,6 +21,8 @@
 from olmo_eval.common.types import LMRequest, RequestType, SamplingParams
 from olmo_eval.evals.external.base import ExternalEval
 from olmo_eval.evals.external.result import ExternalEvalResult
+from olmo_eval.harness.sandbox import SandboxManager
+from olmo_eval.harness.sandbox.config import Capability, SandboxConfig, SandboxMode
 from olmo_eval.inference.base import InferenceProvider
 
 from . import loader as scicode_loader
@@ -36,6 +39,23 @@
 _SANDBOX_DEPS_CONTAINER_DIR = "/opt/scicode-deps"
 
 
+def _scicode_dockerfile_extra() -> tuple[str, ...]:
+    """Bake SciCode sandbox deps into the swerex image at build time.
+
+    Embedding the lockfile contents (base64) makes the image hash sensitive to
+    lockfile changes (see image.py:55-57), so dep bumps invalidate the cache.
+    """
+    pyproject_b64 = base64.b64encode((_SANDBOX_LOCK_DIR / "pyproject.toml").read_bytes()).decode()
+    uvlock_b64 = base64.b64encode((_SANDBOX_LOCK_DIR / "uv.lock").read_bytes()).decode()
+    deps_dir = _SANDBOX_DEPS_CONTAINER_DIR
+    return (
+        f"RUN mkdir -p {deps_dir}",
+        f"RUN echo {pyproject_b64} | base64 -d > {deps_dir}/pyproject.toml",
+        f"RUN echo {uvlock_b64} | base64 -d > {deps_dir}/uv.lock",
+        f"RUN uv sync --active --frozen --no-dev --project {deps_dir}",
+    )
+
+
 @dataclass
 class SciCodeConfig:
     """Arguments for SciCode external evaluation."""
@@ -77,6 +97,10 @@ def description(self) -> str:
     def timeout_seconds(self) -> float:
         return 14400.0
 
+    def prebuild_sandbox_specs(self) -> tuple[tuple[str, tuple[str, ...]], ...]:
+        base_image = SciCodeConfig.__dataclass_fields__["sandbox_image"].default
+        return ((base_image, _scicode_dockerfile_extra()),)
+
     async def execute(
         self,
         provider: InferenceProvider,
@@ -253,13 +277,6 @@ async def _verify(
         if not scorable_indices:
             return []
 
-        from olmo_eval.harness.sandbox import SandboxManager
-        from olmo_eval.harness.sandbox.config import (
-            Capability,
-            SandboxConfig,
-            SandboxMode,
-        )
-
         sandbox_config = SandboxConfig(
             image=sc_args.sandbox_image,
             mode=SandboxMode.DOCKER,
@@ -268,32 +285,14 @@ async def _verify(
             startup_timeout=sc_args.startup_timeout,
             command_timeout=sc_args.command_timeout,
             inject_swerex=True,
-            volumes=(
-                (sc_args.h5py_host_path, sc_args.h5py_container_path),
-                (
-                    str(_SANDBOX_LOCK_DIR / "pyproject.toml"),
-                    f"{_SANDBOX_DEPS_CONTAINER_DIR}/pyproject.toml",
-                ),
-                (
-                    str(_SANDBOX_LOCK_DIR / "uv.lock"),
-                    f"{_SANDBOX_DEPS_CONTAINER_DIR}/uv.lock",
-                ),
-            ),
+            dockerfile_extra=_scicode_dockerfile_extra(),
+            volumes=((sc_args.h5py_host_path, sc_args.h5py_container_path),),
         )
         sandbox_manager = SandboxManager([sandbox_config], owner=f"scicode-{problem.problem_id}")
 
         results: list[bool] = []
         try:
             await sandbox_manager.start()
-            # --active installs into the venv that swerex's image bakes at /root/venv
-            # (exported via VIRTUAL_ENV); without it, uv would create a fresh project venv.
-            sync_result = await sandbox_manager.execute_code(
-                f"uv sync --active --frozen --no-dev --project {_SANDBOX_DEPS_CONTAINER_DIR}",
-                language="bash",
-                timeout=sc_args.startup_timeout,
-            )
-            if not sync_result.success:
-                raise RuntimeError(f"SciCode sandbox dep install failed: {sync_result.output}")
             for idx in scorable_indices:
                 step = problem.sub_steps[idx]
                 script = scicode_verifier.build_step_script(

src/olmo_eval/runners/common/mixins.py

@@ -12,7 +12,7 @@
 
 from typing import TYPE_CHECKING, Any
 
-from olmo_eval.cli.utils import console
+from olmo_eval.common.console import console
 from olmo_eval.common.logging import get_logger
 from olmo_eval.runners.common.models import (
     MetricsOutput,

src/olmo_eval/runners/io/storage.py

@@ -6,7 +6,7 @@
 from pathlib import Path
 from typing import TYPE_CHECKING, Any
 
-from olmo_eval.cli.utils import console
+from olmo_eval.common.console import console
 from olmo_eval.common.logging import get_logger
 from olmo_eval.runners.common.models import S3Config
 from olmo_eval.runners.io.formatting import build_s3_prefix

tests/cli/test_arg_round_trip.py

@@ -0,0 +1,42 @@
+"""Round-trip tests for eval-arg / provider-kwarg serialization.
+
+The Beaker launcher serializes parsed args back into ``-a key=value`` strings
+that the inner ``run-external`` CLI re-parses. ``json.dumps`` on the way out and
+``json.loads`` (via ``_coerce_value``) on the way in must agree on every shape
+we care about: strings, bools, ints, floats, lists, dicts, None.
+"""
+
+from __future__ import annotations
+
+import json
+import unittest
+from typing import Any
+
+from parameterized import parameterized
+
+from olmo_eval.cli import utils as cli_utils
+
+
+class TestArgRoundTrip(unittest.TestCase):
+    @parameterized.expand(
+        [
+            ("string", "hello"),
+            ("string_with_spaces", "hello world"),
+            ("bool_true", True),
+            ("bool_false", False),
+            ("int", 123),
+            ("float", 3.14),
+            ("list_of_strings", ["2", "11"]),
+            ("list_of_ints", [1, 2, 3]),
+            ("dict", {"a": 1, "b": "two"}),
+            ("none", None),
+        ]
+    )
+    def test_round_trip(self, _name: str, value: Any) -> None:
+        encoded = json.dumps(value)
+        parsed = cli_utils.parse_key_value_args((f"k={encoded}",), coerce_types=True)
+        self.assertEqual(parsed["k"], value)
+
+
+if __name__ == "__main__":
+    unittest.main()