From 0667a1411ef1b87268554a518459bbe17e06bebe Mon Sep 17 00:00:00 2001
From: Tyler Murray <tylerm@allenai.org>
Date: Fri, 3 Apr 2026 10:54:27 -0700
Subject: [PATCH 01/29] experimental support for podman service and modal

---
 pyproject.toml                                |  2 +-
 .../external/benchmarks/swebench/eval.py      | 37 +++++++++++++++----
 src/olmo_eval/launch/beaker/launcher.py       | 13 +++++++
 3 files changed, 43 insertions(+), 9 deletions(-)

diff --git a/pyproject.toml b/pyproject.toml
index ed59eb237..e75e7e575 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -80,7 +80,7 @@ clients = [
 swebench = [
     "olmo-eval-internal[litellm]",
     "mini-swe-agent @ git+https://github.com/allenai/mini-swe-agent.git@e614105ca837995765512bfd5aa1d216558f002c",
-    "swebench==4.1.0",
+    "swebench>=4.1.0",
 ]
 storage = [
     "olmo-eval-internal[s3,postgres]",
diff --git a/src/olmo_eval/evals/external/benchmarks/swebench/eval.py b/src/olmo_eval/evals/external/benchmarks/swebench/eval.py
index 3b456f3f3..f88aec822 100644
--- a/src/olmo_eval/evals/external/benchmarks/swebench/eval.py
+++ b/src/olmo_eval/evals/external/benchmarks/swebench/eval.py
@@ -44,6 +44,7 @@ class SWEBenchArgs:
     max_workers_eval: int = 4
     temperature: float = 0.0
     max_turns: int = 30
+    use_modal: bool = False
 
     @classmethod
     def from_dict(cls, data: dict[str, Any]) -> SWEBenchArgs:
@@ -56,6 +57,7 @@ def from_dict(cls, data: dict[str, Any]) -> SWEBenchArgs:
             max_workers_eval=int(data.get("max_workers_eval", 4)),
             temperature=float(data.get("temperature", 0.0)),
             max_turns=int(data.get("max_turns", 30)),
+            use_modal=data.get("use_modal", False) in (True, "true", "True", "1", 1),
         )
 
 
@@ -97,6 +99,7 @@ def arguments(self) -> dict[str, tuple[str, Any | None]]:
             "max_workers_eval": ("Parallel workers for the SWE-bench scoring harness", 4),
             "temperature": ("Sampling temperature", 0.0),
             "max_turns": ("Max agent turns per instance", 30),
+            "use_modal": ("Run scoring harness on Modal cloud", False),
         }
 
     async def execute(
@@ -109,6 +112,18 @@ async def execute(
         start_time = time.time()
         swe_args = SWEBenchArgs.from_dict(args)
 
+        if swe_args.use_modal:
+            missing = []
+            if not os.environ.get("MODAL_TOKEN_ID"):
+                missing.append("MODAL_TOKEN_ID")
+            if not os.environ.get("MODAL_TOKEN_SECRET"):
+                missing.append("MODAL_TOKEN_SECRET")
+            if missing:
+                return self._error_result(
+                    f"Modal credentials missing: {', '.join(missing)}",
+                    start_time,
+                )
+
         tmp_dir = None
         if output_dir is None:
             tmp_dir = tempfile.mkdtemp()
@@ -218,8 +233,9 @@ async def _run_scoring(
         work_dir: Path,
         container_runtime: str,
     ) -> tuple[bool, str]:
-        """Run the SWE-bench evaluation harness to score patches. Returns (success, output)."""
+        """Run the SWE-bench evaluation harness to score patches."""
         dataset_hf = _DATASET_HF_PATHS.get(swe_args.dataset, swe_args.dataset)
+
         cmd = [
             sys.executable,
             "-m",
@@ -228,18 +244,23 @@ async def _run_scoring(
             dataset_hf,
             "--predictions_path",
             str(preds_path),
-            "--max_workers",
-            str(swe_args.max_workers_eval),
             "--run_id",
             run_id,
         ]
 
-        # The SWE-bench harness uses the Docker Python SDK, which reads DOCKER_HOST.
-        # When using podman, point it at the podman socket so the SDK can find it.
         env = os.environ.copy()
-        if container_runtime == "podman" and "DOCKER_HOST" not in env:
-            uid = os.getuid()
-            env["DOCKER_HOST"] = f"unix:///run/user/{uid}/podman/podman.sock"
+
+        if swe_args.use_modal:
+            # Modal handles containers in the cloud
+            cmd.extend(["--modal", "true", "--parallelism", str(swe_args.max_workers_eval)])
+        else:
+            # Use podman service socket (started by Beaker launcher)
+            # DOCKER_HOST should already be set by _build_install_cmd
+            cmd.extend(["--max_workers", str(swe_args.max_workers_eval)])
+            if "DOCKER_HOST" not in env:
+                logger.warning(
+                    "DOCKER_HOST not set. Scoring may fail without podman service or Modal."
+                )
 
         logger.info(f"[{self.name}] Running SWE-bench harness: {shlex.join(cmd)}")
         ok, output = await self._run_subprocess(
diff --git a/src/olmo_eval/launch/beaker/launcher.py b/src/olmo_eval/launch/beaker/launcher.py
index 248d7107d..80b623b0e 100644
--- a/src/olmo_eval/launch/beaker/launcher.py
+++ b/src/olmo_eval/launch/beaker/launcher.py
@@ -430,6 +430,10 @@ class BeakerJobConfig:
     # Use this for external evals that run vLLM as a server subprocess.
     vllm_isolated_venv: bool = False
 
+    # Start podman system service for Docker API compatibility (swebench harness, etc.)
+    # When True, starts `podman system service` and sets DOCKER_HOST env var.
+    enable_podman_service: bool = False
+
 
 def resolve_clusters(cluster: str | list[str]) -> list[str]:
     """Resolve cluster aliases to full cluster names.
@@ -633,6 +637,7 @@ def _build_install_cmd(
         enable_sandbox: bool = False,
         setup_store_secrets: bool = False,
         vllm_isolated_venv: bool = False,
+        enable_podman_service: bool = False,
     ) -> str:
         """Build installation command for gantry's install_cmd parameter.
 
@@ -653,6 +658,7 @@ def _build_install_cmd(
             enable_sandbox: If True, set up /dev/net/tun and Artifact Registry auth.
             setup_store_secrets: If True, run setup_store_secrets to configure database access.
             vllm_isolated_venv: If True, install vLLM in isolated venv for server mode.
+            enable_podman_service: If True, start podman system service and set DOCKER_HOST.
 
         Returns:
             Shell command string for installation.
@@ -683,6 +689,12 @@ def _build_install_cmd(
             script = "/gantry-runtime/src/olmo_eval/launch/beaker/scripts/setup_artifact_registry"
             steps.append(f"source {script}")
 
+        # Start podman system service for Docker API compatibility (swebench harness, etc.)
+        if enable_podman_service:
+            steps.append("podman system service --time=0 unix:///tmp/podman.sock &")
+            steps.append("sleep 2")
+            steps.append("export DOCKER_HOST=unix:///tmp/podman.sock")
+
         # Export additional environment variables (e.g., UV_CACHE_DIR)
         if env_exports:
             for key, value in env_exports.items():
@@ -765,6 +777,7 @@ def launch(self, config: BeakerJobConfig, dry_run: bool = False) -> BeakerExperi
             config.enable_sandbox,
             config.setup_store_secrets,
             config.vllm_isolated_venv,
+            config.enable_podman_service,
         )
 
         # Build weka mounts as tuples: (bucket, mount_path)

From b7efc23a7a1645e91a6e046d0889762107cfcc53 Mon Sep 17 00:00:00 2001
From: Tyler Murray <tylerm@allenai.org>
Date: Fri, 3 Apr 2026 11:03:23 -0700
Subject: [PATCH 02/29] Cleanup podman service flag

---
 src/olmo_eval/cli/beaker/job_assembler.py | 2 ++
 src/olmo_eval/cli/beaker/launch.py        | 9 +++++++++
 2 files changed, 11 insertions(+)

diff --git a/src/olmo_eval/cli/beaker/job_assembler.py b/src/olmo_eval/cli/beaker/job_assembler.py
index 9cd166745..b6896ca8a 100644
--- a/src/olmo_eval/cli/beaker/job_assembler.py
+++ b/src/olmo_eval/cli/beaker/job_assembler.py
@@ -157,6 +157,7 @@ def assemble_external_eval_job(
     retries: int | None = None,
     provider_kind: str | None = None,
     base_url: str | None = None,
+    enable_podman_service: bool = False,
 ) -> Any:
     """Assemble a BeakerJobConfig for running external evaluations.
 
@@ -347,6 +348,7 @@ def assemble_external_eval_job(
         extras=extras,
         provider_packages=provider_packages,
         vllm_isolated_venv=vllm_isolated_venv,
+        enable_podman_service=enable_podman_service,
     )
 
 
diff --git a/src/olmo_eval/cli/beaker/launch.py b/src/olmo_eval/cli/beaker/launch.py
index f59d54b7a..fb36d74f9 100644
--- a/src/olmo_eval/cli/beaker/launch.py
+++ b/src/olmo_eval/cli/beaker/launch.py
@@ -238,6 +238,11 @@
     default=None,
     help="Number of GPUs. Defaults to 1 for GPU providers, 0 otherwise.",
 )
+@click.option(
+    "--enable-podman-service",
+    is_flag=True,
+    help="Start podman system service for Docker API compatibility (e.g., swebench harness).",
+)
 def launch(
     config: str | None,
     name: str | None,
@@ -282,6 +287,7 @@ def launch(
     uv_cache_dir: str,
     secret_env: tuple[str, ...],
     gpus: int | None,
+    enable_podman_service: bool,
 ) -> None:
     """Launch an evaluation job on Beaker.
 
@@ -425,6 +431,7 @@ def launch(
             preemptible=preemptible,
             retries=retries,
             gpus=gpus,
+            enable_podman_service=enable_podman_service,
         )
         return
 
@@ -1004,6 +1011,7 @@ def _launch_external_evals(
     preemptible: bool | None = None,
     retries: int | None = None,
     gpus: int | None = None,
+    enable_podman_service: bool = False,
 ) -> None:
     """Launch external evaluation jobs on Beaker.
 
@@ -1188,6 +1196,7 @@ def _launch_external_evals(
             retries=retries,
             provider_kind=str(provider_config.kind),
             base_url=provider_config.base_url,
+            enable_podman_service=enable_podman_service,
         )
         job_configs.append(job_config)
 

From f837e2528ac9c82ecd93c403dc4517c73d8ce574 Mon Sep 17 00:00:00 2001
From: Tyler Murray <tylerm@allenai.org>
Date: Fri, 3 Apr 2026 11:06:42 -0700
Subject: [PATCH 03/29] tweak podman servic start command

---
 src/olmo_eval/launch/beaker/launcher.py | 9 ++++++---
 1 file changed, 6 insertions(+), 3 deletions(-)

diff --git a/src/olmo_eval/launch/beaker/launcher.py b/src/olmo_eval/launch/beaker/launcher.py
index 80b623b0e..f8ce19182 100644
--- a/src/olmo_eval/launch/beaker/launcher.py
+++ b/src/olmo_eval/launch/beaker/launcher.py
@@ -691,9 +691,12 @@ def _build_install_cmd(
 
         # Start podman system service for Docker API compatibility (swebench harness, etc.)
         if enable_podman_service:
-            steps.append("podman system service --time=0 unix:///tmp/podman.sock &")
-            steps.append("sleep 2")
-            steps.append("export DOCKER_HOST=unix:///tmp/podman.sock")
+            # Use semicolons after backgrounded command since && requires exit status
+            steps.append(
+                "podman system service --time=0 unix:///tmp/podman.sock & "
+                "sleep 2; "
+                "export DOCKER_HOST=unix:///tmp/podman.sock"
+            )
 
         # Export additional environment variables (e.g., UV_CACHE_DIR)
         if env_exports:

From fdce6f46de7e148bb067f6e0083c6dd827d9cab9 Mon Sep 17 00:00:00 2001
From: Tyler Murray <tylerm@allenai.org>
Date: Fri, 3 Apr 2026 11:16:54 -0700
Subject: [PATCH 04/29] Fix argument name

---
 .../evals/external/benchmarks/swebench/eval.py       | 12 +++++-------
 1 file changed, 5 insertions(+), 7 deletions(-)

diff --git a/src/olmo_eval/evals/external/benchmarks/swebench/eval.py b/src/olmo_eval/evals/external/benchmarks/swebench/eval.py
index f88aec822..be53c226a 100644
--- a/src/olmo_eval/evals/external/benchmarks/swebench/eval.py
+++ b/src/olmo_eval/evals/external/benchmarks/swebench/eval.py
@@ -250,17 +250,15 @@ async def _run_scoring(
 
         env = os.environ.copy()
 
+        cmd.extend(["--max_workers", str(swe_args.max_workers_eval)])
+
         if swe_args.use_modal:
             # Modal handles containers in the cloud
-            cmd.extend(["--modal", "true", "--parallelism", str(swe_args.max_workers_eval)])
-        else:
+            cmd.extend(["--modal", "true"])
+        elif "DOCKER_HOST" not in env:
             # Use podman service socket (started by Beaker launcher)
             # DOCKER_HOST should already be set by _build_install_cmd
-            cmd.extend(["--max_workers", str(swe_args.max_workers_eval)])
-            if "DOCKER_HOST" not in env:
-                logger.warning(
-                    "DOCKER_HOST not set. Scoring may fail without podman service or Modal."
-                )
+            logger.warning("DOCKER_HOST not set. Scoring may fail without podman service or Modal.")
 
         logger.info(f"[{self.name}] Running SWE-bench harness: {shlex.join(cmd)}")
         ok, output = await self._run_subprocess(

From 56028cdb4991ca4c6d9fc6c4c1ad2cf4108fb2e6 Mon Sep 17 00:00:00 2001
From: Tyler Murray <tylerm@allenai.org>
Date: Fri, 3 Apr 2026 11:28:39 -0700
Subject: [PATCH 05/29] Dummy api key

---
 src/olmo_eval/evals/external/benchmarks/swebench/eval.py | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/src/olmo_eval/evals/external/benchmarks/swebench/eval.py b/src/olmo_eval/evals/external/benchmarks/swebench/eval.py
index be53c226a..0953a2abe 100644
--- a/src/olmo_eval/evals/external/benchmarks/swebench/eval.py
+++ b/src/olmo_eval/evals/external/benchmarks/swebench/eval.py
@@ -221,6 +221,11 @@ async def _run_agent(
         env = os.environ.copy()
         env["MSWEA_DOCKER_EXECUTABLE"] = container_runtime
 
+        # LiteLLM's OpenAI provider requires an API key even for local servers.
+        # Set a dummy key when using local vLLM to satisfy the requirement.
+        if is_local and "OPENAI_API_KEY" not in env:
+            env["OPENAI_API_KEY"] = "EMPTY"
+
         logger.info(f"[{self.name}] Running mini-swe-agent: {shlex.join(cmd)}")
         # Reserve 20% of total timeout for the scoring phase
         return await self._run_subprocess(cmd, timeout=self.timeout_seconds * 0.8, env=env)

From 72bcbc68270a297d9a6ad335c87315f7bede2924 Mon Sep 17 00:00:00 2001
From: Tyler Murray <tylerm@allenai.org>
Date: Fri, 3 Apr 2026 11:54:08 -0700
Subject: [PATCH 06/29] Fix vllm model prefix

---
 src/olmo_eval/evals/external/benchmarks/swebench/eval.py | 9 ++-------
 1 file changed, 2 insertions(+), 7 deletions(-)

diff --git a/src/olmo_eval/evals/external/benchmarks/swebench/eval.py b/src/olmo_eval/evals/external/benchmarks/swebench/eval.py
index 0953a2abe..05d292b8a 100644
--- a/src/olmo_eval/evals/external/benchmarks/swebench/eval.py
+++ b/src/olmo_eval/evals/external/benchmarks/swebench/eval.py
@@ -184,9 +184,9 @@ async def _run_agent(
         container_runtime: str,
     ) -> tuple[bool, str]:
         """Run mini-swe-agent to generate patches. Returns (success, output)."""
-        # For local vLLM servers, use the openai/ litellm prefix with api_base config.
+        # For local vLLM servers, use the hosted_vllm/ litellm prefix with api_base config.
         # For external APIs (OpenAI etc.), pass the model name as-is.
-        litellm_model = f"openai/{model_name}" if is_local else model_name
+        litellm_model = f"hosted_vllm/{model_name}" if is_local else model_name
 
         cmd = [
             sys.executable,
@@ -221,11 +221,6 @@ async def _run_agent(
         env = os.environ.copy()
         env["MSWEA_DOCKER_EXECUTABLE"] = container_runtime
 
-        # LiteLLM's OpenAI provider requires an API key even for local servers.
-        # Set a dummy key when using local vLLM to satisfy the requirement.
-        if is_local and "OPENAI_API_KEY" not in env:
-            env["OPENAI_API_KEY"] = "EMPTY"
-
         logger.info(f"[{self.name}] Running mini-swe-agent: {shlex.join(cmd)}")
         # Reserve 20% of total timeout for the scoring phase
         return await self._run_subprocess(cmd, timeout=self.timeout_seconds * 0.8, env=env)

From e32244b6a39220b3a490ca093ab7f4898227a81d Mon Sep 17 00:00:00 2001
From: Tyler Murray <tylerm@allenai.org>
Date: Fri, 3 Apr 2026 12:06:03 -0700
Subject: [PATCH 07/29] Try overriding api key

---
 src/olmo_eval/evals/external/benchmarks/swebench/eval.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/src/olmo_eval/evals/external/benchmarks/swebench/eval.py b/src/olmo_eval/evals/external/benchmarks/swebench/eval.py
index 05d292b8a..057bcd5d5 100644
--- a/src/olmo_eval/evals/external/benchmarks/swebench/eval.py
+++ b/src/olmo_eval/evals/external/benchmarks/swebench/eval.py
@@ -212,6 +212,7 @@ async def _run_agent(
         ]
         if is_local:
             cmd += ["-c", f"model.api_base={provider_url}"]
+            cmd += ["-c", "model.api_key=local"]
         if swe_args.instance_filter:
             cmd += ["--filter", swe_args.instance_filter]
         if swe_args.instance_slice:

From a1d43acbcd5ab5bb8db3f586ebc0daf83e66f124 Mon Sep 17 00:00:00 2001
From: Tyler Murray <tylerm@allenai.org>
Date: Fri, 3 Apr 2026 12:31:38 -0700
Subject: [PATCH 08/29] try diff env vars

---
 src/olmo_eval/evals/external/benchmarks/swebench/eval.py | 7 +++----
 1 file changed, 3 insertions(+), 4 deletions(-)

diff --git a/src/olmo_eval/evals/external/benchmarks/swebench/eval.py b/src/olmo_eval/evals/external/benchmarks/swebench/eval.py
index 057bcd5d5..dd1927231 100644
--- a/src/olmo_eval/evals/external/benchmarks/swebench/eval.py
+++ b/src/olmo_eval/evals/external/benchmarks/swebench/eval.py
@@ -212,7 +212,6 @@ async def _run_agent(
         ]
         if is_local:
             cmd += ["-c", f"model.api_base={provider_url}"]
-            cmd += ["-c", "model.api_key=local"]
         if swe_args.instance_filter:
             cmd += ["--filter", swe_args.instance_filter]
         if swe_args.instance_slice:
@@ -221,6 +220,9 @@ async def _run_agent(
         # mini-swe-agent respects MSWEA_DOCKER_EXECUTABLE to switch container runtimes
         env = os.environ.copy()
         env["MSWEA_DOCKER_EXECUTABLE"] = container_runtime
+        if is_local:
+            env["HOSTED_VLLM_API_KEY"] = "local"
+            env["HOSTED_VLLM_API_BASE"] = provider_url
 
         logger.info(f"[{self.name}] Running mini-swe-agent: {shlex.join(cmd)}")
         # Reserve 20% of total timeout for the scoring phase
@@ -254,11 +256,8 @@ async def _run_scoring(
         cmd.extend(["--max_workers", str(swe_args.max_workers_eval)])
 
         if swe_args.use_modal:
-            # Modal handles containers in the cloud
             cmd.extend(["--modal", "true"])
         elif "DOCKER_HOST" not in env:
-            # Use podman service socket (started by Beaker launcher)
-            # DOCKER_HOST should already be set by _build_install_cmd
             logger.warning("DOCKER_HOST not set. Scoring may fail without podman service or Modal.")
 
         logger.info(f"[{self.name}] Running SWE-bench harness: {shlex.join(cmd)}")

From 4f880ca34aed401645069725f9601da9a24aa79c Mon Sep 17 00:00:00 2001
From: Tyler Murray <tylerm@allenai.org>
Date: Fri, 3 Apr 2026 12:48:18 -0700
Subject: [PATCH 09/29] cost reporting error fix

---
 src/olmo_eval/evals/external/benchmarks/swebench/eval.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/src/olmo_eval/evals/external/benchmarks/swebench/eval.py b/src/olmo_eval/evals/external/benchmarks/swebench/eval.py
index dd1927231..c9e37edeb 100644
--- a/src/olmo_eval/evals/external/benchmarks/swebench/eval.py
+++ b/src/olmo_eval/evals/external/benchmarks/swebench/eval.py
@@ -223,6 +223,7 @@ async def _run_agent(
         if is_local:
             env["HOSTED_VLLM_API_KEY"] = "local"
             env["HOSTED_VLLM_API_BASE"] = provider_url
+            env["MSWEA_COST_TRACKING"] = "ignore_errors"
 
         logger.info(f"[{self.name}] Running mini-swe-agent: {shlex.join(cmd)}")
         # Reserve 20% of total timeout for the scoring phase

From e5458b7079750c3f6eff1a712a8ce1253b6f923d Mon Sep 17 00:00:00 2001
From: Tyler Murray <tylerm@allenai.org>
Date: Fri, 3 Apr 2026 13:24:12 -0700
Subject: [PATCH 10/29] Get agent and scoring logs

---
 .../evals/external/benchmarks/swebench/eval.py       | 12 ++++++++++++
 1 file changed, 12 insertions(+)

diff --git a/src/olmo_eval/evals/external/benchmarks/swebench/eval.py b/src/olmo_eval/evals/external/benchmarks/swebench/eval.py
index c9e37edeb..a02f639b4 100644
--- a/src/olmo_eval/evals/external/benchmarks/swebench/eval.py
+++ b/src/olmo_eval/evals/external/benchmarks/swebench/eval.py
@@ -145,6 +145,8 @@ async def execute(
                 provider_url, model_name, is_local, swe_args, work_dir, container_runtime
             )
             logger.info(f"[{self.name}] Patch generation exit: {'ok' if gen_ok else 'failed'}")
+            if output_dir:
+                self._write_log_file(work_dir, "mini_swe_agent", gen_output)
 
             if not gen_ok or not preds_path.exists():
                 return self._error_result(
@@ -156,6 +158,8 @@ async def execute(
                 swe_args, preds_path, run_id, work_dir, container_runtime
             )
             logger.info(f"[{self.name}] Scoring exit: {'ok' if score_ok else 'failed'}")
+            if output_dir:
+                self._write_log_file(work_dir, "swebench_scoring", score_output)
 
             all_output = gen_output + "\n" + score_output
             result = self._parse_results(work_dir, run_id, score_ok, all_output, start_time)
@@ -294,6 +298,14 @@ async def _run_subprocess(
         output = stdout.decode(errors="replace")
         return proc.returncode == 0, output
 
+    def _write_log_file(self, output_dir: Path, name: str, content: str) -> None:
+        """Write subprocess output to a log file."""
+        logs_dir = output_dir / "logs"
+        logs_dir.mkdir(parents=True, exist_ok=True)
+        log_path = logs_dir / f"{name}.log"
+        log_path.write_text(content)
+        logger.info(f"[{self.name}] Log saved to {log_path}")
+
     def _parse_results(
         self,
         work_dir: Path,

From 268a1eda80d76868b8100e2ccd705c8c302e35bb Mon Sep 17 00:00:00 2001
From: Tyler Murray <tylerm@allenai.org>
Date: Fri, 3 Apr 2026 13:52:30 -0700
Subject: [PATCH 11/29] more logging tweaks

---
 .../external/benchmarks/swebench/eval.py      | 82 ++++++++++++++-----
 1 file changed, 62 insertions(+), 20 deletions(-)

diff --git a/src/olmo_eval/evals/external/benchmarks/swebench/eval.py b/src/olmo_eval/evals/external/benchmarks/swebench/eval.py
index a02f639b4..0a33dda64 100644
--- a/src/olmo_eval/evals/external/benchmarks/swebench/eval.py
+++ b/src/olmo_eval/evals/external/benchmarks/swebench/eval.py
@@ -139,14 +139,23 @@ async def execute(
         model_name = provider.model_name
         is_local = self._is_local_provider(provider, provider_url)
 
+        # Set up real-time log files if output_dir is provided
+        logs_dir = work_dir / "logs" if output_dir else None
+        agent_log = logs_dir / "mini_swe_agent.log" if logs_dir else None
+        scoring_log = logs_dir / "swebench_scoring.log" if logs_dir else None
+
         try:
             # Phase 1: generate patches with mini-swe-agent
             gen_ok, gen_output = await self._run_agent(
-                provider_url, model_name, is_local, swe_args, work_dir, container_runtime
+                provider_url,
+                model_name,
+                is_local,
+                swe_args,
+                work_dir,
+                container_runtime,
+                log_file=agent_log,
             )
             logger.info(f"[{self.name}] Patch generation exit: {'ok' if gen_ok else 'failed'}")
-            if output_dir:
-                self._write_log_file(work_dir, "mini_swe_agent", gen_output)
 
             if not gen_ok or not preds_path.exists():
                 return self._error_result(
@@ -155,11 +164,14 @@ async def execute(
 
             # Phase 2: score patches with the SWE-bench harness
             score_ok, score_output = await self._run_scoring(
-                swe_args, preds_path, run_id, work_dir, container_runtime
+                swe_args,
+                preds_path,
+                run_id,
+                work_dir,
+                container_runtime,
+                log_file=scoring_log,
             )
             logger.info(f"[{self.name}] Scoring exit: {'ok' if score_ok else 'failed'}")
-            if output_dir:
-                self._write_log_file(work_dir, "swebench_scoring", score_output)
 
             all_output = gen_output + "\n" + score_output
             result = self._parse_results(work_dir, run_id, score_ok, all_output, start_time)
@@ -186,6 +198,7 @@ async def _run_agent(
         swe_args: SWEBenchArgs,
         work_dir: Path,
         container_runtime: str,
+        log_file: Path | None = None,
     ) -> tuple[bool, str]:
         """Run mini-swe-agent to generate patches. Returns (success, output)."""
         # For local vLLM servers, use the hosted_vllm/ litellm prefix with api_base config.
@@ -231,7 +244,9 @@ async def _run_agent(
 
         logger.info(f"[{self.name}] Running mini-swe-agent: {shlex.join(cmd)}")
         # Reserve 20% of total timeout for the scoring phase
-        return await self._run_subprocess(cmd, timeout=self.timeout_seconds * 0.8, env=env)
+        return await self._run_subprocess(
+            cmd, timeout=self.timeout_seconds * 0.8, env=env, log_file=log_file
+        )
 
     async def _run_scoring(
         self,
@@ -240,6 +255,7 @@ async def _run_scoring(
         run_id: str,
         work_dir: Path,
         container_runtime: str,
+        log_file: Path | None = None,
     ) -> tuple[bool, str]:
         """Run the SWE-bench evaluation harness to score patches."""
         dataset_hf = _DATASET_HF_PATHS.get(swe_args.dataset, swe_args.dataset)
@@ -267,7 +283,11 @@ async def _run_scoring(
 
         logger.info(f"[{self.name}] Running SWE-bench harness: {shlex.join(cmd)}")
         ok, output = await self._run_subprocess(
-            cmd, timeout=self.timeout_seconds * 0.2, cwd=str(work_dir), env=env
+            cmd,
+            timeout=self.timeout_seconds * 0.2,
+            cwd=str(work_dir),
+            env=env,
+            log_file=log_file,
         )
         if not ok:
             logger.warning(f"[{self.name}] Scoring subprocess output:\n{output}")
@@ -279,8 +299,12 @@ async def _run_subprocess(
         timeout: float,
         cwd: str | None = None,
         env: dict[str, str] | None = None,
+        log_file: Path | None = None,
     ) -> tuple[bool, str]:
-        """Run a subprocess and return (success, combined stdout+stderr)."""
+        """Run a subprocess and return (success, combined stdout+stderr).
+
+        If log_file is provided, output is streamed to the file in real-time.
+        """
         proc = await asyncio.create_subprocess_exec(
             *cmd,
             stdout=asyncio.subprocess.PIPE,
@@ -288,23 +312,41 @@ async def _run_subprocess(
             cwd=cwd,
             env=env if env is not None else os.environ.copy(),
         )
+
+        output_chunks: list[str] = []
+        log_handle = None
+
+        if log_file:
+            log_file.parent.mkdir(parents=True, exist_ok=True)
+            log_handle = log_file.open("w")
+
         try:
-            stdout, _ = await asyncio.wait_for(proc.communicate(), timeout=timeout)
+
+            async def stream_output() -> None:
+                assert proc.stdout is not None
+                while True:
+                    chunk = await proc.stdout.read(4096)
+                    if not chunk:
+                        break
+                    decoded = chunk.decode(errors="replace")
+                    output_chunks.append(decoded)
+                    if log_handle:
+                        log_handle.write(decoded)
+                        log_handle.flush()
+
+            await asyncio.wait_for(stream_output(), timeout=timeout)
+            await proc.wait()
         except TimeoutError:
             proc.kill()
             await proc.communicate()
+            if log_handle:
+                log_handle.write(f"\n[Subprocess timed out after {timeout:.0f}s]\n")
             return False, f"[Subprocess timed out after {timeout:.0f}s]"
+        finally:
+            if log_handle:
+                log_handle.close()
 
-        output = stdout.decode(errors="replace")
-        return proc.returncode == 0, output
-
-    def _write_log_file(self, output_dir: Path, name: str, content: str) -> None:
-        """Write subprocess output to a log file."""
-        logs_dir = output_dir / "logs"
-        logs_dir.mkdir(parents=True, exist_ok=True)
-        log_path = logs_dir / f"{name}.log"
-        log_path.write_text(content)
-        logger.info(f"[{self.name}] Log saved to {log_path}")
+        return proc.returncode == 0, "".join(output_chunks)
 
     def _parse_results(
         self,

From f296e8d179837861c2cd006b22fc15a6bf1075e8 Mon Sep 17 00:00:00 2001
From: Tyler Murray <tylerm@allenai.org>
Date: Fri, 3 Apr 2026 14:26:01 -0700
Subject: [PATCH 12/29] Add dummy modal.toml

---
 .../evals/external/benchmarks/swebench/eval.py  | 17 +++++++++++++++++
 1 file changed, 17 insertions(+)

diff --git a/src/olmo_eval/evals/external/benchmarks/swebench/eval.py b/src/olmo_eval/evals/external/benchmarks/swebench/eval.py
index 0a33dda64..3c442b4e5 100644
--- a/src/olmo_eval/evals/external/benchmarks/swebench/eval.py
+++ b/src/olmo_eval/evals/external/benchmarks/swebench/eval.py
@@ -32,6 +32,20 @@
 }
 
 
+def _ensure_modal_config_exists() -> None:
+    """Create empty ~/.modal.toml if missing.
+
+    SWE-bench's validate_modal_credentials() checks for this file's existence,
+    but Modal SDK uses MODAL_TOKEN_ID/MODAL_TOKEN_SECRET env vars for auth
+    (which take precedence over the file). We create an empty file to satisfy
+    the validation while keeping credentials in env vars only.
+    """
+    modal_toml = Path.home() / ".modal.toml"
+    if not modal_toml.exists():
+        modal_toml.touch()
+        logger.info("Created empty ~/.modal.toml to satisfy SWE-bench validation")
+
+
 @dataclass
 class SWEBenchArgs:
     """Arguments for the swe_bench evaluation."""
@@ -258,6 +272,9 @@ async def _run_scoring(
         log_file: Path | None = None,
     ) -> tuple[bool, str]:
         """Run the SWE-bench evaluation harness to score patches."""
+        if swe_args.use_modal:
+            _ensure_modal_config_exists()
+
         dataset_hf = _DATASET_HF_PATHS.get(swe_args.dataset, swe_args.dataset)
 
         cmd = [

From f91a663779d4fb3ec99d2021e27951b8bfc8f520 Mon Sep 17 00:00:00 2001
From: Tyler Murray <tylerm@allenai.org>
Date: Fri, 3 Apr 2026 15:02:24 -0700
Subject: [PATCH 13/29] Modify results file name matching

---
 .../evals/external/benchmarks/swebench/eval.py    | 15 ++++++++-------
 1 file changed, 8 insertions(+), 7 deletions(-)

diff --git a/src/olmo_eval/evals/external/benchmarks/swebench/eval.py b/src/olmo_eval/evals/external/benchmarks/swebench/eval.py
index 3c442b4e5..e57e91848 100644
--- a/src/olmo_eval/evals/external/benchmarks/swebench/eval.py
+++ b/src/olmo_eval/evals/external/benchmarks/swebench/eval.py
@@ -374,13 +374,14 @@ def _parse_results(
         start_time: float,
     ) -> ExternalEvalResult:
         """Parse the SWE-bench harness results JSON into an ExternalEvalResult."""
-        # SWE-bench writes <run_id>.json to the working directory; some versions
-        # use an evaluation_results/ subdirectory instead.
-        candidates = [
-            work_dir / f"{run_id}.json",
-            work_dir / "evaluation_results" / f"{run_id}.json",
-        ]
-        results_file = next((p for p in candidates if p.exists()), None)
+        # SWE-bench writes results with pattern {model_name}.{run_id}.json where
+        # model slashes become double underscores. Search for any file ending
+        # in the run_id pattern.
+        pattern = f"*{run_id}.json"
+        matches = list(work_dir.glob(pattern))
+        # Also check evaluation_results/ subdirectory (older harness versions)
+        matches.extend(work_dir.glob(f"evaluation_results/{pattern}"))
+        results_file = matches[0] if matches else None
 
         if results_file is None:
             return ExternalEvalResult(

From e892875efb05120f7e050dbcee55ce47fd108ba3 Mon Sep 17 00:00:00 2001
From: Tyler Murray <tylerm@allenai.org>
Date: Fri, 3 Apr 2026 15:29:33 -0700
Subject: [PATCH 14/29] Use correct arg for max turns

---
 src/olmo_eval/evals/external/benchmarks/swebench/eval.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/olmo_eval/evals/external/benchmarks/swebench/eval.py b/src/olmo_eval/evals/external/benchmarks/swebench/eval.py
index e57e91848..d210a78d7 100644
--- a/src/olmo_eval/evals/external/benchmarks/swebench/eval.py
+++ b/src/olmo_eval/evals/external/benchmarks/swebench/eval.py
@@ -237,7 +237,7 @@ async def _run_agent(
             "-c",
             "swebench.yaml",
             "-c",
-            f"agent.max_iterations={swe_args.max_turns}",
+            f"agent.step_limit={swe_args.max_turns}",
             "-c",
             f"model.model_kwargs.temperature={swe_args.temperature}",
         ]

From abead866c6f58307c9493d5f3c785a4dddb1c8cf Mon Sep 17 00:00:00 2001
From: Tyler Murray <tylerm@allenai.org>
Date: Fri, 3 Apr 2026 16:11:16 -0700
Subject: [PATCH 15/29] Try bumping vllm

---
 pyproject.toml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/pyproject.toml b/pyproject.toml
index e75e7e575..dcc396a76 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -42,7 +42,7 @@ hf = [
     "transformers~=4.57.3",
 ]
 vllm = [
-    "vllm[runai]==0.13.0",  # runai extras enable S3 model loading
+    "vllm[runai]==0.17.1",  # runai extras enable S3 model loading
     "opencv-python-headless==4.13.0.92",  # Required to avoid libxcb import error
     "torch-c-dlpack-ext",  # Required for TVM tensor allocation in vLLM
 ]

From f7ed4cf476d8026ff528d7258a4baf1f5a2fd6ff Mon Sep 17 00:00:00 2001
From: Tyler Murray <tylerm@allenai.org>
Date: Fri, 3 Apr 2026 16:14:28 -0700
Subject: [PATCH 16/29] Print runtime in external eval as well

---
 src/olmo_eval/cli/run_external.py | 10 +++++++++-
 1 file changed, 9 insertions(+), 1 deletion(-)

diff --git a/src/olmo_eval/cli/run_external.py b/src/olmo_eval/cli/run_external.py
index b2e58fcc0..f1b16cfe5 100644
--- a/src/olmo_eval/cli/run_external.py
+++ b/src/olmo_eval/cli/run_external.py
@@ -7,7 +7,12 @@
 
 import click
 
-from olmo_eval.cli.utils import ConfiguredExternalEval, console, parse_key_value_args
+from olmo_eval.cli.utils import (
+    ConfiguredExternalEval,
+    console,
+    parse_key_value_args,
+    print_runtime_environment,
+)
 from olmo_eval.common.constants.infrastructure import BEAKER_RESULT_DIR
 
 
@@ -200,6 +205,9 @@ def run_external(
 
     configure_logging(level="INFO")
 
+    # Print runtime environment summary
+    print_runtime_environment()
+
     # Build provider config
     from olmo_eval.common.configs import get_provider_config
 

From 3dce1b2bbb0840c571b271ac3db0009dd9329470 Mon Sep 17 00:00:00 2001
From: Tyler Murray <tylerm@allenai.org>
Date: Fri, 3 Apr 2026 16:16:58 -0700
Subject: [PATCH 17/29] bump transformers as well

---
 pyproject.toml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/pyproject.toml b/pyproject.toml
index dcc396a76..e9e434d2a 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -39,7 +39,7 @@ Issues = "https://github.com/allenai/olmo-eval/issues"
 
 [project.optional-dependencies]
 hf = [
-    "transformers~=4.57.3",
+    "transformers>=5.5.0",
 ]
 vllm = [
     "vllm[runai]==0.17.1",  # runai extras enable S3 model loading

From c0fc5f3efa2064ede4217fc089ea63153c1829ab Mon Sep 17 00:00:00 2001
From: Tyler Murray <tylerm@allenai.org>
Date: Fri, 3 Apr 2026 16:22:02 -0700
Subject: [PATCH 18/29] Revert transformers bump

---
 pyproject.toml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/pyproject.toml b/pyproject.toml
index e9e434d2a..dcc396a76 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -39,7 +39,7 @@ Issues = "https://github.com/allenai/olmo-eval/issues"
 
 [project.optional-dependencies]
 hf = [
-    "transformers>=5.5.0",
+    "transformers~=4.57.3",
 ]
 vllm = [
     "vllm[runai]==0.17.1",  # runai extras enable S3 model loading

From 9ff7dd1a1d28b9b73f9410230773761aa4584158 Mon Sep 17 00:00:00 2001
From: Tyler Murray <tylerm@allenai.org>
Date: Fri, 3 Apr 2026 16:25:05 -0700
Subject: [PATCH 19/29] Try 0.19

---
 pyproject.toml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/pyproject.toml b/pyproject.toml
index dcc396a76..da9f4b975 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -42,7 +42,7 @@ hf = [
     "transformers~=4.57.3",
 ]
 vllm = [
-    "vllm[runai]==0.17.1",  # runai extras enable S3 model loading
+    "vllm[runai]==0.19.0",  # runai extras enable S3 model loading
     "opencv-python-headless==4.13.0.92",  # Required to avoid libxcb import error
     "torch-c-dlpack-ext",  # Required for TVM tensor allocation in vLLM
 ]

From 85f45b8b65dc3c076aa8d25044b1f6ae12d6a25a Mon Sep 17 00:00:00 2001
From: Tyler Murray <tylerm@allenai.org>
Date: Fri, 3 Apr 2026 16:30:19 -0700
Subject: [PATCH 20/29] Fix vllm install

---
 src/olmo_eval/launch/beaker/launcher.py | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/src/olmo_eval/launch/beaker/launcher.py b/src/olmo_eval/launch/beaker/launcher.py
index f8ce19182..1e7cd36e4 100644
--- a/src/olmo_eval/launch/beaker/launcher.py
+++ b/src/olmo_eval/launch/beaker/launcher.py
@@ -717,10 +717,10 @@ def _build_install_cmd(
                 f"/opt/venv/lib/python*/site-packages/nvidia*; do "
                 f'ln -sf "$pkg" {vllm_venv}/lib/python*/site-packages/; done'
             )
-            # Install vLLM (torch already available via symlink)
+            # Install vLLM from package extras (torch already available via symlink)
             steps.append(
-                f"VIRTUAL_ENV={vllm_venv} uv pip install "
-                f"--cache-dir \"$UV_CACHE_DIR\" 'vllm[runai]==0.13.0'"
+                f"cd /gantry-runtime && VIRTUAL_ENV={vllm_venv} uv pip install "
+                f"--cache-dir \"$UV_CACHE_DIR\" -e '.[vllm]' -c {constraints}"
             )
             # Set VLLM_PYTHON so VLLMServerProcess uses the isolated venv
             steps.append(f"export VLLM_PYTHON={vllm_venv}/bin/python")

From 82247d7e9a722b8a211c86cd71336589ffae0f2d Mon Sep 17 00:00:00 2001
From: Tyler Murray <tylerm@allenai.org>
Date: Fri, 3 Apr 2026 16:32:10 -0700
Subject: [PATCH 21/29] Downgrade vllm

---
 pyproject.toml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/pyproject.toml b/pyproject.toml
index da9f4b975..dcc396a76 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -42,7 +42,7 @@ hf = [
     "transformers~=4.57.3",
 ]
 vllm = [
-    "vllm[runai]==0.19.0",  # runai extras enable S3 model loading
+    "vllm[runai]==0.17.1",  # runai extras enable S3 model loading
     "opencv-python-headless==4.13.0.92",  # Required to avoid libxcb import error
     "torch-c-dlpack-ext",  # Required for TVM tensor allocation in vLLM
 ]

From e61206abdd198601e19b9cda4fe741838c0ec63c Mon Sep 17 00:00:00 2001
From: Tyler Murray <tylerm@allenai.org>
Date: Fri, 3 Apr 2026 16:34:25 -0700
Subject: [PATCH 22/29] Back to 0.13 until we vet torch 2.10

---
 pyproject.toml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/pyproject.toml b/pyproject.toml
index dcc396a76..e75e7e575 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -42,7 +42,7 @@ hf = [
     "transformers~=4.57.3",
 ]
 vllm = [
-    "vllm[runai]==0.17.1",  # runai extras enable S3 model loading
+    "vllm[runai]==0.13.0",  # runai extras enable S3 model loading
     "opencv-python-headless==4.13.0.92",  # Required to avoid libxcb import error
     "torch-c-dlpack-ext",  # Required for TVM tensor allocation in vLLM
 ]

From 693dd52bb91aaeb50e014f6dd215fd9e5de31d02 Mon Sep 17 00:00:00 2001
From: Tyler Murray <tylerm@allenai.org>
Date: Fri, 3 Apr 2026 16:39:09 -0700
Subject: [PATCH 23/29] Try parsing to avoid reresolution

---
 src/olmo_eval/launch/beaker/launcher.py | 10 +++++++---
 1 file changed, 7 insertions(+), 3 deletions(-)

diff --git a/src/olmo_eval/launch/beaker/launcher.py b/src/olmo_eval/launch/beaker/launcher.py
index 1e7cd36e4..8ab3e5828 100644
--- a/src/olmo_eval/launch/beaker/launcher.py
+++ b/src/olmo_eval/launch/beaker/launcher.py
@@ -717,10 +717,14 @@ def _build_install_cmd(
                 f"/opt/venv/lib/python*/site-packages/nvidia*; do "
                 f'ln -sf "$pkg" {vllm_venv}/lib/python*/site-packages/; done'
             )
-            # Install vLLM from package extras (torch already available via symlink)
+            # Extract all vllm deps from pyproject.toml and install directly
+            # (installing via .[vllm] extra causes uv to re-resolve torch, ignoring symlinks)
             steps.append(
-                f"cd /gantry-runtime && VIRTUAL_ENV={vllm_venv} uv pip install "
-                f"--cache-dir \"$UV_CACHE_DIR\" -e '.[vllm]' -c {constraints}"
+                f'VLLM_DEPS=$(python -c "import tomllib; '
+                f"print(' '.join(tomllib.load(open('/gantry-runtime/pyproject.toml','rb'))"
+                f"['project']['optional-dependencies']['vllm']))\") && "
+                f"VIRTUAL_ENV={vllm_venv} uv pip install "
+                f'--cache-dir "$UV_CACHE_DIR" $VLLM_DEPS -c {constraints}'
             )
             # Set VLLM_PYTHON so VLLMServerProcess uses the isolated venv
             steps.append(f"export VLLM_PYTHON={vllm_venv}/bin/python")

From 11ed6950d173d4ac2cf9cdb39d49f97efee619af Mon Sep 17 00:00:00 2001
From: Tyler Murray <tylerm@allenai.org>
Date: Fri, 3 Apr 2026 16:41:12 -0700
Subject: [PATCH 24/29] Try again

---
 src/olmo_eval/launch/beaker/launcher.py | 18 +++++++++++++-----
 1 file changed, 13 insertions(+), 5 deletions(-)

diff --git a/src/olmo_eval/launch/beaker/launcher.py b/src/olmo_eval/launch/beaker/launcher.py
index 8ab3e5828..2930167b0 100644
--- a/src/olmo_eval/launch/beaker/launcher.py
+++ b/src/olmo_eval/launch/beaker/launcher.py
@@ -717,14 +717,22 @@ def _build_install_cmd(
                 f"/opt/venv/lib/python*/site-packages/nvidia*; do "
                 f'ln -sf "$pkg" {vllm_venv}/lib/python*/site-packages/; done'
             )
-            # Extract all vllm deps from pyproject.toml and install directly
-            # (installing via .[vllm] extra causes uv to re-resolve torch, ignoring symlinks)
+            # Install vllm deps from pyproject.toml:
+            # - vllm itself with --no-deps to skip torch resolution (torch is symlinked)
+            # - other deps normally
             steps.append(
                 f'VLLM_DEPS=$(python -c "import tomllib; '
-                f"print(' '.join(tomllib.load(open('/gantry-runtime/pyproject.toml','rb'))"
-                f"['project']['optional-dependencies']['vllm']))\") && "
+                f"deps = tomllib.load(open('/gantry-runtime/pyproject.toml','rb'))"
+                f"['project']['optional-dependencies']['vllm']; "
+                f"vllm_pkg = next(d for d in deps if d.startswith('vllm')); "
+                f"other_deps = [d for d in deps if not d.startswith('vllm')]; "
+                f"print(vllm_pkg + '\\n' + ' '.join(other_deps))\") && "
+                f'VLLM_PKG=$(echo "$VLLM_DEPS" | head -1) && '
+                f'OTHER_DEPS=$(echo "$VLLM_DEPS" | tail -1) && '
                 f"VIRTUAL_ENV={vllm_venv} uv pip install "
-                f'--cache-dir "$UV_CACHE_DIR" $VLLM_DEPS -c {constraints}'
+                f'--cache-dir "$UV_CACHE_DIR" --no-deps "$VLLM_PKG" -c {constraints} && '
+                f"VIRTUAL_ENV={vllm_venv} uv pip install "
+                f'--cache-dir "$UV_CACHE_DIR" $OTHER_DEPS -c {constraints}'
             )
             # Set VLLM_PYTHON so VLLMServerProcess uses the isolated venv
             steps.append(f"export VLLM_PYTHON={vllm_venv}/bin/python")

From 724ef0bb880ac1189b2c6e9a5cf9f4a08734762a Mon Sep 17 00:00:00 2001
From: Tyler Murray <tylerm@allenai.org>
Date: Fri, 3 Apr 2026 16:50:35 -0700
Subject: [PATCH 25/29] Revert

---
 src/olmo_eval/launch/beaker/launcher.py | 18 +++---------------
 1 file changed, 3 insertions(+), 15 deletions(-)

diff --git a/src/olmo_eval/launch/beaker/launcher.py b/src/olmo_eval/launch/beaker/launcher.py
index 2930167b0..f25d3f141 100644
--- a/src/olmo_eval/launch/beaker/launcher.py
+++ b/src/olmo_eval/launch/beaker/launcher.py
@@ -717,22 +717,10 @@ def _build_install_cmd(
                 f"/opt/venv/lib/python*/site-packages/nvidia*; do "
                 f'ln -sf "$pkg" {vllm_venv}/lib/python*/site-packages/; done'
             )
-            # Install vllm deps from pyproject.toml:
-            # - vllm itself with --no-deps to skip torch resolution (torch is symlinked)
-            # - other deps normally
+            # Install vLLM extra from project (no torch constraint - it's symlinked)
             steps.append(
-                f'VLLM_DEPS=$(python -c "import tomllib; '
-                f"deps = tomllib.load(open('/gantry-runtime/pyproject.toml','rb'))"
-                f"['project']['optional-dependencies']['vllm']; "
-                f"vllm_pkg = next(d for d in deps if d.startswith('vllm')); "
-                f"other_deps = [d for d in deps if not d.startswith('vllm')]; "
-                f"print(vllm_pkg + '\\n' + ' '.join(other_deps))\") && "
-                f'VLLM_PKG=$(echo "$VLLM_DEPS" | head -1) && '
-                f'OTHER_DEPS=$(echo "$VLLM_DEPS" | tail -1) && '
-                f"VIRTUAL_ENV={vllm_venv} uv pip install "
-                f'--cache-dir "$UV_CACHE_DIR" --no-deps "$VLLM_PKG" -c {constraints} && '
-                f"VIRTUAL_ENV={vllm_venv} uv pip install "
-                f'--cache-dir "$UV_CACHE_DIR" $OTHER_DEPS -c {constraints}'
+                f"cd /gantry-runtime && VIRTUAL_ENV={vllm_venv} uv pip install "
+                f"--cache-dir \"$UV_CACHE_DIR\" -e '.[vllm]'"
             )
             # Set VLLM_PYTHON so VLLMServerProcess uses the isolated venv
             steps.append(f"export VLLM_PYTHON={vllm_venv}/bin/python")

From 8314a05baccc9addddc3c2d1f504a493b28a4b29 Mon Sep 17 00:00:00 2001
From: Tyler Murray <tylerm@allenai.org>
Date: Fri, 3 Apr 2026 17:12:02 -0700
Subject: [PATCH 26/29] Tweaks

---
 .../external/benchmarks/swebench/eval.py      | 19 +++++++++++--------
 1 file changed, 11 insertions(+), 8 deletions(-)

diff --git a/src/olmo_eval/evals/external/benchmarks/swebench/eval.py b/src/olmo_eval/evals/external/benchmarks/swebench/eval.py
index d210a78d7..3ad0d579c 100644
--- a/src/olmo_eval/evals/external/benchmarks/swebench/eval.py
+++ b/src/olmo_eval/evals/external/benchmarks/swebench/eval.py
@@ -403,17 +403,20 @@ def _parse_results(
                 duration_seconds=time.time() - start_time,
             )
 
-        resolved = data.get("resolved_instances", data.get("resolved", []))
-        unresolved = data.get("unresolved_instances", data.get("unresolved", []))
-        total = len(resolved) + len(unresolved)
-        resolve_rate = len(resolved) / total if total > 0 else 0.0
+        resolved_ids = data.get("resolved_ids", [])
+        unresolved_ids = data.get("unresolved_ids", [])
+        resolved_count = data.get("resolved", 0)
+        unresolved_count = data.get("unresolved", 0)
+
+        total = resolved_count + unresolved_count
+        resolve_rate = resolved_count / total if total > 0 else 0.0
 
         predictions = [
             {"native_id": iid, "instance_metrics": {"resolved": {"external": 1.0}}}
-            for iid in resolved
+            for iid in resolved_ids
         ] + [
             {"native_id": iid, "instance_metrics": {"resolved": {"external": 0.0}}}
-            for iid in unresolved
+            for iid in unresolved_ids
         ]
 
         return ExternalEvalResult(
@@ -421,10 +424,10 @@ def _parse_results(
             success=score_ok and total > 0,
             metrics={
                 "resolve_rate": resolve_rate,
-                "resolved": float(len(resolved)),
+                "resolved": float(resolved_count),
                 "total": float(total),
             },
-            metadata={"run_id": run_id, "resolved_instances": resolved},
+            metadata={"run_id": run_id, "resolved_instances": resolved_ids},
             raw_output=raw_output,
             predictions=predictions if predictions else None,
         )

From acf559121023aa92b3168b52e151c31c5768b625 Mon Sep 17 00:00:00 2001
From: Tyler Murray <tylerm@allenai.org>
Date: Mon, 6 Apr 2026 09:28:44 -0700
Subject: [PATCH 27/29] Fix some results handling and parallelism flag for
 Modal

---
 src/olmo_eval/cli/run_external.py             |   2 +-
 .../external/benchmarks/swebench/eval.py      | 293 ++++++++++++++++--
 src/olmo_eval/evals/external/result.py        |  21 ++
 src/olmo_eval/runners/external/runner.py      |   2 +-
 4 files changed, 298 insertions(+), 20 deletions(-)

diff --git a/src/olmo_eval/cli/run_external.py b/src/olmo_eval/cli/run_external.py
index f1b16cfe5..9a0c2a2e6 100644
--- a/src/olmo_eval/cli/run_external.py
+++ b/src/olmo_eval/cli/run_external.py
@@ -367,7 +367,7 @@ def run_external(
             metrics = "\n".join(metrics_lines)
         else:
             status = "[red]Failed[/red]"
-            metrics = result.error or "Unknown error"
+            metrics = result.failure_reason()
 
         results_table.add_row(name, status, metrics)
 
diff --git a/src/olmo_eval/evals/external/benchmarks/swebench/eval.py b/src/olmo_eval/evals/external/benchmarks/swebench/eval.py
index 3ad0d579c..5a4876492 100644
--- a/src/olmo_eval/evals/external/benchmarks/swebench/eval.py
+++ b/src/olmo_eval/evals/external/benchmarks/swebench/eval.py
@@ -23,6 +23,16 @@
 
 logger = logging.getLogger(__name__)
 
+_INSTANCE_REPORT_KEYS = frozenset(
+    {
+        "patch_is_None",
+        "patch_exists",
+        "patch_successfully_applied",
+        "resolved",
+        "tests_status",
+    }
+)
+
 # Maps short aliases to HuggingFace dataset paths used by the SWE-bench harness.
 # mini-swe-agent has its own alias table; we pass aliases through to it directly.
 _DATASET_HF_PATHS = {
@@ -291,12 +301,15 @@ async def _run_scoring(
 
         env = os.environ.copy()
 
-        cmd.extend(["--max_workers", str(swe_args.max_workers_eval)])
-
         if swe_args.use_modal:
+            cmd.extend(["--parallelism", str(swe_args.max_workers_eval)])
             cmd.extend(["--modal", "true"])
-        elif "DOCKER_HOST" not in env:
-            logger.warning("DOCKER_HOST not set. Scoring may fail without podman service or Modal.")
+        else:
+            cmd.extend(["--max_workers", str(swe_args.max_workers_eval)])
+            if "DOCKER_HOST" not in env:
+                logger.warning(
+                    "DOCKER_HOST not set. Scoring may fail without podman service or Modal."
+                )
 
         logger.info(f"[{self.name}] Running SWE-bench harness: {shlex.join(cmd)}")
         ok, output = await self._run_subprocess(
@@ -403,21 +416,52 @@ def _parse_results(
                 duration_seconds=time.time() - start_time,
             )
 
-        resolved_ids = data.get("resolved_ids", [])
-        unresolved_ids = data.get("unresolved_ids", [])
-        resolved_count = data.get("resolved", 0)
-        unresolved_count = data.get("unresolved", 0)
+        resolved_ids = self._coerce_instance_ids(data.get("resolved_ids", []))
+        unresolved_ids = self._coerce_instance_ids(data.get("unresolved_ids", []))
+        submitted_ids = self._coerce_instance_ids(data.get("submitted_ids", []))
+        instance_reports = self._find_instance_reports(work_dir, results_file)
+        exit_statuses = self._find_exit_statuses(work_dir)
+        predictions = self._build_predictions(
+            submitted_ids=submitted_ids,
+            resolved_ids=resolved_ids,
+            unresolved_ids=unresolved_ids,
+            instance_reports=instance_reports,
+            exit_statuses=exit_statuses,
+        )
 
-        total = resolved_count + unresolved_count
-        resolve_rate = resolved_count / total if total > 0 else 0.0
+        if predictions:
+            resolved_instances = [
+                pred["native_id"]
+                for pred in predictions
+                if pred["instance_metrics"]["resolved"]["external"] == 1.0
+            ]
+            resolved_count = len(resolved_instances)
+            total = len(predictions)
+        else:
+            resolved_count = self._read_int(
+                data.get("resolved"),
+                data.get("resolved_instances"),
+                len(resolved_ids),
+            )
+            unresolved_count = self._read_int(
+                data.get("unresolved"),
+                data.get("unresolved_instances"),
+                len(unresolved_ids),
+            )
+            submitted_count = self._read_int(
+                data.get("submitted_instances"),
+                len(submitted_ids),
+            )
+            total = submitted_count if submitted_count > 0 else resolved_count + unresolved_count
+            resolved_instances = resolved_ids
 
-        predictions = [
-            {"native_id": iid, "instance_metrics": {"resolved": {"external": 1.0}}}
-            for iid in resolved_ids
-        ] + [
-            {"native_id": iid, "instance_metrics": {"resolved": {"external": 0.0}}}
-            for iid in unresolved_ids
-        ]
+        resolve_rate = resolved_count / total if total > 0 else 0.0
+        metadata: dict[str, Any] = {
+            "run_id": run_id,
+            "resolved_instances": resolved_instances,
+        }
+        if exit_statuses:
+            metadata["instance_exit_statuses"] = exit_statuses
 
         return ExternalEvalResult(
             name=self.name,
@@ -427,7 +471,220 @@ def _parse_results(
                 "resolved": float(resolved_count),
                 "total": float(total),
             },
-            metadata={"run_id": run_id, "resolved_instances": resolved_ids},
+            metadata=metadata,
             raw_output=raw_output,
             predictions=predictions if predictions else None,
         )
+
+    def _build_predictions(
+        self,
+        submitted_ids: list[str],
+        resolved_ids: list[str],
+        unresolved_ids: list[str],
+        instance_reports: dict[str, dict[str, Any]],
+        exit_statuses: dict[str, str],
+    ) -> list[dict[str, Any]]:
+        """Build per-instance predictions from any available SWE-bench artifacts."""
+        instance_ids = self._ordered_unique_ids(
+            resolved_ids,
+            unresolved_ids,
+            submitted_ids,
+            instance_reports.keys(),
+            exit_statuses.keys(),
+        )
+        return [
+            {
+                "native_id": instance_id,
+                "instance_metrics": {
+                    "resolved": {
+                        "external": self._resolve_prediction_score(
+                            instance_id,
+                            resolved_ids=resolved_ids,
+                            unresolved_ids=unresolved_ids,
+                            instance_reports=instance_reports,
+                        )
+                    }
+                },
+            }
+            for instance_id in instance_ids
+        ]
+
+    def _coerce_instance_ids(self, value: Any) -> list[str]:
+        if not isinstance(value, list):
+            return []
+        return [str(item) for item in value if item is not None]
+
+    def _ordered_unique_ids(self, *groups: Any) -> list[str]:
+        ordered_ids: list[str] = []
+        seen: set[str] = set()
+
+        for group in groups:
+            for instance_id in group:
+                if instance_id in seen:
+                    continue
+                seen.add(instance_id)
+                ordered_ids.append(str(instance_id))
+
+        return ordered_ids
+
+    def _resolve_prediction_score(
+        self,
+        instance_id: str,
+        resolved_ids: list[str],
+        unresolved_ids: list[str],
+        instance_reports: dict[str, dict[str, Any]],
+    ) -> float:
+        report_resolved = instance_reports.get(instance_id, {}).get("resolved")
+        if isinstance(report_resolved, bool):
+            return 1.0 if report_resolved else 0.0
+
+        if instance_id in resolved_ids:
+            return 1.0
+
+        if instance_id in unresolved_ids:
+            return 0.0
+
+        return 0.0
+
+    def _read_int(self, *values: Any) -> int:
+        for value in values:
+            try:
+                if value is None:
+                    continue
+                return int(value)
+            except (TypeError, ValueError):
+                continue
+        return 0
+
+    def _find_instance_reports(
+        self, work_dir: Path, results_file: Path
+    ) -> dict[str, dict[str, Any]]:
+        """Locate a per-instance report JSON if the harness emitted one."""
+        for candidate in sorted(work_dir.rglob("*.json")):
+            if candidate == results_file:
+                continue
+            payload = self._load_json(candidate)
+            if payload is None:
+                continue
+            reports = self._extract_instance_reports(payload)
+            if reports:
+                return reports
+        return {}
+
+    def _load_json(self, path: Path) -> Any | None:
+        try:
+            return json.loads(path.read_text())
+        except (OSError, json.JSONDecodeError):
+            return None
+
+    def _extract_instance_reports(self, payload: Any) -> dict[str, dict[str, Any]]:
+        if isinstance(payload, dict):
+            if (
+                payload
+                and all(isinstance(v, dict) for v in payload.values())
+                and any(self._looks_like_instance_report(v) for v in payload.values())
+            ):
+                return {str(k): v for k, v in payload.items() if isinstance(v, dict)}
+
+            for value in payload.values():
+                nested = self._extract_instance_reports(value)
+                if nested:
+                    return nested
+
+        if isinstance(payload, list):
+            for value in payload:
+                nested = self._extract_instance_reports(value)
+                if nested:
+                    return nested
+
+        return {}
+
+    def _looks_like_instance_report(self, value: dict[str, Any]) -> bool:
+        return bool(_INSTANCE_REPORT_KEYS.intersection(value))
+
+    def _find_exit_statuses(self, work_dir: Path) -> dict[str, str]:
+        """Locate and parse instances_by_exit_status if present."""
+        for candidate in sorted(work_dir.rglob("*")):
+            if not candidate.is_file():
+                continue
+            try:
+                if candidate.suffix == ".json":
+                    payload = self._load_json(candidate)
+                    statuses = self._extract_exit_statuses_from_json(payload)
+                    if statuses:
+                        return statuses
+
+                text = candidate.read_text(errors="ignore")
+            except OSError:
+                continue
+
+            statuses = self._parse_instances_by_exit_status(text)
+            if statuses:
+                return statuses
+
+        return {}
+
+    def _extract_exit_statuses_from_json(self, payload: Any) -> dict[str, str]:
+        if isinstance(payload, dict):
+            grouped = payload.get("instances_by_exit_status")
+            if isinstance(grouped, dict):
+                return self._coerce_exit_status_mapping(grouped)
+
+            for value in payload.values():
+                nested = self._extract_exit_statuses_from_json(value)
+                if nested:
+                    return nested
+
+        if isinstance(payload, list):
+            for value in payload:
+                nested = self._extract_exit_statuses_from_json(value)
+                if nested:
+                    return nested
+
+        return {}
+
+    def _coerce_exit_status_mapping(self, grouped: dict[str, Any]) -> dict[str, str]:
+        statuses: dict[str, str] = {}
+        for status, instance_ids in grouped.items():
+            if not isinstance(instance_ids, list):
+                continue
+            for instance_id in instance_ids:
+                if instance_id is not None:
+                    statuses[str(instance_id)] = str(status)
+        return statuses
+
+    def _parse_instances_by_exit_status(self, text: str) -> dict[str, str]:
+        marker = "instances_by_exit_status:"
+        if marker not in text:
+            return {}
+
+        statuses: dict[str, str] = {}
+        in_section = False
+        section_indent = 0
+        current_status: str | None = None
+
+        for raw_line in text.splitlines():
+            stripped = raw_line.strip()
+            if not in_section:
+                if stripped == marker:
+                    in_section = True
+                    section_indent = len(raw_line) - len(raw_line.lstrip())
+                continue
+
+            if not stripped:
+                continue
+
+            indent = len(raw_line) - len(raw_line.lstrip())
+            if indent <= section_indent and not stripped.startswith("-"):
+                break
+
+            if stripped.endswith(":") and not stripped.startswith("-"):
+                current_status = stripped[:-1]
+                continue
+
+            if stripped.startswith("-") and current_status is not None:
+                instance_id = stripped[1:].strip()
+                if instance_id:
+                    statuses[instance_id] = current_status
+
+        return statuses
diff --git a/src/olmo_eval/evals/external/result.py b/src/olmo_eval/evals/external/result.py
index 2ed34a22d..5bbd82a99 100644
--- a/src/olmo_eval/evals/external/result.py
+++ b/src/olmo_eval/evals/external/result.py
@@ -29,6 +29,19 @@ class ExternalEvalResult:
     raw_output: str | None = None
     predictions: list[dict[str, Any]] | None = None
 
+    def failure_reason(self) -> str:
+        """Return a user-facing failure reason."""
+        if self.error:
+            return self.error
+
+        if self.metrics:
+            metrics = ", ".join(
+                f"{name}={self._format_metric_value(value)}" for name, value in self.metrics.items()
+            )
+            return f"Unknown error (metrics: {metrics})"
+
+        return "Unknown error"
+
     def to_dict(self) -> dict[str, Any]:
         """Convert to dictionary for serialization."""
         result: dict[str, Any] = {
@@ -69,3 +82,11 @@ def from_error(cls, name: str, error: str) -> ExternalEvalResult:
             success=False,
             error=error,
         )
+
+    @staticmethod
+    def _format_metric_value(value: Any) -> str:
+        if isinstance(value, float):
+            if value == int(value):
+                return str(int(value))
+            return f"{value:.4f}"
+        return str(value)
diff --git a/src/olmo_eval/runners/external/runner.py b/src/olmo_eval/runners/external/runner.py
index 30b761749..d76331b3d 100644
--- a/src/olmo_eval/runners/external/runner.py
+++ b/src/olmo_eval/runners/external/runner.py
@@ -174,7 +174,7 @@ async def run_async(self) -> dict[str, ExternalEvalResult]:
                         for metric, value in result.metrics.items():
                             logger.info(f"  {metric}: {value}")
                     else:
-                        logger.error(f"[{eval_name}] Failed: {result.error}")
+                        logger.error(f"[{eval_name}] Failed: {result.failure_reason()}")
 
                 except Exception as e:
                     logger.exception(f"[{eval_name}] Unexpected error")

From 0a948c7e0ef5b3c9c6eaaa5982691cc8b98a21c1 Mon Sep 17 00:00:00 2001
From: Tyler Murray <tylerm@allenai.org>
Date: Mon, 6 Apr 2026 09:28:59 -0700
Subject: [PATCH 28/29] Some tests

---
 .../external/benchmarks/test_swebench.py      | 235 ++++++++++++++++++
 tests/runners/test_external_runner.py         |  37 +++
 2 files changed, 272 insertions(+)
 create mode 100644 tests/evals/external/benchmarks/test_swebench.py
 create mode 100644 tests/runners/test_external_runner.py

diff --git a/tests/evals/external/benchmarks/test_swebench.py b/tests/evals/external/benchmarks/test_swebench.py
new file mode 100644
index 000000000..0cd5f3079
--- /dev/null
+++ b/tests/evals/external/benchmarks/test_swebench.py
@@ -0,0 +1,235 @@
+"""Tests for SWE-bench external result parsing."""
+
+import json
+import sys
+import time
+
+import pytest
+
+from olmo_eval.evals.external.benchmarks.swebench.eval import SWEBenchExternalEval
+
+
+def test_parse_results_includes_instances_from_exit_statuses(tmp_path):
+    eval_obj = SWEBenchExternalEval()
+    run_id = "swe_bench_test1234"
+
+    summary = {
+        "resolved_ids": ["django__django-10914"],
+        "unresolved_ids": ["astropy__astropy-14182"],
+        "resolved": 1,
+        "unresolved": 1,
+    }
+    (tmp_path / f"model.{run_id}.json").write_text(json.dumps(summary))
+
+    report = {
+        "django__django-10914": {
+            "patch_is_None": False,
+            "patch_exists": True,
+            "patch_successfully_applied": True,
+            "resolved": True,
+            "tests_status": {},
+        },
+        "astropy__astropy-14182": {
+            "patch_is_None": False,
+            "patch_exists": True,
+            "patch_successfully_applied": True,
+            "resolved": False,
+            "tests_status": {},
+        },
+    }
+    (tmp_path / "report.json").write_text(json.dumps(report))
+
+    exit_statuses = """
+instances_by_exit_status:
+    ContextWindowExceededError:
+    - astropy__astropy-12907
+    LimitsExceeded:
+    - astropy__astropy-14995
+    - astropy__astropy-7746
+    - astropy__astropy-14365
+    - django__django-10924
+    Submitted:
+    - django__django-10914
+    - astropy__astropy-14182
+    - astropy__astropy-693
+"""
+    (tmp_path / "exit_statuses.yaml").write_text(exit_statuses.strip())
+
+    result = eval_obj._parse_results(
+        work_dir=tmp_path,
+        run_id=run_id,
+        score_ok=True,
+        raw_output="",
+        start_time=time.time(),
+    )
+
+    assert result.success is True
+    assert result.metrics == {
+        "resolve_rate": 1 / 8,
+        "resolved": 1.0,
+        "total": 8.0,
+    }
+    assert result.predictions is not None
+    assert [pred["native_id"] for pred in result.predictions] == [
+        "django__django-10914",
+        "astropy__astropy-14182",
+        "astropy__astropy-12907",
+        "astropy__astropy-14995",
+        "astropy__astropy-7746",
+        "astropy__astropy-14365",
+        "django__django-10924",
+        "astropy__astropy-693",
+    ]
+
+    resolved_by_instance = {
+        pred["native_id"]: pred["instance_metrics"]["resolved"]["external"]
+        for pred in result.predictions
+    }
+    assert resolved_by_instance["django__django-10914"] == 1.0
+    assert resolved_by_instance["astropy__astropy-14182"] == 0.0
+    assert resolved_by_instance["astropy__astropy-693"] == 0.0
+    assert resolved_by_instance["astropy__astropy-12907"] == 0.0
+
+    assert result.metadata["instance_exit_statuses"] == {
+        "astropy__astropy-12907": "ContextWindowExceededError",
+        "astropy__astropy-14995": "LimitsExceeded",
+        "astropy__astropy-7746": "LimitsExceeded",
+        "astropy__astropy-14365": "LimitsExceeded",
+        "django__django-10924": "LimitsExceeded",
+        "django__django-10914": "Submitted",
+        "astropy__astropy-14182": "Submitted",
+        "astropy__astropy-693": "Submitted",
+    }
+
+
+def test_parse_results_handles_schema_v2_submitted_ids(tmp_path):
+    eval_obj = SWEBenchExternalEval()
+    run_id = "swe_bench_test5678"
+
+    summary = {
+        "total_instances": 300,
+        "submitted_instances": 8,
+        "completed_instances": 2,
+        "resolved_instances": 1,
+        "unresolved_instances": 1,
+        "empty_patch_instances": 6,
+        "error_instances": 0,
+        "completed_ids": [
+            "astropy__astropy-14182",
+            "django__django-10914",
+        ],
+        "empty_patch_ids": [
+            "astropy__astropy-12907",
+            "astropy__astropy-14365",
+            "astropy__astropy-14995",
+            "astropy__astropy-6938",
+            "astropy__astropy-7746",
+            "django__django-10924",
+        ],
+        "submitted_ids": [
+            "astropy__astropy-12907",
+            "astropy__astropy-14182",
+            "astropy__astropy-14365",
+            "astropy__astropy-14995",
+            "astropy__astropy-6938",
+            "astropy__astropy-7746",
+            "django__django-10914",
+            "django__django-10924",
+        ],
+        "resolved_ids": ["django__django-10914"],
+        "unresolved_ids": ["astropy__astropy-14182"],
+        "error_ids": [],
+        "schema_version": 2,
+    }
+    (tmp_path / f"model.{run_id}.json").write_text(json.dumps(summary))
+
+    result = eval_obj._parse_results(
+        work_dir=tmp_path,
+        run_id=run_id,
+        score_ok=True,
+        raw_output="",
+        start_time=time.time(),
+    )
+
+    assert result.success is True
+    assert result.metrics == {
+        "resolve_rate": 1 / 8,
+        "resolved": 1.0,
+        "total": 8.0,
+    }
+    assert result.predictions is not None
+    assert [pred["native_id"] for pred in result.predictions] == [
+        "django__django-10914",
+        "astropy__astropy-14182",
+        "astropy__astropy-12907",
+        "astropy__astropy-14365",
+        "astropy__astropy-14995",
+        "astropy__astropy-6938",
+        "astropy__astropy-7746",
+        "django__django-10924",
+    ]
+
+
+@pytest.mark.anyio
+async def test_run_scoring_uses_max_workers_without_modal(tmp_path, monkeypatch):
+    eval_obj = SWEBenchExternalEval()
+    captured: dict[str, object] = {}
+
+    async def fake_run_subprocess(cmd, **kwargs):
+        captured["cmd"] = cmd
+        captured.update(kwargs)
+        return True, ""
+
+    monkeypatch.setattr(eval_obj, "_run_subprocess", fake_run_subprocess)
+    from olmo_eval.evals.external.benchmarks.swebench.eval import SWEBenchArgs
+
+    await eval_obj._run_scoring(
+        swe_args=SWEBenchArgs(max_workers_eval=7, use_modal=False),
+        preds_path=tmp_path / "preds.json",
+        run_id="run123",
+        work_dir=tmp_path,
+        container_runtime="podman",
+    )
+
+    cmd = captured["cmd"]
+    assert isinstance(cmd, list)
+    assert cmd[:3] == [sys.executable, "-m", "swebench.harness.run_evaluation"]
+    assert "--max_workers" in cmd
+    assert cmd[cmd.index("--max_workers") + 1] == "7"
+    assert "--parallelism" not in cmd
+    assert "--modal" not in cmd
+
+
+@pytest.mark.anyio
+async def test_run_scoring_uses_parallelism_with_modal(tmp_path, monkeypatch):
+    eval_obj = SWEBenchExternalEval()
+    captured: dict[str, object] = {}
+
+    async def fake_run_subprocess(cmd, **kwargs):
+        captured["cmd"] = cmd
+        captured.update(kwargs)
+        return True, ""
+
+    monkeypatch.setattr(eval_obj, "_run_subprocess", fake_run_subprocess)
+    monkeypatch.setattr(
+        "olmo_eval.evals.external.benchmarks.swebench.eval._ensure_modal_config_exists",
+        lambda: None,
+    )
+
+    from olmo_eval.evals.external.benchmarks.swebench.eval import SWEBenchArgs
+
+    await eval_obj._run_scoring(
+        swe_args=SWEBenchArgs(max_workers_eval=7, use_modal=True),
+        preds_path=tmp_path / "preds.json",
+        run_id="run123",
+        work_dir=tmp_path,
+        container_runtime="podman",
+    )
+
+    cmd = captured["cmd"]
+    assert isinstance(cmd, list)
+    assert "--parallelism" in cmd
+    assert cmd[cmd.index("--parallelism") + 1] == "7"
+    assert "--modal" in cmd
+    assert cmd[cmd.index("--modal") + 1] == "true"
+    assert "--max_workers" not in cmd
diff --git a/tests/runners/test_external_runner.py b/tests/runners/test_external_runner.py
new file mode 100644
index 000000000..5b25bae3f
--- /dev/null
+++ b/tests/runners/test_external_runner.py
@@ -0,0 +1,37 @@
+"""Tests for external evaluation runner UX."""
+
+import logging
+
+from olmo_eval.evals.external.result import ExternalEvalResult
+from olmo_eval.inference.providers.config import ProviderConfig
+from olmo_eval.runners.external.runner import ExternalEvalRunner
+
+
+class _DummyEval:
+    async def execute_with_provider(self, **_: object) -> ExternalEvalResult:
+        return ExternalEvalResult(
+            name="dummy_eval",
+            success=False,
+            metrics={"resolve_rate": 0.125, "resolved": 1.0, "total": 8.0},
+        )
+
+
+def test_external_runner_logs_useful_failure_reason(monkeypatch, tmp_path, caplog):
+    monkeypatch.setattr(
+        "olmo_eval.runners.external.runner.get_external_eval",
+        lambda name: _DummyEval(),
+    )
+
+    runner = ExternalEvalRunner(
+        provider_config=ProviderConfig(kind="mock", model="test-model"),
+        external_eval_names=["dummy_eval"],
+        output_dir=str(tmp_path),
+    )
+
+    with caplog.at_level(logging.ERROR):
+        results = runner.run()
+
+    assert results["dummy_eval"].success is False
+    assert "Failed: Unknown error (metrics: resolve_rate=0.1250, resolved=1, total=8)" in (
+        caplog.text
+    )

From d1bfa92d0972e47dd354824f4d1fac50c9cd8023 Mon Sep 17 00:00:00 2001
From: Tyler Murray <tylerm@allenai.org>
Date: Mon, 6 Apr 2026 09:49:29 -0700
Subject: [PATCH 29/29] Wrong arg for concurrency

---
 .../evals/external/benchmarks/swebench/eval.py        | 11 ++++-------
 tests/evals/external/benchmarks/test_swebench.py      |  8 +++-----
 2 files changed, 7 insertions(+), 12 deletions(-)

diff --git a/src/olmo_eval/evals/external/benchmarks/swebench/eval.py b/src/olmo_eval/evals/external/benchmarks/swebench/eval.py
index 5a4876492..b6c18b230 100644
--- a/src/olmo_eval/evals/external/benchmarks/swebench/eval.py
+++ b/src/olmo_eval/evals/external/benchmarks/swebench/eval.py
@@ -301,15 +301,12 @@ async def _run_scoring(
 
         env = os.environ.copy()
 
+        cmd.extend(["--max_workers", str(swe_args.max_workers_eval)])
+
         if swe_args.use_modal:
-            cmd.extend(["--parallelism", str(swe_args.max_workers_eval)])
             cmd.extend(["--modal", "true"])
-        else:
-            cmd.extend(["--max_workers", str(swe_args.max_workers_eval)])
-            if "DOCKER_HOST" not in env:
-                logger.warning(
-                    "DOCKER_HOST not set. Scoring may fail without podman service or Modal."
-                )
+        elif "DOCKER_HOST" not in env:
+            logger.warning("DOCKER_HOST not set. Scoring may fail without podman service or Modal.")
 
         logger.info(f"[{self.name}] Running SWE-bench harness: {shlex.join(cmd)}")
         ok, output = await self._run_subprocess(
diff --git a/tests/evals/external/benchmarks/test_swebench.py b/tests/evals/external/benchmarks/test_swebench.py
index 0cd5f3079..0efd68351 100644
--- a/tests/evals/external/benchmarks/test_swebench.py
+++ b/tests/evals/external/benchmarks/test_swebench.py
@@ -196,12 +196,11 @@ async def fake_run_subprocess(cmd, **kwargs):
     assert cmd[:3] == [sys.executable, "-m", "swebench.harness.run_evaluation"]
     assert "--max_workers" in cmd
     assert cmd[cmd.index("--max_workers") + 1] == "7"
-    assert "--parallelism" not in cmd
     assert "--modal" not in cmd
 
 
 @pytest.mark.anyio
-async def test_run_scoring_uses_parallelism_with_modal(tmp_path, monkeypatch):
+async def test_run_scoring_uses_max_workers_with_modal(tmp_path, monkeypatch):
     eval_obj = SWEBenchExternalEval()
     captured: dict[str, object] = {}
 
@@ -228,8 +227,7 @@ async def fake_run_subprocess(cmd, **kwargs):
 
     cmd = captured["cmd"]
     assert isinstance(cmd, list)
-    assert "--parallelism" in cmd
-    assert cmd[cmd.index("--parallelism") + 1] == "7"
+    assert "--max_workers" in cmd
+    assert cmd[cmd.index("--max_workers") + 1] == "7"
     assert "--modal" in cmd
     assert cmd[cmd.index("--modal") + 1] == "true"
-    assert "--max_workers" not in cmd