DNM: Tweaks to support MSWEA + swe-bench

pyproject.toml

@@ -80,7 +80,7 @@ clients = [
 swebench = [
     "olmo-eval-internal[litellm]",
     "mini-swe-agent @ git+https://github.com/allenai/mini-swe-agent.git@e614105ca837995765512bfd5aa1d216558f002c",
-    "swebench==4.1.0",
+    "swebench>=4.1.0",
 ]
 storage = [
     "olmo-eval-internal[s3,postgres]",

src/olmo_eval/cli/beaker/job_assembler.py

@@ -157,6 +157,7 @@ def assemble_external_eval_job(
     retries: int | None = None,
     provider_kind: str | None = None,
     base_url: str | None = None,
+    enable_podman_service: bool = False,
 ) -> Any:
     """Assemble a BeakerJobConfig for running external evaluations.
 
@@ -347,6 +348,7 @@ def assemble_external_eval_job(
         extras=extras,
         provider_packages=provider_packages,
         vllm_isolated_venv=vllm_isolated_venv,
+        enable_podman_service=enable_podman_service,
     )
 
 

src/olmo_eval/cli/beaker/launch.py

@@ -238,6 +238,11 @@
     default=None,
     help="Number of GPUs. Defaults to 1 for GPU providers, 0 otherwise.",
 )
+@click.option(
+    "--enable-podman-service",
+    is_flag=True,
+    help="Start podman system service for Docker API compatibility (e.g., swebench harness).",
+)
 def launch(
     config: str | None,
     name: str | None,
@@ -282,6 +287,7 @@ def launch(
     uv_cache_dir: str,
     secret_env: tuple[str, ...],
     gpus: int | None,
+    enable_podman_service: bool,
 ) -> None:
     """Launch an evaluation job on Beaker.
 
@@ -425,6 +431,7 @@ def launch(
             preemptible=preemptible,
             retries=retries,
             gpus=gpus,
+            enable_podman_service=enable_podman_service,
         )
         return
 
@@ -1004,6 +1011,7 @@ def _launch_external_evals(
     preemptible: bool | None = None,
     retries: int | None = None,
     gpus: int | None = None,
+    enable_podman_service: bool = False,
 ) -> None:
     """Launch external evaluation jobs on Beaker.
 
@@ -1188,6 +1196,7 @@ def _launch_external_evals(
             retries=retries,
             provider_kind=str(provider_config.kind),
             base_url=provider_config.base_url,
+            enable_podman_service=enable_podman_service,
         )
         job_configs.append(job_config)
 

src/olmo_eval/cli/run_external.py

@@ -7,7 +7,12 @@
 
 import click
 
-from olmo_eval.cli.utils import ConfiguredExternalEval, console, parse_key_value_args
+from olmo_eval.cli.utils import (
+    ConfiguredExternalEval,
+    console,
+    parse_key_value_args,
+    print_runtime_environment,
+)
 from olmo_eval.common.constants.infrastructure import BEAKER_RESULT_DIR
 
 
@@ -200,6 +205,9 @@ def run_external(
 
     configure_logging(level="INFO")
 
+    # Print runtime environment summary
+    print_runtime_environment()
+
     # Build provider config
     from olmo_eval.common.configs import get_provider_config
 
@@ -359,7 +367,7 @@ def run_external(
             metrics = "\n".join(metrics_lines)
         else:
             status = "[red]Failed[/red]"
-            metrics = result.error or "Unknown error"
+            metrics = result.failure_reason()
 
         results_table.add_row(name, status, metrics)