Skip to content

Commit e8783f6

Browse files
chore: improvements to command line arguments (#781)
# Motivation Makes `run_eval` command require explicit arguments # Content <!-- Please include a summary of the change --> # Testing <!-- How was the change tested? --> # Please check the following before marking your PR as ready for review - [ ] I have added tests for my changes - [ ] I have updated the documentation or added new documentation as needed
1 parent 0d600cf commit e8783f6

File tree

1 file changed

+7
-7
lines changed
  • codegen-examples/examples/swebench_agent_run

1 file changed

+7
-7
lines changed

codegen-examples/examples/swebench_agent_run/run_eval.py

Lines changed: 7 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -269,14 +269,14 @@ def process_batch_local(examples: list[SweBenchExample], model: str, num_workers
269269

270270

271271
async def run_eval(
272-
use_existing_preds: str | None,
273272
dataset: str,
274-
length: int,
273+
use_existing_preds: str | None = None,
274+
length: int | None = None,
275275
instance_id: str | None = None,
276276
local: bool = False,
277277
codebases: dict[str, Codebase] = {},
278278
repo: str | None = None,
279-
num_workers: int = 5,
279+
num_workers: int = 2,
280280
model: str = "claude-3-7-sonnet-latest",
281281
):
282282
run_id = use_existing_preds or str(uuid.uuid4())
@@ -353,20 +353,20 @@ async def run_eval(
353353

354354

355355
@click.command()
356-
@click.option("--use-existing-preds", help="The run ID of the existing predictions to use.", type=str, default=None)
357356
@click.option("--dataset", help="The dataset to use.", type=click.Choice(["lite", "full", "verified", "lite_small", "lite_medium", "lite_large"]), default="lite")
358-
@click.option("--length", help="The number of examples to process.", type=int, default=10)
357+
@click.option("--use-existing-preds", help="The run ID of the existing predictions to use.", type=str, default=None)
358+
@click.option("--length", help="The number of examples to process.", type=int, default=None)
359359
@click.option("--instance-id", help="The instance ID of the example to process.", type=str, default=None)
360360
@click.option("--local", help="Run the evaluation locally.", is_flag=True, default=False)
361361
@click.option("--repo", help="The repo to use.", type=str, default=None)
362362
@click.option(
363363
"--num-workers", help="The number of workers to use. This is the number of examples that will be processed concurrently. A large number may lead to rate limiting issues.", type=int, default=5
364364
)
365365
@click.option("--model", help="The model to use.", type=str, default="claude-3-7-sonnet-latest")
366-
def run_eval_command(use_existing_preds, dataset, length, instance_id, local, repo, num_workers, model):
366+
def run_eval_command(dataset, use_existing_preds, length, instance_id, local, repo, num_workers, model):
367367
print(f"Repo: {repo}")
368368
print(f"Model: {model}")
369-
asyncio.run(run_eval(use_existing_preds=use_existing_preds, dataset=dataset, length=length, instance_id=instance_id, codebases=None, local=local, repo=repo, num_workers=num_workers, model=model))
369+
asyncio.run(run_eval(dataset=dataset, use_existing_preds=use_existing_preds, length=length, instance_id=instance_id, local=local, repo=repo, num_workers=num_workers, model=model))
370370

371371

372372
if __name__ == "__main__":

0 commit comments

Comments
 (0)