diff --git a/.gitignore b/.gitignore index b43656d722..0016d0def2 100644 --- a/.gitignore +++ b/.gitignore @@ -1,6 +1,7 @@ # Dependencies .sisyphus/ node_modules/ +__pycache__ # Build output dist/ @@ -18,6 +19,7 @@ Thumbs.db # Logs *.log npm-debug.log* +jobs # Lock files (use bun.lockb instead) package-lock.json diff --git a/benchmark/__init__.py b/benchmark/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/benchmark/install-sisyphus.sh.j2 b/benchmark/install-sisyphus.sh.j2 new file mode 100644 index 0000000000..f323353301 --- /dev/null +++ b/benchmark/install-sisyphus.sh.j2 @@ -0,0 +1,46 @@ +#!/bin/bash +set -e + +apt-get update +apt-get install -y curl unzip + +# Install bun +curl -fsSL https://bun.sh/install | bash +export BUN_INSTALL="$HOME/.bun" +export PATH="$BUN_INSTALL/bin:$PATH" +bun --version + +# Install OpenCode +{% if version %} +bun install -g opencode-ai@{{ version }} +{% else %} +bun install -g opencode-ai@latest +{% endif %} + +# Pre-create oh-my-opencode config BEFORE install to disable problematic hooks +# This prevents hooks from initializing during plugin load +# - comment-checker: Downloads Go binary from GitHub (rate limiting with multiple containers) +# - auto-update-checker: Checks for updates (unnecessary in benchmarks) +# - session-notification: OS notifications (no display in containers) +# - background-notification: OS notifications (no display in containers) +mkdir -p ~/.config/opencode +cat > ~/.config/opencode/oh-my-opencode.json << 'EOF' +{ + "disabled_hooks": [ + "auto-update-checker", + "session-notification", + "background-notification" + ] +} +EOF + +# Install oh-my-opencode plugin (provides Sisyphus agent) +# --claude=no uses opencode/zen provider (free) instead of Anthropic +{% if omo_version %} +bunx oh-my-opencode@{{ omo_version }} install --no-tui --claude=no --chatgpt=no --gemini=no +{% else %} +bunx oh-my-opencode@latest install --no-tui --claude=no --chatgpt=no --gemini=no +{% endif %} + +opencode --version +echo "Sisyphus agent ready" diff --git a/benchmark/sisyphus_agent.py b/benchmark/sisyphus_agent.py new file mode 100644 index 0000000000..efbf42abf0 --- /dev/null +++ b/benchmark/sisyphus_agent.py @@ -0,0 +1,83 @@ +import os +import shlex +from pathlib import Path + +from harbor.agents.installed.base import BaseInstalledAgent, ExecInput +from harbor.models.agent.context import AgentContext + + +class SisyphusAgent(BaseInstalledAgent): + """ + Sisyphus agent uses OpenCode with oh-my-opencode plugin. + """ + + @staticmethod + def name() -> str: + return "sisyphus" + + @property + def _install_agent_template_path(self) -> Path: + return Path(__file__).parent / "install-sisyphus.sh.j2" + + def populate_context_post_run(self, context: AgentContext) -> None: + pass + + def create_run_agent_commands(self, instruction: str) -> list[ExecInput]: + escaped_instruction = shlex.quote(instruction) + + if not self.model_name or "/" not in self.model_name: + raise ValueError("Model name must be in the format provider/model_name") + + provider, _ = self.model_name.split("/", 1) + + env = self._get_provider_env(provider) + env["OPENCODE_FAKE_VCS"] = "git" + + return [ + ExecInput( + command=( + f"opencode --model {self.model_name} run " + f"--agent Sisyphus --format=json {escaped_instruction} " + f"2>&1 | tee /logs/agent/sisyphus.txt" + ), + env=env, + ) + ] + + def _get_provider_env(self, provider: str) -> dict[str, str]: + env = {} + provider_keys = { + "amazon-bedrock": [ + "AWS_ACCESS_KEY_ID", + "AWS_SECRET_ACCESS_KEY", + "AWS_REGION", + ], + "anthropic": ["ANTHROPIC_API_KEY"], + "azure": ["AZURE_RESOURCE_NAME", "AZURE_API_KEY"], + "deepseek": ["DEEPSEEK_API_KEY"], + "github-copilot": ["GITHUB_TOKEN"], + "google": [ + "GEMINI_API_KEY", + "GOOGLE_GENERATIVE_AI_API_KEY", + "GOOGLE_APPLICATION_CREDENTIALS", + "GOOGLE_CLOUD_PROJECT", + "GOOGLE_CLOUD_LOCATION", + "GOOGLE_GENAI_USE_VERTEXAI", + "GOOGLE_API_KEY", + ], + "groq": ["GROQ_API_KEY"], + "huggingface": ["HF_TOKEN"], + "llama": ["LLAMA_API_KEY"], + "mistral": ["MISTRAL_API_KEY"], + "openai": ["OPENAI_API_KEY"], + "opencode": [], # opencode/zen - no API key required + "xai": ["XAI_API_KEY"], + } + + keys = provider_keys.get(provider, []) + + for key in keys: + if key in os.environ: + env[key] = os.environ[key] + + return env