Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
270 changes: 168 additions & 102 deletions Cargo.lock

Large diffs are not rendered by default.

19 changes: 9 additions & 10 deletions Cargo.toml
Original file line number Diff line number Diff line change
Expand Up @@ -13,15 +13,14 @@ members = [
resolver = "2"

[workspace.dependencies]
sacp = "3.0.0"
sacp-tokio = "3.0.0"
sacp-conductor = "5.0.0"
sacp-rmcp = "0.9.0"
sacp-tee = "0.2.0"
sacp-test = "1.0.0"
sacp-acp-client = "0.1.0"
yopo = "2.0.0"
elizacp = "4.0.0"
sacp = "9.0.0"
sacp-tokio = "9.0.0"
sacp-conductor = "9.0.0"
sacp-rmcp = "9.0.0"
sacp-tee = "9.0.0"
sacp-test = "9.0.0"
yopo = "9.0.0"
elizacp = "9.0.0"
jsonrpcmsg = "0.1.2"

# Core async runtime
Expand Down Expand Up @@ -49,7 +48,7 @@ tracing-subscriber = { version = "0.3", features = ["env-filter"] }
rmcp = { version = "0.9", features = ["server", "transport-io", "schemars"] }

# Sparkle integration
sparkle = { package = "sparkle-mcp", version = "0.1.7" }
sparkle = { package = "sparkle-mcp", version = "0.2.1" }

# CLI parsing
clap = { version = "4.0", features = ["derive"] }
Expand Down
1 change: 1 addition & 0 deletions benchmark-output/serde_from_value_expected.txt
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
The response should describe that serde_json::from_value takes a serde_json::Value and deserializes it into a type T. It should mention that it returns a Result<T, Error>.
1 change: 1 addition & 0 deletions benchmark-output/serde_from_value_prompt.txt
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
Please use the `rust_crate_query` tool from the `rust-crate-sources` MCP server to research the signature of the serde_json::from_value API and describe what inputs it accepts. Do not try to read files from disk - use the MCP tool.
24 changes: 24 additions & 0 deletions benchmark-output/serde_from_value_response.txt
Original file line number Diff line number Diff line change
@@ -0,0 +1,24 @@
## `serde_json::from_value` Signature

```rust
pub fn from_value<T>(value: Value) -> Result<T, Error>
where
T: DeserializeOwned,
```

### Inputs

- **`value: Value`** - A `serde_json::Value` (the JSON value type). Takes ownership of the value.

### Type Parameter

- **`T: DeserializeOwned`** - The target type must implement `DeserializeOwned` (equivalent to `T: for<'de> Deserialize<'de>`), meaning it can deserialize without borrowing from input data.

### Return Value

- **`Result<T, Error>`** - Returns `Ok(T)` on success, or `Err(serde_json::Error)` on failure.

### When It Fails

1. Structure mismatch (e.g., `T` expects a struct but `Value` is not an object)
2. `T`'s `Deserialize` implementation rejects the data (missing required fields, number overflow, etc.)
6 changes: 6 additions & 0 deletions benchmark-output/serde_from_value_validation.txt
Original file line number Diff line number Diff line change
@@ -0,0 +1,6 @@
PASS

The actual response correctly describes that `serde_json::from_value` takes a `serde_json::Value` and deserializes it into a type `T`, and that it returns a `Result<T, Error>`. The response actually exceeds the expected requirements by also including:
- The full function signature with the trait bound
- Explanation of the `DeserializeOwned` constraint
- Examples of when deserialization can fail
68 changes: 36 additions & 32 deletions setup/src/acp.rs
Original file line number Diff line number Diff line change
Expand Up @@ -4,19 +4,18 @@ use anyhow::{Context, Result, anyhow};
use std::path::{Path, PathBuf};
use std::process::Command;

/// Install ACP binaries: sacp-conductor, elizacp, sacp-tee from crates.io,
/// and symposium-acp-proxy from local repository
/// Install ACP binaries from local repository
pub fn install_acp_binaries(repo_root: &Path, dry_run: bool) -> Result<()> {
println!("📦 Installing ACP binaries...");

// Verify we're in the symposium repository
verify_symposium_repo(repo_root)?;

// Install from crates.io
install_from_crates_io(&["sacp-conductor", "elizacp", "sacp-tee"], dry_run)?;
install_from_crates_io(&["elizacp"], dry_run)?;

// Install symposium-acp-proxy from local repository
install_symposium_acp_proxy(repo_root, dry_run)?;
// Install symposium-acp-agent from local repository
install_local_binaries(repo_root, dry_run)?;

if !dry_run {
println!("✅ ACP binaries installed successfully!");
Expand Down Expand Up @@ -74,39 +73,44 @@ fn install_from_crates_io(crates: &[&str], dry_run: bool) -> Result<()> {
Ok(())
}

/// Install symposium-acp-proxy from local repository
fn install_symposium_acp_proxy(repo_root: &Path, dry_run: bool) -> Result<()> {
let symposium_acp_proxy_dir = repo_root.join("src/symposium-acp-proxy");
/// Install local symposium binaries from the repository
fn install_local_binaries(repo_root: &Path, dry_run: bool) -> Result<()> {
for binary_name in ["symposium-acp-agent"] {
let binary_dir = repo_root.join("src").join(binary_name);

if !symposium_acp_proxy_dir.exists() {
return Err(anyhow!(
"❌ symposium-acp-proxy directory not found at: {}",
symposium_acp_proxy_dir.display()
));
}
if !binary_dir.exists() {
return Err(anyhow!(
"❌ {} directory not found at: {}",
binary_name,
binary_dir.display()
));
}

println!(" Path: {}", symposium_acp_proxy_dir.display());
if dry_run {
println!(" Would install {} from local repository", binary_name);
} else {
println!(" Installing {} from local repository...", binary_name);

if dry_run {
println!(" Would install symposium-acp-proxy from local repository");
} else {
println!(" Installing symposium-acp-proxy from local repository...");
let output = Command::new("cargo")
.args(["install", "--path", ".", "--force"])
.current_dir(&binary_dir)
.output()
.context(format!(
"Failed to execute cargo install for {}",
binary_name
))?;

let output = Command::new("cargo")
.args(["install", "--path", ".", "--force"])
.current_dir(&symposium_acp_proxy_dir)
.output()
.context("Failed to execute cargo install for symposium-acp-proxy")?;
if !output.status.success() {
let stderr = String::from_utf8_lossy(&output.stderr);
return Err(anyhow!(
"❌ Failed to install {}:\n Error: {}",
binary_name,
stderr.trim()
));
}

if !output.status.success() {
let stderr = String::from_utf8_lossy(&output.stderr);
return Err(anyhow!(
"❌ Failed to install symposium-acp-proxy:\n Error: {}",
stderr.trim()
));
println!(" ✅ {} installed", binary_name);
}

println!(" ✅ symposium-acp-proxy installed");
}
Ok(())
}
Expand Down
9 changes: 3 additions & 6 deletions setup/src/main.rs
Original file line number Diff line number Diff line change
Expand Up @@ -102,9 +102,8 @@ fn main() -> Result<()> {
}

if configure_zed {
let conductor_path = acp::get_binary_path("sacp-conductor")?;
let symposium_acp_proxy_path = acp::get_binary_path("symposium-acp-proxy")?;
zed::configure_zed(&conductor_path, &symposium_acp_proxy_path, args.dry_run)?;
let symposium_acp_agent_path = acp::get_binary_path("symposium-acp-agent")?;
zed::configure_zed(&symposium_acp_agent_path, args.dry_run)?;
println!();
}

Expand Down Expand Up @@ -143,10 +142,8 @@ fn print_completion_message(

if installed_acp {
println!("📦 ACP binaries installed to ~/.cargo/bin/:");
println!(" • sacp-conductor");
println!(" • elizacp");
println!(" • sacp-tee");
println!(" • symposium-acp-proxy");
println!(" • symposium-acp-agent");
println!();
}

Expand Down
25 changes: 6 additions & 19 deletions setup/src/zed.rs
Original file line number Diff line number Diff line change
Expand Up @@ -62,11 +62,7 @@ fn is_command_available(cmd: &str) -> bool {
}

/// Configure Zed with detected agents
pub fn configure_zed(
conductor_path: &Path,
symposium_acp_path: &Path,
dry_run: bool,
) -> Result<()> {
pub fn configure_zed(symposium_acp_agent_path: &Path, dry_run: bool) -> Result<()> {
let zed_config_path = get_zed_config_path()?;

if !zed_config_path.exists() {
Expand Down Expand Up @@ -105,8 +101,7 @@ pub fn configure_zed(
for agent in &agents {
let config_name = agent.config_name();

let agent_config =
create_agent_config(conductor_path, symposium_acp_path, agent.npx_package());
let agent_config = create_agent_config(symposium_acp_agent_path, agent.npx_package());

if dry_run {
println!(" Would add configuration for: {}", config_name);
Expand Down Expand Up @@ -137,19 +132,11 @@ pub fn configure_zed(
}

/// Create an agent server configuration entry
fn create_agent_config(
conductor_path: &Path,
symposium_acp_path: &Path,
npx_package: &str,
) -> Value {
fn create_agent_config(symposium_acp_agent_path: &Path, npx_package: &str) -> Value {
json!({
"default_mode": "bypassPermissions",
"command": conductor_path.to_string_lossy(),
"args": [
"agent",
symposium_acp_path.to_string_lossy(),
format!("npx -y '{}'", npx_package)
],
"type": "custom",
"command": symposium_acp_agent_path.to_string_lossy(),
"args": ["--", "npx", "-y", npx_package],
"env": {}
})
}
Expand Down
94 changes: 24 additions & 70 deletions src/symposium-benchmark/src/main.rs
Original file line number Diff line number Diff line change
Expand Up @@ -5,13 +5,11 @@

use anyhow::Result;
use clap::Parser;
use sacp::{ByteStreams, Component, DynComponent};
use sacp::DynComponent;
use sacp_conductor::Conductor;
use sacp_tokio::AcpAgent;
use std::path::PathBuf;
use std::str::FromStr;
use tokio::io::duplex;
use tokio_util::compat::{TokioAsyncReadCompatExt, TokioAsyncWriteCompatExt};

#[derive(Parser, Debug)]
#[command(name = "symposium-benchmark")]
Expand Down Expand Up @@ -56,19 +54,8 @@ async fn main() -> Result<()> {

// Initialize tracing based on --log argument
if let Some(log_targets) = &args.log {
let mut filter = tracing_subscriber::EnvFilter::from_default_env();
for target in log_targets.split(',') {
let target = target.trim();
if !target.is_empty() {
// If target already has a level (contains '='), use as-is; otherwise default to debug
let directive = if target.contains('=') {
target.to_string()
} else {
format!("{}=debug", target)
};
filter = filter.add_directive(directive.parse().unwrap());
}
}
let filter =
tracing_subscriber::EnvFilter::try_new(log_targets).expect("invalid log filter syntax");
tracing_subscriber::fmt()
.with_env_filter(filter)
.with_writer(std::io::stderr)
Expand Down Expand Up @@ -115,66 +102,37 @@ async fn run_benchmark(benchmark: &Benchmark, output_dir: &PathBuf) -> Result<()
let research_prompt = benchmark.prompt;
let expected_result = benchmark.expected;

// Create components: rust-crate-sources-proxy + Claude Code
let proxy = symposium_crate_sources_proxy::CrateSourcesProxy;
let claude_agent = AcpAgent::from_str("npx -y '@zed-industries/claude-code-acp'")?;

// Create duplex streams for editor <-> conductor communication
let (editor_write, conductor_read) = duplex(8192);
let (conductor_write, editor_read) = duplex(8192);

// Spawn conductor with proxy + agent chain (with tees for debugging)
let conductor_handle = tokio::spawn(async move {
Conductor::new(
"benchmark-conductor".to_string(),
vec![DynComponent::new(proxy), DynComponent::new(claude_agent)],
Default::default(),
)
.trace_to_path("killme.jsons")
.map_err(sacp::util::internal_error)?
.run(ByteStreams::new(
conductor_write.compat_write(),
conductor_read.compat(),
))
.await
});

// Send prompt using yopo
let response = yopo::prompt(
ByteStreams::new(editor_write.compat_write(), editor_read.compat()),
research_prompt,
// Build conductor with crate sources proxy and agent
let conductor = Conductor::new(
"benchmark-conductor".to_string(),
vec![
DynComponent::new(symposium_crate_sources_proxy::CrateSourcesProxy),
DynComponent::new(AcpAgent::from_str(
"npx -y '@zed-industries/claude-code-acp'",
)?),
],
Default::default(),
)
.await?;
.trace_to_path("killme.jsons")
.map_err(sacp::util::internal_error)?;

// Run prompt
let response = yopo::prompt(conductor, research_prompt).await?;

tracing::info!("Research response received: {} chars", response.len());

// Validate response using another Claude Code instance
tracing::info!("Validating response");

let validator_agent = AcpAgent::from_str("npx -y '@zed-industries/claude-code-acp'")?;
let (validator_write, validator_read) = duplex(8192);
let (validator_out_write, validator_out_read) = duplex(8192);

let validator_handle = tokio::spawn(async move {
validator_agent
.serve(ByteStreams::new(
validator_out_write.compat_write(),
validator_read.compat(),
))
.await
});

let validation_prompt = format!(
"Compare this response to the expected result and respond with PASS or FAIL. \
let validation_result = yopo::prompt(
AcpAgent::from_str("npx -y '@zed-industries/claude-code-acp'")?,
&format!(
"Compare this response to the expected result and respond with PASS or FAIL. \
If FAIL, explain what's missing.\n\n\
Expected: {}\n\n\
Actual response:\n{}",
expected_result, response
);

let validation_result = yopo::prompt(
ByteStreams::new(validator_write.compat_write(), validator_out_read.compat()),
&validation_prompt,
expected_result, response
),
)
.await?;

Expand All @@ -199,9 +157,5 @@ async fn run_benchmark(benchmark: &Benchmark, output_dir: &PathBuf) -> Result<()
println!("VALIDATION RESULT:\n{}", validation_result);
println!("========================\n");

// Clean up
validator_handle.await??;
conductor_handle.await??;

Ok(())
}
Loading