diff --git a/POC-TO-MVP-PLAN.md b/POC-TO-MVP-PLAN.md index 0fb5d19f9..6757a6706 100644 --- a/POC-TO-MVP-PLAN.md +++ b/POC-TO-MVP-PLAN.md @@ -253,17 +253,17 @@ Status: - Relay config pass-through exists for Hermes profiles. - Native harness outputs are preserved separately from Relay outputs. - SDK, CLI, and Harbor-facing paths expose ArtifactManifest data. +- Relay artifact discovery is hardened for ATOF/ATIF outputs when telemetry is + enabled. +- Relay-enabled profiles have tests for inspectable telemetry outputs or clear + telemetry references. +- ArtifactManifest remains populated with output, logs, patch/status, native + harness artifacts, and telemetry references where available. Next steps: -- Harden Relay artifact discovery for ATOF/ATIF outputs when telemetry is - enabled. -- Add tests that verify Relay-enabled profiles produce inspectable telemetry - outputs or clear telemetry references. - Add tests that verify Relay-disabled profiles still produce native output, harness events where available, and logs. -- Keep ArtifactManifest populated with output, logs, patch/status, native - harness artifacts, and telemetry references where available. - Confirm these artifacts are visible through SDK, CLI, and Harbor consumers. ### 6. Consumer Proof: Harbor diff --git a/crates/fabric-core/src/runtime.rs b/crates/fabric-core/src/runtime.rs index b18b2e0cb..347c0292c 100644 --- a/crates/fabric-core/src/runtime.rs +++ b/crates/fabric-core/src/runtime.rs @@ -751,6 +751,9 @@ fn run_process_adapter( None }; + let parsed_output = parse_stdout_output(&stdout); + promote_relay_artifacts_to_manifest(&parsed_output, &mut artifacts); + Ok(RunResult { agent_name: plan.agent_name.clone(), profile: plan.profile.clone(), @@ -761,7 +764,7 @@ fn run_process_adapter( invocation_id: invocation.invocation_id, request_id: request.request_id, status, - output: parse_stdout_output(&stdout), + output: parsed_output, error, artifacts, telemetry: telemetry_ref(plan, relay_config.as_ref()), @@ -959,6 +962,9 @@ fn run_python_adapter( None }; + let parsed_output = parse_stdout_output(&stdout); + promote_relay_artifacts_to_manifest(&parsed_output, &mut artifacts); + Ok(RunResult { agent_name: plan.agent_name.clone(), profile: plan.profile.clone(), @@ -969,7 +975,7 @@ fn run_python_adapter( invocation_id: invocation.invocation_id, request_id: request.request_id, status, - output: parse_stdout_output(&stdout), + output: parsed_output, error, artifacts, telemetry: telemetry_ref(plan, relay_config.as_ref()), @@ -1190,6 +1196,97 @@ fn parse_stdout_output(stdout: &str) -> Value { serde_json::from_str(stdout).unwrap_or_else(|_| Value::String(stdout.to_string())) } +#[derive(Debug, Default, Deserialize)] +struct RelayArtifactOutput { + #[serde(default)] + relay_artifacts: Vec, +} + +#[derive(Debug, Deserialize)] +struct RelayArtifactCandidate { + kind: String, + path: PathBuf, +} + +fn promote_relay_artifacts_to_manifest(output: &Value, manifest: &mut ArtifactManifest) { + let relay_output: RelayArtifactOutput = + serde_json::from_value(output.clone()).unwrap_or_default(); + + for artifact in relay_output.relay_artifacts { + let Ok(artifact) = serde_json::from_value::(artifact) else { + continue; + }; + let kind = artifact.kind.as_str(); + if !matches!(kind, "atof" | "atif") { + continue; + } + if artifact.path.as_os_str().is_empty() { + continue; + } + + let path = resolve_relay_artifact_path(manifest, &artifact.path); + if !path.exists() + || manifest + .artifacts + .iter() + .any(|artifact| artifact.path == path) + { + continue; + } + + let name = unique_relay_artifact_name(manifest, kind); + manifest.artifacts.push(ArtifactRef { + name, + kind: kind.to_string(), + path, + media_type: relay_artifact_media_type(kind).map(str::to_string), + }); + } +} + +fn resolve_relay_artifact_path(manifest: &ArtifactManifest, path: &Path) -> PathBuf { + if path.is_absolute() { + return path.to_path_buf(); + } + manifest + .root + .as_ref() + .map(|root| root.join(path)) + .unwrap_or_else(|| path.to_path_buf()) +} + +fn relay_artifact_media_type(kind: &str) -> Option<&'static str> { + match kind { + "atof" => Some("application/x-ndjson"), + "atif" => Some("application/json"), + _ => None, + } +} + +fn unique_relay_artifact_name(manifest: &ArtifactManifest, kind: &str) -> String { + let base = format!("relay_{kind}"); + if !manifest + .artifacts + .iter() + .any(|artifact| artifact.name == base) + { + return base; + } + + let mut index = 2; + loop { + let candidate = format!("{base}_{index}"); + if !manifest + .artifacts + .iter() + .any(|artifact| artifact.name == candidate) + { + return candidate; + } + index += 1; + } +} + fn process_command_args(plan: &RunPlan, settings: &ProcessAdapterSettings) -> Vec { let mut args = Vec::new(); if let Some(script) = settings.script.as_ref() { @@ -1712,6 +1809,103 @@ environment: let _ = fs::remove_dir_all(root); } + #[test] + fn run_promotes_relay_artifacts_into_artifact_manifest() { + let root = std::env::temp_dir().join(format!( + "fabric-relay-artifact-manifest-test-{}", + std::process::id() + )); + let _ = fs::remove_dir_all(&root); + fs::create_dir_all(root.join("adapters/process")).expect("create adapters dir"); + let adapter_script = r#" +from pathlib import Path +import json +relay_dir = Path("artifacts/relay").resolve() +relay_dir.mkdir(parents=True, exist_ok=True) +atof = relay_dir / "events.atof.jsonl" +atif = relay_dir / "trajectory-runtime.atif.json" +atif_extra = relay_dir / "trajectory-child.atif.json" +atof.write_text('{"kind":"scope"}\n', encoding="utf-8") +atif.write_text('{"trajectory":true}', encoding="utf-8") +atif_extra.write_text('{"trajectory":"child"}', encoding="utf-8") +print(json.dumps({ + "response": "ok", + "relay_artifacts": [ + {"kind": "atof", "path": str(atof)}, + {"kind": "atif", "path": str(atif)}, + {"kind": "atif", "path": str(atif_extra)} + ] +})) +"#; + let agent_config = serde_json::json!({ + "schema_version": "fabric.agent/v1alpha1", + "metadata": { + "name": "relay-artifact-test-agent", + }, + "harness": { + "adapter_id": "acme.fabric.process", + "settings": { + "command": "python3", + "args": ["-c", adapter_script], + }, + }, + "models": { + "default": { + "provider": "test", + "model": "test-model", + }, + }, + "runtime": { + "mode": "oneshot", + "transport": "cli", + "input_schema": "text", + "output_schema": "text", + "artifacts": "./artifacts", + }, + }); + fs::write( + root.join("agent.yaml"), + serde_yaml::to_string(&agent_config).expect("serialize agent config"), + ) + .expect("write config"); + fs::write( + root.join("adapters/process/fabric-adapter.json"), + process_adapter_descriptor(), + ) + .expect("write adapter descriptor"); + + let plan = resolve_run_plan(&root, None).expect("run plan"); + let result = run_plan(&plan, RunRequest::text("collect relay")).expect("run result"); + + assert_eq!(result.status, RunStatus::Succeeded); + let atof = result + .artifacts + .artifacts + .iter() + .find(|artifact| artifact.name == "relay_atof" && artifact.kind == "atof") + .expect("ATOF artifact promoted to manifest"); + let atif = result + .artifacts + .artifacts + .iter() + .find(|artifact| artifact.name == "relay_atif" && artifact.kind == "atif") + .expect("ATIF artifact promoted to manifest"); + let atif_extra = result + .artifacts + .artifacts + .iter() + .find(|artifact| artifact.name == "relay_atif_2" && artifact.kind == "atif") + .expect("second ATIF artifact promoted to manifest with unique name"); + assert!(atof.path.ends_with("events.atof.jsonl")); + assert_eq!(atof.media_type.as_deref(), Some("application/x-ndjson")); + assert!(atif.path.ends_with("trajectory-runtime.atif.json")); + assert_eq!(atif.media_type.as_deref(), Some("application/json")); + assert!(atif_extra.path.ends_with("trajectory-child.atif.json")); + assert_eq!(atif_extra.media_type.as_deref(), Some("application/json")); + + let _ = fs::remove_dir_all(root); + } + #[test] fn process_adapter_failure_returns_structured_error() { let root = std::env::temp_dir().join(format!( diff --git a/tests/smoke_relay_integration.py b/tests/smoke_relay_integration.py index 64ed8cba1..56a262c63 100644 --- a/tests/smoke_relay_integration.py +++ b/tests/smoke_relay_integration.py @@ -64,6 +64,12 @@ def main() -> None: relay_artifacts = result["output"]["relay_artifacts"] kinds = {artifact["kind"] for artifact in relay_artifacts} assert {"atof", "atif"} <= kinds + manifest_relay_kinds = { + artifact["kind"] + for artifact in result["artifacts"]["artifacts"] + if artifact["name"].startswith("relay_") + } + assert {"atof", "atif"} <= manifest_relay_kinds atof_paths = [Path(artifact["path"]) for artifact in relay_artifacts if artifact["kind"] == "atof"] atif_paths = [Path(artifact["path"]) for artifact in relay_artifacts if artifact["kind"] == "atif"]