From 06cb1ee08cbbb5944165ac2f4b908ec8b9b8c9e4 Mon Sep 17 00:00:00 2001 From: Adam Scerra Date: Mon, 10 Aug 2026 07:42:19 -0400 Subject: [PATCH 01/14] feat(eval): add eval measurements and EM-001 trace_fitness scorer Score wild-run OTEL traces fail-open in the same job as fullsend run, always writing eval-measurements.jsonl. Agents enable scorers via measurement manifests (ADR 0087). Signed-off-by: Adam Scerra Co-authored-by: Cursor Signed-off-by: Adam Scerra --- .github/workflows/reusable-code.yml | 7 + .github/workflows/reusable-fix.yml | 7 + .github/workflows/reusable-prioritize.yml | 7 + .github/workflows/reusable-retro.yml | 7 + .github/workflows/reusable-review.yml | 7 + .github/workflows/reusable-triage.yml | 8 + action.yml | 53 +++++ docs/.vitepress/config.ts | 1 + ...050-distributed-tracing-instrumentation.md | 7 + ...-eval-measurements-online-trace-scoring.md | 119 +++++++++++ docs/architecture.md | 2 + docs/glossary.md | 8 +- .../infrastructure/distributed-tracing.md | 12 ++ .../infrastructure/eval-measurements.md | 149 ++++++++++++++ internal/cli/evalmeasure.go | 83 ++++++++ internal/cli/evalmeasure_test.go | 51 +++++ internal/cli/root.go | 1 + internal/evalmeasure/export_local.go | 83 ++++++++ internal/evalmeasure/export_local_test.go | 67 ++++++ internal/evalmeasure/export_mlflow.go | 189 +++++++++++++++++ internal/evalmeasure/export_mlflow_test.go | 102 ++++++++++ internal/evalmeasure/fitness.go | 190 ++++++++++++++++++ internal/evalmeasure/parse.go | 187 +++++++++++++++++ internal/evalmeasure/parse_test.go | 38 ++++ internal/evalmeasure/registry.go | 81 ++++++++ internal/evalmeasure/registry_test.go | 86 ++++++++ internal/evalmeasure/run.go | 57 ++++++ internal/evalmeasure/run_test.go | 48 +++++ internal/evalmeasure/score_test.go | 48 +++++ internal/evalmeasure/testdata/README.md | 6 + internal/evalmeasure/testdata/complete.jsonl | 1 + .../evalmeasure/testdata/missing-cost.jsonl | 1 + .../testdata/review-unknown-workitem.jsonl | 1 + .../evalmeasure/testdata/sample-registry.yaml | 5 + internal/evalmeasure/testdata/split.jsonl | 2 + internal/evalmeasure/types.go | 147 ++++++++++++++ internal/evalmeasure/types_test.go | 180 +++++++++++++++++ 37 files changed, 2044 insertions(+), 4 deletions(-) create mode 100644 docs/ADRs/0087-eval-measurements-online-trace-scoring.md create mode 100644 docs/guides/infrastructure/eval-measurements.md create mode 100644 internal/cli/evalmeasure.go create mode 100644 internal/cli/evalmeasure_test.go create mode 100644 internal/evalmeasure/export_local.go create mode 100644 internal/evalmeasure/export_local_test.go create mode 100644 internal/evalmeasure/export_mlflow.go create mode 100644 internal/evalmeasure/export_mlflow_test.go create mode 100644 internal/evalmeasure/fitness.go create mode 100644 internal/evalmeasure/parse.go create mode 100644 internal/evalmeasure/parse_test.go create mode 100644 internal/evalmeasure/registry.go create mode 100644 internal/evalmeasure/registry_test.go create mode 100644 internal/evalmeasure/run.go create mode 100644 internal/evalmeasure/run_test.go create mode 100644 internal/evalmeasure/score_test.go create mode 100644 internal/evalmeasure/testdata/README.md create mode 100644 internal/evalmeasure/testdata/complete.jsonl create mode 100644 internal/evalmeasure/testdata/missing-cost.jsonl create mode 100644 internal/evalmeasure/testdata/review-unknown-workitem.jsonl create mode 100644 internal/evalmeasure/testdata/sample-registry.yaml create mode 100644 internal/evalmeasure/testdata/split.jsonl create mode 100644 internal/evalmeasure/types.go create mode 100644 internal/evalmeasure/types_test.go diff --git a/.github/workflows/reusable-code.yml b/.github/workflows/reusable-code.yml index d89289e110..5e8d55bf9b 100644 --- a/.github/workflows/reusable-code.yml +++ b/.github/workflows/reusable-code.yml @@ -50,6 +50,8 @@ on: required: false OTEL_EXPORTER_OTLP_HEADERS: required: false + MLFLOW_TRACKING_PASSWORD: + required: false concurrency: group: fullsend-code-agent-${{ inputs.source_repo }}-${{ fromJSON(inputs.event_payload).issue.number || fromJSON(inputs.event_payload).pull_request.number }} @@ -197,6 +199,11 @@ jobs: OTEL_EXPORTER_OTLP_CERTIFICATE: ${{ vars.OTEL_EXPORTER_OTLP_CERTIFICATE }} OTEL_RESOURCE_ATTRIBUTES: ${{ vars.OTEL_RESOURCE_ATTRIBUTES }} OTEL_SDK_DISABLED: ${{ vars.OTEL_SDK_DISABLED }} + # Eval measurements → MLflow Assessments (Quality UI). Optional — + # OTEL ingest Bearer cannot write assessments; needs tracking Basic auth. + MLFLOW_TRACKING_URI: ${{ vars.MLFLOW_TRACKING_URI }} + MLFLOW_TRACKING_USERNAME: ${{ vars.MLFLOW_TRACKING_USERNAME }} + MLFLOW_TRACKING_PASSWORD: ${{ secrets.MLFLOW_TRACKING_PASSWORD }} with: agent: code version: ${{ inputs.fullsend_version || job.workflow_sha }} diff --git a/.github/workflows/reusable-fix.yml b/.github/workflows/reusable-fix.yml index c7aa2f3478..0eeb41b0d9 100644 --- a/.github/workflows/reusable-fix.yml +++ b/.github/workflows/reusable-fix.yml @@ -62,6 +62,8 @@ on: required: false OTEL_EXPORTER_OTLP_HEADERS: required: false + MLFLOW_TRACKING_PASSWORD: + required: false concurrency: group: >- @@ -394,6 +396,11 @@ jobs: OTEL_EXPORTER_OTLP_CERTIFICATE: ${{ vars.OTEL_EXPORTER_OTLP_CERTIFICATE }} OTEL_RESOURCE_ATTRIBUTES: ${{ vars.OTEL_RESOURCE_ATTRIBUTES }} OTEL_SDK_DISABLED: ${{ vars.OTEL_SDK_DISABLED }} + # Eval measurements → MLflow Assessments (Quality UI). Optional — + # OTEL ingest Bearer cannot write assessments; needs tracking Basic auth. + MLFLOW_TRACKING_URI: ${{ vars.MLFLOW_TRACKING_URI }} + MLFLOW_TRACKING_USERNAME: ${{ vars.MLFLOW_TRACKING_USERNAME }} + MLFLOW_TRACKING_PASSWORD: ${{ secrets.MLFLOW_TRACKING_PASSWORD }} with: agent: fix version: ${{ inputs.fullsend_version || job.workflow_sha }} diff --git a/.github/workflows/reusable-prioritize.yml b/.github/workflows/reusable-prioritize.yml index f53e6d5803..612bb5f096 100644 --- a/.github/workflows/reusable-prioritize.yml +++ b/.github/workflows/reusable-prioritize.yml @@ -54,6 +54,8 @@ on: required: false OTEL_EXPORTER_OTLP_HEADERS: required: false + MLFLOW_TRACKING_PASSWORD: + required: false concurrency: group: fullsend-prioritize-agent-${{ inputs.source_repo }}-${{ fromJSON(inputs.event_payload).issue.number || fromJSON(inputs.event_payload).pull_request.number }} @@ -158,6 +160,11 @@ jobs: OTEL_EXPORTER_OTLP_CERTIFICATE: ${{ vars.OTEL_EXPORTER_OTLP_CERTIFICATE }} OTEL_RESOURCE_ATTRIBUTES: ${{ vars.OTEL_RESOURCE_ATTRIBUTES }} OTEL_SDK_DISABLED: ${{ vars.OTEL_SDK_DISABLED }} + # Eval measurements → MLflow Assessments (Quality UI). Optional — + # OTEL ingest Bearer cannot write assessments; needs tracking Basic auth. + MLFLOW_TRACKING_URI: ${{ vars.MLFLOW_TRACKING_URI }} + MLFLOW_TRACKING_USERNAME: ${{ vars.MLFLOW_TRACKING_USERNAME }} + MLFLOW_TRACKING_PASSWORD: ${{ secrets.MLFLOW_TRACKING_PASSWORD }} with: agent: prioritize version: ${{ inputs.fullsend_version || job.workflow_sha }} diff --git a/.github/workflows/reusable-retro.yml b/.github/workflows/reusable-retro.yml index e02a86bdb6..384c55c674 100644 --- a/.github/workflows/reusable-retro.yml +++ b/.github/workflows/reusable-retro.yml @@ -50,6 +50,8 @@ on: required: false OTEL_EXPORTER_OTLP_HEADERS: required: false + MLFLOW_TRACKING_PASSWORD: + required: false concurrency: group: fullsend-retro-agent-${{ inputs.source_repo }}-${{ fromJSON(inputs.event_payload).pull_request.number || fromJSON(inputs.event_payload).issue.number }} @@ -169,6 +171,11 @@ jobs: OTEL_EXPORTER_OTLP_CERTIFICATE: ${{ vars.OTEL_EXPORTER_OTLP_CERTIFICATE }} OTEL_RESOURCE_ATTRIBUTES: ${{ vars.OTEL_RESOURCE_ATTRIBUTES }} OTEL_SDK_DISABLED: ${{ vars.OTEL_SDK_DISABLED }} + # Eval measurements → MLflow Assessments (Quality UI). Optional — + # OTEL ingest Bearer cannot write assessments; needs tracking Basic auth. + MLFLOW_TRACKING_URI: ${{ vars.MLFLOW_TRACKING_URI }} + MLFLOW_TRACKING_USERNAME: ${{ vars.MLFLOW_TRACKING_USERNAME }} + MLFLOW_TRACKING_PASSWORD: ${{ secrets.MLFLOW_TRACKING_PASSWORD }} with: agent: retro version: ${{ inputs.fullsend_version || job.workflow_sha }} diff --git a/.github/workflows/reusable-review.yml b/.github/workflows/reusable-review.yml index 18983d99de..390cd705a9 100644 --- a/.github/workflows/reusable-review.yml +++ b/.github/workflows/reusable-review.yml @@ -50,6 +50,8 @@ on: required: false OTEL_EXPORTER_OTLP_HEADERS: required: false + MLFLOW_TRACKING_PASSWORD: + required: false concurrency: group: fullsend-review-agent-${{ inputs.source_repo }}-${{ fromJSON(inputs.event_payload).pull_request.number || fromJSON(inputs.event_payload).issue.number }} @@ -185,6 +187,11 @@ jobs: OTEL_EXPORTER_OTLP_CERTIFICATE: ${{ vars.OTEL_EXPORTER_OTLP_CERTIFICATE }} OTEL_RESOURCE_ATTRIBUTES: ${{ vars.OTEL_RESOURCE_ATTRIBUTES }} OTEL_SDK_DISABLED: ${{ vars.OTEL_SDK_DISABLED }} + # Eval measurements → MLflow Assessments (Quality UI). Optional — + # OTEL ingest Bearer cannot write assessments; needs tracking Basic auth. + MLFLOW_TRACKING_URI: ${{ vars.MLFLOW_TRACKING_URI }} + MLFLOW_TRACKING_USERNAME: ${{ vars.MLFLOW_TRACKING_USERNAME }} + MLFLOW_TRACKING_PASSWORD: ${{ secrets.MLFLOW_TRACKING_PASSWORD }} with: agent: review fullsend-dir: ${{ inputs.install_mode == 'per-repo' && '.fullsend' || '' }} diff --git a/.github/workflows/reusable-triage.yml b/.github/workflows/reusable-triage.yml index aa1a1f870b..c388030fd3 100644 --- a/.github/workflows/reusable-triage.yml +++ b/.github/workflows/reusable-triage.yml @@ -55,6 +55,8 @@ on: required: false OTEL_EXPORTER_OTLP_HEADERS: required: false + MLFLOW_TRACKING_PASSWORD: + required: false concurrency: group: fullsend-triage-agent-${{ inputs.source_repo }}-${{ fromJSON(inputs.event_payload).issue.number || fromJSON(inputs.event_payload).pull_request.number }} @@ -175,6 +177,12 @@ jobs: OTEL_EXPORTER_OTLP_CERTIFICATE: ${{ vars.OTEL_EXPORTER_OTLP_CERTIFICATE }} OTEL_RESOURCE_ATTRIBUTES: ${{ vars.OTEL_RESOURCE_ATTRIBUTES }} OTEL_SDK_DISABLED: ${{ vars.OTEL_SDK_DISABLED }} + # Eval measurements → MLflow Assessments (Quality UI). Optional — + # OTEL ingest Bearer cannot write assessments; needs tracking Basic + # auth. URI defaults from OTEL endpoint when unset. + MLFLOW_TRACKING_URI: ${{ vars.MLFLOW_TRACKING_URI }} + MLFLOW_TRACKING_USERNAME: ${{ vars.MLFLOW_TRACKING_USERNAME }} + MLFLOW_TRACKING_PASSWORD: ${{ secrets.MLFLOW_TRACKING_PASSWORD }} with: agent: triage version: ${{ inputs.fullsend_version || job.workflow_sha }} diff --git a/action.yml b/action.yml index fe1a5f826a..da070a3b5f 100644 --- a/action.yml +++ b/action.yml @@ -482,3 +482,56 @@ runs: fullsend reconcile-status "${RECONCILE_FLAGS[@]+"${RECONCILE_FLAGS[@]}"}" || { echo "::warning::Could not reconcile orphaned status comment" } + + # Eval measurements (fail-open): score run-telemetry.jsonl with the agents + # measurement manifest. Same job as fullsend run; never fails the agent. + # Scores always land in eval-measurements.jsonl (tool-agnostic artifact). + - name: Eval measurements + if: always() && inputs.agent != '__install_only__' + continue-on-error: true + shell: bash + env: + AGENT: ${{ inputs.agent }} + FULLSEND_DIR: ${{ inputs.fullsend-dir }} + GH_TOKEN: ${{ inputs.github_token }} + run: | + set -euo pipefail + FULLSEND_DIR="${FULLSEND_DIR:-${GITHUB_WORKSPACE}}" + TELEMETRY=$(find "${GITHUB_WORKSPACE}/output" -name run-telemetry.jsonl -print -quit 2>/dev/null || true) + if [[ -z "${TELEMETRY}" ]]; then + echo "No run-telemetry.jsonl under output/; skipping eval measurements" + exit 0 + fi + + REGISTRY="" + LOCAL_REG="${FULLSEND_DIR}/eval/measurements/${AGENT}.yaml" + if [[ -f "${LOCAL_REG}" ]]; then + REGISTRY="${LOCAL_REG}" + else + # Same pin as agents-repo harness fallback (config.DefaultUpstreamRef=v0). + URL="https://raw.githubusercontent.com/fullsend-ai/agents/v0/eval/measurements/${AGENT}.yaml" + TMP="$(mktemp)" + CURL_ARGS=( -fsSL -o "${TMP}" ) + if [[ -n "${GH_TOKEN:-}" ]]; then + CURL_ARGS+=( -H "Authorization: Bearer ${GH_TOKEN}" ) + fi + if curl "${CURL_ARGS[@]}" "${URL}"; then + REGISTRY="${TMP}" + else + echo "No eval measurement manifest for ${AGENT} at ${LOCAL_REG} or ${URL}; skipping" + rm -f "${TMP}" + exit 0 + fi + fi + + fullsend eval-measure \ + --telemetry "${TELEMETRY}" \ + --registry "${REGISTRY}" \ + --out-dir "$(dirname "${TELEMETRY}")" + + - name: Upload fullsend artifacts + if: always() && inputs.agent != '__install_only__' + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: fullsend-${{ inputs.agent }} + path: ${{ github.workspace }}/output diff --git a/docs/.vitepress/config.ts b/docs/.vitepress/config.ts index 573f98c511..da6f9f3a70 100644 --- a/docs/.vitepress/config.ts +++ b/docs/.vitepress/config.ts @@ -259,6 +259,7 @@ export default defineConfig({ { text: "Standalone Mint", link: "/guides/infrastructure/standalone-mint" }, { text: "Private Repositories", link: "/guides/infrastructure/private-repositories" }, { text: "Tracing Reference", link: "/guides/infrastructure/distributed-tracing" }, + { text: "Eval Measurements", link: "/guides/infrastructure/eval-measurements" }, { text: "Advanced Setup", link: "/guides/infrastructure/advanced-setup" }, { text: "Layered Config Reference", diff --git a/docs/ADRs/0050-distributed-tracing-instrumentation.md b/docs/ADRs/0050-distributed-tracing-instrumentation.md index f6859bbf6b..d877e0c622 100644 --- a/docs/ADRs/0050-distributed-tracing-instrumentation.md +++ b/docs/ADRs/0050-distributed-tracing-instrumentation.md @@ -155,3 +155,10 @@ and agent spans in `run-telemetry.jsonl`, which became the sole Level 1 artifact. OTLP export also changed from post-hoc directory upload to live span export via the OTel SDK's batch processor. The core decision (three-level opt-in, OTel-native, W3C propagation) is unchanged. + +**2026-08-10 — Eval measurements ([ADR 0087](0087-eval-measurements-online-trace-scoring.md)):** +online scoring of wild-run traces writes `eval-measurements.jsonl` beside +telemetry; portable remote score export is designed to follow the same OTLP +configuration as this ADR (MLflow Assessments adapter available now; OTLP +path planned). Backend-specific UI adapters remain optional. Distinct from +functional eval fixtures ([ADR 0051](0051-agent-eval-harness-for-test-infrastructure.md)). diff --git a/docs/ADRs/0087-eval-measurements-online-trace-scoring.md b/docs/ADRs/0087-eval-measurements-online-trace-scoring.md new file mode 100644 index 0000000000..77cca96a82 --- /dev/null +++ b/docs/ADRs/0087-eval-measurements-online-trace-scoring.md @@ -0,0 +1,119 @@ +--- +title: "87. Eval measurements as online trace scoring with portable export" +status: Accepted +relates_to: + - operational-observability + - testing-agents +topics: + - observability + - evaluation + - opentelemetry +--- + +# 87. Eval measurements as online trace scoring with portable export + +Date: 2026-08-10 + +## Status + +Accepted + +Visual companion: [explainer diagrams](0087-eval-measurements-explainer.html). + +## Context + +Agent runs already emit OpenTelemetry traces as `run-telemetry.jsonl`, with +optional live OTLP export when `OTEL_EXPORTER_OTLP_*` is set +([ADR 0050](0050-distributed-tracing-instrumentation.md)). Separately, +[ADR 0051](0051-agent-eval-harness-for-test-infrastructure.md) owns the +**functional** eval harness: curated fixtures / scenarios in +`fullsend-ai/agents` `eval//` that gate agent PRs. Those fixtures do +not score wild production runs. + +Operators also need an **online / trend** layer on wild traces (completeness +first; quality signals later) without a second, product-specific export stack. +Dogfood showed MLflow Assessments improve one backend's Quality UI, but +requiring `MLFLOW_TRACKING_*` breaks ADR 0050's portability promise for scores. + +Adjacent telemetry proposals (not competing with this score path): + +- **Level 3 content capture** ([#5947](https://github.com/fullsend-ai/fullsend/pull/5947) + — proposed ADRs 0084/0085): richer span content enables later scorers; + v1 reads Level 1/2 metadata in `run-telemetry.jsonl`. Content is OTLP-only + under that proposal, so content-aware scorers will need OTLP/backend access + or a widened local contract. Measure CLI is host-side after sandbox exit + (outside the sandbox OTEL denylist surface). +- **Span status from run outcome** ([#5944](https://github.com/fullsend-ai/fullsend/pull/5944)): + OTLP Status (and `fullsend.transcript_error`) become the reliable + success/failure signal. EM-001 only checks that `exit_code` is **present** + (fitness). Outcome scorers must key on Status, not `exit_code == 0`. +- **Observer / lessons → fixtures** ([#2423](https://github.com/fullsend-ai/fullsend/pull/2423)): + narrative analysis and golden-set promotion. This ADR is same-job + deterministic scoring on traces. + +## Options + +1. **Local JSONL only** — simplest; no remote scores on any backend. +2. **Backend-native APIs only** (e.g. MLflow Assessments) — good UX on one + host; every new backend needs a new adapter and secrets. +3. **Same OTLP path as agent traces, plus optional adapters** — scores travel + with the same endpoint/headers orgs already configure; adapters are + opt-in for product UIs. + +## Decision + +Introduce **eval measurements**: deterministic scorers that read +`run-telemetry.jsonl` after `fullsend run` in the **same** managed job +(`fullsend eval-measure` in `action.yml`), **fail-open**. + +**Not fixtures:** functional eval scenarios remain ADR 0051 / `eval//`. +Measurements never block delivery. Prefer the glossary terms *eval +measurement* (score) and *eval scenario* (fixture). + +**Ownership:** `fullsend` owns parser, scorers, CLI, and job wiring. +`fullsend-ai/agents` owns per-agent **measurement manifests** at +`eval/measurements/.yaml` (which scorers to enable). + +**Persistence (always):** write `eval-measurements.jsonl` (and a small ledger) +next to `run-telemetry.jsonl`. Scores are data, never gates. + +**Remote export:** the portable path will use the same `OTEL_EXPORTER_OTLP_*` +as ADR 0050 (not yet implemented). Backend-specific adapters (e.g. MLflow +Assessments) are **optional** and available now. Adapters must not be required +for scores to leave the runner once the OTLP path lands. + +**First scorer:** `trace_fitness` (catalog id `em-001`) — span-tree / +attribute fitness so later scorers can trust the trace. Further scorers +(budget, outcome/status, content-aware) land via manifests without changing +this export model. + +### Versioning (per measurement, not platform “v1”) + +There is no product-wide “eval measurements v1” switch. “First ship” just +means only one scorer is enabled yet. Each manifest entry carries: + +| Field | Meaning | +|---|---| +| `id` | Stable catalog id (`em-001`). New measurement concept → new id. | +| `scorer` | Go dispatch name (`trace_fitness`). | +| `version` | Integer **contract** version of that measurement’s checks / pass rule. | + +Scores and the idempotency ledger key on `id@version` (e.g. `em-001@1`). +Bump `version` when pass/fail semantics change so trends do not mix eras. +Add a check that does not change the pass definition → same version is fine. +Entirely new signal → new `em-NNN` (and usually a new `scorer` string). + +## Consequences + +- Wild runs produce a reviewable score file beside telemetry with or without a + remote backend. +- Orgs that already set OTEL for traces get a portable score export contract + without a second auth scheme; optional adapters may enrich one product UI. +- Missing manifests skip cleanly; measure failure never fails the agent job. +- Functional scenarios (gate) and eval measurements (trend) stay separate; + retro/observer lesson extraction may still promote production cases into + fixtures without replacing online scoring. +- Retro can recommend a **manifest scorer** or a **scenario fixture** — not + substitutes. +- Richer telemetry (Level 3 / Status fixes) expands what scorers *can* assert; + it does not replace this same-job path. diff --git a/docs/architecture.md b/docs/architecture.md index f93337e126..f27d39c36f 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -287,11 +287,13 @@ Observability is a cross-cutting concern that touches every other component. Eac - JSONL reasoning trace exposure: raw JSONL conversation transcripts are extracted from sandboxes and stored with owner-scoped access. Credential scanning acts as an invariant check on [ADR 0017](ADRs/0017-credential-isolation-for-sandboxed-agents.md)'s isolation model. Agents handling data from protected sources beyond the target repo can opt in to JSONL suppression via configuration ([ADR 0021](ADRs/0021-jsonl-reasoning-trace-exposure.md)). - Event-driven stage dispatch remains traceable end-to-end in the GitHub Actions UI by using synchronous `workflow_call` dispatch (see [ADR 0041](ADRs/0041-synchronous-workflow-call-event-dispatch.md)). - Distributed tracing: framework-native OpenTelemetry instrumentation with zero-configuration baseline. Every run produces `run-telemetry.jsonl` locally; optional live OTLP export to any compatible backend. W3C trace context propagation links multi-agent pipelines into unified traces. OTEL GenAI semantic conventions enable LLM-aware backends ([ADR 0050](ADRs/0050-distributed-tracing-instrumentation.md)). +- Eval measurements: fail-open same-job scoring of wild-run traces into `eval-measurements.jsonl` beside telemetry; portable remote score export is designed to use the same OTLP configuration as agent traces (MLflow Assessments adapter available now; OTLP path planned); backend-specific UI adapters are optional ([ADR 0087](ADRs/0087-eval-measurements-online-trace-scoring.md)). See [Eval Measurements](guides/infrastructure/eval-measurements.md). **Open questions:** - What signals matter most — cost, latency, token usage, action logs, decision traces, or something else? - ~~How do we balance detailed tracing (useful for debugging) with the volume of data agents will produce?~~ Decided in [ADR 0050](ADRs/0050-distributed-tracing-instrumentation.md): instrument all lifecycle steps comprehensively; volume is managed by backends not by suppressing data at the source. +- ~~How do we score wild agent traces for trends without a second export stack?~~ Decided in [ADR 0087](ADRs/0087-eval-measurements-online-trace-scoring.md): eval measurements write local JSONL beside telemetry; portable remote export is designed to use the same OTLP config as traces (OTLP path planned; MLflow adapter available now); product UI adapters are optional. - What is the retention and access model for agent logs? Who can see what? (JSONL trace access model decided in [ADR 0021](ADRs/0021-jsonl-reasoning-trace-exposure.md); retention policy and broader log access remain open.) - How does observability interact with the security requirement that "every action is logged, attributable, and reviewable"? (See [security-threat-model.md](problems/security-threat-model.md).) - Is there a real-time monitoring requirement (agent is stuck, agent is behaving anomalously), or is observability primarily forensic? diff --git a/docs/glossary.md b/docs/glossary.md index 29bdc5541e..98c2a6227b 100644 --- a/docs/glossary.md +++ b/docs/glossary.md @@ -88,13 +88,13 @@ See [autonomy-spectrum.md](problems/autonomy-spectrum.md) and [agent-architectur ### Eval Measurement -A score, judge, or metric applied to agent (or agent-chain) behavior — for example cost per run, whether the code agent later passes review, or whether a review agent recommends merge and a human still intervenes. Measurements are not the inputs under test; they are what you score. The same measurement can be applied to curated [eval scenarios](#eval-scenario) or to live ("wild") production traffic, at agent scope or across the platform chain. Prefer this term (or synonyms *eval score* / *eval judge*) over the bare word "evals," which is ambiguous with [eval scenarios](#eval-scenario). -See [testing-agents.md](problems/testing-agents.md) and [Observability](#observability). +A score, judge, or metric applied to agent (or agent-chain) behavior — for example cost per run, whether the code agent later passes review, or whether a review agent recommends merge and a human still intervenes. Measurements are not the inputs under test; they are what you score. Online / trend scoring of wild-run traces is decided in [ADR 0087](ADRs/0087-eval-measurements-online-trace-scoring.md) (`fullsend eval-measure`, `eval-measurements.jsonl`). The same measurement *idea* can also be applied to curated [eval scenarios](#eval-scenario), but those PR-gate fixtures are a separate path ([ADR 0051](ADRs/0051-agent-eval-harness-for-test-infrastructure.md)). Prefer this term (or synonyms *eval score* / *eval judge*) over the bare word "evals," which is ambiguous with [eval scenarios](#eval-scenario). +See [Eval Measurements](guides/infrastructure/eval-measurements.md), [testing-agents.md](problems/testing-agents.md), and [Observability](#observability). ### Eval Scenario -A fixed, reproducible test case — a concrete input with an expected outcome that you re-run when an agent changes. Example: triage is presented with an issue asking to add a cheeseburger to the README and is expected to reject and close it. Scenarios are maintained like tests: if intentional agent behavior changes, update the scenario expectations. They answer "did this change make the agent better or worse on known cases?" and can later grow by promoting interesting production cases from telemetry into the curated set. Distinct from [eval measurements](#eval-measurement) (the scores/judges applied to a scenario or to wild traffic). Prefer this term over the bare word "evals." -See [testing-agents.md](problems/testing-agents.md) (golden-set evaluation). +A fixed, reproducible test case — a concrete input with an expected outcome that you re-run when an agent changes. Example: triage is presented with an issue asking to add a cheeseburger to the README and is expected to reject and close it. Scenarios are maintained like tests: if intentional agent behavior changes, update the scenario expectations. They answer "did this change make the agent better or worse on known cases?" and can later grow by promoting interesting production cases from telemetry into the curated set. Distinct from [eval measurements](#eval-measurement) (online/trend scores on wild traces, or judges applied to a scenario). Prefer this term over the bare word "evals." Also called a *functional eval fixture* in agent CI. +See [ADR 0051](ADRs/0051-agent-eval-harness-for-test-infrastructure.md) and [testing-agents.md](problems/testing-agents.md) (golden-set evaluation). ### Evergreen diff --git a/docs/guides/infrastructure/distributed-tracing.md b/docs/guides/infrastructure/distributed-tracing.md index b299341039..3c674dcca5 100644 --- a/docs/guides/infrastructure/distributed-tracing.md +++ b/docs/guides/infrastructure/distributed-tracing.md @@ -244,7 +244,19 @@ Any variable and secret names work here; the values reach the exporter as-is. Consult your backend's documentation for the endpoint URL and authentication mechanism. +## Eval measurements + +After each managed agent run, `fullsend eval-measure` scores +`run-telemetry.jsonl` in the same job (fail-open). Scores always land in +`eval-measurements.jsonl` beside telemetry. Portable remote export uses the +same `OTEL_EXPORTER_OTLP_*` configuration as agent traces; product UI +adapters (e.g. MLflow Assessments) are optional. Measurements read the +Level 1/2 metadata contract of `run-telemetry.jsonl` (not Level 3 content). +See [Eval Measurements](./eval-measurements.md) and +[ADR 0087](../../ADRs/0087-eval-measurements-online-trace-scoring.md). + ## See also - [How To Emit Traces](../user/how-to-emit-traces.md): step-by-step setup guide - [Tracing Development Guide](../dev/tracing.md): implementation details for contributors +- [Eval Measurements](./eval-measurements.md): online scoring of wild-run traces diff --git a/docs/guides/infrastructure/eval-measurements.md b/docs/guides/infrastructure/eval-measurements.md new file mode 100644 index 0000000000..a54f14831a --- /dev/null +++ b/docs/guides/infrastructure/eval-measurements.md @@ -0,0 +1,149 @@ +# Eval Measurements + +Eval measurements score agent runs from their **OpenTelemetry traces** for +trends over time. They are **not** functional evals (PR-gate fixtures under +`fullsend-ai/agents` `eval//`). + +**Decided:** [ADR 0087](../../ADRs/0087-eval-measurements-online-trace-scoring.md). +Telemetry baseline: [Distributed Tracing](./distributed-tracing.md) +([ADR 0050](../../ADRs/0050-distributed-tracing-instrumentation.md)). + +## Architecture (read this first) + +Every managed agent job that produces telemetry gets the same three layers. +Only the first is mandatory; the others reuse existing OTEL config or are +optional product adapters. + +```text +fullsend run + └─ always writes output/**/run-telemetry.jsonl + └─ if OTEL_EXPORTER_OTLP_* set → live OTLP export of agent spans + (any compatible backend — ADR 0050) + +fullsend eval-measure (same GHA job, fail-open, after run) + └─ always writes output/**/eval-measurements.jsonl + (+ eval-measure-ledger.jsonl for idempotency) + └─ portable remote: same OTEL_EXPORTER_OTLP_* as the agent run + (scores follow traces — no second required endpoint) + └─ optional adapter: MLflow Assessments when MLFLOW_TRACKING_* is set + (Quality / Assessments UI only — not required for portability) +``` + +| Artifact | When | Purpose | +|---|---|---| +| `run-telemetry.jsonl` | Every run | OTLP JSON TracesData lines (local source of truth for spans) | +| `eval-measurements.jsonl` | Every measured run | One JSON object per score (`name`, `label`, `value`, `explanation`, `trace_id`, …) | +| Remote agent spans | OTEL configured | Same spans the local file holds | +| Remote scores (portable) | OTEL configured | Scores on the OTLP path — works with any OTLP backend | +| MLflow Assessments | `MLFLOW_TRACKING_PASSWORD` (+ URI) set | Extra UX on MLflow only | + +**Do not** treat MLflow Assessments as the primary export. Telemetry already +goes “anywhere OTLP points”; scores must follow that model. Assessments are an +optional adapter for one backend’s Quality panel. + +## Measurements vs functional evals + +| | Functional evals | Eval measurements | +|---|---|---| +| **Repo path** | `agents/eval//` | `agents/eval/measurements/.yaml` | +| **When** | PR / CI fixture gates | After each managed agent run | +| **Input** | Case fixtures + judge harness | `run-telemetry.jsonl` only | +| **Blocks delivery?** | Yes (when wired as a check) | Never (fail-open) | + +## Ownership + +| Concern | Repo | +|---|---| +| Parser, scorers, CLI, GHA post-step | `fullsend-ai/fullsend` | +| Per-agent **measurement manifests** (which scorers) | `fullsend-ai/agents` | + +First scorer: **`trace_fitness`** (catalog id `em-001`) — span tree + expected +attributes so later scorers can trust the trace. + +Manifest shape: + +```yaml +agent: review +measurements: + - id: em-001 + scorer: trace_fitness + version: 1 +``` + +### Versioning + +Not a platform “v1.” Each entry versions its own contract: + +- **`id`** — stable catalog id (`em-001`). New concept → new id. +- **`scorer`** — which Go scorer to run (`trace_fitness`). +- **`version`** — bump when pass/fail semantics change; scores store + `em-001@1`. Ledger is idempotent per `(trace_id, name, id@version)`. + +The EM-001 `exit` check only requires that `exit_code` is **present** on the +run span (instrumentation fitness). It does **not** treat `exit_code == 0` as +success. After [#5944](https://github.com/fullsend-ai/fullsend/pull/5944), +run/agent **OTLP Status** (and `fullsend.transcript_error`) are the +success/failure signal for outcome scorers. + +## Adjacent telemetry work + +| Proposal | Relationship to measurements | +|---|---| +| [#5947](https://github.com/fullsend-ai/fullsend/pull/5947) Level 3 activation + sandbox OTEL denylist | Richer traces fuel later scorers. v1 reads Level 1/2 local JSONL; content-aware scorers need OTLP/backend (or a widened file contract) under the proposed L3 rules. Measure CLI is host-side after the sandbox exits. | +| [#5944](https://github.com/fullsend-ai/fullsend/pull/5944) Span status from run outcome | Unblocks outcome scorers keyed on Status, not raw exit alone. | +| [#2423](https://github.com/fullsend-ai/fullsend/pull/2423) Semantic observability / observer / lessons | Observer + lessons → fixtures; measurements are the online score path. | + +## Same-job timing + +```text +GitHub Actions job +├── fullsend run +├── fullsend eval-measure # reads run-telemetry.jsonl; never fails the job +└── upload-artifact # includes both JSONL files under output/ +``` + +## Manifest resolution in CI + +`action.yml` resolves the measurement manifest for `inputs.agent`: + +1. `${FULLSEND_DIR}/eval/measurements/${AGENT}.yaml` if present, else +2. `https://raw.githubusercontent.com/fullsend-ai/agents/v0/eval/measurements/${AGENT}.yaml` + +Missing manifest → log and exit `0` (skip). The CLI flag remains +`--registry` for now (path to the YAML); rename is cosmetic follow-up. + +## CLI + +```bash +fullsend eval-measure \ + --telemetry path/to/run-telemetry.jsonl \ + --registry path/to/agents/eval/measurements/review.yaml \ + --out-dir path/to/output +``` + +- `--registry` is required (manifests live in `fullsend-ai/agents`, not in + the fullsend binary). +- Exit `0` when a score is `fail` — scores are data. +- Export failures warn and still exit `0`; local JSONL is kept. + +### Env for remote export + +| Variable | Role | +|---|---| +| `OTEL_EXPORTER_OTLP_TRACES_ENDPOINT` / `_HEADERS` / … | **Portable** score export (same as agent traces) | +| `MLFLOW_TRACKING_URI` | Optional; derived from OTEL endpoint if unset | +| `MLFLOW_TRACKING_USERNAME` | Optional Assessments adapter (default `admin`) | +| `MLFLOW_TRACKING_PASSWORD` | Optional Assessments adapter (Basic auth) | + +Reusable agent workflows forward OTEL and optional `MLFLOW_TRACKING_*`. +The OTEL ingest Bearer token can write `/v1/traces` but cannot create MLflow +Assessments — that adapter needs tracking Basic auth when used. + +## Implementation note + +Local `eval-measurements.jsonl` and the optional MLflow Assessments adapter +are wired in the measure CLI today. Portable OTLP score export (same +`OTEL_*` as traces) is the ADR 0087 remote contract — keep adapters +optional; do not require `MLFLOW_TRACKING_*` for scores to be useful +outside one vendor UI. Until portable OTLP score export lands, remote +scores need the optional Assessments adapter (or local JSONL alone). diff --git a/internal/cli/evalmeasure.go b/internal/cli/evalmeasure.go new file mode 100644 index 0000000000..71ea1523ae --- /dev/null +++ b/internal/cli/evalmeasure.go @@ -0,0 +1,83 @@ +package cli + +import ( + "errors" + "fmt" + "os" + + "github.com/spf13/cobra" + + "github.com/fullsend-ai/fullsend/internal/evalmeasure" + "github.com/fullsend-ai/fullsend/internal/ui" +) + +func newEvalMeasureCmd() *cobra.Command { + var ( + telemetryPath string + registryPath string + outDir string + ) + + cmd := &cobra.Command{ + Use: "eval-measure", + Short: "Score agent run traces with eval measurements", + Long: `Parse run-telemetry.jsonl, score with an agents measurement manifest, +write eval-measurements.jsonl, and best-effort export scores to MLflow +Assessments (Quality / per-trace Assessments panel). + +MLflow export uses MLFLOW_TRACKING_URI (or derives it from +OTEL_EXPORTER_OTLP_TRACES_ENDPOINT) plus MLFLOW_TRACKING_USERNAME/ +MLFLOW_TRACKING_PASSWORD. The OTEL ingest Bearer token is not sufficient +for the Assessments API. + +Exit 0 when scores fail — measurements are data, not gates. Non-zero only +on hard IO/parse errors. Export failures print a warning and exit 0. + +The --registry path is required (manifests live in fullsend-ai/agents, +e.g. eval/measurements/.yaml).`, + RunE: func(cmd *cobra.Command, args []string) error { + printer := ui.New(os.Stdout) + printer.Header("Eval Measure") + + results, err := evalmeasure.MeasureAndExport(cmd.Context(), telemetryPath, registryPath, outDir) + if err != nil { + if isExportWarnErr(err) { + printer.StepWarn(fmt.Sprintf("Export failed (local measurements kept): %v", err)) + printResults(printer, results) + return nil + } + return err + } + printResults(printer, results) + return nil + }, + } + + cmd.Flags().StringVar(&telemetryPath, "telemetry", "", "path to run-telemetry.jsonl (required)") + cmd.Flags().StringVar(®istryPath, "registry", "", "path to agents measurement manifest YAML (required)") + cmd.Flags().StringVar(&outDir, "out-dir", "", "directory for eval-measurements.jsonl (default: telemetry directory)") + _ = cmd.MarkFlagRequired("telemetry") + _ = cmd.MarkFlagRequired("registry") + return cmd +} + +func isExportWarnErr(err error) bool { + var exportErr *evalmeasure.ExportError + return errors.As(err, &exportErr) +} + +func printResults(printer *ui.Printer, results []evalmeasure.EvaluationResult) { + if len(results) == 0 { + printer.StepDone("No new measurements (already scored or no matching traces)") + return + } + for _, r := range results { + line := fmt.Sprintf("%s %s=%.2f (%s) %s", r.Version, r.Name, r.Value, r.Label, r.Explanation) + if r.Label == "pass" { + printer.StepDone(line) + } else { + printer.StepWarn(line) + } + } + printer.StepDone(fmt.Sprintf("Wrote %d measurement(s)", len(results))) +} diff --git a/internal/cli/evalmeasure_test.go b/internal/cli/evalmeasure_test.go new file mode 100644 index 0000000000..8cbdeffc01 --- /dev/null +++ b/internal/cli/evalmeasure_test.go @@ -0,0 +1,51 @@ +package cli + +import ( + "bytes" + "os" + "path/filepath" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestEvalMeasureCmd_ScoresFixture(t *testing.T) { + t.Setenv("OTEL_EXPORTER_OTLP_TRACES_ENDPOINT", "") + t.Setenv("OTEL_EXPORTER_OTLP_ENDPOINT", "") + t.Setenv("MLFLOW_TRACKING_URI", "") + t.Setenv("MLFLOW_TRACKING_PASSWORD", "") + + out := t.TempDir() + telemetry := filepath.Join("..", "evalmeasure", "testdata", "complete.jsonl") + registry := filepath.Join("..", "evalmeasure", "testdata", "sample-registry.yaml") + + cmd := newRootCmd() + buf := &bytes.Buffer{} + cmd.SetOut(buf) + cmd.SetErr(buf) + cmd.SetArgs([]string{ + "eval-measure", + "--telemetry", telemetry, + "--registry", registry, + "--out-dir", out, + }) + err := cmd.Execute() + require.NoError(t, err) + + b, err := os.ReadFile(filepath.Join(out, "eval-measurements.jsonl")) + require.NoError(t, err) + assert.Contains(t, string(b), `"name":"trace_fitness"`) +} + +func TestRootCommand_HasEvalMeasureSubcommand(t *testing.T) { + cmd := newRootCmd() + found := false + for _, sub := range cmd.Commands() { + if sub.Use == "eval-measure" { + found = true + break + } + } + assert.True(t, found, "expected eval-measure subcommand") +} diff --git a/internal/cli/root.go b/internal/cli/root.go index 58761c0828..36e92d71c3 100644 --- a/internal/cli/root.go +++ b/internal/cli/root.go @@ -54,6 +54,7 @@ func newRootCmd() *cobra.Command { cmd.AddCommand(newPostCommentCmd()) cmd.AddCommand(newReconcileStatusCmd()) cmd.AddCommand(newPollCmd()) + cmd.AddCommand(newEvalMeasureCmd()) return cmd } diff --git a/internal/evalmeasure/export_local.go b/internal/evalmeasure/export_local.go new file mode 100644 index 0000000000..3a14c4ef01 --- /dev/null +++ b/internal/evalmeasure/export_local.go @@ -0,0 +1,83 @@ +package evalmeasure + +import ( + "bufio" + "encoding/json" + "fmt" + "os" + "path/filepath" + "strings" +) + +const ( + MeasurementsFile = "eval-measurements.jsonl" + LedgerFile = "eval-measure-ledger.jsonl" +) + +// AppendMeasurements writes one NDJSON EvaluationResult per line. +func AppendMeasurements(path string, results []EvaluationResult) (retErr error) { + if len(results) == 0 { + return nil + } + if err := os.MkdirAll(filepath.Dir(path), 0o755); err != nil { + return err + } + f, err := os.OpenFile(path, os.O_APPEND|os.O_CREATE|os.O_WRONLY, 0o644) + if err != nil { + return err + } + defer func() { + if cerr := f.Close(); retErr == nil { + retErr = cerr + } + }() + enc := json.NewEncoder(f) + for _, r := range results { + if err := enc.Encode(r); err != nil { + return err + } + } + return nil +} + +func ledgerKey(traceID, evalName, version string) string { + return traceID + "|" + evalName + "|" + version +} + +// AlreadyScored reports whether the ledger contains this measurement. +func AlreadyScored(ledgerPath, traceID, evalName, version string) (bool, error) { + f, err := os.Open(ledgerPath) + if err != nil { + if os.IsNotExist(err) { + return false, nil + } + return false, err + } + defer f.Close() + want := ledgerKey(traceID, evalName, version) + sc := bufio.NewScanner(f) + for sc.Scan() { + if strings.TrimSpace(sc.Text()) == want { + return true, nil + } + } + return false, sc.Err() +} + +// RecordScored appends a ledger entry. +func RecordScored(ledgerPath, traceID, evalName, version string) (retErr error) { + if err := os.MkdirAll(filepath.Dir(ledgerPath), 0o755); err != nil { + return err + } + f, err := os.OpenFile(ledgerPath, os.O_APPEND|os.O_CREATE|os.O_WRONLY, 0o644) + if err != nil { + return err + } + defer func() { + if cerr := f.Close(); retErr == nil { + retErr = cerr + } + }() + _, err = fmt.Fprintln(f, ledgerKey(traceID, evalName, version)) + return err +} diff --git a/internal/evalmeasure/export_local_test.go b/internal/evalmeasure/export_local_test.go new file mode 100644 index 0000000000..be1f062e69 --- /dev/null +++ b/internal/evalmeasure/export_local_test.go @@ -0,0 +1,67 @@ +package evalmeasure + +import ( + "os" + "path/filepath" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestAppendMeasurements_EmptySlice(t *testing.T) { + t.Parallel() + dir := t.TempDir() + path := filepath.Join(dir, "out.jsonl") + require.NoError(t, AppendMeasurements(path, nil)) + _, err := os.Stat(path) + assert.True(t, os.IsNotExist(err), "file should not be created for empty slice") +} + +func TestAppendMeasurements_CreatesParentDirs(t *testing.T) { + t.Parallel() + dir := t.TempDir() + path := filepath.Join(dir, "sub", "deep", "out.jsonl") + err := AppendMeasurements(path, []EvaluationResult{{Name: "test", Value: 1}}) + require.NoError(t, err) + b, err := os.ReadFile(path) + require.NoError(t, err) + assert.Contains(t, string(b), `"name":"test"`) +} + +func TestAlreadyScored_NonExistentLedger(t *testing.T) { + t.Parallel() + done, err := AlreadyScored("/nonexistent/ledger.jsonl", "trace1", "eval1", "v1") + require.NoError(t, err) + assert.False(t, done) +} + +func TestRecordScored_ThenAlreadyScored(t *testing.T) { + t.Parallel() + dir := t.TempDir() + ledger := filepath.Join(dir, "ledger.jsonl") + + done, err := AlreadyScored(ledger, "trace1", "eval1", "v1") + require.NoError(t, err) + assert.False(t, done) + + require.NoError(t, RecordScored(ledger, "trace1", "eval1", "v1")) + + done, err = AlreadyScored(ledger, "trace1", "eval1", "v1") + require.NoError(t, err) + assert.True(t, done) + + done, err = AlreadyScored(ledger, "trace2", "eval1", "v1") + require.NoError(t, err) + assert.False(t, done, "different trace should not match") +} + +func TestRecordScored_CreatesParentDirs(t *testing.T) { + t.Parallel() + dir := t.TempDir() + ledger := filepath.Join(dir, "sub", "ledger.jsonl") + require.NoError(t, RecordScored(ledger, "trace1", "eval1", "v1")) + b, err := os.ReadFile(ledger) + require.NoError(t, err) + assert.Contains(t, string(b), "trace1|eval1|v1") +} diff --git a/internal/evalmeasure/export_mlflow.go b/internal/evalmeasure/export_mlflow.go new file mode 100644 index 0000000000..97147e1378 --- /dev/null +++ b/internal/evalmeasure/export_mlflow.go @@ -0,0 +1,189 @@ +package evalmeasure + +import ( + "bytes" + "context" + "encoding/json" + "fmt" + "io" + "net/http" + "net/url" + "os" + "strings" + "time" +) + +// ExportError wraps errors from remote score export (MLflow Assessments). +// The CLI uses errors.As to classify these as warnings (fail-open). +type ExportError struct{ Err error } + +func (e *ExportError) Error() string { return e.Err.Error() } +func (e *ExportError) Unwrap() error { return e.Err } + +// assessmentHTTPDoer is a seam for tests. Not safe for t.Parallel() — +// tests that swap this must not run concurrently. +var assessmentHTTPDo = func(req *http.Request) (*http.Response, error) { + client := &http.Client{Timeout: 15 * time.Second} + return client.Do(req) +} + +type mlflowAssessmentRequest struct { + Assessment mlflowAssessment `json:"assessment"` +} + +type mlflowAssessment struct { + AssessmentName string `json:"assessment_name"` + TraceID string `json:"trace_id"` + Source mlflowSource `json:"source"` + Feedback mlflowFeedback `json:"feedback"` + Rationale string `json:"rationale,omitempty"` + Metadata map[string]string `json:"metadata,omitempty"` + SpanID string `json:"span_id,omitempty"` +} + +type mlflowSource struct { + SourceType string `json:"source_type"` + SourceID string `json:"source_id"` +} + +type mlflowFeedback struct { + Value any `json:"value"` +} + +// ExportMLflowAssessments attaches measurement scores to existing MLflow +// traces via the Assessments REST API (same surface as mlflow.log_feedback). +// +// No-op when MLFLOW_TRACKING_URI is unset and cannot be derived from the OTEL +// traces endpoint, or when tracking basic auth is unset. +// +// Auth: MLFLOW_TRACKING_USERNAME (default "admin") + MLFLOW_TRACKING_PASSWORD. +// The OTEL ingest Bearer token is not accepted by this API on the dogfood host. +func ExportMLflowAssessments(ctx context.Context, results []EvaluationResult) error { + if len(results) == 0 { + return nil + } + user, pass, hasAuth := trackingBasicAuthFromEnv() + explicit := strings.TrimSpace(os.Getenv("MLFLOW_TRACKING_URI")) != "" + base, err := trackingURIFromEnv() + if err != nil { + return err + } + if base == "" { + return nil + } + if !hasAuth { + if explicit { + return &ExportError{Err: fmt.Errorf("MLFLOW_TRACKING_URI is set but MLFLOW_TRACKING_PASSWORD is empty")} + } + return nil + } + + for _, r := range results { + if err := postAssessment(ctx, base, user, pass, r); err != nil { + return &ExportError{Err: err} + } + } + return nil +} + +func postAssessment(ctx context.Context, base, user, pass string, r EvaluationResult) error { + tid := mlflowTraceID(r.TraceID) + payload := mlflowAssessmentRequest{ + Assessment: mlflowAssessment{ + AssessmentName: r.Name, + TraceID: tid, + Source: mlflowSource{ + SourceType: "CODE", + SourceID: "fullsend-evalmeasure/" + r.Version, + }, + Feedback: mlflowFeedback{ + // Label is what Quality / Assessments panels display for pass/fail scorers. + Value: r.Label, + }, + Rationale: r.Explanation, + Metadata: map[string]string{ + "fullsend.measurement.version": r.Version, + "fullsend.evaluation.score.value": fmt.Sprintf("%g", r.Value), + "fullsend.evaluation.score.label": r.Label, + "gen_ai.agent.name": r.Agent, + "fullsend.work_item_id": r.WorkItemID, + "fullsend.source_trace_id": r.TraceID, + // Full observed-vs-limit text also lives in rationale; keep a + // copy in metadata for API consumers / UI detail panes. + "fullsend.evaluation.explanation": r.Explanation, + }, + SpanID: r.SpanID, + }, + } + body, err := json.Marshal(payload) + if err != nil { + return err + } + endpoint := strings.TrimRight(base, "/") + "/ajax-api/3.0/mlflow/traces/" + url.PathEscape(tid) + "/assessments" + req, err := http.NewRequestWithContext(ctx, http.MethodPost, endpoint, bytes.NewReader(body)) + if err != nil { + return err + } + req.Header.Set("Content-Type", "application/json") + req.SetBasicAuth(user, pass) + + resp, err := assessmentHTTPDo(req) + if err != nil { + return fmt.Errorf("post assessment %s: %w", r.Name, err) + } + defer resp.Body.Close() + respBody, _ := io.ReadAll(io.LimitReader(resp.Body, 4096)) + if resp.StatusCode < 200 || resp.StatusCode >= 300 { + return fmt.Errorf("post assessment %s: HTTP %d: %s", r.Name, resp.StatusCode, strings.TrimSpace(string(respBody))) + } + return nil +} + +func mlflowTraceID(hexID string) string { + hexID = strings.TrimSpace(hexID) + if hexID == "" { + return hexID + } + if strings.HasPrefix(hexID, "tr-") { + return hexID + } + return "tr-" + hexID +} + +func trackingURIFromEnv() (string, error) { + if v := strings.TrimSpace(os.Getenv("MLFLOW_TRACKING_URI")); v != "" { + return strings.TrimRight(v, "/"), nil + } + // Derive from OTEL traces endpoint: https://host/v1/traces → https://host + ep := strings.TrimSpace(os.Getenv("OTEL_EXPORTER_OTLP_TRACES_ENDPOINT")) + if ep == "" { + ep = strings.TrimSpace(os.Getenv("OTEL_EXPORTER_OTLP_ENDPOINT")) + } + if ep == "" { + return "", nil + } + u, err := url.Parse(ep) + if err != nil { + return "", fmt.Errorf("derive MLFLOW_TRACKING_URI from OTEL endpoint %q: %w", ep, err) + } + if u.Scheme == "" || u.Host == "" { + return "", fmt.Errorf("derive MLFLOW_TRACKING_URI from OTEL endpoint %q: missing scheme or host", ep) + } + u.Path = "" + u.RawQuery = "" + u.Fragment = "" + u.User = nil + return strings.TrimRight(u.String(), "/"), nil +} + +func trackingBasicAuthFromEnv() (user, pass string, ok bool) { + pass = strings.TrimSpace(os.Getenv("MLFLOW_TRACKING_PASSWORD")) + if pass == "" { + return "", "", false + } + user = strings.TrimSpace(os.Getenv("MLFLOW_TRACKING_USERNAME")) + if user == "" { + user = "admin" + } + return user, pass, true +} diff --git a/internal/evalmeasure/export_mlflow_test.go b/internal/evalmeasure/export_mlflow_test.go new file mode 100644 index 0000000000..d1c5ff2ee1 --- /dev/null +++ b/internal/evalmeasure/export_mlflow_test.go @@ -0,0 +1,102 @@ +package evalmeasure + +import ( + "io" + "net/http" + "strings" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestExportMLflowAssessments_NoopWithoutURI(t *testing.T) { + t.Setenv("MLFLOW_TRACKING_URI", "") + t.Setenv("OTEL_EXPORTER_OTLP_TRACES_ENDPOINT", "") + t.Setenv("OTEL_EXPORTER_OTLP_ENDPOINT", "") + t.Setenv("MLFLOW_TRACKING_PASSWORD", "") + err := ExportMLflowAssessments(t.Context(), []EvaluationResult{{ + Name: "trace_fitness", TraceID: "aa", Label: "pass", Value: 1, + }}) + assert.NoError(t, err) +} + +func TestExportMLflowAssessments_PostsPassLabel(t *testing.T) { + t.Setenv("MLFLOW_TRACKING_URI", "https://mlflow.example") + t.Setenv("MLFLOW_TRACKING_USERNAME", "admin") + t.Setenv("MLFLOW_TRACKING_PASSWORD", "secret") + + var gotURL string + var gotBody string + var gotAuth string + orig := assessmentHTTPDo + t.Cleanup(func() { assessmentHTTPDo = orig }) + assessmentHTTPDo = func(req *http.Request) (*http.Response, error) { + gotURL = req.URL.String() + gotAuth = req.Header.Get("Authorization") + b, _ := io.ReadAll(req.Body) + gotBody = string(b) + return &http.Response{ + StatusCode: 200, + Body: io.NopCloser(strings.NewReader(`{"assessment":{"assessment_id":"a-1"}}`)), + Header: make(http.Header), + }, nil + } + + err := ExportMLflowAssessments(t.Context(), []EvaluationResult{{ + Name: "trace_fitness", + TraceID: "76f79b7475310c3a16d306b98076dd6c", + SpanID: "c29fc9fc1877260b", + Label: "pass", + Value: 1.0, + Explanation: "all checks passed", + Version: "em-001@1", + Agent: "triage", + WorkItemID: "https://github.com/fullsend-ai/fullsend/issues/1342", + }}) + require.NoError(t, err) + assert.Equal(t, "https://mlflow.example/ajax-api/3.0/mlflow/traces/tr-76f79b7475310c3a16d306b98076dd6c/assessments", gotURL) + assert.True(t, strings.HasPrefix(gotAuth, "Basic ")) + assert.Contains(t, gotBody, `"assessment_name":"trace_fitness"`) + assert.Contains(t, gotBody, `"value":"pass"`) + assert.Contains(t, gotBody, `"source_type":"CODE"`) + assert.Contains(t, gotBody, `"fullsend-evalmeasure/em-001@1"`) +} + +func TestExportMLflowAssessments_DerivedURIWithoutPasswordIsNoop(t *testing.T) { + t.Setenv("MLFLOW_TRACKING_URI", "") + t.Setenv("OTEL_EXPORTER_OTLP_TRACES_ENDPOINT", "https://mlflow.example/v1/traces") + t.Setenv("OTEL_EXPORTER_OTLP_ENDPOINT", "") + t.Setenv("MLFLOW_TRACKING_PASSWORD", "") + err := ExportMLflowAssessments(t.Context(), []EvaluationResult{{ + Name: "trace_fitness", TraceID: "aa", Label: "pass", Value: 1, + }}) + assert.NoError(t, err, "derived URI without password should be a silent no-op") +} + +func TestExportMLflowAssessments_ExplicitURIWithoutPasswordErrors(t *testing.T) { + t.Setenv("MLFLOW_TRACKING_URI", "https://mlflow.example") + t.Setenv("OTEL_EXPORTER_OTLP_TRACES_ENDPOINT", "") + t.Setenv("OTEL_EXPORTER_OTLP_ENDPOINT", "") + t.Setenv("MLFLOW_TRACKING_PASSWORD", "") + err := ExportMLflowAssessments(t.Context(), []EvaluationResult{{ + Name: "trace_fitness", TraceID: "aa", Label: "pass", Value: 1, + }}) + require.Error(t, err) + var exportErr *ExportError + assert.ErrorAs(t, err, &exportErr) + assert.Contains(t, err.Error(), "MLFLOW_TRACKING_PASSWORD is empty") +} + +func TestTrackingURIFromEnv_DerivesFromOTLP(t *testing.T) { + t.Setenv("MLFLOW_TRACKING_URI", "") + t.Setenv("OTEL_EXPORTER_OTLP_TRACES_ENDPOINT", "https://mlflow.example.test/v1/traces") + uri, err := trackingURIFromEnv() + require.NoError(t, err) + assert.Equal(t, "https://mlflow.example.test", uri) +} + +func TestMlflowTraceID(t *testing.T) { + assert.Equal(t, "tr-abc", mlflowTraceID("abc")) + assert.Equal(t, "tr-abc", mlflowTraceID("tr-abc")) +} diff --git a/internal/evalmeasure/fitness.go b/internal/evalmeasure/fitness.go new file mode 100644 index 0000000000..506ac581b8 --- /dev/null +++ b/internal/evalmeasure/fitness.go @@ -0,0 +1,190 @@ +package evalmeasure + +import ( + "strings" +) + +const ScorerFitness = "trace_fitness" + +// ScoreFitness implements EM-001 Trace Fitness against a single trace. +func ScoreFitness(tr Trace) EvaluationResult { + return ScoreFitnessNamed(tr, ScorerFitness, "em-001@1") +} + +// ScoreFitnessNamed is ScoreFitness with explicit evaluation name/version. +func ScoreFitnessNamed(tr Trace, evalName, version string) EvaluationResult { + if evalName == "" { + evalName = ScorerFitness + } + if version == "" { + version = "em-001@1" + } + + run, hasRun := tr.SpanByName("run") + agents := tr.SpansByName("agent") + _, hasSandbox := tr.SpanByName("sandbox_create") + + type check struct { + id string + pass bool + } + checks := []check{ + {"span_tree", hasRun && hasSandbox && len(agents) >= 1}, + {"identity", identityOK(run, agents)}, + // "unknown" is the CLI sentinel when no ISSUE_*/GITHUB_ISSUE_URL is set + // (common for review, which wires PR_NUMBER / GITHUB_PR_URL instead). + {"work_item", workItemOK(run)}, + {"operation", attrNonEmpty(run, "gen_ai.operation.name")}, + {"model", modelOK(run, agents)}, + {"usage", usageOK(run, agents)}, + {"cost_tools_turns", costToolsTurnsOK(run, agents)}, + {"exit", hasExit(run)}, + } + + passed := 0 + var failed []string + var details []string + for _, c := range checks { + if c.pass { + passed++ + details = append(details, c.id+"=pass") + } else { + failed = append(failed, c.id) + details = append(details, c.id+"=fail") + } + } + total := len(checks) + value := float64(passed) / float64(total) + label := "fail" + if value == 1.0 { + label = "pass" + } + expl := strings.Join(details, ", ") + if len(failed) > 0 { + expl = "missing: " + strings.Join(failed, ", ") + "; " + expl + } + + spanID := "" + workItem := "" + agent := tr.AgentName() + if hasRun { + spanID = run.SpanID + workItem, _ = run.AttrString("fullsend.work_item_id") + } + + return EvaluationResult{ + Name: evalName, + Label: label, + Explanation: expl, + TraceID: tr.TraceID, + SpanID: spanID, + WorkItemID: workItem, + Agent: agent, + Version: version, + Value: value, + } +} + +func identityOK(run Span, agents []Span) bool { + fa, okFA := run.AttrString("fullsend.agent") + if !okFA || fa == "" || fa == "unknown" { + return false + } + if ga, ok := run.AttrString("gen_ai.agent.name"); ok && ga == fa { + return true + } + for _, a := range agents { + if ga, ok := a.AttrString("gen_ai.agent.name"); ok && ga == fa { + return true + } + } + return false +} + +func workItemOK(run Span) bool { + v, ok := run.AttrString("fullsend.work_item_id") + return ok && v != "" && v != "unknown" +} + +func attrNonEmpty(s Span, key string) bool { + v, ok := s.AttrString(key) + return ok && v != "" +} + +func modelOK(run Span, agents []Span) bool { + hasModel := false + if _, ok := run.AttrString("gen_ai.request.model"); ok { + hasModel = true + } + if !hasModel { + for _, a := range agents { + if _, ok := a.AttrString("gen_ai.request.model"); ok { + hasModel = true + break + } + } + } + hasSystem := false + for _, a := range agents { + if _, ok := a.AttrString("gen_ai.system"); ok { + hasSystem = true + break + } + } + return hasModel && hasSystem +} + +func usageOK(run Span, agents []Span) bool { + if _, ok := run.AttrInt("gen_ai.usage.input_tokens"); ok { + if _, ok := run.AttrInt("gen_ai.usage.output_tokens"); ok { + return true + } + } + for _, a := range agents { + _, inOK := a.AttrInt("gen_ai.usage.input_tokens") + _, outOK := a.AttrInt("gen_ai.usage.output_tokens") + if inOK && outOK { + return true + } + } + return false +} + +func costToolsTurnsOK(run Span, agents []Span) bool { + hasCost := false + hasTools := false + if _, ok := run.AttrFloat("fullsend.cost_usd"); ok { + hasCost = true + } + if _, ok := run.AttrInt("fullsend.tool_calls"); ok { + hasTools = true + } + for _, a := range agents { + if !hasCost { + if _, ok := a.AttrFloat("fullsend.cost_usd"); ok { + hasCost = true + } + } + if !hasTools { + if _, ok := a.AttrInt("fullsend.tool_calls"); ok { + hasTools = true + } + } + } + if !hasCost || !hasTools { + return false + } + if iters, ok := run.AttrInt("fullsend.iterations"); ok && iters > 0 { + if _, ok := run.AttrInt("fullsend.num_turns"); !ok { + return false + } + } + return true +} + +func hasExit(run Span) bool { + // Fitness: attribute present. Do not treat exit_code==0 as success — + // after #5944, OTLP Status / transcript_error carry outcome. + _, ok := run.AttrInt("exit_code") + return ok +} diff --git a/internal/evalmeasure/parse.go b/internal/evalmeasure/parse.go new file mode 100644 index 0000000000..d42ea9be9b --- /dev/null +++ b/internal/evalmeasure/parse.go @@ -0,0 +1,187 @@ +package evalmeasure + +import ( + "bufio" + "encoding/json" + "fmt" + "os" + "strconv" +) + +type otlpTracesData struct { + ResourceSpans []otlpResourceSpans `json:"resourceSpans"` +} + +type otlpResourceSpans struct { + ScopeSpans []otlpScopeSpans `json:"scopeSpans"` +} + +type otlpScopeSpans struct { + Spans []otlpSpan `json:"spans"` +} + +type otlpSpan struct { + TraceID string `json:"traceId"` + SpanID string `json:"spanId"` + ParentSpanID string `json:"parentSpanId"` + Name string `json:"name"` + StartTimeUnixNano string `json:"startTimeUnixNano"` + EndTimeUnixNano string `json:"endTimeUnixNano"` + Attributes []otlpKeyValue `json:"attributes"` + Status *otlpStatus `json:"status"` +} + +type otlpStatus struct { + Code int `json:"code"` +} + +type otlpKeyValue struct { + Key string `json:"key"` + Value map[string]any `json:"value"` +} + +// ParseTelemetryFile reads OTLP JSON TracesData lines from run-telemetry.jsonl +// and merges spans by trace id. +func ParseTelemetryFile(path string) ([]Trace, error) { + f, err := os.Open(path) + if err != nil { + return nil, err + } + defer f.Close() + + byID := make(map[string]*Trace) + var order []string + + sc := bufio.NewScanner(f) + // Spans can be large; raise buffer. + buf := make([]byte, 0, 64*1024) + sc.Buffer(buf, 10*1024*1024) + + lineNo := 0 + for sc.Scan() { + lineNo++ + line := sc.Bytes() + if len(line) == 0 { + continue + } + var doc otlpTracesData + if err := json.Unmarshal(line, &doc); err != nil { + return nil, fmt.Errorf("line %d: %w", lineNo, err) + } + for _, rs := range doc.ResourceSpans { + for _, ss := range rs.ScopeSpans { + for _, raw := range ss.Spans { + sp, err := convertSpan(raw) + if err != nil { + return nil, fmt.Errorf("line %d span %s: %w", lineNo, raw.SpanID, err) + } + tr, ok := byID[sp.TraceID] + if !ok { + tr = &Trace{TraceID: sp.TraceID} + byID[sp.TraceID] = tr + order = append(order, sp.TraceID) + } + tr.Spans = append(tr.Spans, sp) + } + } + } + } + if err := sc.Err(); err != nil { + return nil, err + } + + out := make([]Trace, 0, len(order)) + for _, id := range order { + out = append(out, *byID[id]) + } + return out, nil +} + +func convertSpan(raw otlpSpan) (Span, error) { + start, err := parseUint(raw.StartTimeUnixNano) + if err != nil { + return Span{}, fmt.Errorf("startTimeUnixNano: %w", err) + } + end, err := parseUint(raw.EndTimeUnixNano) + if err != nil { + return Span{}, fmt.Errorf("endTimeUnixNano: %w", err) + } + attrs := make(map[string]any, len(raw.Attributes)) + for _, kv := range raw.Attributes { + if kv.Key == "" { + continue + } + if v, ok := decodeAny(kv.Value); ok { + attrs[kv.Key] = v + } + } + status := 0 + if raw.Status != nil { + status = raw.Status.Code + } + return Span{ + TraceID: raw.TraceID, + SpanID: raw.SpanID, + ParentSpanID: raw.ParentSpanID, + Name: raw.Name, + StartUnixNano: start, + EndUnixNano: end, + StatusCode: status, + Attrs: attrs, + }, nil +} + +func parseUint(s string) (uint64, error) { + if s == "" { + return 0, nil + } + return strconv.ParseUint(s, 10, 64) +} + +// decodeAny extracts scalar OTLP attribute values. arrayValue and kvlistValue +// are intentionally unsupported — fitness scoring uses only scalar attributes. +func decodeAny(m map[string]any) (any, bool) { + if m == nil { + return nil, false + } + if v, ok := m["stringValue"]; ok { + return fmt.Sprint(v), true + } + if v, ok := m["boolValue"]; ok { + switch t := v.(type) { + case bool: + return t, true + default: + return nil, false + } + } + if v, ok := m["doubleValue"]; ok { + switch t := v.(type) { + case float64: + return t, true + case string: + n, err := strconv.ParseFloat(t, 64) + if err != nil { + return nil, false + } + return n, true + default: + return nil, false + } + } + if v, ok := m["intValue"]; ok { + switch t := v.(type) { + case string: + n, err := strconv.ParseInt(t, 10, 64) + if err != nil { + return nil, false + } + return n, true + case float64: + return int64(t), true + default: + return nil, false + } + } + return nil, false +} diff --git a/internal/evalmeasure/parse_test.go b/internal/evalmeasure/parse_test.go new file mode 100644 index 0000000000..1d6d8f22ba --- /dev/null +++ b/internal/evalmeasure/parse_test.go @@ -0,0 +1,38 @@ +package evalmeasure + +import ( + "path/filepath" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestParseTelemetryFile_Complete(t *testing.T) { + t.Parallel() + traces, err := ParseTelemetryFile(filepath.Join("testdata", "complete.jsonl")) + require.NoError(t, err) + require.Len(t, traces, 1) + tr := traces[0] + assert.Equal(t, "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", tr.TraceID) + run, ok := tr.SpanByName("run") + require.True(t, ok) + got, ok := run.AttrString("fullsend.agent") + require.True(t, ok) + assert.Equal(t, "triage", got) + cost, ok := run.AttrFloat("fullsend.cost_usd") + require.True(t, ok) + assert.InDelta(t, 0.54, cost, 1e-9) + assert.Len(t, tr.SpansByName("agent"), 1) + assert.InDelta(t, 6.0, run.DurationSeconds(), 1e-9) +} + +func TestParseTelemetryFile_MergesLinesSameTrace(t *testing.T) { + t.Parallel() + traces, err := ParseTelemetryFile(filepath.Join("testdata", "split.jsonl")) + require.NoError(t, err) + require.Len(t, traces, 1) + assert.Len(t, traces[0].Spans, 3) + _, ok := traces[0].SpanByName("sandbox_create") + assert.True(t, ok) +} diff --git a/internal/evalmeasure/registry.go b/internal/evalmeasure/registry.go new file mode 100644 index 0000000000..c83bb9d738 --- /dev/null +++ b/internal/evalmeasure/registry.go @@ -0,0 +1,81 @@ +package evalmeasure + +import ( + "fmt" + "os" + "strings" + + "gopkg.in/yaml.v3" +) + +// Registry is an agent-specific measurement manifest (owned by agents repo). +type Registry struct { + Agent string `yaml:"agent"` + Measurements []MeasurementSpec `yaml:"measurements"` +} + +// MeasurementSpec selects a framework scorer. +type MeasurementSpec struct { + ID string `yaml:"id"` + Scorer string `yaml:"scorer"` + Name string `yaml:"name"` // optional display override; default = Scorer + Version int `yaml:"version"` +} + +// LoadRegistry loads a measurement manifest YAML file. +func LoadRegistry(path string) (Registry, error) { + b, err := os.ReadFile(path) + if err != nil { + return Registry{}, err + } + var reg Registry + if err := yaml.Unmarshal(b, ®); err != nil { + return Registry{}, err + } + if reg.Agent == "" { + return Registry{}, fmt.Errorf("manifest %s: agent is required", path) + } + for i, m := range reg.Measurements { + if m.ID == "" { + return Registry{}, fmt.Errorf("manifest %s: measurements[%d].id is required", path, i) + } + if m.Scorer == "" { + return Registry{}, fmt.Errorf("manifest %s: measurements[%d].scorer is required", path, i) + } + if m.Version <= 0 { + return Registry{}, fmt.Errorf("manifest %s: measurements[%d].version must be >= 1", path, i) + } + if strings.ContainsAny(m.ID, "|\n") || strings.ContainsAny(m.Scorer, "|\n") || strings.ContainsAny(m.Name, "|\n") { + return Registry{}, fmt.Errorf("manifest %s: measurements[%d].id, .scorer, and .name must not contain pipe or newline", path, i) + } + } + return reg, nil +} + +func (m MeasurementSpec) versionString() string { + return fmt.Sprintf("%s@%d", m.ID, m.Version) +} + +func (m MeasurementSpec) evalName() string { + if m.Name != "" { + return m.Name + } + return m.Scorer +} + +// ScoreTrace runs enabled measurements for traces matching the manifest agent. +func ScoreTrace(tr Trace, reg Registry) []EvaluationResult { + if tr.AgentName() != reg.Agent { + return nil + } + var out []EvaluationResult + for _, m := range reg.Measurements { + switch m.Scorer { + case ScorerFitness: + out = append(out, ScoreFitnessNamed(tr, m.evalName(), m.versionString())) + default: + // Unknown scorers are skipped (forward-compatible). + } + } + return out +} diff --git a/internal/evalmeasure/registry_test.go b/internal/evalmeasure/registry_test.go new file mode 100644 index 0000000000..75600459ac --- /dev/null +++ b/internal/evalmeasure/registry_test.go @@ -0,0 +1,86 @@ +package evalmeasure + +import ( + "os" + "path/filepath" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestLoadRegistry_Valid(t *testing.T) { + t.Parallel() + reg, err := LoadRegistry(filepath.Join("testdata", "sample-registry.yaml")) + require.NoError(t, err) + assert.Equal(t, "triage", reg.Agent) + require.Len(t, reg.Measurements, 1) + assert.Equal(t, "em-001", reg.Measurements[0].ID) + assert.Equal(t, "trace_fitness", reg.Measurements[0].Scorer) + assert.Equal(t, 1, reg.Measurements[0].Version) +} + +func TestLoadRegistry_MissingAgent(t *testing.T) { + t.Parallel() + dir := t.TempDir() + path := filepath.Join(dir, "bad.yaml") + require.NoError(t, os.WriteFile(path, []byte("measurements:\n - id: em-001\n scorer: trace_fitness\n version: 1\n"), 0o644)) + _, err := LoadRegistry(path) + require.Error(t, err) + assert.Contains(t, err.Error(), "agent is required") +} + +func TestLoadRegistry_MissingID(t *testing.T) { + t.Parallel() + dir := t.TempDir() + path := filepath.Join(dir, "bad.yaml") + require.NoError(t, os.WriteFile(path, []byte("agent: test\nmeasurements:\n - scorer: trace_fitness\n version: 1\n"), 0o644)) + _, err := LoadRegistry(path) + require.Error(t, err) + assert.Contains(t, err.Error(), "id is required") +} + +func TestLoadRegistry_MissingScorer(t *testing.T) { + t.Parallel() + dir := t.TempDir() + path := filepath.Join(dir, "bad.yaml") + require.NoError(t, os.WriteFile(path, []byte("agent: test\nmeasurements:\n - id: em-001\n version: 1\n"), 0o644)) + _, err := LoadRegistry(path) + require.Error(t, err) + assert.Contains(t, err.Error(), "scorer is required") +} + +func TestLoadRegistry_ZeroVersion(t *testing.T) { + t.Parallel() + dir := t.TempDir() + path := filepath.Join(dir, "bad.yaml") + require.NoError(t, os.WriteFile(path, []byte("agent: test\nmeasurements:\n - id: em-001\n scorer: trace_fitness\n version: 0\n"), 0o644)) + _, err := LoadRegistry(path) + require.Error(t, err) + assert.Contains(t, err.Error(), "version must be >= 1") +} + +func TestLoadRegistry_PipeInID(t *testing.T) { + t.Parallel() + dir := t.TempDir() + path := filepath.Join(dir, "bad.yaml") + require.NoError(t, os.WriteFile(path, []byte("agent: test\nmeasurements:\n - id: \"em|001\"\n scorer: trace_fitness\n version: 1\n"), 0o644)) + _, err := LoadRegistry(path) + require.Error(t, err) + assert.Contains(t, err.Error(), "must not contain pipe or newline") +} + +func TestLoadRegistry_InvalidYAML(t *testing.T) { + t.Parallel() + dir := t.TempDir() + path := filepath.Join(dir, "bad.yaml") + require.NoError(t, os.WriteFile(path, []byte("{{invalid yaml"), 0o644)) + _, err := LoadRegistry(path) + require.Error(t, err) +} + +func TestLoadRegistry_FileNotFound(t *testing.T) { + t.Parallel() + _, err := LoadRegistry("/nonexistent/path.yaml") + require.Error(t, err) +} diff --git a/internal/evalmeasure/run.go b/internal/evalmeasure/run.go new file mode 100644 index 0000000000..cdc4a4006c --- /dev/null +++ b/internal/evalmeasure/run.go @@ -0,0 +1,57 @@ +package evalmeasure + +import ( + "context" + "fmt" + "path/filepath" +) + +// MeasureFile parses telemetry, scores with the registry, writes local JSONL, +// and best-effort exports MLflow Assessments. Idempotent per ledger. +func MeasureFile(telemetryPath, registryPath, outDir string) ([]EvaluationResult, error) { + return MeasureAndExport(context.Background(), telemetryPath, registryPath, outDir) +} + +// MeasureAndExport is MeasureFile with an explicit context. +func MeasureAndExport(ctx context.Context, telemetryPath, registryPath, outDir string) ([]EvaluationResult, error) { + if outDir == "" { + outDir = filepath.Dir(telemetryPath) + } + reg, err := LoadRegistry(registryPath) + if err != nil { + return nil, fmt.Errorf("load registry: %w", err) + } + traces, err := ParseTelemetryFile(telemetryPath) + if err != nil { + return nil, fmt.Errorf("parse telemetry: %w", err) + } + + ledgerPath := filepath.Join(outDir, LedgerFile) + measPath := filepath.Join(outDir, MeasurementsFile) + var all []EvaluationResult + + for _, tr := range traces { + results := ScoreTrace(tr, reg) + for _, r := range results { + done, err := AlreadyScored(ledgerPath, r.TraceID, r.Name, r.Version) + if err != nil { + return all, fmt.Errorf("check ledger: %w", err) + } + if done { + continue + } + all = append(all, r) + if err := RecordScored(ledgerPath, r.TraceID, r.Name, r.Version); err != nil { + return all, fmt.Errorf("record scored: %w", err) + } + if err := AppendMeasurements(measPath, []EvaluationResult{r}); err != nil { + return all, fmt.Errorf("append measurements: %w", err) + } + } + } + + if err := ExportMLflowAssessments(ctx, all); err != nil { + return all, fmt.Errorf("export mlflow: %w", err) + } + return all, nil +} diff --git a/internal/evalmeasure/run_test.go b/internal/evalmeasure/run_test.go new file mode 100644 index 0000000000..345ae7db78 --- /dev/null +++ b/internal/evalmeasure/run_test.go @@ -0,0 +1,48 @@ +package evalmeasure + +import ( + "os" + "path/filepath" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestLoadRegistryAndScoreTrace(t *testing.T) { + t.Parallel() + reg, err := LoadRegistry(filepath.Join("testdata", "sample-registry.yaml")) + require.NoError(t, err) + assert.Equal(t, "triage", reg.Agent) + + traces, err := ParseTelemetryFile(filepath.Join("testdata", "complete.jsonl")) + require.NoError(t, err) + results := ScoreTrace(traces[0], reg) + require.Len(t, results, 1) + assert.Equal(t, "trace_fitness", results[0].Name) + assert.Equal(t, "em-001@1", results[0].Version) + assert.Equal(t, "pass", results[0].Label) +} + +func TestMeasureFile_Idempotent(t *testing.T) { + t.Setenv("MLFLOW_TRACKING_URI", "") + t.Setenv("MLFLOW_TRACKING_PASSWORD", "") + t.Setenv("OTEL_EXPORTER_OTLP_TRACES_ENDPOINT", "") + t.Setenv("OTEL_EXPORTER_OTLP_ENDPOINT", "") + + out := t.TempDir() + telemetry := filepath.Join("testdata", "complete.jsonl") + registry := filepath.Join("testdata", "sample-registry.yaml") + + first, err := MeasureFile(telemetry, registry, out) + require.NoError(t, err) + require.Len(t, first, 1) + + second, err := MeasureFile(telemetry, registry, out) + require.NoError(t, err) + assert.Empty(t, second) + + b, err := os.ReadFile(filepath.Join(out, MeasurementsFile)) + require.NoError(t, err) + assert.Contains(t, string(b), `"name":"trace_fitness"`) +} diff --git a/internal/evalmeasure/score_test.go b/internal/evalmeasure/score_test.go new file mode 100644 index 0000000000..e341a47ff7 --- /dev/null +++ b/internal/evalmeasure/score_test.go @@ -0,0 +1,48 @@ +package evalmeasure + +import ( + "path/filepath" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestScoreFitness_CompletePass(t *testing.T) { + t.Parallel() + traces, err := ParseTelemetryFile(filepath.Join("testdata", "complete.jsonl")) + require.NoError(t, err) + r := ScoreFitness(traces[0]) + assert.Equal(t, ScorerFitness, r.Name) + assert.Equal(t, "em-001@1", r.Version) + assert.Equal(t, "pass", r.Label) + assert.Equal(t, 1.0, r.Value) + assert.Equal(t, "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", r.TraceID) + assert.Contains(t, r.Explanation, "span_tree=pass") + assert.Contains(t, r.Explanation, "cost_tools_turns=pass") + assert.Contains(t, r.Explanation, "exit=pass") + assert.NotContains(t, r.Explanation, "=fail") +} + +func TestScoreFitness_MissingCostFails(t *testing.T) { + t.Parallel() + traces, err := ParseTelemetryFile(filepath.Join("testdata", "missing-cost.jsonl")) + require.NoError(t, err) + r := ScoreFitness(traces[0]) + assert.Equal(t, "fail", r.Label) + assert.Less(t, r.Value, 1.0) + assert.Contains(t, r.Explanation, "cost_tools_turns") + assert.Contains(t, r.Explanation, "cost_tools_turns=fail") +} + +func TestScoreFitness_ReviewUnknownWorkItemFails(t *testing.T) { + t.Parallel() + traces, err := ParseTelemetryFile(filepath.Join("testdata", "review-unknown-workitem.jsonl")) + require.NoError(t, err) + r := ScoreFitness(traces[0]) + assert.Equal(t, "review", r.Agent) + assert.Equal(t, "fail", r.Label) + assert.Contains(t, r.Explanation, "identity=pass") + assert.Contains(t, r.Explanation, "work_item=fail") + assert.Contains(t, r.Explanation, "missing: work_item") +} diff --git a/internal/evalmeasure/testdata/README.md b/internal/evalmeasure/testdata/README.md new file mode 100644 index 0000000000..a49d56f033 --- /dev/null +++ b/internal/evalmeasure/testdata/README.md @@ -0,0 +1,6 @@ +# evalmeasure testdata + +Synthetic OTLP JSONL fixtures and a sample registry for unit tests only. + +Production registries live in `fullsend-ai/agents` at +`eval/measurements/.yaml`. diff --git a/internal/evalmeasure/testdata/complete.jsonl b/internal/evalmeasure/testdata/complete.jsonl new file mode 100644 index 0000000000..8e96a8c202 --- /dev/null +++ b/internal/evalmeasure/testdata/complete.jsonl @@ -0,0 +1 @@ +{"resourceSpans":[{"resource":{"attributes":[{"key":"service.name","value":{"stringValue":"fullsend"}}]},"scopeSpans":[{"scope":{"name":"github.com/fullsend-ai/fullsend/internal/telemetry","version":"test"},"spans":[{"traceId":"aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa","spanId":"1111111111111111","name":"run","kind":1,"startTimeUnixNano":"1000000000","endTimeUnixNano":"7000000000","attributes":[{"key":"fullsend.agent","value":{"stringValue":"triage"}},{"key":"gen_ai.agent.name","value":{"stringValue":"triage"}},{"key":"gen_ai.operation.name","value":{"stringValue":"invoke_agent"}},{"key":"fullsend.work_item_id","value":{"stringValue":"acme/demo#1"}},{"key":"exit_code","value":{"intValue":"0"}},{"key":"gen_ai.request.model","value":{"stringValue":"claude-opus-4-6"}},{"key":"gen_ai.usage.input_tokens","value":{"intValue":"100"}},{"key":"gen_ai.usage.output_tokens","value":{"intValue":"20"}},{"key":"fullsend.cost_usd","value":{"doubleValue":0.54}},{"key":"fullsend.tool_calls","value":{"intValue":"12"}},{"key":"fullsend.num_turns","value":{"intValue":"8"}},{"key":"fullsend.iterations","value":{"intValue":"1"}}],"status":{"code":1}},{"traceId":"aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa","spanId":"2222222222222222","parentSpanId":"1111111111111111","name":"sandbox_create","kind":1,"startTimeUnixNano":"1100000000","endTimeUnixNano":"2000000000","attributes":[{"key":"gen_ai.operation.name","value":{"stringValue":"create_agent"}}],"status":{"code":1}},{"traceId":"aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa","spanId":"3333333333333333","parentSpanId":"1111111111111111","name":"agent","kind":1,"startTimeUnixNano":"2100000000","endTimeUnixNano":"6900000000","attributes":[{"key":"iteration","value":{"intValue":"1"}},{"key":"gen_ai.agent.name","value":{"stringValue":"triage"}},{"key":"gen_ai.operation.name","value":{"stringValue":"invoke_agent"}},{"key":"gen_ai.system","value":{"stringValue":"anthropic"}},{"key":"gen_ai.request.model","value":{"stringValue":"claude-opus-4-6"}},{"key":"gen_ai.usage.input_tokens","value":{"intValue":"100"}},{"key":"gen_ai.usage.output_tokens","value":{"intValue":"20"}},{"key":"fullsend.cost_usd","value":{"doubleValue":0.54}},{"key":"fullsend.tool_calls","value":{"intValue":"12"}},{"key":"exit_code","value":{"intValue":"0"}}],"status":{"code":1}}]}]}]} diff --git a/internal/evalmeasure/testdata/missing-cost.jsonl b/internal/evalmeasure/testdata/missing-cost.jsonl new file mode 100644 index 0000000000..fb84a58319 --- /dev/null +++ b/internal/evalmeasure/testdata/missing-cost.jsonl @@ -0,0 +1 @@ +{"resourceSpans":[{"resource":{"attributes":[{"key":"service.name","value":{"stringValue":"fullsend"}}]},"scopeSpans":[{"scope":{"name":"github.com/fullsend-ai/fullsend/internal/telemetry","version":"test"},"spans":[{"traceId":"aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa","spanId":"1111111111111111","name":"run","kind":1,"startTimeUnixNano":"1000000000","endTimeUnixNano":"7000000000","attributes":[{"key":"fullsend.agent","value":{"stringValue":"triage"}},{"key":"gen_ai.agent.name","value":{"stringValue":"triage"}},{"key":"gen_ai.operation.name","value":{"stringValue":"invoke_agent"}},{"key":"fullsend.work_item_id","value":{"stringValue":"acme/demo#1"}},{"key":"exit_code","value":{"intValue":"0"}},{"key":"gen_ai.request.model","value":{"stringValue":"claude-opus-4-6"}},{"key":"gen_ai.usage.input_tokens","value":{"intValue":"100"}},{"key":"gen_ai.usage.output_tokens","value":{"intValue":"20"}},{"key":"fullsend.tool_calls","value":{"intValue":"12"}},{"key":"fullsend.num_turns","value":{"intValue":"8"}},{"key":"fullsend.iterations","value":{"intValue":"1"}}],"status":{"code":1}},{"traceId":"aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa","spanId":"2222222222222222","parentSpanId":"1111111111111111","name":"sandbox_create","kind":1,"startTimeUnixNano":"1100000000","endTimeUnixNano":"2000000000","attributes":[{"key":"gen_ai.operation.name","value":{"stringValue":"create_agent"}}],"status":{"code":1}},{"traceId":"aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa","spanId":"3333333333333333","parentSpanId":"1111111111111111","name":"agent","kind":1,"startTimeUnixNano":"2100000000","endTimeUnixNano":"6900000000","attributes":[{"key":"iteration","value":{"intValue":"1"}},{"key":"gen_ai.agent.name","value":{"stringValue":"triage"}},{"key":"gen_ai.operation.name","value":{"stringValue":"invoke_agent"}},{"key":"gen_ai.system","value":{"stringValue":"anthropic"}},{"key":"gen_ai.request.model","value":{"stringValue":"claude-opus-4-6"}},{"key":"gen_ai.usage.input_tokens","value":{"intValue":"100"}},{"key":"gen_ai.usage.output_tokens","value":{"intValue":"20"}},{"key":"fullsend.tool_calls","value":{"intValue":"12"}},{"key":"exit_code","value":{"intValue":"0"}}],"status":{"code":1}}]}]}]} diff --git a/internal/evalmeasure/testdata/review-unknown-workitem.jsonl b/internal/evalmeasure/testdata/review-unknown-workitem.jsonl new file mode 100644 index 0000000000..61678b2fae --- /dev/null +++ b/internal/evalmeasure/testdata/review-unknown-workitem.jsonl @@ -0,0 +1 @@ +{"resourceSpans":[{"resource":{"attributes":[{"key":"service.name","value":{"stringValue":"fullsend"}}]},"scopeSpans":[{"scope":{"name":"github.com/fullsend-ai/fullsend/internal/telemetry","version":"test"},"spans":[{"traceId":"bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb","spanId":"1111111111111111","name":"run","kind":1,"startTimeUnixNano":"1000000000","endTimeUnixNano":"7000000000","attributes":[{"key":"fullsend.agent","value":{"stringValue":"review"}},{"key":"gen_ai.agent.name","value":{"stringValue":"review"}},{"key":"gen_ai.operation.name","value":{"stringValue":"invoke_agent"}},{"key":"fullsend.work_item_id","value":{"stringValue":"unknown"}},{"key":"exit_code","value":{"intValue":"0"}},{"key":"gen_ai.request.model","value":{"stringValue":"claude-opus-4-6"}},{"key":"gen_ai.usage.input_tokens","value":{"intValue":"100"}},{"key":"gen_ai.usage.output_tokens","value":{"intValue":"20"}},{"key":"fullsend.cost_usd","value":{"doubleValue":0.54}},{"key":"fullsend.tool_calls","value":{"intValue":"12"}},{"key":"fullsend.num_turns","value":{"intValue":"8"}},{"key":"fullsend.iterations","value":{"intValue":"1"}}],"status":{"code":1}},{"traceId":"bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb","spanId":"2222222222222222","parentSpanId":"1111111111111111","name":"sandbox_create","kind":1,"startTimeUnixNano":"1100000000","endTimeUnixNano":"2000000000","attributes":[{"key":"gen_ai.operation.name","value":{"stringValue":"create_agent"}}],"status":{"code":1}},{"traceId":"bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb","spanId":"3333333333333333","parentSpanId":"1111111111111111","name":"agent","kind":1,"startTimeUnixNano":"2100000000","endTimeUnixNano":"6900000000","attributes":[{"key":"iteration","value":{"intValue":"1"}},{"key":"gen_ai.agent.name","value":{"stringValue":"review"}},{"key":"gen_ai.operation.name","value":{"stringValue":"invoke_agent"}},{"key":"gen_ai.system","value":{"stringValue":"anthropic"}},{"key":"gen_ai.request.model","value":{"stringValue":"claude-opus-4-6"}},{"key":"gen_ai.usage.input_tokens","value":{"intValue":"100"}},{"key":"gen_ai.usage.output_tokens","value":{"intValue":"20"}},{"key":"fullsend.cost_usd","value":{"doubleValue":0.54}},{"key":"fullsend.tool_calls","value":{"intValue":"12"}},{"key":"exit_code","value":{"intValue":"0"}}],"status":{"code":1}}]}]}]} diff --git a/internal/evalmeasure/testdata/sample-registry.yaml b/internal/evalmeasure/testdata/sample-registry.yaml new file mode 100644 index 0000000000..ef8492df09 --- /dev/null +++ b/internal/evalmeasure/testdata/sample-registry.yaml @@ -0,0 +1,5 @@ +agent: triage +measurements: + - id: em-001 + scorer: trace_fitness + version: 1 diff --git a/internal/evalmeasure/testdata/split.jsonl b/internal/evalmeasure/testdata/split.jsonl new file mode 100644 index 0000000000..434f0b0351 --- /dev/null +++ b/internal/evalmeasure/testdata/split.jsonl @@ -0,0 +1,2 @@ +{"resourceSpans":[{"resource":{"attributes":[{"key":"service.name","value":{"stringValue":"fullsend"}}]},"scopeSpans":[{"scope":{"name":"github.com/fullsend-ai/fullsend/internal/telemetry","version":"test"},"spans":[{"traceId":"aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa","spanId":"1111111111111111","name":"run","kind":1,"startTimeUnixNano":"1000000000","endTimeUnixNano":"7000000000","attributes":[{"key":"fullsend.agent","value":{"stringValue":"triage"}},{"key":"gen_ai.agent.name","value":{"stringValue":"triage"}},{"key":"gen_ai.operation.name","value":{"stringValue":"invoke_agent"}},{"key":"fullsend.work_item_id","value":{"stringValue":"acme/demo#1"}},{"key":"exit_code","value":{"intValue":"0"}},{"key":"gen_ai.request.model","value":{"stringValue":"claude-opus-4-6"}},{"key":"gen_ai.usage.input_tokens","value":{"intValue":"100"}},{"key":"gen_ai.usage.output_tokens","value":{"intValue":"20"}},{"key":"fullsend.cost_usd","value":{"doubleValue":0.54}},{"key":"fullsend.tool_calls","value":{"intValue":"12"}},{"key":"fullsend.num_turns","value":{"intValue":"8"}},{"key":"fullsend.iterations","value":{"intValue":"1"}}],"status":{"code":1}}]}]}]} +{"resourceSpans":[{"resource":{"attributes":[{"key":"service.name","value":{"stringValue":"fullsend"}}]},"scopeSpans":[{"scope":{"name":"github.com/fullsend-ai/fullsend/internal/telemetry","version":"test"},"spans":[{"traceId":"aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa","spanId":"2222222222222222","parentSpanId":"1111111111111111","name":"sandbox_create","kind":1,"startTimeUnixNano":"1100000000","endTimeUnixNano":"2000000000","attributes":[{"key":"gen_ai.operation.name","value":{"stringValue":"create_agent"}}],"status":{"code":1}},{"traceId":"aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa","spanId":"3333333333333333","parentSpanId":"1111111111111111","name":"agent","kind":1,"startTimeUnixNano":"2100000000","endTimeUnixNano":"6900000000","attributes":[{"key":"iteration","value":{"intValue":"1"}},{"key":"gen_ai.agent.name","value":{"stringValue":"triage"}},{"key":"gen_ai.operation.name","value":{"stringValue":"invoke_agent"}},{"key":"gen_ai.system","value":{"stringValue":"anthropic"}},{"key":"gen_ai.request.model","value":{"stringValue":"claude-opus-4-6"}},{"key":"gen_ai.usage.input_tokens","value":{"intValue":"100"}},{"key":"gen_ai.usage.output_tokens","value":{"intValue":"20"}},{"key":"fullsend.cost_usd","value":{"doubleValue":0.54}},{"key":"fullsend.tool_calls","value":{"intValue":"12"}},{"key":"exit_code","value":{"intValue":"0"}}],"status":{"code":1}}]}]}]} diff --git a/internal/evalmeasure/types.go b/internal/evalmeasure/types.go new file mode 100644 index 0000000000..ff3180b84d --- /dev/null +++ b/internal/evalmeasure/types.go @@ -0,0 +1,147 @@ +// Package evalmeasure scores agent run traces for online eval measurements. +package evalmeasure + +import ( + "fmt" + "strconv" +) + +// Span is a portable view of one OTEL span from run-telemetry.jsonl. +type Span struct { + TraceID string + SpanID string + ParentSpanID string + Name string + StartUnixNano uint64 + EndUnixNano uint64 + StatusCode int // 0 unset, 1 OK, 2 ERROR (OTLP) + Attrs map[string]any +} + +// Trace groups spans that share a trace id. +type Trace struct { + TraceID string + Spans []Span +} + +// EvaluationResult is a portable measurement score (no vendor types). +type EvaluationResult struct { + Name string `json:"name"` + Label string `json:"label"` + Explanation string `json:"explanation"` + TraceID string `json:"trace_id"` + SpanID string `json:"span_id"` + WorkItemID string `json:"work_item_id,omitempty"` + Agent string `json:"agent"` + Version string `json:"version"` + Value float64 `json:"value"` +} + +// AttrString returns a string attribute. +func (s Span) AttrString(key string) (string, bool) { + v, ok := s.Attrs[key] + if !ok || v == nil { + return "", false + } + switch t := v.(type) { + case string: + return t, true + default: + return fmt.Sprint(t), true + } +} + +// AttrInt returns an int64 attribute. +func (s Span) AttrInt(key string) (int64, bool) { + v, ok := s.Attrs[key] + if !ok || v == nil { + return 0, false + } + switch t := v.(type) { + case int64: + return t, true + case int: + return int64(t), true + case float64: + return int64(t), true + case string: + n, err := strconv.ParseInt(t, 10, 64) + if err != nil { + return 0, false + } + return n, true + default: + return 0, false + } +} + +// AttrFloat returns a float64 attribute. +func (s Span) AttrFloat(key string) (float64, bool) { + v, ok := s.Attrs[key] + if !ok || v == nil { + return 0, false + } + switch t := v.(type) { + case float64: + return t, true + case int64: + return float64(t), true + case int: + return float64(t), true + case string: + n, err := strconv.ParseFloat(t, 64) + if err != nil { + return 0, false + } + return n, true + default: + return 0, false + } +} + +// DurationSeconds returns end-start in seconds. +func (s Span) DurationSeconds() float64 { + if s.EndUnixNano <= s.StartUnixNano { + return 0 + } + return float64(s.EndUnixNano-s.StartUnixNano) / 1e9 +} + +// SpanByName returns the first span with the given name. +func (t Trace) SpanByName(name string) (Span, bool) { + for _, s := range t.Spans { + if s.Name == name { + return s, true + } + } + return Span{}, false +} + +// SpansByName returns all spans with the given name. +func (t Trace) SpansByName(name string) []Span { + var out []Span + for _, s := range t.Spans { + if s.Name == name { + out = append(out, s) + } + } + return out +} + +// AgentName returns the agent identity from run or agent spans. +func (t Trace) AgentName() string { + if run, ok := t.SpanByName("run"); ok { + if a, ok := run.AttrString("fullsend.agent"); ok && a != "" { + return a + } + if a, ok := run.AttrString("gen_ai.agent.name"); ok && a != "" { + return a + } + } + for _, a := range t.SpansByName("agent") { + if name, ok := a.AttrString("gen_ai.agent.name"); ok && name != "" { + return name + } + } + return "" +} diff --git a/internal/evalmeasure/types_test.go b/internal/evalmeasure/types_test.go new file mode 100644 index 0000000000..917333aad1 --- /dev/null +++ b/internal/evalmeasure/types_test.go @@ -0,0 +1,180 @@ +package evalmeasure + +import ( + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestAttrString(t *testing.T) { + t.Parallel() + tests := []struct { + name string + attrs map[string]any + key string + want string + wantOK bool + }{ + {"string value", map[string]any{"k": "v"}, "k", "v", true}, + {"int value coerced", map[string]any{"k": 42}, "k", "42", true}, + {"missing key", map[string]any{}, "k", "", false}, + {"nil value", map[string]any{"k": nil}, "k", "", false}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + s := Span{Attrs: tt.attrs} + got, ok := s.AttrString(tt.key) + assert.Equal(t, tt.wantOK, ok) + assert.Equal(t, tt.want, got) + }) + } +} + +func TestAttrInt(t *testing.T) { + t.Parallel() + tests := []struct { + name string + attrs map[string]any + key string + want int64 + wantOK bool + }{ + {"int64", map[string]any{"k": int64(42)}, "k", 42, true}, + {"int", map[string]any{"k": 42}, "k", 42, true}, + {"float64", map[string]any{"k": float64(42)}, "k", 42, true}, + {"string parseable", map[string]any{"k": "42"}, "k", 42, true}, + {"string unparseable", map[string]any{"k": "abc"}, "k", 0, false}, + {"missing key", map[string]any{}, "k", 0, false}, + {"nil value", map[string]any{"k": nil}, "k", 0, false}, + {"bool not int", map[string]any{"k": true}, "k", 0, false}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + s := Span{Attrs: tt.attrs} + got, ok := s.AttrInt(tt.key) + assert.Equal(t, tt.wantOK, ok) + assert.Equal(t, tt.want, got) + }) + } +} + +func TestAttrFloat(t *testing.T) { + t.Parallel() + tests := []struct { + name string + attrs map[string]any + key string + want float64 + wantOK bool + }{ + {"float64", map[string]any{"k": float64(3.14)}, "k", 3.14, true}, + {"int64", map[string]any{"k": int64(42)}, "k", 42, true}, + {"int", map[string]any{"k": 42}, "k", 42, true}, + {"string parseable", map[string]any{"k": "3.14"}, "k", 3.14, true}, + {"string unparseable", map[string]any{"k": "abc"}, "k", 0, false}, + {"missing key", map[string]any{}, "k", 0, false}, + {"nil value", map[string]any{"k": nil}, "k", 0, false}, + {"bool not float", map[string]any{"k": true}, "k", 0, false}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + s := Span{Attrs: tt.attrs} + got, ok := s.AttrFloat(tt.key) + assert.Equal(t, tt.wantOK, ok) + assert.InDelta(t, tt.want, got, 1e-9) + }) + } +} + +func TestDurationSeconds(t *testing.T) { + t.Parallel() + tests := []struct { + name string + start uint64 + end uint64 + want float64 + }{ + {"normal", 1000000000, 7000000000, 6.0}, + {"zero duration", 100, 100, 0}, + {"end before start", 200, 100, 0}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + s := Span{StartUnixNano: tt.start, EndUnixNano: tt.end} + assert.InDelta(t, tt.want, s.DurationSeconds(), 1e-9) + }) + } +} + +func TestSpanByName(t *testing.T) { + t.Parallel() + tr := Trace{ + Spans: []Span{ + {Name: "run"}, + {Name: "agent"}, + }, + } + s, ok := tr.SpanByName("run") + assert.True(t, ok) + assert.Equal(t, "run", s.Name) + + _, ok = tr.SpanByName("missing") + assert.False(t, ok) +} + +func TestSpansByName(t *testing.T) { + t.Parallel() + tr := Trace{ + Spans: []Span{ + {Name: "agent"}, + {Name: "run"}, + {Name: "agent"}, + }, + } + assert.Len(t, tr.SpansByName("agent"), 2) + assert.Empty(t, tr.SpansByName("missing")) +} + +func TestAgentName(t *testing.T) { + t.Parallel() + tests := []struct { + name string + spans []Span + want string + }{ + { + "from run fullsend.agent", + []Span{{Name: "run", Attrs: map[string]any{"fullsend.agent": "triage"}}}, + "triage", + }, + { + "from run gen_ai.agent.name", + []Span{{Name: "run", Attrs: map[string]any{"gen_ai.agent.name": "review"}}}, + "review", + }, + { + "from agent span", + []Span{ + {Name: "run", Attrs: map[string]any{}}, + {Name: "agent", Attrs: map[string]any{"gen_ai.agent.name": "code"}}, + }, + "code", + }, + { + "empty when no identity", + []Span{{Name: "run", Attrs: map[string]any{}}}, + "", + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + tr := Trace{Spans: tt.spans} + assert.Equal(t, tt.want, tr.AgentName()) + }) + } +} From 16e82de863eaec092a3754000e732bc36b692ce8 Mon Sep 17 00:00:00 2001 From: Adam Scerra Date: Mon, 10 Aug 2026 09:32:21 -0400 Subject: [PATCH 02/14] refactor(eval): keep measurements tool-agnostic and document ownership Drop the MLflow Assessments adapter and MLFLOW_* workflow wiring so core does not pick a score backend. Clarify engine-vs-manifest ownership, agents@v0 default resolution, and the change guide in ADR 0087 / docs. Signed-off-by: Adam Scerra Co-authored-by: Cursor Signed-off-by: Adam Scerra --- .github/workflows/reusable-code.yml | 15 -- .github/workflows/reusable-fix.yml | 15 -- .github/workflows/reusable-prioritize.yml | 12 -- .github/workflows/reusable-retro.yml | 15 -- .github/workflows/reusable-review.yml | 15 -- .github/workflows/reusable-triage.yml | 16 -- ...050-distributed-tracing-instrumentation.md | 10 +- ...-eval-measurements-online-trace-scoring.md | 85 +++++--- docs/architecture.md | 4 +- .../infrastructure/distributed-tracing.md | 10 +- .../infrastructure/eval-measurements.md | 99 +++++---- internal/cli/evalmeasure.go | 23 +-- internal/cli/evalmeasure_test.go | 5 - internal/evalmeasure/export_mlflow.go | 189 ------------------ internal/evalmeasure/export_mlflow_test.go | 102 ---------- internal/evalmeasure/run.go | 11 +- internal/evalmeasure/run_test.go | 5 - 17 files changed, 138 insertions(+), 493 deletions(-) delete mode 100644 internal/evalmeasure/export_mlflow.go delete mode 100644 internal/evalmeasure/export_mlflow_test.go diff --git a/.github/workflows/reusable-code.yml b/.github/workflows/reusable-code.yml index 5e8d55bf9b..f59cf0beec 100644 --- a/.github/workflows/reusable-code.yml +++ b/.github/workflows/reusable-code.yml @@ -50,8 +50,6 @@ on: required: false OTEL_EXPORTER_OTLP_HEADERS: required: false - MLFLOW_TRACKING_PASSWORD: - required: false concurrency: group: fullsend-code-agent-${{ inputs.source_repo }}-${{ fromJSON(inputs.event_payload).issue.number || fromJSON(inputs.event_payload).pull_request.number }} @@ -199,16 +197,3 @@ jobs: OTEL_EXPORTER_OTLP_CERTIFICATE: ${{ vars.OTEL_EXPORTER_OTLP_CERTIFICATE }} OTEL_RESOURCE_ATTRIBUTES: ${{ vars.OTEL_RESOURCE_ATTRIBUTES }} OTEL_SDK_DISABLED: ${{ vars.OTEL_SDK_DISABLED }} - # Eval measurements → MLflow Assessments (Quality UI). Optional — - # OTEL ingest Bearer cannot write assessments; needs tracking Basic auth. - MLFLOW_TRACKING_URI: ${{ vars.MLFLOW_TRACKING_URI }} - MLFLOW_TRACKING_USERNAME: ${{ vars.MLFLOW_TRACKING_USERNAME }} - MLFLOW_TRACKING_PASSWORD: ${{ secrets.MLFLOW_TRACKING_PASSWORD }} - with: - agent: code - version: ${{ inputs.fullsend_version || job.workflow_sha }} - fullsend-dir: ${{ inputs.install_mode == 'per-repo' && '.fullsend' || '' }} - run-url: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} - status-repo: ${{ inputs.source_repo }} - status-number: ${{ fromJSON(inputs.event_payload).issue.number }} - mint-url: ${{ inputs.mint_url }} diff --git a/.github/workflows/reusable-fix.yml b/.github/workflows/reusable-fix.yml index 0eeb41b0d9..82cb724e47 100644 --- a/.github/workflows/reusable-fix.yml +++ b/.github/workflows/reusable-fix.yml @@ -62,8 +62,6 @@ on: required: false OTEL_EXPORTER_OTLP_HEADERS: required: false - MLFLOW_TRACKING_PASSWORD: - required: false concurrency: group: >- @@ -396,16 +394,3 @@ jobs: OTEL_EXPORTER_OTLP_CERTIFICATE: ${{ vars.OTEL_EXPORTER_OTLP_CERTIFICATE }} OTEL_RESOURCE_ATTRIBUTES: ${{ vars.OTEL_RESOURCE_ATTRIBUTES }} OTEL_SDK_DISABLED: ${{ vars.OTEL_SDK_DISABLED }} - # Eval measurements → MLflow Assessments (Quality UI). Optional — - # OTEL ingest Bearer cannot write assessments; needs tracking Basic auth. - MLFLOW_TRACKING_URI: ${{ vars.MLFLOW_TRACKING_URI }} - MLFLOW_TRACKING_USERNAME: ${{ vars.MLFLOW_TRACKING_USERNAME }} - MLFLOW_TRACKING_PASSWORD: ${{ secrets.MLFLOW_TRACKING_PASSWORD }} - with: - agent: fix - version: ${{ inputs.fullsend_version || job.workflow_sha }} - fullsend-dir: ${{ inputs.install_mode == 'per-repo' && '.fullsend' || '' }} - run-url: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} - status-repo: ${{ inputs.source_repo }} - status-number: ${{ steps.context.outputs.pr_number }} - mint-url: ${{ inputs.mint_url }} diff --git a/.github/workflows/reusable-prioritize.yml b/.github/workflows/reusable-prioritize.yml index 612bb5f096..4ae4a65759 100644 --- a/.github/workflows/reusable-prioritize.yml +++ b/.github/workflows/reusable-prioritize.yml @@ -54,8 +54,6 @@ on: required: false OTEL_EXPORTER_OTLP_HEADERS: required: false - MLFLOW_TRACKING_PASSWORD: - required: false concurrency: group: fullsend-prioritize-agent-${{ inputs.source_repo }}-${{ fromJSON(inputs.event_payload).issue.number || fromJSON(inputs.event_payload).pull_request.number }} @@ -160,13 +158,3 @@ jobs: OTEL_EXPORTER_OTLP_CERTIFICATE: ${{ vars.OTEL_EXPORTER_OTLP_CERTIFICATE }} OTEL_RESOURCE_ATTRIBUTES: ${{ vars.OTEL_RESOURCE_ATTRIBUTES }} OTEL_SDK_DISABLED: ${{ vars.OTEL_SDK_DISABLED }} - # Eval measurements → MLflow Assessments (Quality UI). Optional — - # OTEL ingest Bearer cannot write assessments; needs tracking Basic auth. - MLFLOW_TRACKING_URI: ${{ vars.MLFLOW_TRACKING_URI }} - MLFLOW_TRACKING_USERNAME: ${{ vars.MLFLOW_TRACKING_USERNAME }} - MLFLOW_TRACKING_PASSWORD: ${{ secrets.MLFLOW_TRACKING_PASSWORD }} - with: - agent: prioritize - version: ${{ inputs.fullsend_version || job.workflow_sha }} - fullsend-dir: ${{ inputs.install_mode == 'per-repo' && '.fullsend' || '' }} - mint-url: ${{ inputs.mint_url }} diff --git a/.github/workflows/reusable-retro.yml b/.github/workflows/reusable-retro.yml index 384c55c674..d628e6e708 100644 --- a/.github/workflows/reusable-retro.yml +++ b/.github/workflows/reusable-retro.yml @@ -50,8 +50,6 @@ on: required: false OTEL_EXPORTER_OTLP_HEADERS: required: false - MLFLOW_TRACKING_PASSWORD: - required: false concurrency: group: fullsend-retro-agent-${{ inputs.source_repo }}-${{ fromJSON(inputs.event_payload).pull_request.number || fromJSON(inputs.event_payload).issue.number }} @@ -171,16 +169,3 @@ jobs: OTEL_EXPORTER_OTLP_CERTIFICATE: ${{ vars.OTEL_EXPORTER_OTLP_CERTIFICATE }} OTEL_RESOURCE_ATTRIBUTES: ${{ vars.OTEL_RESOURCE_ATTRIBUTES }} OTEL_SDK_DISABLED: ${{ vars.OTEL_SDK_DISABLED }} - # Eval measurements → MLflow Assessments (Quality UI). Optional — - # OTEL ingest Bearer cannot write assessments; needs tracking Basic auth. - MLFLOW_TRACKING_URI: ${{ vars.MLFLOW_TRACKING_URI }} - MLFLOW_TRACKING_USERNAME: ${{ vars.MLFLOW_TRACKING_USERNAME }} - MLFLOW_TRACKING_PASSWORD: ${{ secrets.MLFLOW_TRACKING_PASSWORD }} - with: - agent: retro - version: ${{ inputs.fullsend_version || job.workflow_sha }} - fullsend-dir: ${{ inputs.install_mode == 'per-repo' && '.fullsend' || '' }} - run-url: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} - status-repo: ${{ inputs.source_repo }} - status-number: ${{ fromJSON(inputs.event_payload).pull_request.number || fromJSON(inputs.event_payload).issue.number }} - mint-url: ${{ inputs.mint_url }} diff --git a/.github/workflows/reusable-review.yml b/.github/workflows/reusable-review.yml index 390cd705a9..7025add6a0 100644 --- a/.github/workflows/reusable-review.yml +++ b/.github/workflows/reusable-review.yml @@ -50,8 +50,6 @@ on: required: false OTEL_EXPORTER_OTLP_HEADERS: required: false - MLFLOW_TRACKING_PASSWORD: - required: false concurrency: group: fullsend-review-agent-${{ inputs.source_repo }}-${{ fromJSON(inputs.event_payload).pull_request.number || fromJSON(inputs.event_payload).issue.number }} @@ -187,16 +185,3 @@ jobs: OTEL_EXPORTER_OTLP_CERTIFICATE: ${{ vars.OTEL_EXPORTER_OTLP_CERTIFICATE }} OTEL_RESOURCE_ATTRIBUTES: ${{ vars.OTEL_RESOURCE_ATTRIBUTES }} OTEL_SDK_DISABLED: ${{ vars.OTEL_SDK_DISABLED }} - # Eval measurements → MLflow Assessments (Quality UI). Optional — - # OTEL ingest Bearer cannot write assessments; needs tracking Basic auth. - MLFLOW_TRACKING_URI: ${{ vars.MLFLOW_TRACKING_URI }} - MLFLOW_TRACKING_USERNAME: ${{ vars.MLFLOW_TRACKING_USERNAME }} - MLFLOW_TRACKING_PASSWORD: ${{ secrets.MLFLOW_TRACKING_PASSWORD }} - with: - agent: review - fullsend-dir: ${{ inputs.install_mode == 'per-repo' && '.fullsend' || '' }} - version: ${{ inputs.fullsend_version || job.workflow_sha }} - run-url: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} - status-repo: ${{ inputs.source_repo }} - status-number: ${{ fromJSON(inputs.event_payload).pull_request.number || fromJSON(inputs.event_payload).issue.number }} - mint-url: ${{ inputs.mint_url }} diff --git a/.github/workflows/reusable-triage.yml b/.github/workflows/reusable-triage.yml index c388030fd3..4cb1f931e1 100644 --- a/.github/workflows/reusable-triage.yml +++ b/.github/workflows/reusable-triage.yml @@ -55,8 +55,6 @@ on: required: false OTEL_EXPORTER_OTLP_HEADERS: required: false - MLFLOW_TRACKING_PASSWORD: - required: false concurrency: group: fullsend-triage-agent-${{ inputs.source_repo }}-${{ fromJSON(inputs.event_payload).issue.number || fromJSON(inputs.event_payload).pull_request.number }} @@ -177,17 +175,3 @@ jobs: OTEL_EXPORTER_OTLP_CERTIFICATE: ${{ vars.OTEL_EXPORTER_OTLP_CERTIFICATE }} OTEL_RESOURCE_ATTRIBUTES: ${{ vars.OTEL_RESOURCE_ATTRIBUTES }} OTEL_SDK_DISABLED: ${{ vars.OTEL_SDK_DISABLED }} - # Eval measurements → MLflow Assessments (Quality UI). Optional — - # OTEL ingest Bearer cannot write assessments; needs tracking Basic - # auth. URI defaults from OTEL endpoint when unset. - MLFLOW_TRACKING_URI: ${{ vars.MLFLOW_TRACKING_URI }} - MLFLOW_TRACKING_USERNAME: ${{ vars.MLFLOW_TRACKING_USERNAME }} - MLFLOW_TRACKING_PASSWORD: ${{ secrets.MLFLOW_TRACKING_PASSWORD }} - with: - agent: triage - version: ${{ inputs.fullsend_version || job.workflow_sha }} - fullsend-dir: ${{ inputs.install_mode == 'per-repo' && '.fullsend' || '' }} - run-url: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} - status-repo: ${{ inputs.source_repo }} - status-number: ${{ fromJSON(inputs.event_payload).issue.number }} - mint-url: ${{ inputs.mint_url }} diff --git a/docs/ADRs/0050-distributed-tracing-instrumentation.md b/docs/ADRs/0050-distributed-tracing-instrumentation.md index d877e0c622..ea09007eda 100644 --- a/docs/ADRs/0050-distributed-tracing-instrumentation.md +++ b/docs/ADRs/0050-distributed-tracing-instrumentation.md @@ -157,8 +157,8 @@ span export via the OTel SDK's batch processor. The core decision (three-level opt-in, OTel-native, W3C propagation) is unchanged. **2026-08-10 — Eval measurements ([ADR 0087](0087-eval-measurements-online-trace-scoring.md)):** -online scoring of wild-run traces writes `eval-measurements.jsonl` beside -telemetry; portable remote score export is designed to follow the same OTLP -configuration as this ADR (MLflow Assessments adapter available now; OTLP -path planned). Backend-specific UI adapters remain optional. Distinct from -functional eval fixtures ([ADR 0051](0051-agent-eval-harness-for-test-infrastructure.md)). +online scoring of wild-run traces always writes `eval-measurements.jsonl` +beside telemetry (tool-agnostic). Portable remote score export follows the +same OTLP configuration as this ADR when implemented — no vendor score +adapters in core. Distinct from functional eval fixtures +([ADR 0051](0051-agent-eval-harness-for-test-infrastructure.md)). diff --git a/docs/ADRs/0087-eval-measurements-online-trace-scoring.md b/docs/ADRs/0087-eval-measurements-online-trace-scoring.md index 77cca96a82..0951265a22 100644 --- a/docs/ADRs/0087-eval-measurements-online-trace-scoring.md +++ b/docs/ADRs/0087-eval-measurements-online-trace-scoring.md @@ -18,8 +18,6 @@ Date: 2026-08-10 Accepted -Visual companion: [explainer diagrams](0087-eval-measurements-explainer.html). - ## Context Agent runs already emit OpenTelemetry traces as `run-telemetry.jsonl`, with @@ -31,18 +29,17 @@ optional live OTLP export when `OTEL_EXPORTER_OTLP_*` is set not score wild production runs. Operators also need an **online / trend** layer on wild traces (completeness -first; quality signals later) without a second, product-specific export stack. -Dogfood showed MLflow Assessments improve one backend's Quality UI, but -requiring `MLFLOW_TRACKING_*` breaks ADR 0050's portability promise for scores. +first; quality signals later). Fullsend must stay **backend-agnostic**: orgs +already choose Phoenix, MLflow, Jaeger, or another OTLP collector for traces. +Baking a single product’s Assessments/Quality API into the core CLI or managed +workflows would force a tool decision on every install. Adjacent telemetry proposals (not competing with this score path): - **Level 3 content capture** ([#5947](https://github.com/fullsend-ai/fullsend/pull/5947) — proposed ADRs 0084/0085): richer span content enables later scorers; - v1 reads Level 1/2 metadata in `run-telemetry.jsonl`. Content is OTLP-only - under that proposal, so content-aware scorers will need OTLP/backend access - or a widened local contract. Measure CLI is host-side after sandbox exit - (outside the sandbox OTEL denylist surface). + first ship reads Level 1/2 metadata in `run-telemetry.jsonl`. Measure CLI is + host-side after sandbox exit. - **Span status from run outcome** ([#5944](https://github.com/fullsend-ai/fullsend/pull/5944)): OTLP Status (and `fullsend.transcript_error`) become the reliable success/failure signal. EM-001 only checks that `exit_code` is **present** @@ -53,12 +50,13 @@ Adjacent telemetry proposals (not competing with this score path): ## Options -1. **Local JSONL only** — simplest; no remote scores on any backend. -2. **Backend-native APIs only** (e.g. MLflow Assessments) — good UX on one - host; every new backend needs a new adapter and secrets. -3. **Same OTLP path as agent traces, plus optional adapters** — scores travel - with the same endpoint/headers orgs already configure; adapters are - opt-in for product UIs. +1. **Local JSONL only** — portable offline artifact; no remote scores from + fullsend itself. +2. **Backend-native APIs in core** (e.g. one vendor’s Assessments API) — + couples every managed workflow to that product’s auth and schema. +3. **Local JSONL + same OTLP path as agent traces for remote** — scores travel + with the endpoint/headers orgs already configure for ADR 0050; no second + vendor stack in core. ## Decision @@ -70,22 +68,47 @@ Introduce **eval measurements**: deterministic scorers that read Measurements never block delivery. Prefer the glossary terms *eval measurement* (score) and *eval scenario* (fixture). -**Ownership:** `fullsend` owns parser, scorers, CLI, and job wiring. -`fullsend-ai/agents` owns per-agent **measurement manifests** at -`eval/measurements/.yaml` (which scorers to enable). +**Ownership (engine vs default policy):** + +| Concern | Repo | +|---|---| +| Parser, scorer **implementations**, CLI, GHA post-step | `fullsend-ai/fullsend` | +| Default measurement manifests for stock agents | `fullsend-ai/agents` (`eval/measurements/.yaml`) | +| Org/repo overrides and BYOA agent manifests | Consumer repo (`FULLSEND_DIR`) | + +Stock agents ship defaults the same way as other agent content: managed jobs +resolve local `${FULLSEND_DIR}/eval/measurements/${AGENT}.yaml` if present, +else fetch `fullsend-ai/agents@v0/...`. Users who only use stock agents do +**not** copy manifests into their repos. Local files are for override, opt-out, +or custom agents — not for receiving defaults. + +Executable scorer logic stays in fullsend because `fullsend eval-measure` is +the released binary that reads `run-telemetry.jsonl` (which fullsend writes). +Agents is content (YAML/prompts), not a library linked into that binary. +EM-001 (`trace_fitness`) is a platform fitness check on that telemetry +contract, so its Go implementation belongs in fullsend even though every +stock agent enables it via agents manifests. + +**Future — logic-as-config:** agent-specific *policy* (thresholds, attribute +checks) should move into declarative manifest fields evaluated by a generic +engine in fullsend. Until that lands, new imperative scorers are Go in +fullsend; wiring a default for a stock agent is an agents manifest change. +New assert primitive → fullsend PR; new id/thresholds on existing primitives → +agents-only (or consumer-repo override). **Persistence (always):** write `eval-measurements.jsonl` (and a small ledger) -next to `run-telemetry.jsonl`. Scores are data, never gates. +next to `run-telemetry.jsonl`. This JSONL is the portable, tool-agnostic +contract — backends and dashboards are chosen by the org, not by fullsend. -**Remote export:** the portable path will use the same `OTEL_EXPORTER_OTLP_*` -as ADR 0050 (not yet implemented). Backend-specific adapters (e.g. MLflow -Assessments) are **optional** and available now. Adapters must not be required -for scores to leave the runner once the OTLP path lands. +**Remote export:** use the same `OTEL_EXPORTER_OTLP_*` configuration as +ADR 0050 when score export is implemented. Do **not** ship vendor-specific +score adapters or `MLFLOW_*` (or similar) wiring in core action/workflows. +Orgs that want a product UI can consume the local JSONL or an OTLP backend +outside fullsend. **First scorer:** `trace_fitness` (catalog id `em-001`) — span-tree / attribute fitness so later scorers can trust the trace. Further scorers -(budget, outcome/status, content-aware) land via manifests without changing -this export model. +land via manifests without changing this export model. ### Versioning (per measurement, not platform “v1”) @@ -105,14 +128,12 @@ Entirely new signal → new `em-NNN` (and usually a new `scorer` string). ## Consequences -- Wild runs produce a reviewable score file beside telemetry with or without a - remote backend. -- Orgs that already set OTEL for traces get a portable score export contract - without a second auth scheme; optional adapters may enrich one product UI. +- Every measured run produces a reviewable, backend-agnostic score file beside + telemetry. +- Core stays tool-agnostic: no product-specific score env vars in managed + workflows; remote scores follow OTEL when that path lands. - Missing manifests skip cleanly; measure failure never fails the agent job. -- Functional scenarios (gate) and eval measurements (trend) stay separate; - retro/observer lesson extraction may still promote production cases into - fixtures without replacing online scoring. +- Functional scenarios (gate) and eval measurements (trend) stay separate. - Retro can recommend a **manifest scorer** or a **scenario fixture** — not substitutes. - Richer telemetry (Level 3 / Status fixes) expands what scorers *can* assert; diff --git a/docs/architecture.md b/docs/architecture.md index f27d39c36f..885215f238 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -287,13 +287,13 @@ Observability is a cross-cutting concern that touches every other component. Eac - JSONL reasoning trace exposure: raw JSONL conversation transcripts are extracted from sandboxes and stored with owner-scoped access. Credential scanning acts as an invariant check on [ADR 0017](ADRs/0017-credential-isolation-for-sandboxed-agents.md)'s isolation model. Agents handling data from protected sources beyond the target repo can opt in to JSONL suppression via configuration ([ADR 0021](ADRs/0021-jsonl-reasoning-trace-exposure.md)). - Event-driven stage dispatch remains traceable end-to-end in the GitHub Actions UI by using synchronous `workflow_call` dispatch (see [ADR 0041](ADRs/0041-synchronous-workflow-call-event-dispatch.md)). - Distributed tracing: framework-native OpenTelemetry instrumentation with zero-configuration baseline. Every run produces `run-telemetry.jsonl` locally; optional live OTLP export to any compatible backend. W3C trace context propagation links multi-agent pipelines into unified traces. OTEL GenAI semantic conventions enable LLM-aware backends ([ADR 0050](ADRs/0050-distributed-tracing-instrumentation.md)). -- Eval measurements: fail-open same-job scoring of wild-run traces into `eval-measurements.jsonl` beside telemetry; portable remote score export is designed to use the same OTLP configuration as agent traces (MLflow Assessments adapter available now; OTLP path planned); backend-specific UI adapters are optional ([ADR 0087](ADRs/0087-eval-measurements-online-trace-scoring.md)). See [Eval Measurements](guides/infrastructure/eval-measurements.md). +- Eval measurements: fail-open same-job scoring of wild-run traces into `eval-measurements.jsonl` beside telemetry; portable remote score export uses the same OTLP configuration as agent traces (planned); scores always land in local eval-measurements.jsonl (tool-agnostic) ([ADR 0087](ADRs/0087-eval-measurements-online-trace-scoring.md)). See [Eval Measurements](guides/infrastructure/eval-measurements.md). **Open questions:** - What signals matter most — cost, latency, token usage, action logs, decision traces, or something else? - ~~How do we balance detailed tracing (useful for debugging) with the volume of data agents will produce?~~ Decided in [ADR 0050](ADRs/0050-distributed-tracing-instrumentation.md): instrument all lifecycle steps comprehensively; volume is managed by backends not by suppressing data at the source. -- ~~How do we score wild agent traces for trends without a second export stack?~~ Decided in [ADR 0087](ADRs/0087-eval-measurements-online-trace-scoring.md): eval measurements write local JSONL beside telemetry; portable remote export is designed to use the same OTLP config as traces (OTLP path planned; MLflow adapter available now); product UI adapters are optional. +- ~~How do we score wild agent traces for trends without a second export stack?~~ Decided in [ADR 0087](ADRs/0087-eval-measurements-online-trace-scoring.md): eval measurements write local JSONL beside telemetry; portable remote export uses the same OTLP config as traces (planned); local eval-measurements.jsonl is always written. - What is the retention and access model for agent logs? Who can see what? (JSONL trace access model decided in [ADR 0021](ADRs/0021-jsonl-reasoning-trace-exposure.md); retention policy and broader log access remain open.) - How does observability interact with the security requirement that "every action is logged, attributable, and reviewable"? (See [security-threat-model.md](problems/security-threat-model.md).) - Is there a real-time monitoring requirement (agent is stuck, agent is behaving anomalously), or is observability primarily forensic? diff --git a/docs/guides/infrastructure/distributed-tracing.md b/docs/guides/infrastructure/distributed-tracing.md index 3c674dcca5..005d874cd5 100644 --- a/docs/guides/infrastructure/distributed-tracing.md +++ b/docs/guides/infrastructure/distributed-tracing.md @@ -248,11 +248,11 @@ authentication mechanism. After each managed agent run, `fullsend eval-measure` scores `run-telemetry.jsonl` in the same job (fail-open). Scores always land in -`eval-measurements.jsonl` beside telemetry. Portable remote export uses the -same `OTEL_EXPORTER_OTLP_*` configuration as agent traces; product UI -adapters (e.g. MLflow Assessments) are optional. Measurements read the -Level 1/2 metadata contract of `run-telemetry.jsonl` (not Level 3 content). -See [Eval Measurements](./eval-measurements.md) and +`eval-measurements.jsonl` beside telemetry (tool-agnostic artifact). Portable +remote export will reuse the same `OTEL_EXPORTER_OTLP_*` configuration as +agent traces when implemented. Measurements read the Level 1/2 metadata +contract of `run-telemetry.jsonl` (not Level 3 content). See +[Eval Measurements](./eval-measurements.md) and [ADR 0087](../../ADRs/0087-eval-measurements-online-trace-scoring.md). ## See also diff --git a/docs/guides/infrastructure/eval-measurements.md b/docs/guides/infrastructure/eval-measurements.md index a54f14831a..ceaf2ea30d 100644 --- a/docs/guides/infrastructure/eval-measurements.md +++ b/docs/guides/infrastructure/eval-measurements.md @@ -10,9 +10,9 @@ Telemetry baseline: [Distributed Tracing](./distributed-tracing.md) ## Architecture (read this first) -Every managed agent job that produces telemetry gets the same three layers. -Only the first is mandatory; the others reuse existing OTEL config or are -optional product adapters. +Fullsend does not pick an observability product for scores. The portable +contract is a local JSONL artifact next to telemetry; remote export (when +implemented) reuses the same OTEL configuration as agent traces. ```text fullsend run @@ -23,10 +23,7 @@ fullsend run fullsend eval-measure (same GHA job, fail-open, after run) └─ always writes output/**/eval-measurements.jsonl (+ eval-measure-ledger.jsonl for idempotency) - └─ portable remote: same OTEL_EXPORTER_OTLP_* as the agent run - (scores follow traces — no second required endpoint) - └─ optional adapter: MLflow Assessments when MLFLOW_TRACKING_* is set - (Quality / Assessments UI only — not required for portability) + └─ portable remote (planned): same OTEL_EXPORTER_OTLP_* as the agent run ``` | Artifact | When | Purpose | @@ -34,12 +31,11 @@ fullsend eval-measure (same GHA job, fail-open, after run) | `run-telemetry.jsonl` | Every run | OTLP JSON TracesData lines (local source of truth for spans) | | `eval-measurements.jsonl` | Every measured run | One JSON object per score (`name`, `label`, `value`, `explanation`, `trace_id`, …) | | Remote agent spans | OTEL configured | Same spans the local file holds | -| Remote scores (portable) | OTEL configured | Scores on the OTLP path — works with any OTLP backend | -| MLflow Assessments | `MLFLOW_TRACKING_PASSWORD` (+ URI) set | Extra UX on MLflow only | +| Remote scores (planned) | OTEL configured | Scores on the OTLP path — any OTLP backend | -**Do not** treat MLflow Assessments as the primary export. Telemetry already -goes “anywhere OTLP points”; scores must follow that model. Assessments are an -optional adapter for one backend’s Quality panel. +Orgs choose Phoenix, MLflow, Jaeger, or another collector independently. +Fullsend does not forward vendor-specific score credentials in managed +workflows. ## Measurements vs functional evals @@ -54,13 +50,36 @@ optional adapter for one backend’s Quality panel. | Concern | Repo | |---|---| -| Parser, scorers, CLI, GHA post-step | `fullsend-ai/fullsend` | -| Per-agent **measurement manifests** (which scorers) | `fullsend-ai/agents` | +| Parser, scorer **implementations**, CLI, GHA post-step | `fullsend-ai/fullsend` | +| Default manifests for stock agents (which scorers / ids) | `fullsend-ai/agents` | +| Overrides, opt-out, BYOA agent manifests | Consumer repo (`FULLSEND_DIR`) | + +Defaults for stock agents live next to those agents in `fullsend-ai/agents`. +Managed jobs fetch them from `agents@v0` when no local file exists — users do +**not** duplicate YAML into every install. Put a file under +`${FULLSEND_DIR}/eval/measurements/` only to change policy or score a custom +agent. + +Scorer *code* stays in fullsend: the measure CLI is the released engine that +understands `run-telemetry.jsonl`. Agents ships **policy** (manifests), not Go. +EM-001 (`trace_fitness`) evaluates fullsend’s telemetry contract across agents; +stock agents opt in via manifests here / in `agents`. + +| Change | Where | +|---|---| +| New Go scorer or new declarative `assert:` primitive | `fullsend` PR | +| New measurement `id` / enable / thresholds for a stock agent (existing scorer) | `agents` PR | +| Org-specific policy for stock or custom agents | Local override in the consumer repo | + +**Planned (not in first ship):** declarative checks in the manifest (attribute +exists, ratio/threshold bands) so most agent-specific policy is YAML-only. +Until then, agent-specific math still lands as a named Go scorer in fullsend, +enabled only for the agents that list it. First scorer: **`trace_fitness`** (catalog id `em-001`) — span tree + expected attributes so later scorers can trust the trace. -Manifest shape: +Manifest shape (first ship — enablement only): ```yaml agent: review @@ -70,6 +89,27 @@ measurements: version: 1 ``` +Illustrative **logic-as-config** (future declarative engine — not wired yet): + +```yaml +agent: code +measurements: + - id: em-001 + scorer: trace_fitness + version: 1 + - id: em-010 + scorer: declarative + version: 1 + where: + span: agent + checks: + - name: turn_token_ratio + assert: ratio_lte + numerator: gen_ai.usage.total_tokens + denominator: fullsend.turns + max: 8000 +``` + ### Versioning Not a platform “v1.” Each entry versions its own contract: @@ -89,7 +129,7 @@ success/failure signal for outcome scorers. | Proposal | Relationship to measurements | |---|---| -| [#5947](https://github.com/fullsend-ai/fullsend/pull/5947) Level 3 activation + sandbox OTEL denylist | Richer traces fuel later scorers. v1 reads Level 1/2 local JSONL; content-aware scorers need OTLP/backend (or a widened file contract) under the proposed L3 rules. Measure CLI is host-side after the sandbox exits. | +| [#5947](https://github.com/fullsend-ai/fullsend/pull/5947) Level 3 activation + sandbox OTEL denylist | Richer traces fuel later scorers. First ship reads Level 1/2 local JSONL; content-aware scorers need OTLP/backend (or a widened file contract) under the proposed L3 rules. Measure CLI is host-side after the sandbox exits. | | [#5944](https://github.com/fullsend-ai/fullsend/pull/5944) Span status from run outcome | Unblocks outcome scorers keyed on Status, not raw exit alone. | | [#2423](https://github.com/fullsend-ai/fullsend/pull/2423) Semantic observability / observer / lessons | Observer + lessons → fixtures; measurements are the online score path. | @@ -109,6 +149,9 @@ GitHub Actions job 1. `${FULLSEND_DIR}/eval/measurements/${AGENT}.yaml` if present, else 2. `https://raw.githubusercontent.com/fullsend-ai/agents/v0/eval/measurements/${AGENT}.yaml` +Step 2 is how stock-agent defaults reach every install (same pin style as +other agents-repo fallbacks). Step 1 is override / BYOA only. + Missing manifest → log and exit `0` (skip). The CLI flag remains `--registry` for now (path to the YAML); rename is cosmetic follow-up. @@ -124,26 +167,10 @@ fullsend eval-measure \ - `--registry` is required (manifests live in `fullsend-ai/agents`, not in the fullsend binary). - Exit `0` when a score is `fail` — scores are data. -- Export failures warn and still exit `0`; local JSONL is kept. - -### Env for remote export - -| Variable | Role | -|---|---| -| `OTEL_EXPORTER_OTLP_TRACES_ENDPOINT` / `_HEADERS` / … | **Portable** score export (same as agent traces) | -| `MLFLOW_TRACKING_URI` | Optional; derived from OTEL endpoint if unset | -| `MLFLOW_TRACKING_USERNAME` | Optional Assessments adapter (default `admin`) | -| `MLFLOW_TRACKING_PASSWORD` | Optional Assessments adapter (Basic auth) | - -Reusable agent workflows forward OTEL and optional `MLFLOW_TRACKING_*`. -The OTEL ingest Bearer token can write `/v1/traces` but cannot create MLflow -Assessments — that adapter needs tracking Basic auth when used. ## Implementation note -Local `eval-measurements.jsonl` and the optional MLflow Assessments adapter -are wired in the measure CLI today. Portable OTLP score export (same -`OTEL_*` as traces) is the ADR 0087 remote contract — keep adapters -optional; do not require `MLFLOW_TRACKING_*` for scores to be useful -outside one vendor UI. Until portable OTLP score export lands, remote -scores need the optional Assessments adapter (or local JSONL alone). +Today the measure CLI always writes local `eval-measurements.jsonl`. Portable +OTLP score export (same `OTEL_*` as traces) is the ADR 0087 remote contract +and is not wired yet — until it lands, consume the JSONL artifact (or your +own pipeline) for remote dashboards. diff --git a/internal/cli/evalmeasure.go b/internal/cli/evalmeasure.go index 71ea1523ae..9731a2bc2b 100644 --- a/internal/cli/evalmeasure.go +++ b/internal/cli/evalmeasure.go @@ -1,7 +1,6 @@ package cli import ( - "errors" "fmt" "os" @@ -22,16 +21,14 @@ func newEvalMeasureCmd() *cobra.Command { Use: "eval-measure", Short: "Score agent run traces with eval measurements", Long: `Parse run-telemetry.jsonl, score with an agents measurement manifest, -write eval-measurements.jsonl, and best-effort export scores to MLflow -Assessments (Quality / per-trace Assessments panel). +and write eval-measurements.jsonl beside the telemetry artifact. -MLflow export uses MLFLOW_TRACKING_URI (or derives it from -OTEL_EXPORTER_OTLP_TRACES_ENDPOINT) plus MLFLOW_TRACKING_USERNAME/ -MLFLOW_TRACKING_PASSWORD. The OTEL ingest Bearer token is not sufficient -for the Assessments API. +Remote backends are not selected by fullsend: scores are a portable local +JSONL artifact. When portable OTLP score export lands, it will reuse the +same OTEL_EXPORTER_OTLP_* configuration as agent traces (ADR 0050 / 0087). Exit 0 when scores fail — measurements are data, not gates. Non-zero only -on hard IO/parse errors. Export failures print a warning and exit 0. +on hard IO/parse errors. The --registry path is required (manifests live in fullsend-ai/agents, e.g. eval/measurements/.yaml).`, @@ -41,11 +38,6 @@ e.g. eval/measurements/.yaml).`, results, err := evalmeasure.MeasureAndExport(cmd.Context(), telemetryPath, registryPath, outDir) if err != nil { - if isExportWarnErr(err) { - printer.StepWarn(fmt.Sprintf("Export failed (local measurements kept): %v", err)) - printResults(printer, results) - return nil - } return err } printResults(printer, results) @@ -61,11 +53,6 @@ e.g. eval/measurements/.yaml).`, return cmd } -func isExportWarnErr(err error) bool { - var exportErr *evalmeasure.ExportError - return errors.As(err, &exportErr) -} - func printResults(printer *ui.Printer, results []evalmeasure.EvaluationResult) { if len(results) == 0 { printer.StepDone("No new measurements (already scored or no matching traces)") diff --git a/internal/cli/evalmeasure_test.go b/internal/cli/evalmeasure_test.go index 8cbdeffc01..5019a2a647 100644 --- a/internal/cli/evalmeasure_test.go +++ b/internal/cli/evalmeasure_test.go @@ -11,11 +11,6 @@ import ( ) func TestEvalMeasureCmd_ScoresFixture(t *testing.T) { - t.Setenv("OTEL_EXPORTER_OTLP_TRACES_ENDPOINT", "") - t.Setenv("OTEL_EXPORTER_OTLP_ENDPOINT", "") - t.Setenv("MLFLOW_TRACKING_URI", "") - t.Setenv("MLFLOW_TRACKING_PASSWORD", "") - out := t.TempDir() telemetry := filepath.Join("..", "evalmeasure", "testdata", "complete.jsonl") registry := filepath.Join("..", "evalmeasure", "testdata", "sample-registry.yaml") diff --git a/internal/evalmeasure/export_mlflow.go b/internal/evalmeasure/export_mlflow.go deleted file mode 100644 index 97147e1378..0000000000 --- a/internal/evalmeasure/export_mlflow.go +++ /dev/null @@ -1,189 +0,0 @@ -package evalmeasure - -import ( - "bytes" - "context" - "encoding/json" - "fmt" - "io" - "net/http" - "net/url" - "os" - "strings" - "time" -) - -// ExportError wraps errors from remote score export (MLflow Assessments). -// The CLI uses errors.As to classify these as warnings (fail-open). -type ExportError struct{ Err error } - -func (e *ExportError) Error() string { return e.Err.Error() } -func (e *ExportError) Unwrap() error { return e.Err } - -// assessmentHTTPDoer is a seam for tests. Not safe for t.Parallel() — -// tests that swap this must not run concurrently. -var assessmentHTTPDo = func(req *http.Request) (*http.Response, error) { - client := &http.Client{Timeout: 15 * time.Second} - return client.Do(req) -} - -type mlflowAssessmentRequest struct { - Assessment mlflowAssessment `json:"assessment"` -} - -type mlflowAssessment struct { - AssessmentName string `json:"assessment_name"` - TraceID string `json:"trace_id"` - Source mlflowSource `json:"source"` - Feedback mlflowFeedback `json:"feedback"` - Rationale string `json:"rationale,omitempty"` - Metadata map[string]string `json:"metadata,omitempty"` - SpanID string `json:"span_id,omitempty"` -} - -type mlflowSource struct { - SourceType string `json:"source_type"` - SourceID string `json:"source_id"` -} - -type mlflowFeedback struct { - Value any `json:"value"` -} - -// ExportMLflowAssessments attaches measurement scores to existing MLflow -// traces via the Assessments REST API (same surface as mlflow.log_feedback). -// -// No-op when MLFLOW_TRACKING_URI is unset and cannot be derived from the OTEL -// traces endpoint, or when tracking basic auth is unset. -// -// Auth: MLFLOW_TRACKING_USERNAME (default "admin") + MLFLOW_TRACKING_PASSWORD. -// The OTEL ingest Bearer token is not accepted by this API on the dogfood host. -func ExportMLflowAssessments(ctx context.Context, results []EvaluationResult) error { - if len(results) == 0 { - return nil - } - user, pass, hasAuth := trackingBasicAuthFromEnv() - explicit := strings.TrimSpace(os.Getenv("MLFLOW_TRACKING_URI")) != "" - base, err := trackingURIFromEnv() - if err != nil { - return err - } - if base == "" { - return nil - } - if !hasAuth { - if explicit { - return &ExportError{Err: fmt.Errorf("MLFLOW_TRACKING_URI is set but MLFLOW_TRACKING_PASSWORD is empty")} - } - return nil - } - - for _, r := range results { - if err := postAssessment(ctx, base, user, pass, r); err != nil { - return &ExportError{Err: err} - } - } - return nil -} - -func postAssessment(ctx context.Context, base, user, pass string, r EvaluationResult) error { - tid := mlflowTraceID(r.TraceID) - payload := mlflowAssessmentRequest{ - Assessment: mlflowAssessment{ - AssessmentName: r.Name, - TraceID: tid, - Source: mlflowSource{ - SourceType: "CODE", - SourceID: "fullsend-evalmeasure/" + r.Version, - }, - Feedback: mlflowFeedback{ - // Label is what Quality / Assessments panels display for pass/fail scorers. - Value: r.Label, - }, - Rationale: r.Explanation, - Metadata: map[string]string{ - "fullsend.measurement.version": r.Version, - "fullsend.evaluation.score.value": fmt.Sprintf("%g", r.Value), - "fullsend.evaluation.score.label": r.Label, - "gen_ai.agent.name": r.Agent, - "fullsend.work_item_id": r.WorkItemID, - "fullsend.source_trace_id": r.TraceID, - // Full observed-vs-limit text also lives in rationale; keep a - // copy in metadata for API consumers / UI detail panes. - "fullsend.evaluation.explanation": r.Explanation, - }, - SpanID: r.SpanID, - }, - } - body, err := json.Marshal(payload) - if err != nil { - return err - } - endpoint := strings.TrimRight(base, "/") + "/ajax-api/3.0/mlflow/traces/" + url.PathEscape(tid) + "/assessments" - req, err := http.NewRequestWithContext(ctx, http.MethodPost, endpoint, bytes.NewReader(body)) - if err != nil { - return err - } - req.Header.Set("Content-Type", "application/json") - req.SetBasicAuth(user, pass) - - resp, err := assessmentHTTPDo(req) - if err != nil { - return fmt.Errorf("post assessment %s: %w", r.Name, err) - } - defer resp.Body.Close() - respBody, _ := io.ReadAll(io.LimitReader(resp.Body, 4096)) - if resp.StatusCode < 200 || resp.StatusCode >= 300 { - return fmt.Errorf("post assessment %s: HTTP %d: %s", r.Name, resp.StatusCode, strings.TrimSpace(string(respBody))) - } - return nil -} - -func mlflowTraceID(hexID string) string { - hexID = strings.TrimSpace(hexID) - if hexID == "" { - return hexID - } - if strings.HasPrefix(hexID, "tr-") { - return hexID - } - return "tr-" + hexID -} - -func trackingURIFromEnv() (string, error) { - if v := strings.TrimSpace(os.Getenv("MLFLOW_TRACKING_URI")); v != "" { - return strings.TrimRight(v, "/"), nil - } - // Derive from OTEL traces endpoint: https://host/v1/traces → https://host - ep := strings.TrimSpace(os.Getenv("OTEL_EXPORTER_OTLP_TRACES_ENDPOINT")) - if ep == "" { - ep = strings.TrimSpace(os.Getenv("OTEL_EXPORTER_OTLP_ENDPOINT")) - } - if ep == "" { - return "", nil - } - u, err := url.Parse(ep) - if err != nil { - return "", fmt.Errorf("derive MLFLOW_TRACKING_URI from OTEL endpoint %q: %w", ep, err) - } - if u.Scheme == "" || u.Host == "" { - return "", fmt.Errorf("derive MLFLOW_TRACKING_URI from OTEL endpoint %q: missing scheme or host", ep) - } - u.Path = "" - u.RawQuery = "" - u.Fragment = "" - u.User = nil - return strings.TrimRight(u.String(), "/"), nil -} - -func trackingBasicAuthFromEnv() (user, pass string, ok bool) { - pass = strings.TrimSpace(os.Getenv("MLFLOW_TRACKING_PASSWORD")) - if pass == "" { - return "", "", false - } - user = strings.TrimSpace(os.Getenv("MLFLOW_TRACKING_USERNAME")) - if user == "" { - user = "admin" - } - return user, pass, true -} diff --git a/internal/evalmeasure/export_mlflow_test.go b/internal/evalmeasure/export_mlflow_test.go deleted file mode 100644 index d1c5ff2ee1..0000000000 --- a/internal/evalmeasure/export_mlflow_test.go +++ /dev/null @@ -1,102 +0,0 @@ -package evalmeasure - -import ( - "io" - "net/http" - "strings" - "testing" - - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" -) - -func TestExportMLflowAssessments_NoopWithoutURI(t *testing.T) { - t.Setenv("MLFLOW_TRACKING_URI", "") - t.Setenv("OTEL_EXPORTER_OTLP_TRACES_ENDPOINT", "") - t.Setenv("OTEL_EXPORTER_OTLP_ENDPOINT", "") - t.Setenv("MLFLOW_TRACKING_PASSWORD", "") - err := ExportMLflowAssessments(t.Context(), []EvaluationResult{{ - Name: "trace_fitness", TraceID: "aa", Label: "pass", Value: 1, - }}) - assert.NoError(t, err) -} - -func TestExportMLflowAssessments_PostsPassLabel(t *testing.T) { - t.Setenv("MLFLOW_TRACKING_URI", "https://mlflow.example") - t.Setenv("MLFLOW_TRACKING_USERNAME", "admin") - t.Setenv("MLFLOW_TRACKING_PASSWORD", "secret") - - var gotURL string - var gotBody string - var gotAuth string - orig := assessmentHTTPDo - t.Cleanup(func() { assessmentHTTPDo = orig }) - assessmentHTTPDo = func(req *http.Request) (*http.Response, error) { - gotURL = req.URL.String() - gotAuth = req.Header.Get("Authorization") - b, _ := io.ReadAll(req.Body) - gotBody = string(b) - return &http.Response{ - StatusCode: 200, - Body: io.NopCloser(strings.NewReader(`{"assessment":{"assessment_id":"a-1"}}`)), - Header: make(http.Header), - }, nil - } - - err := ExportMLflowAssessments(t.Context(), []EvaluationResult{{ - Name: "trace_fitness", - TraceID: "76f79b7475310c3a16d306b98076dd6c", - SpanID: "c29fc9fc1877260b", - Label: "pass", - Value: 1.0, - Explanation: "all checks passed", - Version: "em-001@1", - Agent: "triage", - WorkItemID: "https://github.com/fullsend-ai/fullsend/issues/1342", - }}) - require.NoError(t, err) - assert.Equal(t, "https://mlflow.example/ajax-api/3.0/mlflow/traces/tr-76f79b7475310c3a16d306b98076dd6c/assessments", gotURL) - assert.True(t, strings.HasPrefix(gotAuth, "Basic ")) - assert.Contains(t, gotBody, `"assessment_name":"trace_fitness"`) - assert.Contains(t, gotBody, `"value":"pass"`) - assert.Contains(t, gotBody, `"source_type":"CODE"`) - assert.Contains(t, gotBody, `"fullsend-evalmeasure/em-001@1"`) -} - -func TestExportMLflowAssessments_DerivedURIWithoutPasswordIsNoop(t *testing.T) { - t.Setenv("MLFLOW_TRACKING_URI", "") - t.Setenv("OTEL_EXPORTER_OTLP_TRACES_ENDPOINT", "https://mlflow.example/v1/traces") - t.Setenv("OTEL_EXPORTER_OTLP_ENDPOINT", "") - t.Setenv("MLFLOW_TRACKING_PASSWORD", "") - err := ExportMLflowAssessments(t.Context(), []EvaluationResult{{ - Name: "trace_fitness", TraceID: "aa", Label: "pass", Value: 1, - }}) - assert.NoError(t, err, "derived URI without password should be a silent no-op") -} - -func TestExportMLflowAssessments_ExplicitURIWithoutPasswordErrors(t *testing.T) { - t.Setenv("MLFLOW_TRACKING_URI", "https://mlflow.example") - t.Setenv("OTEL_EXPORTER_OTLP_TRACES_ENDPOINT", "") - t.Setenv("OTEL_EXPORTER_OTLP_ENDPOINT", "") - t.Setenv("MLFLOW_TRACKING_PASSWORD", "") - err := ExportMLflowAssessments(t.Context(), []EvaluationResult{{ - Name: "trace_fitness", TraceID: "aa", Label: "pass", Value: 1, - }}) - require.Error(t, err) - var exportErr *ExportError - assert.ErrorAs(t, err, &exportErr) - assert.Contains(t, err.Error(), "MLFLOW_TRACKING_PASSWORD is empty") -} - -func TestTrackingURIFromEnv_DerivesFromOTLP(t *testing.T) { - t.Setenv("MLFLOW_TRACKING_URI", "") - t.Setenv("OTEL_EXPORTER_OTLP_TRACES_ENDPOINT", "https://mlflow.example.test/v1/traces") - uri, err := trackingURIFromEnv() - require.NoError(t, err) - assert.Equal(t, "https://mlflow.example.test", uri) -} - -func TestMlflowTraceID(t *testing.T) { - assert.Equal(t, "tr-abc", mlflowTraceID("abc")) - assert.Equal(t, "tr-abc", mlflowTraceID("tr-abc")) -} diff --git a/internal/evalmeasure/run.go b/internal/evalmeasure/run.go index cdc4a4006c..2723581dd0 100644 --- a/internal/evalmeasure/run.go +++ b/internal/evalmeasure/run.go @@ -6,14 +6,16 @@ import ( "path/filepath" ) -// MeasureFile parses telemetry, scores with the registry, writes local JSONL, -// and best-effort exports MLflow Assessments. Idempotent per ledger. +// MeasureFile parses telemetry, scores with the manifest, and writes local +// eval-measurements.jsonl. Idempotent per ledger. func MeasureFile(telemetryPath, registryPath, outDir string) ([]EvaluationResult, error) { return MeasureAndExport(context.Background(), telemetryPath, registryPath, outDir) } -// MeasureAndExport is MeasureFile with an explicit context. +// MeasureAndExport is MeasureFile with an explicit context (reserved for +// future portable OTLP score export on the same OTEL_* path as ADR 0050). func MeasureAndExport(ctx context.Context, telemetryPath, registryPath, outDir string) ([]EvaluationResult, error) { + _ = ctx if outDir == "" { outDir = filepath.Dir(telemetryPath) } @@ -50,8 +52,5 @@ func MeasureAndExport(ctx context.Context, telemetryPath, registryPath, outDir s } } - if err := ExportMLflowAssessments(ctx, all); err != nil { - return all, fmt.Errorf("export mlflow: %w", err) - } return all, nil } diff --git a/internal/evalmeasure/run_test.go b/internal/evalmeasure/run_test.go index 345ae7db78..c7478058d7 100644 --- a/internal/evalmeasure/run_test.go +++ b/internal/evalmeasure/run_test.go @@ -25,11 +25,6 @@ func TestLoadRegistryAndScoreTrace(t *testing.T) { } func TestMeasureFile_Idempotent(t *testing.T) { - t.Setenv("MLFLOW_TRACKING_URI", "") - t.Setenv("MLFLOW_TRACKING_PASSWORD", "") - t.Setenv("OTEL_EXPORTER_OTLP_TRACES_ENDPOINT", "") - t.Setenv("OTEL_EXPORTER_OTLP_ENDPOINT", "") - out := t.TempDir() telemetry := filepath.Join("testdata", "complete.jsonl") registry := filepath.Join("testdata", "sample-registry.yaml") From 2add5e9f19f52ccf175de233e4ad22c384e3125f Mon Sep 17 00:00:00 2001 From: Adam Scerra Date: Mon, 10 Aug 2026 21:03:33 -0400 Subject: [PATCH 03/14] fix(eval): address review feedback on PR #6036 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Fix data-loss bug: swap AppendMeasurements before RecordScored so ledger only marks scored after measurement is persisted - Fix broken curl header args in action.yml using bash array instead of parameter expansion (word-split safe) - Add Prerequisites section to eval-measurements guide - Add OTEL/OTLP inline definitions on first use in guide - Use > **Planned:** callout format for unimplemented OTLP export - Update docs/guides/README.md with eval measurements entry - Consolidate ADR 0087 Decision into single paragraph (was multiple sub-decisions); trim Consequences to 5 bullets - ADR 0050 cross-reference uses Planned callout format - Rename printResults → printMeasurementResults Co-Authored-By: Claude Opus 4.6 Signed-off-by: Adam Scerra --- ...050-distributed-tracing-instrumentation.md | 7 +- ...-eval-measurements-online-trace-scoring.md | 73 ++++++------------- docs/architecture.md | 2 +- docs/guides/README.md | 1 + .../infrastructure/eval-measurements.md | 31 ++++++-- internal/cli/evalmeasure.go | 4 +- internal/evalmeasure/run.go | 6 +- 7 files changed, 55 insertions(+), 69 deletions(-) diff --git a/docs/ADRs/0050-distributed-tracing-instrumentation.md b/docs/ADRs/0050-distributed-tracing-instrumentation.md index ea09007eda..9bab3708f9 100644 --- a/docs/ADRs/0050-distributed-tracing-instrumentation.md +++ b/docs/ADRs/0050-distributed-tracing-instrumentation.md @@ -158,7 +158,8 @@ opt-in, OTel-native, W3C propagation) is unchanged. **2026-08-10 — Eval measurements ([ADR 0087](0087-eval-measurements-online-trace-scoring.md)):** online scoring of wild-run traces always writes `eval-measurements.jsonl` -beside telemetry (tool-agnostic). Portable remote score export follows the -same OTLP configuration as this ADR when implemented — no vendor score -adapters in core. Distinct from functional eval fixtures +beside telemetry (tool-agnostic). Distinct from functional eval fixtures ([ADR 0051](0051-agent-eval-harness-for-test-infrastructure.md)). + +> **Planned:** portable remote score export follows the same OTLP +> configuration as this ADR — no vendor score adapters in core. diff --git a/docs/ADRs/0087-eval-measurements-online-trace-scoring.md b/docs/ADRs/0087-eval-measurements-online-trace-scoring.md index 0951265a22..7e79249c88 100644 --- a/docs/ADRs/0087-eval-measurements-online-trace-scoring.md +++ b/docs/ADRs/0087-eval-measurements-online-trace-scoring.md @@ -62,53 +62,21 @@ Adjacent telemetry proposals (not competing with this score path): Introduce **eval measurements**: deterministic scorers that read `run-telemetry.jsonl` after `fullsend run` in the **same** managed job -(`fullsend eval-measure` in `action.yml`), **fail-open**. - -**Not fixtures:** functional eval scenarios remain ADR 0051 / `eval//`. -Measurements never block delivery. Prefer the glossary terms *eval -measurement* (score) and *eval scenario* (fixture). - -**Ownership (engine vs default policy):** - -| Concern | Repo | -|---|---| -| Parser, scorer **implementations**, CLI, GHA post-step | `fullsend-ai/fullsend` | -| Default measurement manifests for stock agents | `fullsend-ai/agents` (`eval/measurements/.yaml`) | -| Org/repo overrides and BYOA agent manifests | Consumer repo (`FULLSEND_DIR`) | - -Stock agents ship defaults the same way as other agent content: managed jobs -resolve local `${FULLSEND_DIR}/eval/measurements/${AGENT}.yaml` if present, -else fetch `fullsend-ai/agents@v0/...`. Users who only use stock agents do -**not** copy manifests into their repos. Local files are for override, opt-out, -or custom agents — not for receiving defaults. - -Executable scorer logic stays in fullsend because `fullsend eval-measure` is -the released binary that reads `run-telemetry.jsonl` (which fullsend writes). -Agents is content (YAML/prompts), not a library linked into that binary. -EM-001 (`trace_fitness`) is a platform fitness check on that telemetry -contract, so its Go implementation belongs in fullsend even though every -stock agent enables it via agents manifests. - -**Future — logic-as-config:** agent-specific *policy* (thresholds, attribute -checks) should move into declarative manifest fields evaluated by a generic -engine in fullsend. Until that lands, new imperative scorers are Go in -fullsend; wiring a default for a stock agent is an agents manifest change. -New assert primitive → fullsend PR; new id/thresholds on existing primitives → -agents-only (or consumer-repo override). - -**Persistence (always):** write `eval-measurements.jsonl` (and a small ledger) -next to `run-telemetry.jsonl`. This JSONL is the portable, tool-agnostic -contract — backends and dashboards are chosen by the org, not by fullsend. - -**Remote export:** use the same `OTEL_EXPORTER_OTLP_*` configuration as -ADR 0050 when score export is implemented. Do **not** ship vendor-specific -score adapters or `MLFLOW_*` (or similar) wiring in core action/workflows. -Orgs that want a product UI can consume the local JSONL or an OTLP backend -outside fullsend. - -**First scorer:** `trace_fitness` (catalog id `em-001`) — span-tree / -attribute fitness so later scorers can trust the trace. Further scorers -land via manifests without changing this export model. +(`fullsend eval-measure` in `action.yml`), **fail-open**. Functional eval +scenarios remain ADR 0051 / `eval//`; measurements never block +delivery. + +Scores always land in a tool-agnostic `eval-measurements.jsonl` (plus a +small idempotency ledger) next to `run-telemetry.jsonl`. Remote score export +will use the same `OTEL_EXPORTER_OTLP_*` configuration as ADR 0050 — no +vendor-specific score adapters in core. `fullsend` owns the parser, scorers, +CLI, and GHA step; `fullsend-ai/agents` owns per-agent measurement manifests +(`eval/measurements/.yaml`) that declare which scorers to enable. +Stock-agent defaults resolve from `agents@v0` at runtime; local files are for +override, opt-out, or custom agents only. + +The first scorer is `trace_fitness` (catalog id `em-001`) — span-tree and +attribute fitness so later scorers can trust the trace. ### Versioning (per measurement, not platform “v1”) @@ -129,12 +97,13 @@ Entirely new signal → new `em-NNN` (and usually a new `scorer` string). ## Consequences - Every measured run produces a reviewable, backend-agnostic score file beside - telemetry. + telemetry; missing manifests skip cleanly and measure failure never fails + the agent job. - Core stays tool-agnostic: no product-specific score env vars in managed workflows; remote scores follow OTEL when that path lands. -- Missing manifests skip cleanly; measure failure never fails the agent job. -- Functional scenarios (gate) and eval measurements (trend) stay separate. -- Retro can recommend a **manifest scorer** or a **scenario fixture** — not - substitutes. +- Functional scenarios (gate) and eval measurements (trend) stay separate; + retro can recommend either a manifest scorer or a scenario fixture. - Richer telemetry (Level 3 / Status fixes) expands what scorers *can* assert; it does not replace this same-job path. +- Per-measurement versioning (`id@version`) lets pass/fail semantics evolve + without mixing trend eras. diff --git a/docs/architecture.md b/docs/architecture.md index 885215f238..1a16690c32 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -287,7 +287,7 @@ Observability is a cross-cutting concern that touches every other component. Eac - JSONL reasoning trace exposure: raw JSONL conversation transcripts are extracted from sandboxes and stored with owner-scoped access. Credential scanning acts as an invariant check on [ADR 0017](ADRs/0017-credential-isolation-for-sandboxed-agents.md)'s isolation model. Agents handling data from protected sources beyond the target repo can opt in to JSONL suppression via configuration ([ADR 0021](ADRs/0021-jsonl-reasoning-trace-exposure.md)). - Event-driven stage dispatch remains traceable end-to-end in the GitHub Actions UI by using synchronous `workflow_call` dispatch (see [ADR 0041](ADRs/0041-synchronous-workflow-call-event-dispatch.md)). - Distributed tracing: framework-native OpenTelemetry instrumentation with zero-configuration baseline. Every run produces `run-telemetry.jsonl` locally; optional live OTLP export to any compatible backend. W3C trace context propagation links multi-agent pipelines into unified traces. OTEL GenAI semantic conventions enable LLM-aware backends ([ADR 0050](ADRs/0050-distributed-tracing-instrumentation.md)). -- Eval measurements: fail-open same-job scoring of wild-run traces into `eval-measurements.jsonl` beside telemetry; portable remote score export uses the same OTLP configuration as agent traces (planned); scores always land in local eval-measurements.jsonl (tool-agnostic) ([ADR 0087](ADRs/0087-eval-measurements-online-trace-scoring.md)). See [Eval Measurements](guides/infrastructure/eval-measurements.md). +- Eval measurements: fail-open same-job scoring of wild-run traces into `eval-measurements.jsonl` beside telemetry; scores always land in a local, tool-agnostic JSONL artifact. Portable remote score export via the same OTLP configuration as agent traces is planned but not yet implemented ([ADR 0087](ADRs/0087-eval-measurements-online-trace-scoring.md)). See [Eval Measurements](guides/infrastructure/eval-measurements.md). **Open questions:** diff --git a/docs/guides/README.md b/docs/guides/README.md index a837d27f56..c26d7dfe12 100644 --- a/docs/guides/README.md +++ b/docs/guides/README.md @@ -27,6 +27,7 @@ Advanced guides for platform operators who deploy and manage the GCP-side infras - [Infrastructure reference](infrastructure/infrastructure-reference.md) — Token mint, WIF, and secrets deployment details - [Enabling fullsend on private repositories](infrastructure/private-repositories.md) — Additional guardrails and configuration for private repos - [Tracing reference](infrastructure/distributed-tracing.md) — Telemetry levels, environment variables, span hierarchy, and attributes +- [Eval measurements](infrastructure/eval-measurements.md) — Online trace scoring with `eval-measurements.jsonl` and measurement manifests ## User guides diff --git a/docs/guides/infrastructure/eval-measurements.md b/docs/guides/infrastructure/eval-measurements.md index ceaf2ea30d..3f632dce00 100644 --- a/docs/guides/infrastructure/eval-measurements.md +++ b/docs/guides/infrastructure/eval-measurements.md @@ -8,11 +8,23 @@ trends over time. They are **not** functional evals (PR-gate fixtures under Telemetry baseline: [Distributed Tracing](./distributed-tracing.md) ([ADR 0050](../../ADRs/0050-distributed-tracing-instrumentation.md)). +## Prerequisites + +- A repository with fullsend installed and producing `run-telemetry.jsonl` + (see [Distributed Tracing](./distributed-tracing.md)). +- A measurement manifest for the agent (stock agents get one from + `fullsend-ai/agents@v0`; custom agents need a local YAML under + `${FULLSEND_DIR}/eval/measurements/`). + ## Architecture (read this first) Fullsend does not pick an observability product for scores. The portable -contract is a local JSONL artifact next to telemetry; remote export (when -implemented) reuses the same OTEL configuration as agent traces. +contract is a local JSONL artifact next to telemetry; remote export reuses +the same [OTEL](../../glossary.md) (OpenTelemetry) configuration as agent +traces when implemented. + +OTLP (OpenTelemetry Protocol) is the wire format that carries spans and +scores to any compatible backend — Phoenix, MLflow, Jaeger, etc. ```text fullsend run @@ -23,15 +35,17 @@ fullsend run fullsend eval-measure (same GHA job, fail-open, after run) └─ always writes output/**/eval-measurements.jsonl (+ eval-measure-ledger.jsonl for idempotency) - └─ portable remote (planned): same OTEL_EXPORTER_OTLP_* as the agent run ``` +> **Planned:** portable remote score export via the same `OTEL_EXPORTER_OTLP_*` +> path as agent traces. Not yet implemented. + | Artifact | When | Purpose | |---|---|---| | `run-telemetry.jsonl` | Every run | OTLP JSON TracesData lines (local source of truth for spans) | | `eval-measurements.jsonl` | Every measured run | One JSON object per score (`name`, `label`, `value`, `explanation`, `trace_id`, …) | | Remote agent spans | OTEL configured | Same spans the local file holds | -| Remote scores (planned) | OTEL configured | Scores on the OTLP path — any OTLP backend | +| Remote scores *(planned)* | OTEL configured | Scores on the OTLP path — any OTLP backend | Orgs choose Phoenix, MLflow, Jaeger, or another collector independently. Fullsend does not forward vendor-specific score credentials in managed @@ -170,7 +184,8 @@ fullsend eval-measure \ ## Implementation note -Today the measure CLI always writes local `eval-measurements.jsonl`. Portable -OTLP score export (same `OTEL_*` as traces) is the ADR 0087 remote contract -and is not wired yet — until it lands, consume the JSONL artifact (or your -own pipeline) for remote dashboards. +Today the measure CLI always writes local `eval-measurements.jsonl`. + +> **Planned:** portable OTLP score export (same `OTEL_*` as traces) is the +> ADR 0087 remote contract and is not wired yet. Until it lands, consume the +> JSONL artifact (or your own pipeline) for remote dashboards. diff --git a/internal/cli/evalmeasure.go b/internal/cli/evalmeasure.go index 9731a2bc2b..2a64743d6a 100644 --- a/internal/cli/evalmeasure.go +++ b/internal/cli/evalmeasure.go @@ -40,7 +40,7 @@ e.g. eval/measurements/.yaml).`, if err != nil { return err } - printResults(printer, results) + printMeasurementResults(printer, results) return nil }, } @@ -53,7 +53,7 @@ e.g. eval/measurements/.yaml).`, return cmd } -func printResults(printer *ui.Printer, results []evalmeasure.EvaluationResult) { +func printMeasurementResults(printer *ui.Printer, results []evalmeasure.EvaluationResult) { if len(results) == 0 { printer.StepDone("No new measurements (already scored or no matching traces)") return diff --git a/internal/evalmeasure/run.go b/internal/evalmeasure/run.go index 2723581dd0..b358df447e 100644 --- a/internal/evalmeasure/run.go +++ b/internal/evalmeasure/run.go @@ -43,12 +43,12 @@ func MeasureAndExport(ctx context.Context, telemetryPath, registryPath, outDir s continue } all = append(all, r) - if err := RecordScored(ledgerPath, r.TraceID, r.Name, r.Version); err != nil { - return all, fmt.Errorf("record scored: %w", err) - } if err := AppendMeasurements(measPath, []EvaluationResult{r}); err != nil { return all, fmt.Errorf("append measurements: %w", err) } + if err := RecordScored(ledgerPath, r.TraceID, r.Name, r.Version); err != nil { + return all, fmt.Errorf("record scored: %w", err) + } } } From 71e411b42e008206b9898d32de1f154a1d0ac990 Mon Sep 17 00:00:00 2001 From: Adam Scerra Date: Mon, 10 Aug 2026 21:14:11 -0400 Subject: [PATCH 04/14] fix(eval): restore with: blocks and rename ledger extension MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The refactoring commit (49f0d198) accidentally deleted the `with:` input blocks from all 6 reusable workflow files when removing the adjacent MLFLOW_* env vars. Without these blocks, `inputs.agent` resolves to empty in action.yml, breaking all agent execution. Also rename eval-measure-ledger.jsonl → .txt since the ledger contains pipe-delimited plain text, not JSONL. Co-Authored-By: Claude Opus 4.6 Signed-off-by: Adam Scerra --- .github/workflows/reusable-code.yml | 8 ++++++++ .github/workflows/reusable-fix.yml | 8 ++++++++ .github/workflows/reusable-prioritize.yml | 5 +++++ .github/workflows/reusable-retro.yml | 8 ++++++++ .github/workflows/reusable-review.yml | 8 ++++++++ .github/workflows/reusable-triage.yml | 8 ++++++++ docs/guides/infrastructure/eval-measurements.md | 2 +- internal/evalmeasure/export_local.go | 2 +- 8 files changed, 47 insertions(+), 2 deletions(-) diff --git a/.github/workflows/reusable-code.yml b/.github/workflows/reusable-code.yml index f59cf0beec..d89289e110 100644 --- a/.github/workflows/reusable-code.yml +++ b/.github/workflows/reusable-code.yml @@ -197,3 +197,11 @@ jobs: OTEL_EXPORTER_OTLP_CERTIFICATE: ${{ vars.OTEL_EXPORTER_OTLP_CERTIFICATE }} OTEL_RESOURCE_ATTRIBUTES: ${{ vars.OTEL_RESOURCE_ATTRIBUTES }} OTEL_SDK_DISABLED: ${{ vars.OTEL_SDK_DISABLED }} + with: + agent: code + version: ${{ inputs.fullsend_version || job.workflow_sha }} + fullsend-dir: ${{ inputs.install_mode == 'per-repo' && '.fullsend' || '' }} + run-url: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} + status-repo: ${{ inputs.source_repo }} + status-number: ${{ fromJSON(inputs.event_payload).issue.number }} + mint-url: ${{ inputs.mint_url }} diff --git a/.github/workflows/reusable-fix.yml b/.github/workflows/reusable-fix.yml index 82cb724e47..c7aa2f3478 100644 --- a/.github/workflows/reusable-fix.yml +++ b/.github/workflows/reusable-fix.yml @@ -394,3 +394,11 @@ jobs: OTEL_EXPORTER_OTLP_CERTIFICATE: ${{ vars.OTEL_EXPORTER_OTLP_CERTIFICATE }} OTEL_RESOURCE_ATTRIBUTES: ${{ vars.OTEL_RESOURCE_ATTRIBUTES }} OTEL_SDK_DISABLED: ${{ vars.OTEL_SDK_DISABLED }} + with: + agent: fix + version: ${{ inputs.fullsend_version || job.workflow_sha }} + fullsend-dir: ${{ inputs.install_mode == 'per-repo' && '.fullsend' || '' }} + run-url: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} + status-repo: ${{ inputs.source_repo }} + status-number: ${{ steps.context.outputs.pr_number }} + mint-url: ${{ inputs.mint_url }} diff --git a/.github/workflows/reusable-prioritize.yml b/.github/workflows/reusable-prioritize.yml index 4ae4a65759..f53e6d5803 100644 --- a/.github/workflows/reusable-prioritize.yml +++ b/.github/workflows/reusable-prioritize.yml @@ -158,3 +158,8 @@ jobs: OTEL_EXPORTER_OTLP_CERTIFICATE: ${{ vars.OTEL_EXPORTER_OTLP_CERTIFICATE }} OTEL_RESOURCE_ATTRIBUTES: ${{ vars.OTEL_RESOURCE_ATTRIBUTES }} OTEL_SDK_DISABLED: ${{ vars.OTEL_SDK_DISABLED }} + with: + agent: prioritize + version: ${{ inputs.fullsend_version || job.workflow_sha }} + fullsend-dir: ${{ inputs.install_mode == 'per-repo' && '.fullsend' || '' }} + mint-url: ${{ inputs.mint_url }} diff --git a/.github/workflows/reusable-retro.yml b/.github/workflows/reusable-retro.yml index d628e6e708..e02a86bdb6 100644 --- a/.github/workflows/reusable-retro.yml +++ b/.github/workflows/reusable-retro.yml @@ -169,3 +169,11 @@ jobs: OTEL_EXPORTER_OTLP_CERTIFICATE: ${{ vars.OTEL_EXPORTER_OTLP_CERTIFICATE }} OTEL_RESOURCE_ATTRIBUTES: ${{ vars.OTEL_RESOURCE_ATTRIBUTES }} OTEL_SDK_DISABLED: ${{ vars.OTEL_SDK_DISABLED }} + with: + agent: retro + version: ${{ inputs.fullsend_version || job.workflow_sha }} + fullsend-dir: ${{ inputs.install_mode == 'per-repo' && '.fullsend' || '' }} + run-url: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} + status-repo: ${{ inputs.source_repo }} + status-number: ${{ fromJSON(inputs.event_payload).pull_request.number || fromJSON(inputs.event_payload).issue.number }} + mint-url: ${{ inputs.mint_url }} diff --git a/.github/workflows/reusable-review.yml b/.github/workflows/reusable-review.yml index 7025add6a0..18983d99de 100644 --- a/.github/workflows/reusable-review.yml +++ b/.github/workflows/reusable-review.yml @@ -185,3 +185,11 @@ jobs: OTEL_EXPORTER_OTLP_CERTIFICATE: ${{ vars.OTEL_EXPORTER_OTLP_CERTIFICATE }} OTEL_RESOURCE_ATTRIBUTES: ${{ vars.OTEL_RESOURCE_ATTRIBUTES }} OTEL_SDK_DISABLED: ${{ vars.OTEL_SDK_DISABLED }} + with: + agent: review + fullsend-dir: ${{ inputs.install_mode == 'per-repo' && '.fullsend' || '' }} + version: ${{ inputs.fullsend_version || job.workflow_sha }} + run-url: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} + status-repo: ${{ inputs.source_repo }} + status-number: ${{ fromJSON(inputs.event_payload).pull_request.number || fromJSON(inputs.event_payload).issue.number }} + mint-url: ${{ inputs.mint_url }} diff --git a/.github/workflows/reusable-triage.yml b/.github/workflows/reusable-triage.yml index 4cb1f931e1..aa1a1f870b 100644 --- a/.github/workflows/reusable-triage.yml +++ b/.github/workflows/reusable-triage.yml @@ -175,3 +175,11 @@ jobs: OTEL_EXPORTER_OTLP_CERTIFICATE: ${{ vars.OTEL_EXPORTER_OTLP_CERTIFICATE }} OTEL_RESOURCE_ATTRIBUTES: ${{ vars.OTEL_RESOURCE_ATTRIBUTES }} OTEL_SDK_DISABLED: ${{ vars.OTEL_SDK_DISABLED }} + with: + agent: triage + version: ${{ inputs.fullsend_version || job.workflow_sha }} + fullsend-dir: ${{ inputs.install_mode == 'per-repo' && '.fullsend' || '' }} + run-url: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} + status-repo: ${{ inputs.source_repo }} + status-number: ${{ fromJSON(inputs.event_payload).issue.number }} + mint-url: ${{ inputs.mint_url }} diff --git a/docs/guides/infrastructure/eval-measurements.md b/docs/guides/infrastructure/eval-measurements.md index 3f632dce00..b704652353 100644 --- a/docs/guides/infrastructure/eval-measurements.md +++ b/docs/guides/infrastructure/eval-measurements.md @@ -34,7 +34,7 @@ fullsend run fullsend eval-measure (same GHA job, fail-open, after run) └─ always writes output/**/eval-measurements.jsonl - (+ eval-measure-ledger.jsonl for idempotency) + (+ eval-measure-ledger.txt for idempotency) ``` > **Planned:** portable remote score export via the same `OTEL_EXPORTER_OTLP_*` diff --git a/internal/evalmeasure/export_local.go b/internal/evalmeasure/export_local.go index 3a14c4ef01..1ded235fb7 100644 --- a/internal/evalmeasure/export_local.go +++ b/internal/evalmeasure/export_local.go @@ -11,7 +11,7 @@ import ( const ( MeasurementsFile = "eval-measurements.jsonl" - LedgerFile = "eval-measure-ledger.jsonl" + LedgerFile = "eval-measure-ledger.txt" ) // AppendMeasurements writes one NDJSON EvaluationResult per line. From 930db1ae7a7ab97995b8409398208a99569c9614 Mon Sep 17 00:00:00 2001 From: Adam Scerra Date: Wed, 12 Aug 2026 06:51:29 -0400 Subject: [PATCH 05/14] fix(eval): address review feedback and rebase onto main Rebase onto main (action.yml conflict resolved with tool-agnostic measure step). Print partial CLI results on write errors, add ledger ordering test, use Planned callout in architecture.md, and cite #5524 as adjacent in ADR 0087 / guide. Signed-off-by: Adam Scerra Co-authored-by: Cursor --- ...-eval-measurements-online-trace-scoring.md | 5 +++++ docs/architecture.md | 4 +++- .../infrastructure/eval-measurements.md | 1 + internal/cli/evalmeasure.go | 1 + internal/evalmeasure/run_test.go | 22 +++++++++++++++++++ 5 files changed, 32 insertions(+), 1 deletion(-) diff --git a/docs/ADRs/0087-eval-measurements-online-trace-scoring.md b/docs/ADRs/0087-eval-measurements-online-trace-scoring.md index 7e79249c88..d1d594edca 100644 --- a/docs/ADRs/0087-eval-measurements-online-trace-scoring.md +++ b/docs/ADRs/0087-eval-measurements-online-trace-scoring.md @@ -47,6 +47,11 @@ Adjacent telemetry proposals (not competing with this score path): - **Observer / lessons → fixtures** ([#2423](https://github.com/fullsend-ai/fullsend/pull/2423)): narrative analysis and golden-set promotion. This ADR is same-job deterministic scoring on traces. +- **Harness snapshot / forge join keys** ([#5524](https://github.com/fullsend-ai/fullsend/pull/5524) + — proposed ADR 0075): sibling artifact for harness fingerprint and + forge/CI pointers beside telemetry. Complementary join/identity layer; + primary run facts belong on the OTEL trace (Level 1), while measurements + stay a derived sibling file. ## Options diff --git a/docs/architecture.md b/docs/architecture.md index 1a16690c32..eddaee40b3 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -287,7 +287,9 @@ Observability is a cross-cutting concern that touches every other component. Eac - JSONL reasoning trace exposure: raw JSONL conversation transcripts are extracted from sandboxes and stored with owner-scoped access. Credential scanning acts as an invariant check on [ADR 0017](ADRs/0017-credential-isolation-for-sandboxed-agents.md)'s isolation model. Agents handling data from protected sources beyond the target repo can opt in to JSONL suppression via configuration ([ADR 0021](ADRs/0021-jsonl-reasoning-trace-exposure.md)). - Event-driven stage dispatch remains traceable end-to-end in the GitHub Actions UI by using synchronous `workflow_call` dispatch (see [ADR 0041](ADRs/0041-synchronous-workflow-call-event-dispatch.md)). - Distributed tracing: framework-native OpenTelemetry instrumentation with zero-configuration baseline. Every run produces `run-telemetry.jsonl` locally; optional live OTLP export to any compatible backend. W3C trace context propagation links multi-agent pipelines into unified traces. OTEL GenAI semantic conventions enable LLM-aware backends ([ADR 0050](ADRs/0050-distributed-tracing-instrumentation.md)). -- Eval measurements: fail-open same-job scoring of wild-run traces into `eval-measurements.jsonl` beside telemetry; scores always land in a local, tool-agnostic JSONL artifact. Portable remote score export via the same OTLP configuration as agent traces is planned but not yet implemented ([ADR 0087](ADRs/0087-eval-measurements-online-trace-scoring.md)). See [Eval Measurements](guides/infrastructure/eval-measurements.md). +- Eval measurements: fail-open same-job scoring of wild-run traces into `eval-measurements.jsonl` beside telemetry; scores always land in a local, tool-agnostic JSONL artifact ([ADR 0087](ADRs/0087-eval-measurements-online-trace-scoring.md)). See [Eval Measurements](guides/infrastructure/eval-measurements.md). + + > **Planned:** portable remote score export via the same OTLP configuration as agent traces ([ADR 0087](ADRs/0087-eval-measurements-online-trace-scoring.md)). Not yet implemented. **Open questions:** diff --git a/docs/guides/infrastructure/eval-measurements.md b/docs/guides/infrastructure/eval-measurements.md index b704652353..f225ce267d 100644 --- a/docs/guides/infrastructure/eval-measurements.md +++ b/docs/guides/infrastructure/eval-measurements.md @@ -146,6 +146,7 @@ success/failure signal for outcome scorers. | [#5947](https://github.com/fullsend-ai/fullsend/pull/5947) Level 3 activation + sandbox OTEL denylist | Richer traces fuel later scorers. First ship reads Level 1/2 local JSONL; content-aware scorers need OTLP/backend (or a widened file contract) under the proposed L3 rules. Measure CLI is host-side after the sandbox exits. | | [#5944](https://github.com/fullsend-ai/fullsend/pull/5944) Span status from run outcome | Unblocks outcome scorers keyed on Status, not raw exit alone. | | [#2423](https://github.com/fullsend-ai/fullsend/pull/2423) Semantic observability / observer / lessons | Observer + lessons → fixtures; measurements are the online score path. | +| [#5524](https://github.com/fullsend-ai/fullsend/pull/5524) Harness snapshot / forge join keys (ADR 0075) | Complementary join/identity proposal beside telemetry; measurements are derived scores, not primary run facts. | ## Same-job timing diff --git a/internal/cli/evalmeasure.go b/internal/cli/evalmeasure.go index 2a64743d6a..41119ce4a5 100644 --- a/internal/cli/evalmeasure.go +++ b/internal/cli/evalmeasure.go @@ -38,6 +38,7 @@ e.g. eval/measurements/.yaml).`, results, err := evalmeasure.MeasureAndExport(cmd.Context(), telemetryPath, registryPath, outDir) if err != nil { + printMeasurementResults(printer, results) return err } printMeasurementResults(printer, results) diff --git a/internal/evalmeasure/run_test.go b/internal/evalmeasure/run_test.go index c7478058d7..37f667424c 100644 --- a/internal/evalmeasure/run_test.go +++ b/internal/evalmeasure/run_test.go @@ -1,6 +1,7 @@ package evalmeasure import ( + "bytes" "os" "path/filepath" "testing" @@ -41,3 +42,24 @@ func TestMeasureFile_Idempotent(t *testing.T) { require.NoError(t, err) assert.Contains(t, string(b), `"name":"trace_fitness"`) } + +func TestMeasureFile_AppendBeforeLedger(t *testing.T) { + out := t.TempDir() + telemetry := filepath.Join("testdata", "complete.jsonl") + registry := filepath.Join("testdata", "sample-registry.yaml") + ledgerPath := filepath.Join(out, LedgerFile) + + first, err := MeasureFile(telemetry, registry, out) + require.NoError(t, err) + require.Len(t, first, 1) + + require.NoError(t, os.Remove(ledgerPath)) + + second, err := MeasureFile(telemetry, registry, out) + require.NoError(t, err) + require.Len(t, second, 1, "retry after missing ledger should re-score") + + lines, err := os.ReadFile(filepath.Join(out, MeasurementsFile)) + require.NoError(t, err) + assert.Equal(t, 2, bytes.Count(lines, []byte("\n")), "missing ledger may duplicate JSONL lines; consumers dedupe on trace_id+version") +} From 615dd691c22d1896f83b237a5edf865baf4ecc30 Mon Sep 17 00:00:00 2001 From: Adam Scerra Date: Thu, 13 Aug 2026 07:24:03 -0400 Subject: [PATCH 06/14] fix(eval): move measure before upload and raise test coverage Run eval-measure before the single upload-artifact step so eval-measurements.jsonl is included and duplicate artifact names are avoided (fixes e2e harness failures). Quote agent in skip logs, honor cancelled context, document eval-measure in cli-internals, and extend unit tests for error paths and scorer dispatch. Signed-off-by: Adam Scerra Co-authored-by: Cursor --- action.yml | 99 ++++++++++++++---------------- docs/guides/dev/cli-internals.md | 4 ++ internal/cli/evalmeasure_test.go | 10 +++ internal/evalmeasure/parse_test.go | 16 +++++ internal/evalmeasure/run.go | 4 +- internal/evalmeasure/run_test.go | 31 ++++++++++ internal/evalmeasure/score_test.go | 24 ++++++++ 7 files changed, 134 insertions(+), 54 deletions(-) diff --git a/action.yml b/action.yml index da070a3b5f..1b35834784 100644 --- a/action.yml +++ b/action.yml @@ -409,6 +409,52 @@ runs: "${STATUS_FLAGS[@]+"${STATUS_FLAGS[@]}"}" \ "${MINT_FLAGS[@]+"${MINT_FLAGS[@]}"}" + # Eval measurements (fail-open): score run-telemetry.jsonl with the agents + # measurement manifest. Same job as fullsend run; never fails the agent. + # Scores always land in eval-measurements.jsonl (tool-agnostic artifact). + - name: Eval measurements + if: always() && inputs.agent != '__install_only__' + continue-on-error: true + shell: bash + env: + AGENT: ${{ inputs.agent }} + FULLSEND_DIR: ${{ inputs.fullsend-dir }} + GH_TOKEN: ${{ inputs.github_token }} + run: | + set -euo pipefail + FULLSEND_DIR="${FULLSEND_DIR:-${GITHUB_WORKSPACE}}" + TELEMETRY=$(find "${GITHUB_WORKSPACE}/output" -name run-telemetry.jsonl -print -quit 2>/dev/null || true) + if [[ -z "${TELEMETRY}" ]]; then + echo "No run-telemetry.jsonl under output/; skipping eval measurements" + exit 0 + fi + + REGISTRY="" + LOCAL_REG="${FULLSEND_DIR}/eval/measurements/${AGENT}.yaml" + if [[ -f "${LOCAL_REG}" ]]; then + REGISTRY="${LOCAL_REG}" + else + # Same pin as agents-repo harness fallback (config.DefaultUpstreamRef=v0). + URL="https://raw.githubusercontent.com/fullsend-ai/agents/v0/eval/measurements/${AGENT}.yaml" + TMP="$(mktemp)" + CURL_ARGS=( -fsSL -o "${TMP}" ) + if [[ -n "${GH_TOKEN:-}" ]]; then + CURL_ARGS+=( -H "Authorization: Bearer ${GH_TOKEN}" ) + fi + if curl "${CURL_ARGS[@]}" "${URL}"; then + REGISTRY="${TMP}" + else + printf 'No eval measurement manifest for %q at %s or %s; skipping\n' "${AGENT}" "${LOCAL_REG}" "${URL}" + rm -f "${TMP}" + exit 0 + fi + fi + + fullsend eval-measure \ + --telemetry "${TELEMETRY}" \ + --registry "${REGISTRY}" \ + --out-dir "$(dirname "${TELEMETRY}")" + - name: Upload fullsend artifacts if: always() && inputs.agent != '__install_only__' uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 @@ -482,56 +528,3 @@ runs: fullsend reconcile-status "${RECONCILE_FLAGS[@]+"${RECONCILE_FLAGS[@]}"}" || { echo "::warning::Could not reconcile orphaned status comment" } - - # Eval measurements (fail-open): score run-telemetry.jsonl with the agents - # measurement manifest. Same job as fullsend run; never fails the agent. - # Scores always land in eval-measurements.jsonl (tool-agnostic artifact). - - name: Eval measurements - if: always() && inputs.agent != '__install_only__' - continue-on-error: true - shell: bash - env: - AGENT: ${{ inputs.agent }} - FULLSEND_DIR: ${{ inputs.fullsend-dir }} - GH_TOKEN: ${{ inputs.github_token }} - run: | - set -euo pipefail - FULLSEND_DIR="${FULLSEND_DIR:-${GITHUB_WORKSPACE}}" - TELEMETRY=$(find "${GITHUB_WORKSPACE}/output" -name run-telemetry.jsonl -print -quit 2>/dev/null || true) - if [[ -z "${TELEMETRY}" ]]; then - echo "No run-telemetry.jsonl under output/; skipping eval measurements" - exit 0 - fi - - REGISTRY="" - LOCAL_REG="${FULLSEND_DIR}/eval/measurements/${AGENT}.yaml" - if [[ -f "${LOCAL_REG}" ]]; then - REGISTRY="${LOCAL_REG}" - else - # Same pin as agents-repo harness fallback (config.DefaultUpstreamRef=v0). - URL="https://raw.githubusercontent.com/fullsend-ai/agents/v0/eval/measurements/${AGENT}.yaml" - TMP="$(mktemp)" - CURL_ARGS=( -fsSL -o "${TMP}" ) - if [[ -n "${GH_TOKEN:-}" ]]; then - CURL_ARGS+=( -H "Authorization: Bearer ${GH_TOKEN}" ) - fi - if curl "${CURL_ARGS[@]}" "${URL}"; then - REGISTRY="${TMP}" - else - echo "No eval measurement manifest for ${AGENT} at ${LOCAL_REG} or ${URL}; skipping" - rm -f "${TMP}" - exit 0 - fi - fi - - fullsend eval-measure \ - --telemetry "${TELEMETRY}" \ - --registry "${REGISTRY}" \ - --out-dir "$(dirname "${TELEMETRY}")" - - - name: Upload fullsend artifacts - if: always() && inputs.agent != '__install_only__' - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 - with: - name: fullsend-${{ inputs.agent }} - path: ${{ github.workspace }}/output diff --git a/docs/guides/dev/cli-internals.md b/docs/guides/dev/cli-internals.md index fcedb260b3..91a1336ed4 100644 --- a/docs/guides/dev/cli-internals.md +++ b/docs/guides/dev/cli-internals.md @@ -109,6 +109,10 @@ fullsend │ └── url # Validate URLs against SSRF attacks ├── post-review # Post PR review comments to GitHub ├── post-comment # Post issue/PR comments to GitHub +├── eval-measure # Score wild-run traces (eval measurements) +│ ├── --telemetry # Path to run-telemetry.jsonl (required) +│ ├── --registry # Agents measurement manifest YAML (required) +│ └── --out-dir # Output dir (default: telemetry directory) └── reconcile-status # Finalize orphaned status comments ├── --repo # Repository in owner/repo format ├── --number # Issue/PR number diff --git a/internal/cli/evalmeasure_test.go b/internal/cli/evalmeasure_test.go index 5019a2a647..f5fc781b2b 100644 --- a/internal/cli/evalmeasure_test.go +++ b/internal/cli/evalmeasure_test.go @@ -44,3 +44,13 @@ func TestRootCommand_HasEvalMeasureSubcommand(t *testing.T) { } assert.True(t, found, "expected eval-measure subcommand") } + +func TestEvalMeasureCmd_MissingRequiredFlags(t *testing.T) { + cmd := newRootCmd() + buf := &bytes.Buffer{} + cmd.SetOut(buf) + cmd.SetErr(buf) + cmd.SetArgs([]string{"eval-measure"}) + err := cmd.Execute() + require.Error(t, err) +} diff --git a/internal/evalmeasure/parse_test.go b/internal/evalmeasure/parse_test.go index 1d6d8f22ba..129c0760d9 100644 --- a/internal/evalmeasure/parse_test.go +++ b/internal/evalmeasure/parse_test.go @@ -1,6 +1,7 @@ package evalmeasure import ( + "os" "path/filepath" "testing" @@ -36,3 +37,18 @@ func TestParseTelemetryFile_MergesLinesSameTrace(t *testing.T) { _, ok := traces[0].SpanByName("sandbox_create") assert.True(t, ok) } + +func TestParseTelemetryFile_InvalidLine(t *testing.T) { + t.Parallel() + dir := t.TempDir() + path := filepath.Join(dir, "bad.jsonl") + require.NoError(t, os.WriteFile(path, []byte("not-json\n"), 0o644)) + _, err := ParseTelemetryFile(path) + require.Error(t, err) +} + +func TestParseTelemetryFile_MissingFile(t *testing.T) { + t.Parallel() + _, err := ParseTelemetryFile(filepath.Join(t.TempDir(), "missing.jsonl")) + require.Error(t, err) +} diff --git a/internal/evalmeasure/run.go b/internal/evalmeasure/run.go index b358df447e..0e98a22386 100644 --- a/internal/evalmeasure/run.go +++ b/internal/evalmeasure/run.go @@ -15,7 +15,9 @@ func MeasureFile(telemetryPath, registryPath, outDir string) ([]EvaluationResult // MeasureAndExport is MeasureFile with an explicit context (reserved for // future portable OTLP score export on the same OTEL_* path as ADR 0050). func MeasureAndExport(ctx context.Context, telemetryPath, registryPath, outDir string) ([]EvaluationResult, error) { - _ = ctx + if err := ctx.Err(); err != nil { + return nil, err + } if outDir == "" { outDir = filepath.Dir(telemetryPath) } diff --git a/internal/evalmeasure/run_test.go b/internal/evalmeasure/run_test.go index 37f667424c..01aa330c54 100644 --- a/internal/evalmeasure/run_test.go +++ b/internal/evalmeasure/run_test.go @@ -2,6 +2,7 @@ package evalmeasure import ( "bytes" + "context" "os" "path/filepath" "testing" @@ -63,3 +64,33 @@ func TestMeasureFile_AppendBeforeLedger(t *testing.T) { require.NoError(t, err) assert.Equal(t, 2, bytes.Count(lines, []byte("\n")), "missing ledger may duplicate JSONL lines; consumers dedupe on trace_id+version") } + +func TestMeasureFile_BadRegistry(t *testing.T) { + _, err := MeasureFile( + filepath.Join("testdata", "complete.jsonl"), + filepath.Join(t.TempDir(), "missing.yaml"), + t.TempDir(), + ) + require.Error(t, err) +} + +func TestMeasureFile_BadTelemetry(t *testing.T) { + _, err := MeasureFile( + filepath.Join(t.TempDir(), "missing.jsonl"), + filepath.Join("testdata", "sample-registry.yaml"), + t.TempDir(), + ) + require.Error(t, err) +} + +func TestMeasureAndExport_CancelledContext(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + cancel() + _, err := MeasureAndExport( + ctx, + filepath.Join("testdata", "complete.jsonl"), + filepath.Join("testdata", "sample-registry.yaml"), + t.TempDir(), + ) + require.Error(t, err) +} diff --git a/internal/evalmeasure/score_test.go b/internal/evalmeasure/score_test.go index e341a47ff7..de51891fc9 100644 --- a/internal/evalmeasure/score_test.go +++ b/internal/evalmeasure/score_test.go @@ -46,3 +46,27 @@ func TestScoreFitness_ReviewUnknownWorkItemFails(t *testing.T) { assert.Contains(t, r.Explanation, "work_item=fail") assert.Contains(t, r.Explanation, "missing: work_item") } + +func TestScoreTrace_AgentMismatchReturnsNil(t *testing.T) { + t.Parallel() + traces, err := ParseTelemetryFile(filepath.Join("testdata", "complete.jsonl")) + require.NoError(t, err) + reg := Registry{Agent: "code", Measurements: []MeasurementSpec{{ID: "em-001", Scorer: ScorerFitness, Version: 1}}} + assert.Empty(t, ScoreTrace(traces[0], reg)) +} + +func TestScoreTrace_UnknownScorerSkipped(t *testing.T) { + t.Parallel() + traces, err := ParseTelemetryFile(filepath.Join("testdata", "complete.jsonl")) + require.NoError(t, err) + reg := Registry{ + Agent: "triage", + Measurements: []MeasurementSpec{ + {ID: "em-999", Scorer: "future_scorer", Version: 1}, + {ID: "em-001", Scorer: ScorerFitness, Version: 1}, + }, + } + results := ScoreTrace(traces[0], reg) + require.Len(t, results, 1) + assert.Equal(t, "trace_fitness", results[0].Name) +} From c8438d83281d5ff860eef17f4e19b770198c2193 Mon Sep 17 00:00:00 2001 From: Adam Scerra Date: Mon, 17 Aug 2026 10:05:56 -0400 Subject: [PATCH 07/14] fix(eval): address review findings on measurements path SHA-pin agents@v0 measurement manifests in the eval-measure binary, score only platform run-telemetry.jsonl, skip unknown scorers and pre-script-skipped runs, and keep GitLab measure fail-open after a failed agent run. Co-authored-by: Cursor Signed-off-by: Adam Scerra --- action.yml | 36 +-- ...-eval-measurements-online-trace-scoring.md | 14 +- docs/architecture.md | 2 +- docs/glossary.md | 17 +- .../infrastructure/eval-measurements.md | 57 +++-- docs/problems/operational-observability.md | 4 +- internal/cli/evalmeasure.go | 185 +++++++++++++-- internal/cli/evalmeasure_test.go | 213 +++++++++++++++++- internal/cli/run.go | 68 ++++-- internal/cli/run_test.go | 127 +++++++++++ internal/cli/telemetry_run_test.go | 3 +- internal/evalmeasure/find.go | 43 ++++ internal/evalmeasure/find_test.go | 74 ++++++ internal/evalmeasure/fitness.go | 67 ++++-- internal/evalmeasure/parse.go | 3 +- internal/evalmeasure/parse_test.go | 25 +- internal/evalmeasure/registry.go | 17 +- internal/evalmeasure/run_test.go | 38 ++++ internal/evalmeasure/score_test.go | 116 +++++++++- .../testdata/prescript-skipped.jsonl | 1 + internal/evalmeasure/types.go | 44 +++- internal/evalmeasure/types_test.go | 29 +++ internal/fetch/fetch.go | 13 +- internal/fetch/fetch_test.go | 4 + .../.gitlab/ci/fullsend-agent.yml | 16 ++ internal/scaffold/scaffold_gitlab_test.go | 4 + 26 files changed, 1099 insertions(+), 121 deletions(-) create mode 100644 internal/evalmeasure/find.go create mode 100644 internal/evalmeasure/find_test.go create mode 100644 internal/evalmeasure/testdata/prescript-skipped.jsonl diff --git a/action.yml b/action.yml index 1b35834784..30ad7f556c 100644 --- a/action.yml +++ b/action.yml @@ -419,41 +419,15 @@ runs: env: AGENT: ${{ inputs.agent }} FULLSEND_DIR: ${{ inputs.fullsend-dir }} - GH_TOKEN: ${{ inputs.github_token }} run: | set -euo pipefail FULLSEND_DIR="${FULLSEND_DIR:-${GITHUB_WORKSPACE}}" - TELEMETRY=$(find "${GITHUB_WORKSPACE}/output" -name run-telemetry.jsonl -print -quit 2>/dev/null || true) - if [[ -z "${TELEMETRY}" ]]; then - echo "No run-telemetry.jsonl under output/; skipping eval measurements" - exit 0 - fi - - REGISTRY="" - LOCAL_REG="${FULLSEND_DIR}/eval/measurements/${AGENT}.yaml" - if [[ -f "${LOCAL_REG}" ]]; then - REGISTRY="${LOCAL_REG}" - else - # Same pin as agents-repo harness fallback (config.DefaultUpstreamRef=v0). - URL="https://raw.githubusercontent.com/fullsend-ai/agents/v0/eval/measurements/${AGENT}.yaml" - TMP="$(mktemp)" - CURL_ARGS=( -fsSL -o "${TMP}" ) - if [[ -n "${GH_TOKEN:-}" ]]; then - CURL_ARGS+=( -H "Authorization: Bearer ${GH_TOKEN}" ) - fi - if curl "${CURL_ARGS[@]}" "${URL}"; then - REGISTRY="${TMP}" - else - printf 'No eval measurement manifest for %q at %s or %s; skipping\n' "${AGENT}" "${LOCAL_REG}" "${URL}" - rm -f "${TMP}" - exit 0 - fi - fi - + # Binary resolves the SHA-pinned agents manifest (allowlist/hash/audit) + # and scores only platform run-telemetry.jsonl at the top of each runDir. fullsend eval-measure \ - --telemetry "${TELEMETRY}" \ - --registry "${REGISTRY}" \ - --out-dir "$(dirname "${TELEMETRY}")" + --agent "${AGENT}" \ + --fullsend-dir "${FULLSEND_DIR}" \ + --output-dir "${GITHUB_WORKSPACE}/output" - name: Upload fullsend artifacts if: always() && inputs.agent != '__install_only__' diff --git a/docs/ADRs/0087-eval-measurements-online-trace-scoring.md b/docs/ADRs/0087-eval-measurements-online-trace-scoring.md index d1d594edca..fa0602337a 100644 --- a/docs/ADRs/0087-eval-measurements-online-trace-scoring.md +++ b/docs/ADRs/0087-eval-measurements-online-trace-scoring.md @@ -71,6 +71,13 @@ Introduce **eval measurements**: deterministic scorers that read scenarios remain ADR 0051 / `eval//`; measurements never block delivery. +In plain terms: eval measurements are the concept of scoring traces. +[OTEL primary facts](../glossary.md#otel-primary-facts) are what happened +on the run (the OTEL trace / `run-telemetry.jsonl`). +[OTEL derived products](../glossary.md#otel-derived-products) are scores +computed from that trace (`eval-measurements.jsonl`). Measurements never +rewrite primary facts, and they are [fail-open](../glossary.md#fail-open). + Scores always land in a tool-agnostic `eval-measurements.jsonl` (plus a small idempotency ledger) next to `run-telemetry.jsonl`. Remote score export will use the same `OTEL_EXPORTER_OTLP_*` configuration as ADR 0050 — no @@ -103,7 +110,9 @@ Entirely new signal → new `em-NNN` (and usually a new `scorer` string). - Every measured run produces a reviewable, backend-agnostic score file beside telemetry; missing manifests skip cleanly and measure failure never fails - the agent job. + the agent job. GitHub Actions is the first-ship managed path (uploads + `output/`). GitLab CI calls the same fail-open `eval-measure` CLI, writing + under `/tmp/fullsend-output` with no `artifacts:` block by default. - Core stays tool-agnostic: no product-specific score env vars in managed workflows; remote scores follow OTEL when that path lands. - Functional scenarios (gate) and eval measurements (trend) stay separate; @@ -112,3 +121,6 @@ Entirely new signal → new `em-NNN` (and usually a new `scorer` string). it does not replace this same-job path. - Per-measurement versioning (`id@version`) lets pass/fail semantics evolve without mixing trend eras. +- Pre-script skipped runs (`fullsend.prescript.skipped=true` on the root span) + are excluded from EM-001: the scorer writes `label: skip` instead of failing + a run that never created a sandbox. diff --git a/docs/architecture.md b/docs/architecture.md index eddaee40b3..db5f617c72 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -287,7 +287,7 @@ Observability is a cross-cutting concern that touches every other component. Eac - JSONL reasoning trace exposure: raw JSONL conversation transcripts are extracted from sandboxes and stored with owner-scoped access. Credential scanning acts as an invariant check on [ADR 0017](ADRs/0017-credential-isolation-for-sandboxed-agents.md)'s isolation model. Agents handling data from protected sources beyond the target repo can opt in to JSONL suppression via configuration ([ADR 0021](ADRs/0021-jsonl-reasoning-trace-exposure.md)). - Event-driven stage dispatch remains traceable end-to-end in the GitHub Actions UI by using synchronous `workflow_call` dispatch (see [ADR 0041](ADRs/0041-synchronous-workflow-call-event-dispatch.md)). - Distributed tracing: framework-native OpenTelemetry instrumentation with zero-configuration baseline. Every run produces `run-telemetry.jsonl` locally; optional live OTLP export to any compatible backend. W3C trace context propagation links multi-agent pipelines into unified traces. OTEL GenAI semantic conventions enable LLM-aware backends ([ADR 0050](ADRs/0050-distributed-tracing-instrumentation.md)). -- Eval measurements: fail-open same-job scoring of wild-run traces into `eval-measurements.jsonl` beside telemetry; scores always land in a local, tool-agnostic JSONL artifact ([ADR 0087](ADRs/0087-eval-measurements-online-trace-scoring.md)). See [Eval Measurements](guides/infrastructure/eval-measurements.md). +- Eval measurements: the concept of scoring traces ([fail-open](glossary.md#fail-open)). [OTEL primary facts](glossary.md#otel-primary-facts) stay on the run trace (`run-telemetry.jsonl`); [OTEL derived products](glossary.md#otel-derived-products) are the scores (`eval-measurements.jsonl`) ([ADR 0087](ADRs/0087-eval-measurements-online-trace-scoring.md)). See [Eval Measurements](guides/infrastructure/eval-measurements.md). > **Planned:** portable remote score export via the same OTLP configuration as agent traces ([ADR 0087](ADRs/0087-eval-measurements-online-trace-scoring.md)). Not yet implemented. diff --git a/docs/glossary.md b/docs/glossary.md index 98c2a6227b..aa59337b72 100644 --- a/docs/glossary.md +++ b/docs/glossary.md @@ -88,7 +88,7 @@ See [autonomy-spectrum.md](problems/autonomy-spectrum.md) and [agent-architectur ### Eval Measurement -A score, judge, or metric applied to agent (or agent-chain) behavior — for example cost per run, whether the code agent later passes review, or whether a review agent recommends merge and a human still intervenes. Measurements are not the inputs under test; they are what you score. Online / trend scoring of wild-run traces is decided in [ADR 0087](ADRs/0087-eval-measurements-online-trace-scoring.md) (`fullsend eval-measure`, `eval-measurements.jsonl`). The same measurement *idea* can also be applied to curated [eval scenarios](#eval-scenario), but those PR-gate fixtures are a separate path ([ADR 0051](ADRs/0051-agent-eval-harness-for-test-infrastructure.md)). Prefer this term (or synonyms *eval score* / *eval judge*) over the bare word "evals," which is ambiguous with [eval scenarios](#eval-scenario). +The concept of **scoring traces** — a score, judge, or metric applied to agent (or agent-chain) behavior. Example signals: cost per run, whether the code agent later passes review, or whether a review agent recommends merge and a human still intervenes. Measurements are not the inputs under test; they are an [OTEL derived product](#otel-derived-products) computed from [OTEL primary facts](#otel-primary-facts). Online / trend scoring of wild-run traces is decided in [ADR 0087](ADRs/0087-eval-measurements-online-trace-scoring.md) (`fullsend eval-measure`, `eval-measurements.jsonl`) and is [fail-open](#fail-open). The same measurement *idea* can also be applied to curated [eval scenarios](#eval-scenario), but those PR-gate fixtures are a separate path ([ADR 0051](ADRs/0051-agent-eval-harness-for-test-infrastructure.md)). Prefer this term (or synonyms *eval score* / *eval judge*) over the bare word "evals," which is ambiguous with [eval scenarios](#eval-scenario). See [Eval Measurements](guides/infrastructure/eval-measurements.md), [testing-agents.md](problems/testing-agents.md), and [Observability](#observability). ### Eval Scenario @@ -102,6 +102,11 @@ A workflow concept where a repository automatically stays up-to-date with depend ## F +### Fail-Open + +When a step's error or a `fail` score must not fail the surrounding job or block delivery. Eval measurements are fail-open: a missing manifest, a scorer `fail`/`skip` label, or a measure-step IO error never fails the agent run. Contrast with fail-closed gates (auth, kill switch) where an error must stop the run. In scripts, fail-open is acceptable for non-critical steps (logging, metrics) and dangerous for gates. +See [ADR 0087](ADRs/0087-eval-measurements-online-trace-scoring.md), [Eval Measurements](guides/infrastructure/eval-measurements.md), and [Shell scripting](contributing/shell-scripting.md). + ### Flapping When agents enter a cycle of conflicting feedback that prevents convergence. Example: the security review agent rejects what the code agent produces to satisfy the correctness review agent, and vice versa, creating an oscillating loop. Flapping is a primary trigger for [escalation](#escalation) — after a configurable number of cycles, the system stops and routes to humans. @@ -147,6 +152,16 @@ See [security-threat-model.md](problems/security-threat-model.md). The logging, tracing, and audit layer for agent actions. Every agent action must be attributable, traceable, and reviewable — both for debugging failures and for security auditability. In practice, this includes capturing agent JSONL logs (including "thinking" traces), converting them to human-readable format, and uploading them as artifacts. Observability is a cross-cutting concern that touches every other component. See [architecture.md](architecture.md). +### OTEL Derived Products + +Values **computed from** a run's OpenTelemetry trace after the fact — scores, fitness checks, later quality signals. They are not a second copy of what happened. First-ship example: `eval-measurements.jsonl` from `fullsend eval-measure` ([eval measurements](#eval-measurement) are the concept of scoring traces). Derived products sit beside telemetry as sibling files; they never replace [OTEL primary facts](#otel-primary-facts). +See [ADR 0087](ADRs/0087-eval-measurements-online-trace-scoring.md) and [Eval Measurements](guides/infrastructure/eval-measurements.md). + +### OTEL Primary Facts + +What **actually happened** on an agent run, recorded as OpenTelemetry (OTEL) spans. The local source of truth is `run-telemetry.jsonl`; when `OTEL_EXPORTER_OTLP_*` is set, the same spans also export live over OTLP ([ADR 0050](ADRs/0050-distributed-tracing-instrumentation.md)). Agent identity, work item, tokens, cost, span tree, and `exit_code` belong here. Sibling files (including [eval measurements](#eval-measurement)) must not become a second source of run truth. +See [Distributed Tracing](guides/infrastructure/distributed-tracing.md) and [ADR 0087](ADRs/0087-eval-measurements-online-trace-scoring.md). + ## P ### Policy Store diff --git a/docs/guides/infrastructure/eval-measurements.md b/docs/guides/infrastructure/eval-measurements.md index f225ce267d..f58a921593 100644 --- a/docs/guides/infrastructure/eval-measurements.md +++ b/docs/guides/infrastructure/eval-measurements.md @@ -18,9 +18,16 @@ Telemetry baseline: [Distributed Tracing](./distributed-tracing.md) ## Architecture (read this first) +Eval measurements are the concept of scoring traces. +[OTEL primary facts](../../glossary.md#otel-primary-facts) are what happened +on the run (`run-telemetry.jsonl`). +[OTEL derived products](../../glossary.md#otel-derived-products) are scores +computed from that trace (`eval-measurements.jsonl`). The step is +[fail-open](../../glossary.md#fail-open): it never blocks delivery. + Fullsend does not pick an observability product for scores. The portable contract is a local JSONL artifact next to telemetry; remote export reuses -the same [OTEL](../../glossary.md) (OpenTelemetry) configuration as agent +the same OpenTelemetry (`OTEL_EXPORTER_OTLP_*`) configuration as agent traces when implemented. OTLP (OpenTelemetry Protocol) is the wire format that carries spans and @@ -28,12 +35,12 @@ scores to any compatible backend — Phoenix, MLflow, Jaeger, etc. ```text fullsend run - └─ always writes output/**/run-telemetry.jsonl + └─ always writes output//run-telemetry.jsonl └─ if OTEL_EXPORTER_OTLP_* set → live OTLP export of agent spans (any compatible backend — ADR 0050) fullsend eval-measure (same GHA job, fail-open, after run) - └─ always writes output/**/eval-measurements.jsonl + └─ always writes output//eval-measurements.jsonl (+ eval-measure-ledger.txt for idempotency) ``` @@ -43,7 +50,7 @@ fullsend eval-measure (same GHA job, fail-open, after run) | Artifact | When | Purpose | |---|---|---| | `run-telemetry.jsonl` | Every run | OTLP JSON TracesData lines (local source of truth for spans) | -| `eval-measurements.jsonl` | Every measured run | One JSON object per score (`name`, `label`, `value`, `explanation`, `trace_id`, …) | +| `eval-measurements.jsonl` | Every measured run | One JSON object per score (`name`, `label`, `value`, `explanation`, `trace_id`, …). On `label: skip`, `value` is unused (serialized as `0`; ignore it). | | Remote agent spans | OTEL configured | Same spans the local file holds | | Remote scores *(planned)* | OTEL configured | Scores on the OTLP path — any OTLP backend | @@ -139,6 +146,13 @@ success. After [#5944](https://github.com/fullsend-ai/fullsend/pull/5944), run/agent **OTLP Status** (and `fullsend.transcript_error`) are the success/failure signal for outcome scorers. +Pre-script **skipped** runs set `fullsend.prescript.skipped=true` on the root +span and never create a sandbox. EM-001 records `label: skip` for those traces +instead of failing the span-tree / model / usage checks. An unknown `scorer:` +string (for example a newer `agents@v0` manifest this binary does not implement +yet) also writes `label: skip`, not `fail`. Trend pass-rate as +`pass / (pass + fail)` and drop `skip`. + ## Adjacent telemetry work | Proposal | Relationship to measurements | @@ -153,35 +167,46 @@ success/failure signal for outcome scorers. ```text GitHub Actions job ├── fullsend run -├── fullsend eval-measure # reads run-telemetry.jsonl; never fails the job +├── fullsend eval-measure # reads output//run-telemetry.jsonl; never fails the job └── upload-artifact # includes both JSONL files under output/ + +GitLab CI agent job +├── fullsend run # --output-dir /tmp/fullsend-output +├── fullsend eval-measure # always (even if run failed); || true +└── (no artifacts: by default — JSONL stays on the runner filesystem) ``` ## Manifest resolution in CI -`action.yml` resolves the measurement manifest for `inputs.agent`: +`fullsend eval-measure` resolves the measurement manifest for the agent: 1. `${FULLSEND_DIR}/eval/measurements/${AGENT}.yaml` if present, else -2. `https://raw.githubusercontent.com/fullsend-ai/agents/v0/eval/measurements/${AGENT}.yaml` +2. SHA-pinned `eval/measurements/${AGENT}.yaml` from `fullsend-ai/agents` + (same `v0` → commit SHA, allowlist, hash, and fetch audit as harness + fallback — not a floating `raw.githubusercontent.com/.../v0/...` curl). + +Step 2 is how stock-agent defaults reach every install. Step 1 is override / +BYOA only. -Step 2 is how stock-agent defaults reach every install (same pin style as -other agents-repo fallbacks). Step 1 is override / BYOA only. +Platform telemetry is `run-telemetry.jsonl` at the top of each run directory. +Nested `iteration-N/output/run-telemetry.jsonl` copies are ignored. -Missing manifest → log and exit `0` (skip). The CLI flag remains -`--registry` for now (path to the YAML); rename is cosmetic follow-up. +Missing manifest or telemetry → log and exit `0` (skip). `--registry` and +`--telemetry` remain for local/debug use. ## CLI ```bash fullsend eval-measure \ - --telemetry path/to/run-telemetry.jsonl \ - --registry path/to/agents/eval/measurements/review.yaml \ - --out-dir path/to/output + --agent review \ + --fullsend-dir "${FULLSEND_DIR}" \ + --output-dir path/to/output ``` -- `--registry` is required (manifests live in `fullsend-ai/agents`, not in - the fullsend binary). +- `--agent` + `--output-dir` is the managed-job form. `--registry` / + `--telemetry` remain for pointing at explicit files. - Exit `0` when a score is `fail` — scores are data. +- Exit `0` when telemetry or the manifest is missing (skip). ## Implementation note diff --git a/docs/problems/operational-observability.md b/docs/problems/operational-observability.md index cacd55639b..92a491ff6e 100644 --- a/docs/problems/operational-observability.md +++ b/docs/problems/operational-observability.md @@ -191,8 +191,8 @@ This works for early experimentation when the volume is low and the operators ar - ~~How do we keep the triggering event and the agent run linked in the GitHub Actions UI for debugging?~~ Decided in [ADR 0041](../ADRs/0041-synchronous-workflow-call-event-dispatch.md) (synchronous `workflow_call` dispatch for the event path). - How should trace access be controlled? (JSONL trace exposure decided in [ADR 0021](../ADRs/0021-jsonl-reasoning-trace-exposure.md): owner-scoped storage with credential scanning as defense-in-depth. Broader question of balancing security and transparency for non-JSONL observability data remains open.) - What retention policy applies to traces? Indefinite retention supports audit requirements but increases storage cost and data sensitivity exposure. Time-bounded retention (e.g., 90 days) limits exposure but may lose traces needed for incident investigation. -- How do we measure "is the system getting better"? What metrics constitute a meaningful quality signal for an autonomous software factory? Merge revert rate? Human override rate? Time-to-review? Cost per decision? Some composite score? The choice of metric shapes what gets optimized. -- At what scale does a dedicated LLM observability platform justify its operational overhead (Postgres, ClickHouse, Redis, S3 for something like Langfuse)? Is there a threshold of agent activity below which structured logging suffices? +- How do we measure "is the system getting better"? What metrics constitute a meaningful quality signal for an autonomous software factory? Merge revert rate? Human override rate? Time-to-review? Cost per decision? Some composite score? The choice of metric shapes what gets optimized. First-ship trend scores (trace fitness on wild runs, local `eval-measurements.jsonl`) are [ADR 0087](../ADRs/0087-eval-measurements-online-trace-scoring.md); richer quality signals remain open. +- At what scale does a dedicated LLM observability platform justify its operational overhead (Postgres, ClickHouse, Redis, S3 for something like Langfuse)? Is there a threshold of agent activity below which structured logging suffices? Score files are local JSONL ([ADR 0087](../ADRs/0087-eval-measurements-online-trace-scoring.md)); remote scores reuse `OTEL_EXPORTER_OTLP_*` when implemented. Platform choice remains open. - ~~How do we handle the bootstrapping problem — the factory needs observability to improve, but building the observability infrastructure is itself work that competes with building the factory?~~ Decided in [ADR 0050](../ADRs/0050-distributed-tracing-instrumentation.md): zero-configuration baseline (local JSONL + summary files) eliminates infrastructure requirements for initial observability; OTLP export adds backends when the org is ready. - Should observability data feed back into agent instructions automatically (e.g., auto-adjusting prompts when false positive rates exceed a threshold), or should it only inform human-driven instruction changes? Automatic feedback creates the risk of instruction oscillation; human-only feedback is slower but more controlled. - How do we build community dashboards that are useful to contributors with different levels of technical depth — from "is the agent doing a good job on my repo" to "show me the trace of this specific review"? diff --git a/internal/cli/evalmeasure.go b/internal/cli/evalmeasure.go index 41119ce4a5..0e9f654547 100644 --- a/internal/cli/evalmeasure.go +++ b/internal/cli/evalmeasure.go @@ -1,12 +1,20 @@ package cli import ( + "context" "fmt" "os" + "path/filepath" + "strings" "github.com/spf13/cobra" + "github.com/fullsend-ai/fullsend/internal/config" "github.com/fullsend-ai/fullsend/internal/evalmeasure" + "github.com/fullsend-ai/fullsend/internal/fetch" + "github.com/fullsend-ai/fullsend/internal/forge" + gh "github.com/fullsend-ai/fullsend/internal/forge/github" + "github.com/fullsend-ai/fullsend/internal/harness" "github.com/fullsend-ai/fullsend/internal/ui" ) @@ -15,6 +23,9 @@ func newEvalMeasureCmd() *cobra.Command { telemetryPath string registryPath string outDir string + outputDir string + agent string + fullsendDir string ) cmd := &cobra.Command{ @@ -23,49 +34,189 @@ func newEvalMeasureCmd() *cobra.Command { Long: `Parse run-telemetry.jsonl, score with an agents measurement manifest, and write eval-measurements.jsonl beside the telemetry artifact. +The binary resolves the manifest (local FULLSEND_DIR override, else a +SHA-pinned fetch from fullsend-ai/agents — same pin, allowlist, hash, and +audit as harness fallback). Platform telemetry is the file at the top of +each run directory; nested iteration-N/output/ copies are ignored. + Remote backends are not selected by fullsend: scores are a portable local JSONL artifact. When portable OTLP score export lands, it will reuse the same OTEL_EXPORTER_OTLP_* configuration as agent traces (ADR 0050 / 0087). Exit 0 when scores fail — measurements are data, not gates. Non-zero only -on hard IO/parse errors. - -The --registry path is required (manifests live in fullsend-ai/agents, -e.g. eval/measurements/.yaml).`, +on hard IO/parse errors. Missing telemetry or manifest is a skip (exit 0).`, RunE: func(cmd *cobra.Command, args []string) error { - printer := ui.New(os.Stdout) + printer := ui.New(cmd.OutOrStdout()) printer.Header("Eval Measure") - results, err := evalmeasure.MeasureAndExport(cmd.Context(), telemetryPath, registryPath, outDir) + results, skipped, err := runEvalMeasure(cmd.Context(), printer, evalMeasureOpts{ + telemetryPath: telemetryPath, + registryPath: registryPath, + outDir: outDir, + outputDir: outputDir, + agent: agent, + fullsendDir: fullsendDir, + }) if err != nil { - printMeasurementResults(printer, results) + printMeasurementResults(printer, results, false) return err } - printMeasurementResults(printer, results) + if skipped { + return nil + } + printMeasurementResults(printer, results, true) return nil }, } - cmd.Flags().StringVar(&telemetryPath, "telemetry", "", "path to run-telemetry.jsonl (required)") - cmd.Flags().StringVar(®istryPath, "registry", "", "path to agents measurement manifest YAML (required)") + cmd.Flags().StringVar(&telemetryPath, "telemetry", "", "path to run-telemetry.jsonl (or use --output-dir)") + cmd.Flags().StringVar(®istryPath, "registry", "", "path to agents measurement manifest YAML (or use --agent)") cmd.Flags().StringVar(&outDir, "out-dir", "", "directory for eval-measurements.jsonl (default: telemetry directory)") - _ = cmd.MarkFlagRequired("telemetry") - _ = cmd.MarkFlagRequired("registry") + cmd.Flags().StringVar(&outputDir, "output-dir", "", "CI output base or runDir; scores only top-of-runDir telemetry") + cmd.Flags().StringVar(&agent, "agent", "", "agent name for manifest resolution") + cmd.Flags().StringVar(&fullsendDir, "fullsend-dir", "", "path to the .fullsend directory (local manifest override + fetch cache)") return cmd } -func printMeasurementResults(printer *ui.Printer, results []evalmeasure.EvaluationResult) { +type evalMeasureOpts struct { + telemetryPath string + registryPath string + outDir string + outputDir string + agent string + fullsendDir string +} + +func runEvalMeasure(ctx context.Context, printer *ui.Printer, opts evalMeasureOpts) ([]evalmeasure.EvaluationResult, bool, error) { + telemPaths, err := resolveEvalMeasureTelemetry(opts) + if err != nil { + return nil, false, err + } + if len(telemPaths) == 0 { + printer.StepInfo("No platform run-telemetry.jsonl at the top of a run directory; skipping eval measurements") + return nil, true, nil + } + + registry, err := resolveEvalMeasureRegistry(ctx, printer, opts) + if err != nil { + return nil, false, err + } + if registry == "" { + printer.StepInfo("No eval measurement manifest; skipping") + return nil, true, nil + } + + var all []evalmeasure.EvaluationResult + for _, p := range telemPaths { + results, err := evalmeasure.MeasureAndExport(ctx, p, registry, opts.outDir) + if err != nil { + return all, false, err + } + all = append(all, results...) + } + return all, false, nil +} + +func resolveEvalMeasureTelemetry(opts evalMeasureOpts) ([]string, error) { + if opts.telemetryPath != "" { + return []string{opts.telemetryPath}, nil + } + if opts.outputDir != "" { + return evalmeasure.FindPlatformTelemetry(opts.outputDir) + } + return nil, fmt.Errorf("either --telemetry or --output-dir is required") +} + +func resolveEvalMeasureRegistry(ctx context.Context, printer *ui.Printer, opts evalMeasureOpts) (string, error) { + if opts.registryPath != "" { + return opts.registryPath, nil + } + if opts.agent == "" { + return "", fmt.Errorf("either --registry or --agent is required") + } + agent, err := sanitizeMeasurementAgentName(opts.agent) + if err != nil { + return "", err + } + if opts.fullsendDir != "" { + local, err := localMeasurementManifest(opts.fullsendDir, agent) + if err != nil { + return "", err + } + if st, err := os.Stat(local); err == nil && !st.IsDir() { + printer.StepInfo("Using local measurement manifest " + local) + return local, nil + } + } + composeOpts, client := evalMeasureFetchContext(opts.fullsendDir, printer) + path, ok := tryAgentsRepoMeasurementManifest(ctx, agent, client, composeOpts, printer) + if !ok { + return "", nil + } + return path, nil +} + +func sanitizeMeasurementAgentName(agent string) (string, error) { + a := strings.ToLower(strings.TrimSpace(agent)) + if a == "" || strings.ContainsAny(a, `/\`) || strings.Contains(a, "..") { + return "", fmt.Errorf("invalid --agent name %q", agent) + } + return a, nil +} + +func localMeasurementManifest(fullsendDir, agent string) (string, error) { + rel := filepath.Join("eval", "measurements", agent+".yaml") + resolved := filepath.Clean(filepath.Join(fullsendDir, rel)) + if r, err := filepath.Rel(fullsendDir, resolved); err != nil || !filepath.IsLocal(r) { + return "", fmt.Errorf("agent name %q escapes fullsend directory", agent) + } + return resolved, nil +} + +func evalMeasureFetchContext(fullsendDir string, printer *ui.Printer) (harness.ComposeOpts, forge.Client) { + workspace := fullsendDir + if workspace == "" { + workspace = os.TempDir() + } + abs, err := filepath.Abs(workspace) + if err != nil { + abs = workspace + } + orgAllowlist := config.DefaultAllowedRemoteResources() + if fullsendDir != "" && printer != nil { + if orgCfg := tryLoadOrgConfig(filepath.Join(abs, "config.yaml"), printer); orgCfg != nil { + orgAllowlist = orgCfg.AllowedResources() + } + } + token, _ := resolveToken() + return harness.ComposeOpts{ + WorkspaceRoot: abs, + FetchPolicy: fetch.DefaultPolicy, + AuditLogPath: filepath.Join(abs, ".fullsend-cache", "fetch-audit.jsonl"), + OrgAllowlist: orgAllowlist, + GitToken: token, + }, gh.New(token) +} + +func printMeasurementResults(printer *ui.Printer, results []evalmeasure.EvaluationResult, wroteOK bool) { if len(results) == 0 { - printer.StepDone("No new measurements (already scored or no matching traces)") + if wroteOK { + printer.StepDone("No new measurements (already scored or no matching traces)") + } return } for _, r := range results { line := fmt.Sprintf("%s %s=%.2f (%s) %s", r.Version, r.Name, r.Value, r.Label, r.Explanation) - if r.Label == "pass" { + switch r.Label { + case evalmeasure.LabelPass: printer.StepDone(line) - } else { + case evalmeasure.LabelSkip: + printer.StepInfo(line) + default: printer.StepWarn(line) } } - printer.StepDone(fmt.Sprintf("Wrote %d measurement(s)", len(results))) + if wroteOK { + printer.StepDone(fmt.Sprintf("Wrote %d measurement(s)", len(results))) + } } diff --git a/internal/cli/evalmeasure_test.go b/internal/cli/evalmeasure_test.go index f5fc781b2b..ae5f8cbaac 100644 --- a/internal/cli/evalmeasure_test.go +++ b/internal/cli/evalmeasure_test.go @@ -2,17 +2,24 @@ package cli import ( "bytes" + "context" + "io" "os" "path/filepath" + "strings" "testing" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" + + "github.com/fullsend-ai/fullsend/internal/evalmeasure" + "github.com/fullsend-ai/fullsend/internal/telemetry" + "github.com/fullsend-ai/fullsend/internal/ui" ) func TestEvalMeasureCmd_ScoresFixture(t *testing.T) { out := t.TempDir() - telemetry := filepath.Join("..", "evalmeasure", "testdata", "complete.jsonl") + telemetryPath := filepath.Join("..", "evalmeasure", "testdata", "complete.jsonl") registry := filepath.Join("..", "evalmeasure", "testdata", "sample-registry.yaml") cmd := newRootCmd() @@ -21,7 +28,7 @@ func TestEvalMeasureCmd_ScoresFixture(t *testing.T) { cmd.SetErr(buf) cmd.SetArgs([]string{ "eval-measure", - "--telemetry", telemetry, + "--telemetry", telemetryPath, "--registry", registry, "--out-dir", out, }) @@ -31,6 +38,7 @@ func TestEvalMeasureCmd_ScoresFixture(t *testing.T) { b, err := os.ReadFile(filepath.Join(out, "eval-measurements.jsonl")) require.NoError(t, err) assert.Contains(t, string(b), `"name":"trace_fitness"`) + assert.Contains(t, buf.String(), "Wrote 1 measurement(s)") } func TestRootCommand_HasEvalMeasureSubcommand(t *testing.T) { @@ -54,3 +62,204 @@ func TestEvalMeasureCmd_MissingRequiredFlags(t *testing.T) { err := cmd.Execute() require.Error(t, err) } + +func TestEvalMeasureCmd_OutputDirIgnoresNestedTelemetry(t *testing.T) { + fsDir := t.TempDir() + outBase := t.TempDir() + runDir := filepath.Join(outBase, "agent-triage-1") + nested := filepath.Join(runDir, "iteration-1", "output") + require.NoError(t, os.MkdirAll(nested, 0o755)) + + good, err := os.ReadFile(filepath.Join("..", "evalmeasure", "testdata", "complete.jsonl")) + require.NoError(t, err) + require.NoError(t, os.WriteFile(filepath.Join(runDir, telemetry.TelemetryFile), good, 0o644)) + // Agent-planted copy: valid JSONL with a different trace id would produce + // a second row if scored. Nested path must be ignored. + planted := bytes.ReplaceAll(good, []byte("aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"), []byte("ffffffffffffffffffffffffffffffff")) + require.NoError(t, os.WriteFile(filepath.Join(nested, telemetry.TelemetryFile), planted, 0o644)) + + reg, err := os.ReadFile(filepath.Join("..", "evalmeasure", "testdata", "sample-registry.yaml")) + require.NoError(t, err) + require.NoError(t, os.MkdirAll(filepath.Join(fsDir, "eval", "measurements"), 0o755)) + require.NoError(t, os.WriteFile(filepath.Join(fsDir, "eval", "measurements", "triage.yaml"), reg, 0o644)) + + cmd := newRootCmd() + buf := &bytes.Buffer{} + cmd.SetOut(buf) + cmd.SetErr(buf) + cmd.SetArgs([]string{ + "eval-measure", + "--agent", "triage", + "--fullsend-dir", fsDir, + "--output-dir", outBase, + }) + require.NoError(t, cmd.Execute()) + + b, err := os.ReadFile(filepath.Join(runDir, evalmeasure.MeasurementsFile)) + require.NoError(t, err) + assert.Contains(t, string(b), `"trace_id":"aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"`) + assert.NotContains(t, string(b), "ffffffffffffffffffffffffffffffff") +} + +func TestEvalMeasureCmd_ErrorDoesNotPrintWrote(t *testing.T) { + out := filepath.Join(t.TempDir(), "not-a-dir") + require.NoError(t, os.WriteFile(out, []byte("x"), 0o644)) + + cmd := newRootCmd() + buf := &bytes.Buffer{} + cmd.SetOut(buf) + cmd.SetErr(buf) + cmd.SetArgs([]string{ + "eval-measure", + "--telemetry", filepath.Join("..", "evalmeasure", "testdata", "complete.jsonl"), + "--registry", filepath.Join("..", "evalmeasure", "testdata", "sample-registry.yaml"), + "--out-dir", out, + }) + err := cmd.Execute() + require.Error(t, err) + assert.NotContains(t, buf.String(), "Wrote") +} + +func TestEvalMeasureCmd_SkipWhenNoTelemetry(t *testing.T) { + cmd := newRootCmd() + buf := &bytes.Buffer{} + cmd.SetOut(buf) + cmd.SetErr(buf) + cmd.SetArgs([]string{ + "eval-measure", + "--agent", "triage", + "--output-dir", t.TempDir(), + }) + require.NoError(t, cmd.Execute()) + assert.Contains(t, buf.String(), "skipping eval measurements") + assert.NotContains(t, buf.String(), "Wrote") +} + +func TestActionYML_EvalMeasureNoFloatingV0Curl(t *testing.T) { + b, err := os.ReadFile(filepath.Join("..", "..", "action.yml")) + require.NoError(t, err) + s := string(b) + assert.NotContains(t, s, "raw.githubusercontent.com/fullsend-ai/agents/v0/eval/measurements") + assert.NotContains(t, s, `find "${GITHUB_WORKSPACE}/output" -name run-telemetry.jsonl`) + idx := strings.Index(s, "name: Eval measurements") + require.Greater(t, idx, 0) + step := s[idx:] + if end := strings.Index(step, "name: Upload fullsend artifacts"); end > 0 { + step = step[:end] + } + assert.Contains(t, step, "eval-measure") + assert.Contains(t, step, "--output-dir") + assert.Contains(t, step, "--agent") + assert.Contains(t, step, "continue-on-error: true") + assert.Contains(t, step, "if: always()") + assert.NotContains(t, step, "curl ") + assert.NotContains(t, step, "Authorization: Bearer") +} + +func TestPlatformTelemetryFileMatchesRecorder(t *testing.T) { + assert.Equal(t, telemetry.TelemetryFile, evalmeasure.PlatformTelemetryFile) +} + +func TestEvalMeasureFetchContext(t *testing.T) { + printer := ui.New(io.Discard) + opts, client := evalMeasureFetchContext("", printer) + require.NotNil(t, client) + assert.NotEmpty(t, opts.OrgAllowlist) + assert.NotEmpty(t, opts.WorkspaceRoot) + + dir := t.TempDir() + opts2, _ := evalMeasureFetchContext(dir, printer) + assert.Contains(t, opts2.AuditLogPath, ".fullsend-cache") + abs, err := filepath.Abs(dir) + require.NoError(t, err) + assert.Equal(t, abs, opts2.WorkspaceRoot) +} + +func TestResolveEvalMeasureRegistry_LocalOverride(t *testing.T) { + fsDir := t.TempDir() + require.NoError(t, os.MkdirAll(filepath.Join(fsDir, "eval", "measurements"), 0o755)) + local := filepath.Join(fsDir, "eval", "measurements", "triage.yaml") + require.NoError(t, os.WriteFile(local, []byte("agent: triage\n"), 0o644)) + got, err := resolveEvalMeasureRegistry(context.Background(), ui.New(io.Discard), evalMeasureOpts{ + agent: "triage", + fullsendDir: fsDir, + }) + require.NoError(t, err) + assert.Equal(t, local, got) +} + +func TestResolveEvalMeasureRegistry_UnknownAgentSkipsRemote(t *testing.T) { + got, err := resolveEvalMeasureRegistry(context.Background(), ui.New(io.Discard), evalMeasureOpts{ + agent: "not-a-stock-agent", + fullsendDir: t.TempDir(), + }) + require.NoError(t, err) + assert.Empty(t, got) +} + +func TestResolveEvalMeasureRegistry_RequiresAgentOrRegistry(t *testing.T) { + _, err := resolveEvalMeasureRegistry(context.Background(), ui.New(io.Discard), evalMeasureOpts{}) + require.Error(t, err) + assert.Contains(t, err.Error(), "--registry or --agent") +} + +func TestSanitizeMeasurementAgentName(t *testing.T) { + got, err := sanitizeMeasurementAgentName(" Triage ") + require.NoError(t, err) + assert.Equal(t, "triage", got) + + for _, bad := range []string{"", " ", "../evil", "foo/bar", `foo\bar`, "a..b"} { + _, err := sanitizeMeasurementAgentName(bad) + require.Error(t, err, "agent %q", bad) + } +} + +func TestLocalMeasurementManifestStaysUnderFullsendDir(t *testing.T) { + fsDir := t.TempDir() + got, err := localMeasurementManifest(fsDir, "triage") + require.NoError(t, err) + assert.Equal(t, filepath.Join(fsDir, "eval", "measurements", "triage.yaml"), got) +} + +func TestResolveEvalMeasureRegistry_RejectsPathAgent(t *testing.T) { + _, err := resolveEvalMeasureRegistry(context.Background(), ui.New(io.Discard), evalMeasureOpts{ + agent: "../../../tmp/evil", + fullsendDir: t.TempDir(), + }) + require.Error(t, err) + assert.Contains(t, err.Error(), "invalid --agent") +} + +func TestEvalMeasureCmd_TelemetryWithoutRegistry(t *testing.T) { + cmd := newRootCmd() + buf := &bytes.Buffer{} + cmd.SetOut(buf) + cmd.SetErr(buf) + cmd.SetArgs([]string{ + "eval-measure", + "--telemetry", filepath.Join("..", "evalmeasure", "testdata", "complete.jsonl"), + }) + err := cmd.Execute() + require.Error(t, err) + assert.Contains(t, err.Error(), "--registry or --agent") +} + +func TestPrintMeasurementResults_SkipAndNoWroteOnError(t *testing.T) { + var buf bytes.Buffer + p := ui.New(&buf) + printMeasurementResults(p, []evalmeasure.EvaluationResult{{ + Name: "trace_fitness", + Version: "em-001@1", + Label: evalmeasure.LabelSkip, + }}, true) + assert.Contains(t, buf.String(), "skip") + assert.Contains(t, buf.String(), "Wrote 1 measurement(s)") + + buf.Reset() + printMeasurementResults(p, []evalmeasure.EvaluationResult{{ + Name: "trace_fitness", + Version: "em-001@1", + Label: evalmeasure.LabelFail, + }}, false) + assert.NotContains(t, buf.String(), "Wrote") +} diff --git a/internal/cli/run.go b/internal/cli/run.go index 628c438af0..526d0f50fd 100644 --- a/internal/cli/run.go +++ b/internal/cli/run.go @@ -26,6 +26,7 @@ import ( "github.com/fullsend-ai/fullsend/internal/binary" "github.com/fullsend-ai/fullsend/internal/config" "github.com/fullsend-ai/fullsend/internal/envfile" + "github.com/fullsend-ai/fullsend/internal/evalmeasure" "github.com/fullsend-ai/fullsend/internal/fetch" "github.com/fullsend-ai/fullsend/internal/fetchsvc" "github.com/fullsend-ai/fullsend/internal/forge" @@ -2391,7 +2392,7 @@ func resolveWorkItemID() string { if prNum := strings.TrimSpace(os.Getenv("PR_NUMBER")); prNum != "" { return prNum } - return "unknown" + return evalmeasure.UnknownSentinel } // telemetryExitCode maps the run's final state to the exit code recorded on @@ -3726,11 +3727,37 @@ func tryAgentsRepoFallback(ctx context.Context, agentName string, forgeClient fo if !defaultAgentsRepoKnownAgents[normalizedName] { return "", nil, false } - if composeOpts.FetchPolicy.Offline { + path, dep, ok := fetchPinnedAgentsRepoFile(ctx, "harness/"+normalizedName+".yaml", forgeClient, composeOpts, printer, "agent "+agentName) + if !ok { return "", nil, false } + return path, []harness.Dependency{dep}, true +} + +// tryAgentsRepoMeasurementManifest SHA-pins eval/measurements/.yaml +// from fullsend-ai/agents (same pin, allowlist, hash, and audit as harness +// fallback). Missing manifests (HTTP 404) skip; network errors warn. +func tryAgentsRepoMeasurementManifest(ctx context.Context, agentName string, forgeClient forge.Client, composeOpts harness.ComposeOpts, printer *ui.Printer) (string, bool) { + normalizedName := strings.ToLower(agentName) + if !defaultAgentsRepoKnownAgents[normalizedName] { + return "", false + } + path, _, ok := fetchPinnedAgentsRepoFile(ctx, "eval/measurements/"+normalizedName+".yaml", forgeClient, composeOpts, printer, "eval measurement manifest for "+agentName) + return path, ok +} + +// fetchPinnedAgentsRepoFile resolves tags/DefaultUpstreamRef to a commit SHA +// and fetches relPath from fullsend-ai/agents. All errors are non-fatal. +func fetchPinnedAgentsRepoFile(ctx context.Context, relPath string, forgeClient forge.Client, composeOpts harness.ComposeOpts, printer *ui.Printer, noun string) (string, harness.Dependency, bool) { + var none harness.Dependency + if strings.Contains(relPath, "..") || strings.HasPrefix(relPath, "/") { + return "", none, false + } + if composeOpts.FetchPolicy.Offline { + return "", none, false + } if forgeClient == nil { - return "", nil, false + return "", none, false } allowlist := composeOpts.OrgAllowlist @@ -3739,30 +3766,34 @@ func tryAgentsRepoFallback(ctx context.Context, agentName string, forgeClient fo tagSHA, err := forgeClient.GetRef(ctx, defaultAgentsRepoOwner, defaultAgentsRepoName, tagRef) if err != nil { printer.StepWarn(fmt.Sprintf("Could not resolve %s/%s@%s: %v", defaultAgentsRepoOwner, defaultAgentsRepoName, config.DefaultUpstreamRef, err)) - return "", nil, false + return "", none, false } if !commitSHAPattern.MatchString(tagSHA) { printer.StepWarn(fmt.Sprintf("Invalid SHA from %s/%s@%s: %q", defaultAgentsRepoOwner, defaultAgentsRepoName, config.DefaultUpstreamRef, tagSHA)) - return "", nil, false + return "", none, false } - rawURL := defaultAgentsRepoURLPrefix + tagSHA + "/harness/" + normalizedName + ".yaml" + rawURL := defaultAgentsRepoURLPrefix + tagSHA + "/" + relPath if harness.MatchingAllowedPrefixInList(rawURL, allowlist) == "" { - printer.StepWarn(fmt.Sprintf("Agents repo fallback skipped for %s: URL not in allowed_remote_resources", agentName)) - return "", nil, false + printer.StepWarn(fmt.Sprintf("Agents repo fallback skipped for %s: URL not in allowed_remote_resources", noun)) + return "", none, false } shortSHA := tagSHA if len(shortSHA) > 12 { shortSHA = shortSHA[:12] } - printer.StepStart(fmt.Sprintf("Fetching agent %s from %s/%s@%s", agentName, defaultAgentsRepoOwner, defaultAgentsRepoName, shortSHA)) + printer.StepStart(fmt.Sprintf("Fetching %s from %s/%s@%s", noun, defaultAgentsRepoOwner, defaultAgentsRepoName, shortSHA)) content, err := fetch.FetchURL(ctx, rawURL, composeOpts.FetchPolicy) if err != nil { - printer.StepWarn(fmt.Sprintf("Failed to fetch agent %s from agents repo: %v", agentName, err)) - return "", nil, false + if isFetchHTTPStatus(err, http.StatusNotFound) { + printer.StepInfo(fmt.Sprintf("No %s at %s/%s@%s (HTTP 404); skipping", noun, defaultAgentsRepoOwner, defaultAgentsRepoName, shortSHA)) + } else { + printer.StepWarn(fmt.Sprintf("Failed to fetch %s from agents repo: %v", noun, err)) + } + return "", none, false } // Content is fetched once and used directly — no self-referential hash @@ -3773,13 +3804,13 @@ func tryAgentsRepoFallback(ctx context.Context, agentName string, forgeClient fo if err := fetch.CachePut(composeOpts.WorkspaceRoot, rawURL, content); err != nil { printer.StepWarn(fmt.Sprintf("Failed to cache agents repo content: %v", err)) - return "", nil, false + return "", none, false } cachePath, err := fetch.CachePath(composeOpts.WorkspaceRoot, contentHash) if err != nil { - printer.StepWarn(fmt.Sprintf("Failed to resolve cache path for agent %s: %v", agentName, err)) - return "", nil, false + printer.StepWarn(fmt.Sprintf("Failed to resolve cache path for %s: %v", noun, err)) + return "", none, false } localPath := filepath.Join(cachePath, "content") @@ -3806,8 +3837,13 @@ func tryAgentsRepoFallback(ctx context.Context, agentName string, forgeClient fo Type: "file", } - printer.StepDone(fmt.Sprintf("Agent %s resolved from %s/%s@%s", agentName, defaultAgentsRepoOwner, defaultAgentsRepoName, config.DefaultUpstreamRef)) - return localPath, []harness.Dependency{dep}, true + printer.StepDone(fmt.Sprintf("%s resolved from %s/%s@%s", noun, defaultAgentsRepoOwner, defaultAgentsRepoName, config.DefaultUpstreamRef)) + return localPath, dep, true +} + +func isFetchHTTPStatus(err error, code int) bool { + var httpErr fetch.HTTPStatusError + return errors.As(err, &httpErr) && httpErr.Status == code } // containedLocalPath resolves a relative source path against baseDir and diff --git a/internal/cli/run_test.go b/internal/cli/run_test.go index 42ae025d29..35d4c8211a 100644 --- a/internal/cli/run_test.go +++ b/internal/cli/run_test.go @@ -1354,6 +1354,133 @@ func TestTryAgentsRepoFallback_SuccessPath(t *testing.T) { assert.NotEmpty(t, deps[0].SHA256) } +func TestTryAgentsRepoMeasurementManifest_Success(t *testing.T) { + manifest := []byte("agent: triage\nmeasurements:\n - id: em-001\n scorer: trace_fitness\n version: 1\n") + fakeSHA := "abcdef1234567890abcdef1234567890abcdef12" + + srv := httptest.NewTLSServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + expectedPath := "/" + fakeSHA + "/eval/measurements/triage.yaml" + if r.URL.Path == expectedPath { + w.WriteHeader(http.StatusOK) + _, _ = w.Write(manifest) + } else { + w.WriteHeader(http.StatusNotFound) + } + })) + t.Cleanup(srv.Close) + + hostPort := strings.TrimPrefix(srv.URL, "https://") + hostname, port, _ := net.SplitHostPort(hostPort) + + tlsCfg := srv.TLS.Clone() + tlsCfg.InsecureSkipVerify = true + policy := fetch.NewTestPolicy(tlsCfg, []string{hostname}, []string{port}) + + orig := defaultAgentsRepoURLPrefix + defaultAgentsRepoURLPrefix = srv.URL + "/" + t.Cleanup(func() { defaultAgentsRepoURLPrefix = orig }) + + fakeClient := forge.NewFakeClient() + fakeClient.Refs["fullsend-ai/agents/tags/v0"] = fakeSHA + + printer := ui.New(io.Discard) + opts := harness.ComposeOpts{ + WorkspaceRoot: t.TempDir(), + FetchPolicy: policy, + OrgAllowlist: []string{srv.URL + "/"}, + } + + path, ok := tryAgentsRepoMeasurementManifest(context.Background(), "triage", fakeClient, opts, printer) + require.True(t, ok) + got, err := os.ReadFile(path) + require.NoError(t, err) + assert.Equal(t, manifest, got) + assert.Contains(t, path, "content") +} + +func TestTryAgentsRepoMeasurementManifest_HTTP404(t *testing.T) { + fakeSHA := "abcdef1234567890abcdef1234567890abcdef12" + srv := httptest.NewTLSServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(http.StatusNotFound) + })) + t.Cleanup(srv.Close) + + hostPort := strings.TrimPrefix(srv.URL, "https://") + hostname, port, _ := net.SplitHostPort(hostPort) + tlsCfg := srv.TLS.Clone() + tlsCfg.InsecureSkipVerify = true + policy := fetch.NewTestPolicy(tlsCfg, []string{hostname}, []string{port}) + + orig := defaultAgentsRepoURLPrefix + defaultAgentsRepoURLPrefix = srv.URL + "/" + t.Cleanup(func() { defaultAgentsRepoURLPrefix = orig }) + + fakeClient := forge.NewFakeClient() + fakeClient.Refs["fullsend-ai/agents/tags/v0"] = fakeSHA + + var buf bytes.Buffer + printer := ui.New(&buf) + opts := harness.ComposeOpts{ + WorkspaceRoot: t.TempDir(), + FetchPolicy: policy, + OrgAllowlist: []string{srv.URL + "/"}, + } + + _, ok := tryAgentsRepoMeasurementManifest(context.Background(), "triage", fakeClient, opts, printer) + assert.False(t, ok) + assert.Contains(t, buf.String(), "HTTP 404") + assert.NotContains(t, buf.String(), "Failed to fetch") +} + +func TestTryAgentsRepoMeasurementManifest_NetworkFailure(t *testing.T) { + fakeSHA := "abcdef1234567890abcdef1234567890abcdef12" + srv := httptest.NewTLSServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + http.Error(w, "boom", http.StatusBadGateway) + })) + t.Cleanup(srv.Close) + + hostPort := strings.TrimPrefix(srv.URL, "https://") + hostname, port, _ := net.SplitHostPort(hostPort) + tlsCfg := srv.TLS.Clone() + tlsCfg.InsecureSkipVerify = true + policy := fetch.NewTestPolicy(tlsCfg, []string{hostname}, []string{port}) + + orig := defaultAgentsRepoURLPrefix + defaultAgentsRepoURLPrefix = srv.URL + "/" + t.Cleanup(func() { defaultAgentsRepoURLPrefix = orig }) + + fakeClient := forge.NewFakeClient() + fakeClient.Refs["fullsend-ai/agents/tags/v0"] = fakeSHA + + var buf bytes.Buffer + printer := ui.New(&buf) + opts := harness.ComposeOpts{ + WorkspaceRoot: t.TempDir(), + FetchPolicy: policy, + OrgAllowlist: []string{srv.URL + "/"}, + } + + _, ok := tryAgentsRepoMeasurementManifest(context.Background(), "triage", fakeClient, opts, printer) + assert.False(t, ok) + assert.Contains(t, buf.String(), "Failed to fetch") + assert.NotContains(t, buf.String(), "HTTP 404") +} + +func TestTryAgentsRepoMeasurementManifest_UnknownAgent(t *testing.T) { + fakeClient := forge.NewFakeClient() + printer := ui.New(io.Discard) + _, ok := tryAgentsRepoMeasurementManifest(context.Background(), "custom-agent", fakeClient, harness.ComposeOpts{}, printer) + assert.False(t, ok) +} + +func TestIsFetchHTTPStatus(t *testing.T) { + assert.True(t, isFetchHTTPStatus(fetch.HTTPStatusError{Status: 404}, 404)) + assert.True(t, isFetchHTTPStatus(fmt.Errorf("wrap: %w", fetch.HTTPStatusError{Status: 404}), 404)) + assert.False(t, isFetchHTTPStatus(fetch.HTTPStatusError{Status: 500}, 404)) + assert.False(t, isFetchHTTPStatus(fmt.Errorf("fetch: request failed: connection refused"), 404)) + assert.False(t, isFetchHTTPStatus(nil, 404)) +} + func TestTryAgentsRepoFallback_AuditLog(t *testing.T) { harnessContent := []byte("agent: agents/triage.md\nrole: test\n") fakeSHA := "abcdef1234567890abcdef1234567890abcdef12" diff --git a/internal/cli/telemetry_run_test.go b/internal/cli/telemetry_run_test.go index d00dbe5f1d..e69b04b5ed 100644 --- a/internal/cli/telemetry_run_test.go +++ b/internal/cli/telemetry_run_test.go @@ -17,6 +17,7 @@ import ( "go.opentelemetry.io/otel/sdk/trace/tracetest" "go.opentelemetry.io/otel/trace" + "github.com/fullsend-ai/fullsend/internal/evalmeasure" agentruntime "github.com/fullsend-ai/fullsend/internal/runtime" "github.com/fullsend-ai/fullsend/internal/security" ) @@ -102,7 +103,7 @@ func TestResolveWorkItemID(t *testing.T) { }, { name: "unknown when nothing is set", - want: "unknown", + want: evalmeasure.UnknownSentinel, }, } for _, tc := range cases { diff --git a/internal/evalmeasure/find.go b/internal/evalmeasure/find.go new file mode 100644 index 0000000000..0c72536a56 --- /dev/null +++ b/internal/evalmeasure/find.go @@ -0,0 +1,43 @@ +package evalmeasure + +import ( + "os" + "path/filepath" +) + +// PlatformTelemetryFile is the host recorder's JSONL at the top of runDir. +// It matches internal/telemetry.TelemetryFile. Nested copies under +// iteration-N/output/ are agent-writable and must not be scored. +const PlatformTelemetryFile = "run-telemetry.jsonl" + +// FindPlatformTelemetry returns run-telemetry.jsonl files that sit at the +// top of outputDir itself (when outputDir is a runDir) or at the top of +// each immediate child directory (when outputDir is the CI output base). +// It never walks deeper, so an agent-planted +// iteration-N/output/run-telemetry.jsonl is ignored. +func FindPlatformTelemetry(outputDir string) ([]string, error) { + direct := filepath.Join(outputDir, PlatformTelemetryFile) + if st, err := os.Stat(direct); err == nil && !st.IsDir() { + // outputDir is a runDir: score only the platform file at the top. + return []string{direct}, nil + } + + entries, err := os.ReadDir(outputDir) + if err != nil { + if os.IsNotExist(err) { + return nil, nil + } + return nil, err + } + var out []string + for _, e := range entries { + if !e.IsDir() { + continue + } + p := filepath.Join(outputDir, e.Name(), PlatformTelemetryFile) + if st, err := os.Stat(p); err == nil && !st.IsDir() { + out = append(out, p) + } + } + return out, nil +} diff --git a/internal/evalmeasure/find_test.go b/internal/evalmeasure/find_test.go new file mode 100644 index 0000000000..a308299b91 --- /dev/null +++ b/internal/evalmeasure/find_test.go @@ -0,0 +1,74 @@ +package evalmeasure + +import ( + "os" + "path/filepath" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestFindPlatformTelemetry_IgnoresNestedIterationCopy(t *testing.T) { + t.Parallel() + outputDir := t.TempDir() + runDir := filepath.Join(outputDir, "agent-review-3311-1") + nested := filepath.Join(runDir, "iteration-1", "output") + require.NoError(t, os.MkdirAll(nested, 0o755)) + + platform := filepath.Join(runDir, PlatformTelemetryFile) + planted := filepath.Join(nested, PlatformTelemetryFile) + require.NoError(t, os.WriteFile(platform, []byte("platform\n"), 0o644)) + require.NoError(t, os.WriteFile(planted, []byte("planted\n"), 0o644)) + + got, err := FindPlatformTelemetry(outputDir) + require.NoError(t, err) + require.Equal(t, []string{platform}, got) +} + +func TestFindPlatformTelemetry_RunDirIgnoresImmediateChild(t *testing.T) { + t.Parallel() + runDir := t.TempDir() + child := filepath.Join(runDir, "planted-child") + require.NoError(t, os.MkdirAll(child, 0o755)) + platform := filepath.Join(runDir, PlatformTelemetryFile) + require.NoError(t, os.WriteFile(platform, []byte("platform\n"), 0o644)) + require.NoError(t, os.WriteFile(filepath.Join(child, PlatformTelemetryFile), []byte("planted\n"), 0o644)) + + got, err := FindPlatformTelemetry(runDir) + require.NoError(t, err) + require.Equal(t, []string{platform}, got) +} + +func TestFindPlatformTelemetry_RunDirDirect(t *testing.T) { + t.Parallel() + runDir := t.TempDir() + nested := filepath.Join(runDir, "iteration-1", "output") + require.NoError(t, os.MkdirAll(nested, 0o755)) + platform := filepath.Join(runDir, PlatformTelemetryFile) + require.NoError(t, os.WriteFile(platform, []byte("platform\n"), 0o644)) + require.NoError(t, os.WriteFile(filepath.Join(nested, PlatformTelemetryFile), []byte("planted\n"), 0o644)) + + got, err := FindPlatformTelemetry(runDir) + require.NoError(t, err) + require.Equal(t, []string{platform}, got) +} + +func TestFindPlatformTelemetry_MissingDir(t *testing.T) { + t.Parallel() + got, err := FindPlatformTelemetry(filepath.Join(t.TempDir(), "nope")) + require.NoError(t, err) + assert.Empty(t, got) +} + +func TestFindPlatformTelemetry_EmptyWhenOnlyNested(t *testing.T) { + t.Parallel() + outputDir := t.TempDir() + nested := filepath.Join(outputDir, "agent-x", "iteration-1", "output") + require.NoError(t, os.MkdirAll(nested, 0o755)) + require.NoError(t, os.WriteFile(filepath.Join(nested, PlatformTelemetryFile), []byte("planted\n"), 0o644)) + + got, err := FindPlatformTelemetry(outputDir) + require.NoError(t, err) + assert.Empty(t, got) +} diff --git a/internal/evalmeasure/fitness.go b/internal/evalmeasure/fitness.go index 506ac581b8..ef1bada745 100644 --- a/internal/evalmeasure/fitness.go +++ b/internal/evalmeasure/fitness.go @@ -4,7 +4,16 @@ import ( "strings" ) -const ScorerFitness = "trace_fitness" +const ( + ScorerFitness = "trace_fitness" + LabelPass = "pass" + LabelFail = "fail" + LabelSkip = "skip" + + // UnknownSentinel is the CLI fallback for missing work-item / agent + // identity (resolveWorkItemID). Scorers must treat it as absent. + UnknownSentinel = "unknown" +) // ScoreFitness implements EM-001 Trace Fitness against a single trace. func ScoreFitness(tr Trace) EvaluationResult { @@ -21,8 +30,29 @@ func ScoreFitnessNamed(tr Trace, evalName, version string) EvaluationResult { } run, hasRun := tr.SpanByName("run") + if hasRun { + if skipped, ok := run.AttrBool("fullsend.prescript.skipped"); ok && skipped { + spanID := run.SpanID + workItem, _ := run.AttrString("fullsend.work_item_id") + reason := "pre-script skipped run; excluded from trace_fitness" + if r, ok := run.AttrString("fullsend.prescript.skip_reason"); ok && r != "" { + reason = reason + ": " + r + } + return EvaluationResult{ + Name: evalName, + Label: LabelSkip, + Explanation: reason, + TraceID: tr.TraceID, + SpanID: spanID, + WorkItemID: workItem, + Agent: tr.AgentName(), + Version: version, + } + } + } agents := tr.SpansByName("agent") _, hasSandbox := tr.SpanByName("sandbox_create") + costOK, costMissing := costToolsTurnsDetail(run, agents) type check struct { id string @@ -31,13 +61,14 @@ func ScoreFitnessNamed(tr Trace, evalName, version string) EvaluationResult { checks := []check{ {"span_tree", hasRun && hasSandbox && len(agents) >= 1}, {"identity", identityOK(run, agents)}, - // "unknown" is the CLI sentinel when no ISSUE_*/GITHUB_ISSUE_URL is set - // (common for review, which wires PR_NUMBER / GITHUB_PR_URL instead). + // UnknownSentinel is the CLI fallback when no issue- or PR-shaped env is set. + // Review jobs that have PR_NUMBER / GITHUB_PR_URL should not hit this + // after #5622; a remaining unknown is a real fitness fail. {"work_item", workItemOK(run)}, {"operation", attrNonEmpty(run, "gen_ai.operation.name")}, {"model", modelOK(run, agents)}, {"usage", usageOK(run, agents)}, - {"cost_tools_turns", costToolsTurnsOK(run, agents)}, + {"cost_tools_turns", costOK}, {"exit", hasExit(run)}, } @@ -50,14 +81,18 @@ func ScoreFitnessNamed(tr Trace, evalName, version string) EvaluationResult { details = append(details, c.id+"=pass") } else { failed = append(failed, c.id) - details = append(details, c.id+"=fail") + if c.id == "cost_tools_turns" && len(costMissing) > 0 { + details = append(details, c.id+"=fail["+strings.Join(costMissing, ",")+"]") + } else { + details = append(details, c.id+"=fail") + } } } total := len(checks) value := float64(passed) / float64(total) - label := "fail" + label := LabelFail if value == 1.0 { - label = "pass" + label = LabelPass } expl := strings.Join(details, ", ") if len(failed) > 0 { @@ -87,7 +122,7 @@ func ScoreFitnessNamed(tr Trace, evalName, version string) EvaluationResult { func identityOK(run Span, agents []Span) bool { fa, okFA := run.AttrString("fullsend.agent") - if !okFA || fa == "" || fa == "unknown" { + if !okFA || fa == "" || fa == UnknownSentinel { return false } if ga, ok := run.AttrString("gen_ai.agent.name"); ok && ga == fa { @@ -103,7 +138,7 @@ func identityOK(run Span, agents []Span) bool { func workItemOK(run Span) bool { v, ok := run.AttrString("fullsend.work_item_id") - return ok && v != "" && v != "unknown" + return ok && v != "" && v != UnknownSentinel } func attrNonEmpty(s Span, key string) bool { @@ -150,7 +185,7 @@ func usageOK(run Span, agents []Span) bool { return false } -func costToolsTurnsOK(run Span, agents []Span) bool { +func costToolsTurnsDetail(run Span, agents []Span) (bool, []string) { hasCost := false hasTools := false if _, ok := run.AttrFloat("fullsend.cost_usd"); ok { @@ -171,15 +206,19 @@ func costToolsTurnsOK(run Span, agents []Span) bool { } } } - if !hasCost || !hasTools { - return false + var missing []string + if !hasCost { + missing = append(missing, "cost") + } + if !hasTools { + missing = append(missing, "tool_calls") } if iters, ok := run.AttrInt("fullsend.iterations"); ok && iters > 0 { if _, ok := run.AttrInt("fullsend.num_turns"); !ok { - return false + missing = append(missing, "num_turns") } } - return true + return len(missing) == 0, missing } func hasExit(run Span) bool { diff --git a/internal/evalmeasure/parse.go b/internal/evalmeasure/parse.go index d42ea9be9b..b94e5d6764 100644 --- a/internal/evalmeasure/parse.go +++ b/internal/evalmeasure/parse.go @@ -66,7 +66,8 @@ func ParseTelemetryFile(path string) ([]Trace, error) { } var doc otlpTracesData if err := json.Unmarshal(line, &doc); err != nil { - return nil, fmt.Errorf("line %d: %w", lineNo, err) + // Skip one truncated or corrupt JSONL line; keep the rest of the file. + continue } for _, rs := range doc.ResourceSpans { for _, ss := range rs.ScopeSpans { diff --git a/internal/evalmeasure/parse_test.go b/internal/evalmeasure/parse_test.go index 129c0760d9..a39dcd1912 100644 --- a/internal/evalmeasure/parse_test.go +++ b/internal/evalmeasure/parse_test.go @@ -38,13 +38,32 @@ func TestParseTelemetryFile_MergesLinesSameTrace(t *testing.T) { assert.True(t, ok) } -func TestParseTelemetryFile_InvalidLine(t *testing.T) { +func TestParseTelemetryFile_InvalidLineSkipped(t *testing.T) { t.Parallel() dir := t.TempDir() path := filepath.Join(dir, "bad.jsonl") require.NoError(t, os.WriteFile(path, []byte("not-json\n"), 0o644)) - _, err := ParseTelemetryFile(path) - require.Error(t, err) + traces, err := ParseTelemetryFile(path) + require.NoError(t, err, "a truncated/corrupt line must not fail the whole file") + assert.Empty(t, traces) +} + +func TestParseTelemetryFile_TruncatedLineDoesNotDiscardFile(t *testing.T) { + t.Parallel() + good, err := os.ReadFile(filepath.Join("testdata", "complete.jsonl")) + require.NoError(t, err) + dir := t.TempDir() + path := filepath.Join(dir, "mixed.jsonl") + body := string(good) + if body != "" && body[len(body)-1] != '\n' { + body += "\n" + } + body += "{\"resourceSpans\":[{\"scopeSpans\":[{\"spans\":[{\"traceId\":\"truncated\n" + require.NoError(t, os.WriteFile(path, []byte(body), 0o644)) + traces, err := ParseTelemetryFile(path) + require.NoError(t, err) + require.Len(t, traces, 1) + assert.Equal(t, "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", traces[0].TraceID) } func TestParseTelemetryFile_MissingFile(t *testing.T) { diff --git a/internal/evalmeasure/registry.go b/internal/evalmeasure/registry.go index c83bb9d738..781b3ce025 100644 --- a/internal/evalmeasure/registry.go +++ b/internal/evalmeasure/registry.go @@ -64,8 +64,14 @@ func (m MeasurementSpec) evalName() string { } // ScoreTrace runs enabled measurements for traces matching the manifest agent. +// An empty or UnknownSentinel agent name still scores so identity=fail is +// recorded (EM-001 exists to catch missing identity). A different, known +// agent name is skipped. Unknown scorer strings write a skip row rather +// than silent-no-op or a fail that mixes into pass-rate (a newer agents@v0 +// manifest can name a scorer this binary does not implement yet). func ScoreTrace(tr Trace, reg Registry) []EvaluationResult { - if tr.AgentName() != reg.Agent { + name := tr.AgentName() + if name != "" && name != UnknownSentinel && name != reg.Agent { return nil } var out []EvaluationResult @@ -74,7 +80,14 @@ func ScoreTrace(tr Trace, reg Registry) []EvaluationResult { case ScorerFitness: out = append(out, ScoreFitnessNamed(tr, m.evalName(), m.versionString())) default: - // Unknown scorers are skipped (forward-compatible). + out = append(out, EvaluationResult{ + Name: m.evalName(), + Label: LabelSkip, + Explanation: "unknown scorer: " + m.Scorer, + TraceID: tr.TraceID, + Agent: name, + Version: m.versionString(), + }) } } return out diff --git a/internal/evalmeasure/run_test.go b/internal/evalmeasure/run_test.go index 01aa330c54..f370f00805 100644 --- a/internal/evalmeasure/run_test.go +++ b/internal/evalmeasure/run_test.go @@ -94,3 +94,41 @@ func TestMeasureAndExport_CancelledContext(t *testing.T) { ) require.Error(t, err) } + +func TestMeasureFile_PrescriptSkippedRecordsSkip(t *testing.T) { + out := t.TempDir() + results, err := MeasureFile( + filepath.Join("testdata", "prescript-skipped.jsonl"), + filepath.Join("testdata", "sample-registry.yaml"), + out, + ) + require.NoError(t, err) + require.Len(t, results, 1) + assert.Equal(t, LabelSkip, results[0].Label) + assert.NotEqual(t, LabelFail, results[0].Label) + + b, err := os.ReadFile(filepath.Join(out, MeasurementsFile)) + require.NoError(t, err) + assert.Contains(t, string(b), `"label":"skip"`) + assert.NotContains(t, string(b), `"label":"fail"`) +} + +func TestMeasureFile_EmptyIdentityPersistsFailRow(t *testing.T) { + dir := t.TempDir() + telem := filepath.Join(dir, "run-telemetry.jsonl") + // Minimal OTLP line: run span with no agent identity. + line := `{"resourceSpans":[{"scopeSpans":[{"spans":[{"traceId":"dddddddddddddddddddddddddddddddd","spanId":"1111111111111111","name":"run","startTimeUnixNano":"1","endTimeUnixNano":"2","attributes":[{"key":"fullsend.work_item_id","value":{"stringValue":"acme/demo#1"}},{"key":"exit_code","value":{"intValue":"0"}}]},{"traceId":"dddddddddddddddddddddddddddddddd","spanId":"2222222222222222","name":"sandbox_create","startTimeUnixNano":"1","endTimeUnixNano":"2"},{"traceId":"dddddddddddddddddddddddddddddddd","spanId":"3333333333333333","name":"agent","startTimeUnixNano":"1","endTimeUnixNano":"2","attributes":[{"key":"gen_ai.system","value":{"stringValue":"anthropic"}}]}]}]}]}` + "\n" + require.NoError(t, os.WriteFile(telem, []byte(line), 0o644)) + + out := t.TempDir() + results, err := MeasureFile(telem, filepath.Join("testdata", "sample-registry.yaml"), out) + require.NoError(t, err) + require.Len(t, results, 1) + assert.Equal(t, LabelFail, results[0].Label) + assert.Contains(t, results[0].Explanation, "identity=fail") + + b, err := os.ReadFile(filepath.Join(out, MeasurementsFile)) + require.NoError(t, err) + assert.Contains(t, string(b), `"label":"fail"`) + assert.Contains(t, string(b), "identity=fail") +} diff --git a/internal/evalmeasure/score_test.go b/internal/evalmeasure/score_test.go index de51891fc9..ace4434c82 100644 --- a/internal/evalmeasure/score_test.go +++ b/internal/evalmeasure/score_test.go @@ -47,6 +47,20 @@ func TestScoreFitness_ReviewUnknownWorkItemFails(t *testing.T) { assert.Contains(t, r.Explanation, "missing: work_item") } +func TestScoreFitness_PrescriptSkippedExcluded(t *testing.T) { + t.Parallel() + traces, err := ParseTelemetryFile(filepath.Join("testdata", "prescript-skipped.jsonl")) + require.NoError(t, err) + r := ScoreFitness(traces[0]) + assert.Equal(t, "triage", r.Agent) + assert.Equal(t, LabelSkip, r.Label) + assert.NotEqual(t, "fail", r.Label) + assert.NotEqual(t, "pass", r.Label) + assert.Contains(t, r.Explanation, "pre-script skipped") + assert.NotContains(t, r.Explanation, "span_tree=fail") + assert.Equal(t, "cccccccccccccccccccccccccccccccc", r.TraceID) +} + func TestScoreTrace_AgentMismatchReturnsNil(t *testing.T) { t.Parallel() traces, err := ParseTelemetryFile(filepath.Join("testdata", "complete.jsonl")) @@ -55,7 +69,61 @@ func TestScoreTrace_AgentMismatchReturnsNil(t *testing.T) { assert.Empty(t, ScoreTrace(traces[0], reg)) } -func TestScoreTrace_UnknownScorerSkipped(t *testing.T) { +func TestScoreTrace_EmptyAgentNameRecordsIdentityFail(t *testing.T) { + t.Parallel() + tr := Trace{ + TraceID: "dddddddddddddddddddddddddddddddd", + Spans: []Span{ + { + Name: "run", + SpanID: "1111111111111111", + Attrs: map[string]any{ + "fullsend.work_item_id": "acme/demo#1", + "exit_code": int64(0), + }, + }, + {Name: "sandbox_create"}, + {Name: "agent", Attrs: map[string]any{"gen_ai.system": "anthropic"}}, + }, + } + assert.Empty(t, tr.AgentName()) + reg := Registry{ + Agent: "triage", + Measurements: []MeasurementSpec{{ID: "em-001", Scorer: ScorerFitness, Version: 1}}, + } + results := ScoreTrace(tr, reg) + require.Len(t, results, 1, "empty identity must still write a row — silent drop is survivorship bias") + assert.Equal(t, LabelFail, results[0].Label) + assert.Contains(t, results[0].Explanation, "identity=fail") +} + +func TestScoreTrace_UnknownAgentSentinelRecordsIdentityFail(t *testing.T) { + t.Parallel() + tr := Trace{ + TraceID: "eeeeeeeeeeeeeeeeeeeeeeeeeeeeeeee", + Spans: []Span{ + { + Name: "run", + Attrs: map[string]any{ + "fullsend.agent": UnknownSentinel, + "fullsend.work_item_id": "acme/demo#1", + "exit_code": int64(0), + }, + }, + {Name: "sandbox_create"}, + {Name: "agent"}, + }, + } + reg := Registry{ + Agent: "review", + Measurements: []MeasurementSpec{{ID: "em-001", Scorer: ScorerFitness, Version: 1}}, + } + results := ScoreTrace(tr, reg) + require.Len(t, results, 1) + assert.Contains(t, results[0].Explanation, "identity=fail") +} + +func TestScoreTrace_UnknownScorerSkipsNotSilent(t *testing.T) { t.Parallel() traces, err := ParseTelemetryFile(filepath.Join("testdata", "complete.jsonl")) require.NoError(t, err) @@ -67,6 +135,48 @@ func TestScoreTrace_UnknownScorerSkipped(t *testing.T) { }, } results := ScoreTrace(traces[0], reg) - require.Len(t, results, 1) - assert.Equal(t, "trace_fitness", results[0].Name) + require.Len(t, results, 2) + assert.Equal(t, "future_scorer", results[0].Name) + assert.Equal(t, LabelSkip, results[0].Label) + assert.Contains(t, results[0].Explanation, "unknown scorer") + assert.Equal(t, 0.0, results[0].Value) + assert.Equal(t, "trace_fitness", results[1].Name) +} + +func TestScoreFitness_CostToolsTurnsNamesSubcheck(t *testing.T) { + t.Parallel() + traces, err := ParseTelemetryFile(filepath.Join("testdata", "missing-cost.jsonl")) + require.NoError(t, err) + r := ScoreFitness(traces[0]) + assert.Equal(t, LabelFail, r.Label) + assert.Contains(t, r.Explanation, "cost_tools_turns=fail[cost]") +} + +func TestScoreFitness_MissingTurnsNamesSubcheck(t *testing.T) { + t.Parallel() + tr := Trace{ + TraceID: "1", + Spans: []Span{ + { + Name: "run", + Attrs: map[string]any{ + "fullsend.agent": "triage", + "gen_ai.agent.name": "triage", + "gen_ai.operation.name": "invoke_agent", + "fullsend.work_item_id": "acme/demo#1", + "exit_code": int64(0), + "gen_ai.request.model": "claude", + "gen_ai.usage.input_tokens": int64(1), + "gen_ai.usage.output_tokens": int64(1), + "fullsend.cost_usd": 0.1, + "fullsend.tool_calls": int64(1), + "fullsend.iterations": int64(1), + }, + }, + {Name: "sandbox_create"}, + {Name: "agent", Attrs: map[string]any{"gen_ai.system": "anthropic", "gen_ai.agent.name": "triage"}}, + }, + } + r := ScoreFitness(tr) + assert.Contains(t, r.Explanation, "cost_tools_turns=fail[num_turns]") } diff --git a/internal/evalmeasure/testdata/prescript-skipped.jsonl b/internal/evalmeasure/testdata/prescript-skipped.jsonl new file mode 100644 index 0000000000..6477768774 --- /dev/null +++ b/internal/evalmeasure/testdata/prescript-skipped.jsonl @@ -0,0 +1 @@ +{"resourceSpans":[{"resource":{"attributes":[{"key":"service.name","value":{"stringValue":"fullsend"}}]},"scopeSpans":[{"scope":{"name":"github.com/fullsend-ai/fullsend/internal/telemetry","version":"test"},"spans":[{"traceId":"cccccccccccccccccccccccccccccccc","spanId":"1111111111111111","name":"run","kind":1,"startTimeUnixNano":"1000000000","endTimeUnixNano":"2000000000","attributes":[{"key":"fullsend.agent","value":{"stringValue":"triage"}},{"key":"gen_ai.agent.name","value":{"stringValue":"triage"}},{"key":"gen_ai.operation.name","value":{"stringValue":"invoke_agent"}},{"key":"fullsend.work_item_id","value":{"stringValue":"acme/demo#1"}},{"key":"fullsend.prescript.skipped","value":{"boolValue":true}},{"key":"fullsend.prescript.skip_reason","value":{"stringValue":"already handled"}},{"key":"exit_code","value":{"intValue":"0"}}],"status":{"code":1}}]}]}]} diff --git a/internal/evalmeasure/types.go b/internal/evalmeasure/types.go index ff3180b84d..9ae9e869df 100644 --- a/internal/evalmeasure/types.go +++ b/internal/evalmeasure/types.go @@ -26,15 +26,18 @@ type Trace struct { // EvaluationResult is a portable measurement score (no vendor types). type EvaluationResult struct { - Name string `json:"name"` - Label string `json:"label"` - Explanation string `json:"explanation"` - TraceID string `json:"trace_id"` - SpanID string `json:"span_id"` - WorkItemID string `json:"work_item_id,omitempty"` - Agent string `json:"agent"` - Version string `json:"version"` - Value float64 `json:"value"` + Name string `json:"name"` + Label string `json:"label"` + Explanation string `json:"explanation"` + TraceID string `json:"trace_id"` + SpanID string `json:"span_id"` + WorkItemID string `json:"work_item_id,omitempty"` + Agent string `json:"agent"` + Version string `json:"version"` + // Value is the numeric score. Skip rows leave this at 0; consumers + // must key off Label, not Value==0. Do not add omitempty: a real + // fail can also be 0.0. + Value float64 `json:"value"` } // AttrString returns a string attribute. @@ -51,6 +54,29 @@ func (s Span) AttrString(key string) (string, bool) { } } +// AttrBool returns a bool attribute. +func (s Span) AttrBool(key string) (bool, bool) { + v, ok := s.Attrs[key] + if !ok || v == nil { + return false, false + } + switch t := v.(type) { + case bool: + return t, true + case string: + switch t { + case "true": + return true, true + case "false": + return false, true + default: + return false, false + } + default: + return false, false + } +} + // AttrInt returns an int64 attribute. func (s Span) AttrInt(key string) (int64, bool) { v, ok := s.Attrs[key] diff --git a/internal/evalmeasure/types_test.go b/internal/evalmeasure/types_test.go index 917333aad1..44bbdb3a9e 100644 --- a/internal/evalmeasure/types_test.go +++ b/internal/evalmeasure/types_test.go @@ -178,3 +178,32 @@ func TestAgentName(t *testing.T) { }) } } + +func TestAttrBool(t *testing.T) { + t.Parallel() + tests := []struct { + name string + attrs map[string]any + key string + want bool + wantOK bool + }{ + {"true", map[string]any{"k": true}, "k", true, true}, + {"false", map[string]any{"k": false}, "k", false, true}, + {"string true", map[string]any{"k": "true"}, "k", true, true}, + {"string false", map[string]any{"k": "false"}, "k", false, true}, + {"string other", map[string]any{"k": "yes"}, "k", false, false}, + {"missing key", map[string]any{}, "k", false, false}, + {"nil value", map[string]any{"k": nil}, "k", false, false}, + {"int not bool", map[string]any{"k": 1}, "k", false, false}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + s := Span{Attrs: tt.attrs} + got, ok := s.AttrBool(tt.key) + assert.Equal(t, tt.wantOK, ok) + assert.Equal(t, tt.want, got) + }) + } +} diff --git a/internal/fetch/fetch.go b/internal/fetch/fetch.go index 09f8b158bc..1c170bbf8c 100644 --- a/internal/fetch/fetch.go +++ b/internal/fetch/fetch.go @@ -87,6 +87,17 @@ var ( errTooLarge = errors.New("fetch: response body exceeds size limit") ) +// HTTPStatusError is returned when FetchURL gets a non-200 response. +type HTTPStatusError struct { + Status int +} + +func (e HTTPStatusError) Error() string { + return fmt.Sprintf("%s: got %d", errNonOK.Error(), e.Status) +} + +func (e HTTPStatusError) Unwrap() error { return errNonOK } + // FetchURL retrieves the content at rawURL subject to the given policy. // It returns the response body bytes or an error describing why the fetch // was rejected or failed. @@ -193,7 +204,7 @@ func FetchURL(ctx context.Context, rawURL string, policy FetchPolicy) ([]byte, e // 9. Only accept 200 OK. if resp.StatusCode != http.StatusOK { - return nil, fmt.Errorf("%w: got %d", errNonOK, resp.StatusCode) + return nil, HTTPStatusError{Status: resp.StatusCode} } // 10. Size limit: read one extra byte to detect overflow. diff --git a/internal/fetch/fetch_test.go b/internal/fetch/fetch_test.go index 382ec8c7ff..265c42eb1c 100644 --- a/internal/fetch/fetch_test.go +++ b/internal/fetch/fetch_test.go @@ -151,6 +151,10 @@ func TestFetchURL(t *testing.T) { if !errors.Is(err, errNonOK) { t.Fatalf("expected errNonOK, got: %v", err) } + var httpErr HTTPStatusError + if !errors.As(err, &httpErr) || httpErr.Status != http.StatusNotFound { + t.Fatalf("expected HTTPStatusError 404, got: %v", err) + } }) t.Run("Success", func(t *testing.T) { diff --git a/internal/scaffold/fullsend-repo-gitlab/.gitlab/ci/fullsend-agent.yml b/internal/scaffold/fullsend-repo-gitlab/.gitlab/ci/fullsend-agent.yml index cee6cd0404..6c0643c933 100644 --- a/internal/scaffold/fullsend-repo-gitlab/.gitlab/ci/fullsend-agent.yml +++ b/internal/scaffold/fullsend-repo-gitlab/.gitlab/ci/fullsend-agent.yml @@ -296,6 +296,9 @@ stages: # Run the agent — fullsend run resolves the harness file, reads # the image field, and creates the sandbox container via Podman. + # Capture the run status so eval-measure still runs after a failed + # agent (failed runs still write run-telemetry.jsonl). + set +e fullsend run "${STAGE}" \ --fullsend-dir .fullsend \ --target-repo . \ @@ -304,6 +307,19 @@ stages: --run-url "${CI_PIPELINE_URL}" \ --status-repo "${CI_PROJECT_PATH}" \ --status-number "${CI_MERGE_REQUEST_IID:-${STATUS_IID:-0}}" + RUN_STATUS=$? + set -e + + # Eval measurements (fail-open): same CLI as GitHub Actions. Never + # fail the agent job. Manifest: local .fullsend override, else + # SHA-pinned fetch from public fullsend-ai/agents (GitHub client). + fullsend eval-measure \ + --agent "${STAGE}" \ + --fullsend-dir .fullsend \ + --output-dir /tmp/fullsend-output \ + || true + + exit "${RUN_STATUS}" resource_group: "fullsend-${STAGE}-${RESOURCE_KEY}" rules: - if: $STAGE diff --git a/internal/scaffold/scaffold_gitlab_test.go b/internal/scaffold/scaffold_gitlab_test.go index fa655f5bda..a26066f2b1 100644 --- a/internal/scaffold/scaffold_gitlab_test.go +++ b/internal/scaffold/scaffold_gitlab_test.go @@ -155,6 +155,10 @@ func TestGitLabAgentTemplateContent(t *testing.T) { assert.Contains(t, s, "- agent") // Generic template parameterized by STAGE assert.Contains(t, s, `fullsend run "${STAGE}"`) + assert.Contains(t, s, `fullsend eval-measure`) + assert.Contains(t, s, "RUN_STATUS=$?") + assert.Contains(t, s, `exit "${RUN_STATUS}"`) + assert.Contains(t, s, "|| true") assert.Contains(t, s, "--fullsend-dir") assert.Contains(t, s, "--target-repo") assert.Contains(t, s, "--output-dir") From 3d5aefc6407a6294a0bc4b78fe8dea17c1d59cb5 Mon Sep 17 00:00:00 2001 From: Adam Scerra Date: Mon, 17 Aug 2026 14:49:37 -0400 Subject: [PATCH 08/14] fix(eval): address second-round measurement review Inject GH_TOKEN for SHA-pinned agents@v0 GetRef, score only the host runDir for --agent, reject empty gen_ai model/system strings, and warn when telemetry JSONL lines are unreadable. Co-authored-by: Cursor Signed-off-by: Adam Scerra --- action.yml | 2 + ...-eval-measurements-online-trace-scoring.md | 8 ++-- .../infrastructure/eval-measurements.md | 11 +++-- internal/cli/evalmeasure.go | 20 +++++++-- internal/cli/evalmeasure_test.go | 22 ++++++++++ internal/evalmeasure/find.go | 37 +++++++++++++--- internal/evalmeasure/find_test.go | 31 ++++++++++--- internal/evalmeasure/fitness.go | 31 ++++++------- internal/evalmeasure/parse.go | 24 +++++++--- internal/evalmeasure/parse_test.go | 14 +++--- internal/evalmeasure/run.go | 22 +++++----- internal/evalmeasure/run_test.go | 4 +- internal/evalmeasure/score_test.go | 44 ++++++++++++++++--- internal/evalmeasure/types.go | 20 +++++++-- .../.gitlab/ci/fullsend-agent.yml | 4 +- internal/scaffold/scaffold_gitlab_test.go | 1 + 16 files changed, 223 insertions(+), 72 deletions(-) diff --git a/action.yml b/action.yml index 30ad7f556c..70f1a913f6 100644 --- a/action.yml +++ b/action.yml @@ -419,6 +419,8 @@ runs: env: AGENT: ${{ inputs.agent }} FULLSEND_DIR: ${{ inputs.fullsend-dir }} + # For GetRef of agents@v0 (SHA pin). Not sent to raw.githubusercontent.com. + GH_TOKEN: ${{ inputs.github_token }} run: | set -euo pipefail FULLSEND_DIR="${FULLSEND_DIR:-${GITHUB_WORKSPACE}}" diff --git a/docs/ADRs/0087-eval-measurements-online-trace-scoring.md b/docs/ADRs/0087-eval-measurements-online-trace-scoring.md index fa0602337a..4c8d3b41a3 100644 --- a/docs/ADRs/0087-eval-measurements-online-trace-scoring.md +++ b/docs/ADRs/0087-eval-measurements-online-trace-scoring.md @@ -47,8 +47,8 @@ Adjacent telemetry proposals (not competing with this score path): - **Observer / lessons → fixtures** ([#2423](https://github.com/fullsend-ai/fullsend/pull/2423)): narrative analysis and golden-set promotion. This ADR is same-job deterministic scoring on traces. -- **Harness snapshot / forge join keys** ([#5524](https://github.com/fullsend-ai/fullsend/pull/5524) - — proposed ADR 0075): sibling artifact for harness fingerprint and +- **Harness snapshot / forge join keys** ([#5524](https://github.com/fullsend-ai/fullsend/pull/5524)): + sibling artifact for harness fingerprint and forge/CI pointers beside telemetry. Complementary join/identity layer; primary run facts belong on the OTEL trace (Level 1), while measurements stay a derived sibling file. @@ -88,7 +88,9 @@ Stock-agent defaults resolve from `agents@v0` at runtime; local files are for override, opt-out, or custom agents only. The first scorer is `trace_fitness` (catalog id `em-001`) — span-tree and -attribute fitness so later scorers can trust the trace. +attribute fitness so later scorers can trust the trace. EM-001 reads +experimental OpenTelemetry GenAI attribute names (`gen_ai.*` constants in +`internal/evalmeasure`); an upstream rename is an `em-001` version bump. ### Versioning (per measurement, not platform “v1”) diff --git a/docs/guides/infrastructure/eval-measurements.md b/docs/guides/infrastructure/eval-measurements.md index f58a921593..f73d442c38 100644 --- a/docs/guides/infrastructure/eval-measurements.md +++ b/docs/guides/infrastructure/eval-measurements.md @@ -160,7 +160,7 @@ yet) also writes `label: skip`, not `fail`. Trend pass-rate as | [#5947](https://github.com/fullsend-ai/fullsend/pull/5947) Level 3 activation + sandbox OTEL denylist | Richer traces fuel later scorers. First ship reads Level 1/2 local JSONL; content-aware scorers need OTLP/backend (or a widened file contract) under the proposed L3 rules. Measure CLI is host-side after the sandbox exits. | | [#5944](https://github.com/fullsend-ai/fullsend/pull/5944) Span status from run outcome | Unblocks outcome scorers keyed on Status, not raw exit alone. | | [#2423](https://github.com/fullsend-ai/fullsend/pull/2423) Semantic observability / observer / lessons | Observer + lessons → fixtures; measurements are the online score path. | -| [#5524](https://github.com/fullsend-ai/fullsend/pull/5524) Harness snapshot / forge join keys (ADR 0075) | Complementary join/identity proposal beside telemetry; measurements are derived scores, not primary run facts. | +| [#5524](https://github.com/fullsend-ai/fullsend/pull/5524) Harness snapshot / forge join keys | Complementary join/identity proposal beside telemetry; measurements are derived scores, not primary run facts. | ## Same-job timing @@ -184,12 +184,17 @@ GitLab CI agent job 2. SHA-pinned `eval/measurements/${AGENT}.yaml` from `fullsend-ai/agents` (same `v0` → commit SHA, allowlist, hash, and fetch audit as harness fallback — not a floating `raw.githubusercontent.com/.../v0/...` curl). + GitHub Actions injects `GH_TOKEN` for that `GetRef`. GitLab CI has no + GitHub token by default, so step 2 skips unless an operator exports + `GH_TOKEN` / `GITHUB_TOKEN`; a local FULLSEND_DIR override still works. Step 2 is how stock-agent defaults reach every install. Step 1 is override / BYOA only. -Platform telemetry is `run-telemetry.jsonl` at the top of each run directory. -Nested `iteration-N/output/run-telemetry.jsonl` copies are ignored. +Platform telemetry is `run-telemetry.jsonl` at the top of the host run +directory (`agent---` under the CI output base). Nested +`iteration-N/output/run-telemetry.jsonl` copies and leftover sibling runDirs +are ignored. Missing manifest or telemetry → log and exit `0` (skip). `--registry` and `--telemetry` remain for local/debug use. diff --git a/internal/cli/evalmeasure.go b/internal/cli/evalmeasure.go index 0e9f654547..49528f1d8a 100644 --- a/internal/cli/evalmeasure.go +++ b/internal/cli/evalmeasure.go @@ -108,7 +108,10 @@ func runEvalMeasure(ctx context.Context, printer *ui.Printer, opts evalMeasureOp var all []evalmeasure.EvaluationResult for _, p := range telemPaths { - results, err := evalmeasure.MeasureAndExport(ctx, p, registry, opts.outDir) + results, stats, err := evalmeasure.MeasureAndExport(ctx, p, registry, opts.outDir) + if stats.SkippedLines > 0 { + printer.StepWarn(fmt.Sprintf("%s: skipped %d of %d unreadable telemetry line(s)", p, stats.SkippedLines, stats.NonEmptyLines)) + } if err != nil { return all, false, err } @@ -122,7 +125,15 @@ func resolveEvalMeasureTelemetry(opts evalMeasureOpts) ([]string, error) { return []string{opts.telemetryPath}, nil } if opts.outputDir != "" { - return evalmeasure.FindPlatformTelemetry(opts.outputDir) + agent := "" + if opts.agent != "" { + a, err := sanitizeMeasurementAgentName(opts.agent) + if err != nil { + return nil, err + } + agent = a + } + return evalmeasure.FindPlatformTelemetry(opts.outputDir, agent) } return nil, fmt.Errorf("either --telemetry or --output-dir is required") } @@ -188,7 +199,10 @@ func evalMeasureFetchContext(fullsendDir string, printer *ui.Printer) (harness.C orgAllowlist = orgCfg.AllowedResources() } } - token, _ := resolveToken() + token, err := resolveToken() + if (err != nil || token == "") && printer != nil { + printer.StepWarn("No GH_TOKEN/GITHUB_TOKEN; agents@v0 GetRef is unauthenticated. GitLab jobs skip stock manifests unless an operator wires a GitHub token.") + } return harness.ComposeOpts{ WorkspaceRoot: abs, FetchPolicy: fetch.DefaultPolicy, diff --git a/internal/cli/evalmeasure_test.go b/internal/cli/evalmeasure_test.go index ae5f8cbaac..92889d53e4 100644 --- a/internal/cli/evalmeasure_test.go +++ b/internal/cli/evalmeasure_test.go @@ -135,6 +135,27 @@ func TestEvalMeasureCmd_SkipWhenNoTelemetry(t *testing.T) { assert.NotContains(t, buf.String(), "Wrote") } +func TestEvalMeasureCmd_WarnsOnCorruptTelemetryLine(t *testing.T) { + out := t.TempDir() + good, err := os.ReadFile(filepath.Join("..", "evalmeasure", "testdata", "complete.jsonl")) + require.NoError(t, err) + telem := filepath.Join(out, "run-telemetry.jsonl") + require.NoError(t, os.WriteFile(telem, append(append([]byte{}, good...), []byte("\nnot-json\n")...), 0o644)) + + cmd := newRootCmd() + buf := &bytes.Buffer{} + cmd.SetOut(buf) + cmd.SetErr(buf) + cmd.SetArgs([]string{ + "eval-measure", + "--telemetry", telem, + "--registry", filepath.Join("..", "evalmeasure", "testdata", "sample-registry.yaml"), + "--out-dir", out, + }) + require.NoError(t, cmd.Execute()) + assert.Contains(t, buf.String(), "unreadable telemetry line") +} + func TestActionYML_EvalMeasureNoFloatingV0Curl(t *testing.T) { b, err := os.ReadFile(filepath.Join("..", "..", "action.yml")) require.NoError(t, err) @@ -152,6 +173,7 @@ func TestActionYML_EvalMeasureNoFloatingV0Curl(t *testing.T) { assert.Contains(t, step, "--agent") assert.Contains(t, step, "continue-on-error: true") assert.Contains(t, step, "if: always()") + assert.Contains(t, step, "GH_TOKEN:") assert.NotContains(t, step, "curl ") assert.NotContains(t, step, "Authorization: Bearer") } diff --git a/internal/evalmeasure/find.go b/internal/evalmeasure/find.go index 0c72536a56..25889d71d6 100644 --- a/internal/evalmeasure/find.go +++ b/internal/evalmeasure/find.go @@ -3,6 +3,8 @@ package evalmeasure import ( "os" "path/filepath" + "strings" + "time" ) // PlatformTelemetryFile is the host recorder's JSONL at the top of runDir. @@ -12,10 +14,15 @@ const PlatformTelemetryFile = "run-telemetry.jsonl" // FindPlatformTelemetry returns run-telemetry.jsonl files that sit at the // top of outputDir itself (when outputDir is a runDir) or at the top of -// each immediate child directory (when outputDir is the CI output base). +// a host-created child runDir (when outputDir is the CI output base). +// +// Host runDirs are named agent--- (see fullsend run). +// When agent is non-empty, only children with that prefix are considered. +// If several match, only the newest platform file is scored — leftover +// sibling directories from a previous job are ignored. // It never walks deeper, so an agent-planted // iteration-N/output/run-telemetry.jsonl is ignored. -func FindPlatformTelemetry(outputDir string) ([]string, error) { +func FindPlatformTelemetry(outputDir, agent string) ([]string, error) { direct := filepath.Join(outputDir, PlatformTelemetryFile) if st, err := os.Stat(direct); err == nil && !st.IsDir() { // outputDir is a runDir: score only the platform file at the top. @@ -29,15 +36,33 @@ func FindPlatformTelemetry(outputDir string) ([]string, error) { } return nil, err } - var out []string + prefix := "" + if agent != "" { + prefix = "agent-" + strings.ToLower(agent) + "-" + } + var bestPath string + var bestMod time.Time + found := false for _, e := range entries { if !e.IsDir() { continue } + if prefix != "" && !strings.HasPrefix(e.Name(), prefix) { + continue + } p := filepath.Join(outputDir, e.Name(), PlatformTelemetryFile) - if st, err := os.Stat(p); err == nil && !st.IsDir() { - out = append(out, p) + st, err := os.Stat(p) + if err != nil || st.IsDir() { + continue + } + if !found || st.ModTime().After(bestMod) { + bestPath = p + bestMod = st.ModTime() + found = true } } - return out, nil + if !found { + return nil, nil + } + return []string{bestPath}, nil } diff --git a/internal/evalmeasure/find_test.go b/internal/evalmeasure/find_test.go index a308299b91..00a54f4d7b 100644 --- a/internal/evalmeasure/find_test.go +++ b/internal/evalmeasure/find_test.go @@ -21,7 +21,7 @@ func TestFindPlatformTelemetry_IgnoresNestedIterationCopy(t *testing.T) { require.NoError(t, os.WriteFile(platform, []byte("platform\n"), 0o644)) require.NoError(t, os.WriteFile(planted, []byte("planted\n"), 0o644)) - got, err := FindPlatformTelemetry(outputDir) + got, err := FindPlatformTelemetry(outputDir, "") require.NoError(t, err) require.Equal(t, []string{platform}, got) } @@ -35,7 +35,7 @@ func TestFindPlatformTelemetry_RunDirIgnoresImmediateChild(t *testing.T) { require.NoError(t, os.WriteFile(platform, []byte("platform\n"), 0o644)) require.NoError(t, os.WriteFile(filepath.Join(child, PlatformTelemetryFile), []byte("planted\n"), 0o644)) - got, err := FindPlatformTelemetry(runDir) + got, err := FindPlatformTelemetry(runDir, "") require.NoError(t, err) require.Equal(t, []string{platform}, got) } @@ -49,14 +49,14 @@ func TestFindPlatformTelemetry_RunDirDirect(t *testing.T) { require.NoError(t, os.WriteFile(platform, []byte("platform\n"), 0o644)) require.NoError(t, os.WriteFile(filepath.Join(nested, PlatformTelemetryFile), []byte("planted\n"), 0o644)) - got, err := FindPlatformTelemetry(runDir) + got, err := FindPlatformTelemetry(runDir, "") require.NoError(t, err) require.Equal(t, []string{platform}, got) } func TestFindPlatformTelemetry_MissingDir(t *testing.T) { t.Parallel() - got, err := FindPlatformTelemetry(filepath.Join(t.TempDir(), "nope")) + got, err := FindPlatformTelemetry(filepath.Join(t.TempDir(), "nope"), "") require.NoError(t, err) assert.Empty(t, got) } @@ -68,7 +68,28 @@ func TestFindPlatformTelemetry_EmptyWhenOnlyNested(t *testing.T) { require.NoError(t, os.MkdirAll(nested, 0o755)) require.NoError(t, os.WriteFile(filepath.Join(nested, PlatformTelemetryFile), []byte("planted\n"), 0o644)) - got, err := FindPlatformTelemetry(outputDir) + got, err := FindPlatformTelemetry(outputDir, "") require.NoError(t, err) assert.Empty(t, got) } + +func TestFindPlatformTelemetry_IgnoresSiblingLeftoverRunDir(t *testing.T) { + t.Parallel() + outputDir := t.TempDir() + leftover := filepath.Join(outputDir, "agent-triage-1-1") + current := filepath.Join(outputDir, "agent-review-9-9") + require.NoError(t, os.MkdirAll(leftover, 0o755)) + require.NoError(t, os.MkdirAll(current, 0o755)) + leftFile := filepath.Join(leftover, PlatformTelemetryFile) + curFile := filepath.Join(current, PlatformTelemetryFile) + require.NoError(t, os.WriteFile(leftFile, []byte("old\n"), 0o644)) + require.NoError(t, os.WriteFile(curFile, []byte("new\n"), 0o644)) + + got, err := FindPlatformTelemetry(outputDir, "review") + require.NoError(t, err) + require.Equal(t, []string{curFile}, got) + + gotAllNewest, err := FindPlatformTelemetry(outputDir, "") + require.NoError(t, err) + require.Len(t, gotAllNewest, 1) +} diff --git a/internal/evalmeasure/fitness.go b/internal/evalmeasure/fitness.go index ef1bada745..9f55f99297 100644 --- a/internal/evalmeasure/fitness.go +++ b/internal/evalmeasure/fitness.go @@ -33,7 +33,7 @@ func ScoreFitnessNamed(tr Trace, evalName, version string) EvaluationResult { if hasRun { if skipped, ok := run.AttrBool("fullsend.prescript.skipped"); ok && skipped { spanID := run.SpanID - workItem, _ := run.AttrString("fullsend.work_item_id") + workItem, _ := run.AttrString(AttrFullsendWorkItemID) reason := "pre-script skipped run; excluded from trace_fitness" if r, ok := run.AttrString("fullsend.prescript.skip_reason"); ok && r != "" { reason = reason + ": " + r @@ -65,7 +65,7 @@ func ScoreFitnessNamed(tr Trace, evalName, version string) EvaluationResult { // Review jobs that have PR_NUMBER / GITHUB_PR_URL should not hit this // after #5622; a remaining unknown is a real fitness fail. {"work_item", workItemOK(run)}, - {"operation", attrNonEmpty(run, "gen_ai.operation.name")}, + {"operation", attrNonEmpty(run, AttrGenAIOperationName)}, {"model", modelOK(run, agents)}, {"usage", usageOK(run, agents)}, {"cost_tools_turns", costOK}, @@ -104,7 +104,7 @@ func ScoreFitnessNamed(tr Trace, evalName, version string) EvaluationResult { agent := tr.AgentName() if hasRun { spanID = run.SpanID - workItem, _ = run.AttrString("fullsend.work_item_id") + workItem, _ = run.AttrString(AttrFullsendWorkItemID) } return EvaluationResult{ @@ -121,15 +121,15 @@ func ScoreFitnessNamed(tr Trace, evalName, version string) EvaluationResult { } func identityOK(run Span, agents []Span) bool { - fa, okFA := run.AttrString("fullsend.agent") + fa, okFA := run.AttrString(AttrFullsendAgent) if !okFA || fa == "" || fa == UnknownSentinel { return false } - if ga, ok := run.AttrString("gen_ai.agent.name"); ok && ga == fa { + if ga, ok := run.AttrString(AttrGenAIAgentName); ok && ga == fa { return true } for _, a := range agents { - if ga, ok := a.AttrString("gen_ai.agent.name"); ok && ga == fa { + if ga, ok := a.AttrString(AttrGenAIAgentName); ok && ga == fa { return true } } @@ -137,7 +137,7 @@ func identityOK(run Span, agents []Span) bool { } func workItemOK(run Span) bool { - v, ok := run.AttrString("fullsend.work_item_id") + v, ok := run.AttrString(AttrFullsendWorkItemID) return ok && v != "" && v != UnknownSentinel } @@ -147,13 +147,10 @@ func attrNonEmpty(s Span, key string) bool { } func modelOK(run Span, agents []Span) bool { - hasModel := false - if _, ok := run.AttrString("gen_ai.request.model"); ok { - hasModel = true - } + hasModel := attrNonEmpty(run, AttrGenAIRequestModel) if !hasModel { for _, a := range agents { - if _, ok := a.AttrString("gen_ai.request.model"); ok { + if attrNonEmpty(a, AttrGenAIRequestModel) { hasModel = true break } @@ -161,7 +158,7 @@ func modelOK(run Span, agents []Span) bool { } hasSystem := false for _, a := range agents { - if _, ok := a.AttrString("gen_ai.system"); ok { + if attrNonEmpty(a, AttrGenAISystem) { hasSystem = true break } @@ -170,14 +167,14 @@ func modelOK(run Span, agents []Span) bool { } func usageOK(run Span, agents []Span) bool { - if _, ok := run.AttrInt("gen_ai.usage.input_tokens"); ok { - if _, ok := run.AttrInt("gen_ai.usage.output_tokens"); ok { + if _, ok := run.AttrInt(AttrGenAIUsageInputTokens); ok { + if _, ok := run.AttrInt(AttrGenAIUsageOutputTokens); ok { return true } } for _, a := range agents { - _, inOK := a.AttrInt("gen_ai.usage.input_tokens") - _, outOK := a.AttrInt("gen_ai.usage.output_tokens") + _, inOK := a.AttrInt(AttrGenAIUsageInputTokens) + _, outOK := a.AttrInt(AttrGenAIUsageOutputTokens) if inOK && outOK { return true } diff --git a/internal/evalmeasure/parse.go b/internal/evalmeasure/parse.go index b94e5d6764..8afbe1e57f 100644 --- a/internal/evalmeasure/parse.go +++ b/internal/evalmeasure/parse.go @@ -40,12 +40,21 @@ type otlpKeyValue struct { Value map[string]any `json:"value"` } +// ParseStats counts telemetry JSONL lines so operators can tell "no traces" +// from "file present but unreadable". +type ParseStats struct { + NonEmptyLines int + SkippedLines int +} + // ParseTelemetryFile reads OTLP JSON TracesData lines from run-telemetry.jsonl -// and merges spans by trace id. -func ParseTelemetryFile(path string) ([]Trace, error) { +// and merges spans by trace id. Truncated or corrupt lines are skipped +// (fail-open); SkippedLines is the count of those lines. +func ParseTelemetryFile(path string) ([]Trace, ParseStats, error) { + var stats ParseStats f, err := os.Open(path) if err != nil { - return nil, err + return nil, stats, err } defer f.Close() @@ -64,9 +73,10 @@ func ParseTelemetryFile(path string) ([]Trace, error) { if len(line) == 0 { continue } + stats.NonEmptyLines++ var doc otlpTracesData if err := json.Unmarshal(line, &doc); err != nil { - // Skip one truncated or corrupt JSONL line; keep the rest of the file. + stats.SkippedLines++ continue } for _, rs := range doc.ResourceSpans { @@ -74,7 +84,7 @@ func ParseTelemetryFile(path string) ([]Trace, error) { for _, raw := range ss.Spans { sp, err := convertSpan(raw) if err != nil { - return nil, fmt.Errorf("line %d span %s: %w", lineNo, raw.SpanID, err) + return nil, stats, fmt.Errorf("line %d span %s: %w", lineNo, raw.SpanID, err) } tr, ok := byID[sp.TraceID] if !ok { @@ -88,14 +98,14 @@ func ParseTelemetryFile(path string) ([]Trace, error) { } } if err := sc.Err(); err != nil { - return nil, err + return nil, stats, err } out := make([]Trace, 0, len(order)) for _, id := range order { out = append(out, *byID[id]) } - return out, nil + return out, stats, nil } func convertSpan(raw otlpSpan) (Span, error) { diff --git a/internal/evalmeasure/parse_test.go b/internal/evalmeasure/parse_test.go index a39dcd1912..8f37462028 100644 --- a/internal/evalmeasure/parse_test.go +++ b/internal/evalmeasure/parse_test.go @@ -11,7 +11,7 @@ import ( func TestParseTelemetryFile_Complete(t *testing.T) { t.Parallel() - traces, err := ParseTelemetryFile(filepath.Join("testdata", "complete.jsonl")) + traces, _, err := ParseTelemetryFile(filepath.Join("testdata", "complete.jsonl")) require.NoError(t, err) require.Len(t, traces, 1) tr := traces[0] @@ -30,7 +30,7 @@ func TestParseTelemetryFile_Complete(t *testing.T) { func TestParseTelemetryFile_MergesLinesSameTrace(t *testing.T) { t.Parallel() - traces, err := ParseTelemetryFile(filepath.Join("testdata", "split.jsonl")) + traces, _, err := ParseTelemetryFile(filepath.Join("testdata", "split.jsonl")) require.NoError(t, err) require.Len(t, traces, 1) assert.Len(t, traces[0].Spans, 3) @@ -43,9 +43,11 @@ func TestParseTelemetryFile_InvalidLineSkipped(t *testing.T) { dir := t.TempDir() path := filepath.Join(dir, "bad.jsonl") require.NoError(t, os.WriteFile(path, []byte("not-json\n"), 0o644)) - traces, err := ParseTelemetryFile(path) + traces, stats, err := ParseTelemetryFile(path) require.NoError(t, err, "a truncated/corrupt line must not fail the whole file") assert.Empty(t, traces) + assert.Equal(t, 1, stats.NonEmptyLines) + assert.Equal(t, 1, stats.SkippedLines) } func TestParseTelemetryFile_TruncatedLineDoesNotDiscardFile(t *testing.T) { @@ -60,14 +62,16 @@ func TestParseTelemetryFile_TruncatedLineDoesNotDiscardFile(t *testing.T) { } body += "{\"resourceSpans\":[{\"scopeSpans\":[{\"spans\":[{\"traceId\":\"truncated\n" require.NoError(t, os.WriteFile(path, []byte(body), 0o644)) - traces, err := ParseTelemetryFile(path) + traces, stats, err := ParseTelemetryFile(path) require.NoError(t, err) require.Len(t, traces, 1) assert.Equal(t, "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", traces[0].TraceID) + assert.GreaterOrEqual(t, stats.SkippedLines, 1) + assert.Greater(t, stats.NonEmptyLines, stats.SkippedLines) } func TestParseTelemetryFile_MissingFile(t *testing.T) { t.Parallel() - _, err := ParseTelemetryFile(filepath.Join(t.TempDir(), "missing.jsonl")) + _, _, err := ParseTelemetryFile(filepath.Join(t.TempDir(), "missing.jsonl")) require.Error(t, err) } diff --git a/internal/evalmeasure/run.go b/internal/evalmeasure/run.go index 0e98a22386..773c98eaef 100644 --- a/internal/evalmeasure/run.go +++ b/internal/evalmeasure/run.go @@ -9,25 +9,27 @@ import ( // MeasureFile parses telemetry, scores with the manifest, and writes local // eval-measurements.jsonl. Idempotent per ledger. func MeasureFile(telemetryPath, registryPath, outDir string) ([]EvaluationResult, error) { - return MeasureAndExport(context.Background(), telemetryPath, registryPath, outDir) + r, _, err := MeasureAndExport(context.Background(), telemetryPath, registryPath, outDir) + return r, err } // MeasureAndExport is MeasureFile with an explicit context (reserved for // future portable OTLP score export on the same OTEL_* path as ADR 0050). -func MeasureAndExport(ctx context.Context, telemetryPath, registryPath, outDir string) ([]EvaluationResult, error) { +func MeasureAndExport(ctx context.Context, telemetryPath, registryPath, outDir string) ([]EvaluationResult, ParseStats, error) { + var stats ParseStats if err := ctx.Err(); err != nil { - return nil, err + return nil, stats, err } if outDir == "" { outDir = filepath.Dir(telemetryPath) } reg, err := LoadRegistry(registryPath) if err != nil { - return nil, fmt.Errorf("load registry: %w", err) + return nil, stats, fmt.Errorf("load registry: %w", err) } - traces, err := ParseTelemetryFile(telemetryPath) + traces, stats, err := ParseTelemetryFile(telemetryPath) if err != nil { - return nil, fmt.Errorf("parse telemetry: %w", err) + return nil, stats, fmt.Errorf("parse telemetry: %w", err) } ledgerPath := filepath.Join(outDir, LedgerFile) @@ -39,20 +41,20 @@ func MeasureAndExport(ctx context.Context, telemetryPath, registryPath, outDir s for _, r := range results { done, err := AlreadyScored(ledgerPath, r.TraceID, r.Name, r.Version) if err != nil { - return all, fmt.Errorf("check ledger: %w", err) + return all, stats, fmt.Errorf("check ledger: %w", err) } if done { continue } all = append(all, r) if err := AppendMeasurements(measPath, []EvaluationResult{r}); err != nil { - return all, fmt.Errorf("append measurements: %w", err) + return all, stats, fmt.Errorf("append measurements: %w", err) } if err := RecordScored(ledgerPath, r.TraceID, r.Name, r.Version); err != nil { - return all, fmt.Errorf("record scored: %w", err) + return all, stats, fmt.Errorf("record scored: %w", err) } } } - return all, nil + return all, stats, nil } diff --git a/internal/evalmeasure/run_test.go b/internal/evalmeasure/run_test.go index f370f00805..fff57a7c2e 100644 --- a/internal/evalmeasure/run_test.go +++ b/internal/evalmeasure/run_test.go @@ -17,7 +17,7 @@ func TestLoadRegistryAndScoreTrace(t *testing.T) { require.NoError(t, err) assert.Equal(t, "triage", reg.Agent) - traces, err := ParseTelemetryFile(filepath.Join("testdata", "complete.jsonl")) + traces, _, err := ParseTelemetryFile(filepath.Join("testdata", "complete.jsonl")) require.NoError(t, err) results := ScoreTrace(traces[0], reg) require.Len(t, results, 1) @@ -86,7 +86,7 @@ func TestMeasureFile_BadTelemetry(t *testing.T) { func TestMeasureAndExport_CancelledContext(t *testing.T) { ctx, cancel := context.WithCancel(context.Background()) cancel() - _, err := MeasureAndExport( + _, _, err := MeasureAndExport( ctx, filepath.Join("testdata", "complete.jsonl"), filepath.Join("testdata", "sample-registry.yaml"), diff --git a/internal/evalmeasure/score_test.go b/internal/evalmeasure/score_test.go index ace4434c82..005bc295a3 100644 --- a/internal/evalmeasure/score_test.go +++ b/internal/evalmeasure/score_test.go @@ -10,7 +10,7 @@ import ( func TestScoreFitness_CompletePass(t *testing.T) { t.Parallel() - traces, err := ParseTelemetryFile(filepath.Join("testdata", "complete.jsonl")) + traces, _, err := ParseTelemetryFile(filepath.Join("testdata", "complete.jsonl")) require.NoError(t, err) r := ScoreFitness(traces[0]) assert.Equal(t, ScorerFitness, r.Name) @@ -26,7 +26,7 @@ func TestScoreFitness_CompletePass(t *testing.T) { func TestScoreFitness_MissingCostFails(t *testing.T) { t.Parallel() - traces, err := ParseTelemetryFile(filepath.Join("testdata", "missing-cost.jsonl")) + traces, _, err := ParseTelemetryFile(filepath.Join("testdata", "missing-cost.jsonl")) require.NoError(t, err) r := ScoreFitness(traces[0]) assert.Equal(t, "fail", r.Label) @@ -37,7 +37,7 @@ func TestScoreFitness_MissingCostFails(t *testing.T) { func TestScoreFitness_ReviewUnknownWorkItemFails(t *testing.T) { t.Parallel() - traces, err := ParseTelemetryFile(filepath.Join("testdata", "review-unknown-workitem.jsonl")) + traces, _, err := ParseTelemetryFile(filepath.Join("testdata", "review-unknown-workitem.jsonl")) require.NoError(t, err) r := ScoreFitness(traces[0]) assert.Equal(t, "review", r.Agent) @@ -49,7 +49,7 @@ func TestScoreFitness_ReviewUnknownWorkItemFails(t *testing.T) { func TestScoreFitness_PrescriptSkippedExcluded(t *testing.T) { t.Parallel() - traces, err := ParseTelemetryFile(filepath.Join("testdata", "prescript-skipped.jsonl")) + traces, _, err := ParseTelemetryFile(filepath.Join("testdata", "prescript-skipped.jsonl")) require.NoError(t, err) r := ScoreFitness(traces[0]) assert.Equal(t, "triage", r.Agent) @@ -63,7 +63,7 @@ func TestScoreFitness_PrescriptSkippedExcluded(t *testing.T) { func TestScoreTrace_AgentMismatchReturnsNil(t *testing.T) { t.Parallel() - traces, err := ParseTelemetryFile(filepath.Join("testdata", "complete.jsonl")) + traces, _, err := ParseTelemetryFile(filepath.Join("testdata", "complete.jsonl")) require.NoError(t, err) reg := Registry{Agent: "code", Measurements: []MeasurementSpec{{ID: "em-001", Scorer: ScorerFitness, Version: 1}}} assert.Empty(t, ScoreTrace(traces[0], reg)) @@ -125,7 +125,7 @@ func TestScoreTrace_UnknownAgentSentinelRecordsIdentityFail(t *testing.T) { func TestScoreTrace_UnknownScorerSkipsNotSilent(t *testing.T) { t.Parallel() - traces, err := ParseTelemetryFile(filepath.Join("testdata", "complete.jsonl")) + traces, _, err := ParseTelemetryFile(filepath.Join("testdata", "complete.jsonl")) require.NoError(t, err) reg := Registry{ Agent: "triage", @@ -145,7 +145,7 @@ func TestScoreTrace_UnknownScorerSkipsNotSilent(t *testing.T) { func TestScoreFitness_CostToolsTurnsNamesSubcheck(t *testing.T) { t.Parallel() - traces, err := ParseTelemetryFile(filepath.Join("testdata", "missing-cost.jsonl")) + traces, _, err := ParseTelemetryFile(filepath.Join("testdata", "missing-cost.jsonl")) require.NoError(t, err) r := ScoreFitness(traces[0]) assert.Equal(t, LabelFail, r.Label) @@ -180,3 +180,33 @@ func TestScoreFitness_MissingTurnsNamesSubcheck(t *testing.T) { r := ScoreFitness(tr) assert.Contains(t, r.Explanation, "cost_tools_turns=fail[num_turns]") } + +func TestScoreFitness_EmptyModelStringFails(t *testing.T) { + t.Parallel() + tr := Trace{ + TraceID: "2", + Spans: []Span{ + { + Name: "run", + Attrs: map[string]any{ + "fullsend.agent": "triage", + "gen_ai.agent.name": "triage", + "gen_ai.operation.name": "invoke_agent", + "fullsend.work_item_id": "acme/demo#1", + "exit_code": int64(0), + "gen_ai.request.model": "", + "gen_ai.usage.input_tokens": int64(1), + "gen_ai.usage.output_tokens": int64(1), + "fullsend.cost_usd": 0.1, + "fullsend.tool_calls": int64(1), + "fullsend.iterations": int64(0), + }, + }, + {Name: "sandbox_create"}, + {Name: "agent", Attrs: map[string]any{"gen_ai.system": "", "gen_ai.agent.name": "triage"}}, + }, + } + r := ScoreFitness(tr) + assert.Equal(t, LabelFail, r.Label) + assert.Contains(t, r.Explanation, "model=fail") +} diff --git a/internal/evalmeasure/types.go b/internal/evalmeasure/types.go index 9ae9e869df..1a633e37c3 100644 --- a/internal/evalmeasure/types.go +++ b/internal/evalmeasure/types.go @@ -6,6 +6,20 @@ import ( "strconv" ) +// Attribute names used by EM-001. gen_ai.* keys follow experimental +// OpenTelemetry GenAI semantic conventions; an upstream rename is an +// em-001 version bump, not a silent fleet fail. +const ( + AttrFullsendAgent = "fullsend.agent" + AttrFullsendWorkItemID = "fullsend.work_item_id" + AttrGenAIAgentName = "gen_ai.agent.name" + AttrGenAISystem = "gen_ai.system" + AttrGenAIRequestModel = "gen_ai.request.model" + AttrGenAIUsageInputTokens = "gen_ai.usage.input_tokens" + AttrGenAIUsageOutputTokens = "gen_ai.usage.output_tokens" + AttrGenAIOperationName = "gen_ai.operation.name" +) + // Span is a portable view of one OTEL span from run-telemetry.jsonl. type Span struct { TraceID string @@ -157,15 +171,15 @@ func (t Trace) SpansByName(name string) []Span { // AgentName returns the agent identity from run or agent spans. func (t Trace) AgentName() string { if run, ok := t.SpanByName("run"); ok { - if a, ok := run.AttrString("fullsend.agent"); ok && a != "" { + if a, ok := run.AttrString(AttrFullsendAgent); ok && a != "" { return a } - if a, ok := run.AttrString("gen_ai.agent.name"); ok && a != "" { + if a, ok := run.AttrString(AttrGenAIAgentName); ok && a != "" { return a } } for _, a := range t.SpansByName("agent") { - if name, ok := a.AttrString("gen_ai.agent.name"); ok && name != "" { + if name, ok := a.AttrString(AttrGenAIAgentName); ok && name != "" { return name } } diff --git a/internal/scaffold/fullsend-repo-gitlab/.gitlab/ci/fullsend-agent.yml b/internal/scaffold/fullsend-repo-gitlab/.gitlab/ci/fullsend-agent.yml index 00f6309744..f2dcf71688 100644 --- a/internal/scaffold/fullsend-repo-gitlab/.gitlab/ci/fullsend-agent.yml +++ b/internal/scaffold/fullsend-repo-gitlab/.gitlab/ci/fullsend-agent.yml @@ -292,7 +292,9 @@ stages: # Eval measurements (fail-open): same CLI as GitHub Actions. Never # fail the agent job. Manifest: local .fullsend override, else - # SHA-pinned fetch from public fullsend-ai/agents (GitHub client). + # SHA-pinned fetch from public fullsend-ai/agents (GitHub GetRef). + # This job has no GH_TOKEN; stock-agent manifests skip unless an + # operator exports GH_TOKEN or GITHUB_TOKEN. Local override still works. fullsend eval-measure \ --agent "${STAGE}" \ --fullsend-dir .fullsend \ diff --git a/internal/scaffold/scaffold_gitlab_test.go b/internal/scaffold/scaffold_gitlab_test.go index 5453ce6e7c..d872a292bf 100644 --- a/internal/scaffold/scaffold_gitlab_test.go +++ b/internal/scaffold/scaffold_gitlab_test.go @@ -159,6 +159,7 @@ func TestGitLabAgentTemplateContent(t *testing.T) { assert.Contains(t, s, "RUN_STATUS=$?") assert.Contains(t, s, `exit "${RUN_STATUS}"`) assert.Contains(t, s, "|| true") + assert.Contains(t, s, "stock-agent manifests skip") assert.Contains(t, s, "--fullsend-dir") assert.Contains(t, s, "--target-repo") assert.Contains(t, s, "--output-dir") From 61b9bae78e6cc4e5b766b4bd36b8b013c0d3bf1f Mon Sep 17 00:00:00 2001 From: Adam Scerra Date: Mon, 17 Aug 2026 21:54:30 -0400 Subject: [PATCH 09/14] fix(eval): keep partial eval-measure rows on persist error Forward MeasureAndExport's already-scored results when a later append/ledger write fails so stdout matches the JSONL on disk. Co-authored-by: Cursor Signed-off-by: Adam Scerra --- internal/cli/evalmeasure.go | 2 +- internal/cli/evalmeasure_test.go | 62 ++++++++++++++++++++++++++++++++ internal/evalmeasure/run.go | 15 ++++++++ internal/evalmeasure/run_test.go | 33 +++++++++++++++++ 4 files changed, 111 insertions(+), 1 deletion(-) diff --git a/internal/cli/evalmeasure.go b/internal/cli/evalmeasure.go index 49528f1d8a..0521f8f655 100644 --- a/internal/cli/evalmeasure.go +++ b/internal/cli/evalmeasure.go @@ -113,7 +113,7 @@ func runEvalMeasure(ctx context.Context, printer *ui.Printer, opts evalMeasureOp printer.StepWarn(fmt.Sprintf("%s: skipped %d of %d unreadable telemetry line(s)", p, stats.SkippedLines, stats.NonEmptyLines)) } if err != nil { - return all, false, err + return append(all, results...), false, err } all = append(all, results...) } diff --git a/internal/cli/evalmeasure_test.go b/internal/cli/evalmeasure_test.go index 92889d53e4..b17e803e30 100644 --- a/internal/cli/evalmeasure_test.go +++ b/internal/cli/evalmeasure_test.go @@ -101,6 +101,68 @@ func TestEvalMeasureCmd_OutputDirIgnoresNestedTelemetry(t *testing.T) { assert.NotContains(t, string(b), "ffffffffffffffffffffffffffffffff") } +func writeTwoTraceTelemetry(t *testing.T) string { + t.Helper() + src, err := os.ReadFile(filepath.Join("..", "evalmeasure", "testdata", "complete.jsonl")) + require.NoError(t, err) + src = bytes.TrimSpace(src) + second := bytes.ReplaceAll(src, []byte("aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"), []byte("bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb")) + p := filepath.Join(t.TempDir(), "run-telemetry.jsonl") + require.NoError(t, os.WriteFile(p, append(append(src, '\n'), second...), 0o644)) + return p +} + +func TestRunEvalMeasure_ErrorIncludesPartialResults(t *testing.T) { + out := t.TempDir() + telem := writeTwoTraceTelemetry(t) + registry := filepath.Join("..", "evalmeasure", "testdata", "sample-registry.yaml") + ctx := evalmeasure.WithPersistHook(context.Background(), func() { + meas := filepath.Join(out, evalmeasure.MeasurementsFile) + require.NoError(t, os.Remove(meas)) + require.NoError(t, os.Mkdir(meas, 0o755)) + }) + var buf bytes.Buffer + results, skipped, err := runEvalMeasure(ctx, ui.New(&buf), evalMeasureOpts{ + telemetryPath: telem, + registryPath: registry, + outDir: out, + }) + require.Error(t, err) + assert.False(t, skipped) + require.GreaterOrEqual(t, len(results), 1) + assert.Equal(t, "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", results[0].TraceID) + + printMeasurementResults(ui.New(&buf), results, false) + assert.Contains(t, buf.String(), "trace_fitness") + assert.NotContains(t, buf.String(), "Wrote") +} + +func TestEvalMeasureCmd_ErrorPrintsPartialFromFailingFile(t *testing.T) { + out := t.TempDir() + telem := writeTwoTraceTelemetry(t) + ctx := evalmeasure.WithPersistHook(context.Background(), func() { + meas := filepath.Join(out, evalmeasure.MeasurementsFile) + require.NoError(t, os.Remove(meas)) + require.NoError(t, os.Mkdir(meas, 0o755)) + }) + + cmd := newRootCmd() + cmd.SetContext(ctx) + buf := &bytes.Buffer{} + cmd.SetOut(buf) + cmd.SetErr(buf) + cmd.SetArgs([]string{ + "eval-measure", + "--telemetry", telem, + "--registry", filepath.Join("..", "evalmeasure", "testdata", "sample-registry.yaml"), + "--out-dir", out, + }) + err := cmd.Execute() + require.Error(t, err) + assert.Contains(t, buf.String(), "trace_fitness") + assert.NotContains(t, buf.String(), "Wrote") +} + func TestEvalMeasureCmd_ErrorDoesNotPrintWrote(t *testing.T) { out := filepath.Join(t.TempDir(), "not-a-dir") require.NoError(t, os.WriteFile(out, []byte("x"), 0o644)) diff --git a/internal/evalmeasure/run.go b/internal/evalmeasure/run.go index 773c98eaef..ace2973786 100644 --- a/internal/evalmeasure/run.go +++ b/internal/evalmeasure/run.go @@ -6,6 +6,17 @@ import ( "path/filepath" ) +type persistHookKey struct{} + +// WithPersistHook runs fn after each successful RecordScored. Tests use it +// to fail a later persist; production callers pass a plain context. +func WithPersistHook(ctx context.Context, fn func()) context.Context { + if ctx == nil { + ctx = context.Background() + } + return context.WithValue(ctx, persistHookKey{}, fn) +} + // MeasureFile parses telemetry, scores with the manifest, and writes local // eval-measurements.jsonl. Idempotent per ledger. func MeasureFile(telemetryPath, registryPath, outDir string) ([]EvaluationResult, error) { @@ -35,6 +46,7 @@ func MeasureAndExport(ctx context.Context, telemetryPath, registryPath, outDir s ledgerPath := filepath.Join(outDir, LedgerFile) measPath := filepath.Join(outDir, MeasurementsFile) var all []EvaluationResult + hook, _ := ctx.Value(persistHookKey{}).(func()) for _, tr := range traces { results := ScoreTrace(tr, reg) @@ -53,6 +65,9 @@ func MeasureAndExport(ctx context.Context, telemetryPath, registryPath, outDir s if err := RecordScored(ledgerPath, r.TraceID, r.Name, r.Version); err != nil { return all, stats, fmt.Errorf("record scored: %w", err) } + if hook != nil { + hook() + } } } diff --git a/internal/evalmeasure/run_test.go b/internal/evalmeasure/run_test.go index fff57a7c2e..165e69b86a 100644 --- a/internal/evalmeasure/run_test.go +++ b/internal/evalmeasure/run_test.go @@ -95,6 +95,39 @@ func TestMeasureAndExport_CancelledContext(t *testing.T) { require.Error(t, err) } +func TestWithPersistHook_NilContext(t *testing.T) { + ctx := WithPersistHook(nil, func() {}) + require.NotNil(t, ctx) + _, ok := ctx.Value(persistHookKey{}).(func()) + assert.True(t, ok) +} + +func writeTwoTraceTelemetry(t *testing.T, completePath string) string { + t.Helper() + src, err := os.ReadFile(completePath) + require.NoError(t, err) + src = bytes.TrimSpace(src) + second := bytes.ReplaceAll(src, []byte("aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"), []byte("bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb")) + p := filepath.Join(t.TempDir(), "run-telemetry.jsonl") + require.NoError(t, os.WriteFile(p, append(append(src, '\n'), second...), 0o644)) + return p +} + +func TestMeasureAndExport_KeepsFirstWhenSecondPersistFails(t *testing.T) { + out := t.TempDir() + telem := writeTwoTraceTelemetry(t, filepath.Join("testdata", "complete.jsonl")) + ctx := WithPersistHook(context.Background(), func() { + meas := filepath.Join(out, MeasurementsFile) + require.NoError(t, os.Remove(meas)) + require.NoError(t, os.Mkdir(meas, 0o755)) + }) + results, _, err := MeasureAndExport(ctx, telem, filepath.Join("testdata", "sample-registry.yaml"), out) + require.Error(t, err) + require.GreaterOrEqual(t, len(results), 1) + assert.Equal(t, "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", results[0].TraceID) + assert.Contains(t, err.Error(), "append measurements") +} + func TestMeasureFile_PrescriptSkippedRecordsSkip(t *testing.T) { out := t.TempDir() results, err := MeasureFile( From 5a21884641a06209d36203fd134d69d129d7c1d6 Mon Sep 17 00:00:00 2001 From: Adam Scerra Date: Tue, 18 Aug 2026 19:25:01 -0400 Subject: [PATCH 10/14] fix(eval): address third-round measurement review findings Align stdout with disk on persist errors, fail-open parse/span skips with Incomplete warns, normalize agent runDir matching, add --offline, fix GitLab artifact retention and work_item URLs, restore exit-78 neutral skip, and accept gen_ai.provider.name for EM-001. Co-authored-by: Cursor Signed-off-by: Adam Scerra --- action.yml | 3 +- ...050-distributed-tracing-instrumentation.md | 4 +- ...-eval-measurements-online-trace-scoring.md | 24 ++- docs/cli/README.md | 1 + docs/guides/dev/cli-internals.md | 1 + .../infrastructure/distributed-tracing.md | 7 +- .../infrastructure/eval-measurements.md | 36 ++-- internal/cli/evalmeasure.go | 26 ++- internal/cli/evalmeasure_test.go | 54 +++++- internal/cli/prescript_run_test.go | 155 ++++++++++++++++++ internal/cli/run.go | 79 ++++++++- internal/cli/telemetry_run_test.go | 36 +++- internal/evalmeasure/find.go | 24 ++- internal/evalmeasure/find_test.go | 30 ++++ internal/evalmeasure/fitness.go | 26 ++- internal/evalmeasure/parse.go | 31 +++- internal/evalmeasure/parse_test.go | 18 ++ internal/evalmeasure/registry.go | 2 +- internal/evalmeasure/run.go | 18 +- internal/evalmeasure/run_test.go | 33 +++- internal/evalmeasure/score_test.go | 48 ++++++ internal/evalmeasure/types.go | 11 +- internal/prescript/prescript.go | 11 ++ .../.gitlab/ci/fullsend-agent.yml | 35 +++- internal/scaffold/scaffold_gitlab_test.go | 9 +- 25 files changed, 632 insertions(+), 90 deletions(-) diff --git a/action.yml b/action.yml index 70f1a913f6..461086a8b7 100644 --- a/action.yml +++ b/action.yml @@ -411,7 +411,8 @@ runs: # Eval measurements (fail-open): score run-telemetry.jsonl with the agents # measurement manifest. Same job as fullsend run; never fails the agent. - # Scores always land in eval-measurements.jsonl (tool-agnostic artifact). + # Writes eval-measurements.jsonl when at least one new score row is produced + # (tool-agnostic artifact); missing telemetry/manifest skips with no file. - name: Eval measurements if: always() && inputs.agent != '__install_only__' continue-on-error: true diff --git a/docs/ADRs/0050-distributed-tracing-instrumentation.md b/docs/ADRs/0050-distributed-tracing-instrumentation.md index 9bab3708f9..c8e3dad671 100644 --- a/docs/ADRs/0050-distributed-tracing-instrumentation.md +++ b/docs/ADRs/0050-distributed-tracing-instrumentation.md @@ -157,8 +157,8 @@ span export via the OTel SDK's batch processor. The core decision (three-level opt-in, OTel-native, W3C propagation) is unchanged. **2026-08-10 — Eval measurements ([ADR 0087](0087-eval-measurements-online-trace-scoring.md)):** -online scoring of wild-run traces always writes `eval-measurements.jsonl` -beside telemetry (tool-agnostic). Distinct from functional eval fixtures +online scoring of wild-run traces writes `eval-measurements.jsonl` +beside telemetry when at least one new score is produced (tool-agnostic). Distinct from functional eval fixtures ([ADR 0051](0051-agent-eval-harness-for-test-infrastructure.md)). > **Planned:** portable remote score export follows the same OTLP diff --git a/docs/ADRs/0087-eval-measurements-online-trace-scoring.md b/docs/ADRs/0087-eval-measurements-online-trace-scoring.md index 4c8d3b41a3..bc69e268fc 100644 --- a/docs/ADRs/0087-eval-measurements-online-trace-scoring.md +++ b/docs/ADRs/0087-eval-measurements-online-trace-scoring.md @@ -78,8 +78,9 @@ on the run (the OTEL trace / `run-telemetry.jsonl`). computed from that trace (`eval-measurements.jsonl`). Measurements never rewrite primary facts, and they are [fail-open](../glossary.md#fail-open). -Scores always land in a tool-agnostic `eval-measurements.jsonl` (plus a -small idempotency ledger) next to `run-telemetry.jsonl`. Remote score export +Scores land in a tool-agnostic `eval-measurements.jsonl` (plus a +small idempotency ledger) next to `run-telemetry.jsonl` whenever at least +one new measurement row is produced (including `label: skip`). Remote score export will use the same `OTEL_EXPORTER_OTLP_*` configuration as ADR 0050 — no vendor-specific score adapters in core. `fullsend` owns the parser, scorers, CLI, and GHA step; `fullsend-ai/agents` owns per-agent measurement manifests @@ -89,8 +90,12 @@ override, opt-out, or custom agents only. The first scorer is `trace_fitness` (catalog id `em-001`) — span-tree and attribute fitness so later scorers can trust the trace. EM-001 reads -experimental OpenTelemetry GenAI attribute names (`gen_ai.*` constants in -`internal/evalmeasure`); an upstream rename is an `em-001` version bump. +OpenTelemetry GenAI attribute names (`gen_ai.*` constants in +`internal/evalmeasure`). `gen_ai.system` was renamed to `gen_ai.provider.name` +in semconv v1.37.0; `modelOK` accepts either so `em-001@1` survives the +emitter migration. Other upstream renames remain an `em-001` version bump. +Pre-script-skipped runs and runs with no `agent` span (never reached an +iteration) record `label: skip` and are excluded from pass/(pass+fail). ### Versioning (per measurement, not platform “v1”) @@ -113,8 +118,10 @@ Entirely new signal → new `em-NNN` (and usually a new `scorer` string). - Every measured run produces a reviewable, backend-agnostic score file beside telemetry; missing manifests skip cleanly and measure failure never fails the agent job. GitHub Actions is the first-ship managed path (uploads - `output/`). GitLab CI calls the same fail-open `eval-measure` CLI, writing - under `/tmp/fullsend-output` with no `artifacts:` block by default. + `output/`). GitLab CI calls the same fail-open `eval-measure` CLI under + `$CI_PROJECT_DIR/output` with `artifacts: when: always`. Stock manifests + fetch from public `agents@v0` even without `GH_TOKEN` (rate-limited); a + token is recommended on shared runners. - Core stays tool-agnostic: no product-specific score env vars in managed workflows; remote scores follow OTEL when that path lands. - Functional scenarios (gate) and eval measurements (trend) stay separate; @@ -124,5 +131,6 @@ Entirely new signal → new `em-NNN` (and usually a new `scorer` string). - Per-measurement versioning (`id@version`) lets pass/fail semantics evolve without mixing trend eras. - Pre-script skipped runs (`fullsend.prescript.skipped=true` on the root span) - are excluded from EM-001: the scorer writes `label: skip` instead of failing - a run that never created a sandbox. + and runs with no `agent` span (never reached an iteration) are excluded from + EM-001: the scorer writes `label: skip` instead of failing a run that never + produced a full telemetry contract. diff --git a/docs/cli/README.md b/docs/cli/README.md index 09b70ed236..d049d7d99a 100644 --- a/docs/cli/README.md +++ b/docs/cli/README.md @@ -27,6 +27,7 @@ Download the latest binary from [GitHub Releases](https://github.com/fullsend-ai | `fullsend run` | Execute an agent locally in a sandbox. See [running agents locally](../guides/user/running-agents-locally.md). | | `fullsend lock [agent-name]` | Pin remote dependencies to `lock.yaml` | | `fullsend scan` | Run security scanners on agent input/output | +| `fullsend eval-measure` | Score wild-run traces into `eval-measurements.jsonl`. See [Eval measurements](../guides/infrastructure/eval-measurements.md). | ## Global flags diff --git a/docs/guides/dev/cli-internals.md b/docs/guides/dev/cli-internals.md index 6cf81fc2ce..e5ec517e86 100644 --- a/docs/guides/dev/cli-internals.md +++ b/docs/guides/dev/cli-internals.md @@ -133,6 +133,7 @@ fullsend │ ├── --registry # Agents measurement manifest YAML (or --agent) │ ├── --agent # Agent name for manifest resolution (managed-job form) │ ├── --fullsend-dir # .fullsend dir (local manifest override + fetch cache) +│ ├── --offline # Reject network fetches (local manifest only) │ └── --out-dir # Output dir (default: telemetry directory) └── reconcile-status # Finalize orphaned status comments ├── --repo # Repository in owner/repo format diff --git a/docs/guides/infrastructure/distributed-tracing.md b/docs/guides/infrastructure/distributed-tracing.md index 005d874cd5..c38db4c1c4 100644 --- a/docs/guides/infrastructure/distributed-tracing.md +++ b/docs/guides/infrastructure/distributed-tracing.md @@ -108,7 +108,7 @@ and are recognized by LLM-aware backends for GenAI dashboards. |-----------|---------|------------| | `gen_ai.operation.name` | `invoke_agent` | `run`, `agent` (`create_agent` on `sandbox_create`) | | `gen_ai.agent.name` | `triage` | `run`, `agent` | -| `gen_ai.system` | `anthropic` | `agent` (the model vendor, from the runtime) | +| `gen_ai.system` / `gen_ai.provider.name` | `anthropic` | `agent` (model vendor; `system` deprecated in OTel GenAI semconv v1.37 — EM-001 accepts either) | | `gen_ai.request.model` | `claude-opus-4-6` | `run` (aggregated), `agent` (resolved model) | | `gen_ai.usage.input_tokens` / `output_tokens` / `cache_*_input_tokens` | `109938` | `run` (aggregated), `agent` | @@ -247,8 +247,9 @@ authentication mechanism. ## Eval measurements After each managed agent run, `fullsend eval-measure` scores -`run-telemetry.jsonl` in the same job (fail-open). Scores always land in -`eval-measurements.jsonl` beside telemetry (tool-agnostic artifact). Portable +`run-telemetry.jsonl` in the same job (fail-open). Scores land in +`eval-measurements.jsonl` beside telemetry when at least one new score is +produced (tool-agnostic artifact). Portable remote export will reuse the same `OTEL_EXPORTER_OTLP_*` configuration as agent traces when implemented. Measurements read the Level 1/2 metadata contract of `run-telemetry.jsonl` (not Level 3 content). See diff --git a/docs/guides/infrastructure/eval-measurements.md b/docs/guides/infrastructure/eval-measurements.md index f73d442c38..e445205368 100644 --- a/docs/guides/infrastructure/eval-measurements.md +++ b/docs/guides/infrastructure/eval-measurements.md @@ -40,8 +40,8 @@ fullsend run (any compatible backend — ADR 0050) fullsend eval-measure (same GHA job, fail-open, after run) - └─ always writes output//eval-measurements.jsonl - (+ eval-measure-ledger.txt for idempotency) + └─ writes output//eval-measurements.jsonl when at least one + new score is produced (+ eval-measure-ledger.txt for idempotency) ``` > **Planned:** portable remote score export via the same `OTEL_EXPORTER_OTLP_*` @@ -148,10 +148,12 @@ success/failure signal for outcome scorers. Pre-script **skipped** runs set `fullsend.prescript.skipped=true` on the root span and never create a sandbox. EM-001 records `label: skip` for those traces -instead of failing the span-tree / model / usage checks. An unknown `scorer:` -string (for example a newer `agents@v0` manifest this binary does not implement -yet) also writes `label: skip`, not `fail`. Trend pass-rate as -`pass / (pass + fail)` and drop `skip`. +instead of failing the span-tree / model / usage checks. Runs with no `agent` +span (never reached an iteration — e.g. sandbox/provider failure) are also +`label: skip`, so pass/(pass+fail) measures the telemetry contract rather than +runner health. An unknown `scorer:` string (for example a newer `agents@v0` +manifest this binary does not implement yet) also writes `label: skip`, not +`fail`. Trend pass-rate as `pass / (pass + fail)` and drop `skip`. ## Adjacent telemetry work @@ -171,22 +173,27 @@ GitHub Actions job └── upload-artifact # includes both JSONL files under output/ GitLab CI agent job -├── fullsend run # --output-dir /tmp/fullsend-output +├── fullsend run # --output-dir $CI_PROJECT_DIR/output ├── fullsend eval-measure # always (even if run failed); || true -└── (no artifacts: by default — JSONL stays on the runner filesystem) +└── artifacts: output/ # when: always (parity with GHA upload of output/) ``` +Add `output/` to the consuming repo's `.gitignore` so local GitLab-checkout +runs do not stage telemetry accidentally. + ## Manifest resolution in CI `fullsend eval-measure` resolves the measurement manifest for the agent: 1. `${FULLSEND_DIR}/eval/measurements/${AGENT}.yaml` if present, else -2. SHA-pinned `eval/measurements/${AGENT}.yaml` from `fullsend-ai/agents` +2. SHA-pinned `eval/measurements/${AGENT}.yaml` from public `fullsend-ai/agents` (same `v0` → commit SHA, allowlist, hash, and fetch audit as harness fallback — not a floating `raw.githubusercontent.com/.../v0/...` curl). GitHub Actions injects `GH_TOKEN` for that `GetRef`. GitLab CI has no - GitHub token by default, so step 2 skips unless an operator exports - `GH_TOKEN` / `GITHUB_TOKEN`; a local FULLSEND_DIR override still works. + GitHub token by default; because `agents` is public, `GetRef` still runs + unauthenticated (~60 req/hr per IP). On busy shared runners, export + `GH_TOKEN` / `GITHUB_TOKEN` to avoid rate-limit skips. A local + FULLSEND_DIR override skips the remote fetch entirely. Step 2 is how stock-agent defaults reach every install. Step 1 is override / BYOA only. @@ -210,12 +217,17 @@ fullsend eval-measure \ - `--agent` + `--output-dir` is the managed-job form. `--registry` / `--telemetry` remain for pointing at explicit files. +- `--offline` rejects the remote `agents@v0` fetch (local FULLSEND_DIR + manifest only), matching `fullsend run --offline`. - Exit `0` when a score is `fail` — scores are data. - Exit `0` when telemetry or the manifest is missing (skip). ## Implementation note -Today the measure CLI always writes local `eval-measurements.jsonl`. +Today the measure CLI writes local `eval-measurements.jsonl` whenever at +least one new measurement row is appended (including `label: skip`). No +file is written when telemetry/manifest is missing, no traces match, or +every candidate row is already in the ledger. > **Planned:** portable OTLP score export (same `OTEL_*` as traces) is the > ADR 0087 remote contract and is not wired yet. Until it lands, consume the diff --git a/internal/cli/evalmeasure.go b/internal/cli/evalmeasure.go index 0521f8f655..8afa33aaeb 100644 --- a/internal/cli/evalmeasure.go +++ b/internal/cli/evalmeasure.go @@ -26,6 +26,7 @@ func newEvalMeasureCmd() *cobra.Command { outputDir string agent string fullsendDir string + offline bool ) cmd := &cobra.Command{ @@ -56,6 +57,7 @@ on hard IO/parse errors. Missing telemetry or manifest is a skip (exit 0).`, outputDir: outputDir, agent: agent, fullsendDir: fullsendDir, + offline: offline, }) if err != nil { printMeasurementResults(printer, results, false) @@ -75,6 +77,7 @@ on hard IO/parse errors. Missing telemetry or manifest is a skip (exit 0).`, cmd.Flags().StringVar(&outputDir, "output-dir", "", "CI output base or runDir; scores only top-of-runDir telemetry") cmd.Flags().StringVar(&agent, "agent", "", "agent name for manifest resolution") cmd.Flags().StringVar(&fullsendDir, "fullsend-dir", "", "path to the .fullsend directory (local manifest override + fetch cache)") + cmd.Flags().BoolVar(&offline, "offline", false, "reject network fetches; only use a local FULLSEND_DIR measurement manifest") return cmd } @@ -85,6 +88,7 @@ type evalMeasureOpts struct { outputDir string agent string fullsendDir string + offline bool } func runEvalMeasure(ctx context.Context, printer *ui.Printer, opts evalMeasureOpts) ([]evalmeasure.EvaluationResult, bool, error) { @@ -109,8 +113,14 @@ func runEvalMeasure(ctx context.Context, printer *ui.Printer, opts evalMeasureOp var all []evalmeasure.EvaluationResult for _, p := range telemPaths { results, stats, err := evalmeasure.MeasureAndExport(ctx, p, registry, opts.outDir) + if stats.Incomplete != "" { + printer.StepWarn(fmt.Sprintf("%s: telemetry parse incomplete (%s); scored available traces", p, stats.Incomplete)) + } if stats.SkippedLines > 0 { - printer.StepWarn(fmt.Sprintf("%s: skipped %d of %d unreadable telemetry line(s)", p, stats.SkippedLines, stats.NonEmptyLines)) + printer.StepWarn(fmt.Sprintf("%s: skipped %d unreadable of %d telemetry line(s)", p, stats.SkippedLines, stats.NonEmptyLines)) + } + if stats.SkippedSpans > 0 { + printer.StepWarn(fmt.Sprintf("%s: skipped %d unreadable span(s) inside otherwise-valid telemetry line(s)", p, stats.SkippedSpans)) } if err != nil { return append(all, results...), false, err @@ -159,7 +169,7 @@ func resolveEvalMeasureRegistry(ctx context.Context, printer *ui.Printer, opts e return local, nil } } - composeOpts, client := evalMeasureFetchContext(opts.fullsendDir, printer) + composeOpts, client := evalMeasureFetchContext(opts.fullsendDir, opts.offline, printer) path, ok := tryAgentsRepoMeasurementManifest(ctx, agent, client, composeOpts, printer) if !ok { return "", nil @@ -184,7 +194,7 @@ func localMeasurementManifest(fullsendDir, agent string) (string, error) { return resolved, nil } -func evalMeasureFetchContext(fullsendDir string, printer *ui.Printer) (harness.ComposeOpts, forge.Client) { +func evalMeasureFetchContext(fullsendDir string, offline bool, printer *ui.Printer) (harness.ComposeOpts, forge.Client) { workspace := fullsendDir if workspace == "" { workspace = os.TempDir() @@ -200,12 +210,16 @@ func evalMeasureFetchContext(fullsendDir string, printer *ui.Printer) (harness.C } } token, err := resolveToken() - if (err != nil || token == "") && printer != nil { - printer.StepWarn("No GH_TOKEN/GITHUB_TOKEN; agents@v0 GetRef is unauthenticated. GitLab jobs skip stock manifests unless an operator wires a GitHub token.") + if (err != nil || token == "") && printer != nil && !offline { + printer.StepWarn("No GH_TOKEN/GITHUB_TOKEN; agents@v0 GetRef runs unauthenticated (public repo, ~60 req/hr per IP). Prefer a token on shared runners; local FULLSEND_DIR override skips the fetch.") + } + policy := fetch.DefaultPolicy + if offline { + policy.Offline = true } return harness.ComposeOpts{ WorkspaceRoot: abs, - FetchPolicy: fetch.DefaultPolicy, + FetchPolicy: policy, AuditLogPath: filepath.Join(abs, ".fullsend-cache", "fetch-audit.jsonl"), OrgAllowlist: orgAllowlist, GitToken: token, diff --git a/internal/cli/evalmeasure_test.go b/internal/cli/evalmeasure_test.go index b17e803e30..577d5edb8c 100644 --- a/internal/cli/evalmeasure_test.go +++ b/internal/cli/evalmeasure_test.go @@ -41,6 +41,13 @@ func TestEvalMeasureCmd_ScoresFixture(t *testing.T) { assert.Contains(t, buf.String(), "Wrote 1 measurement(s)") } +func TestEvalMeasureCmd_HasOfflineFlag(t *testing.T) { + cmd := newEvalMeasureCmd() + f := cmd.Flags().Lookup("offline") + require.NotNil(t, f) + assert.Equal(t, "false", f.DefValue) +} + func TestRootCommand_HasEvalMeasureSubcommand(t *testing.T) { cmd := newRootCmd() found := false @@ -66,7 +73,7 @@ func TestEvalMeasureCmd_MissingRequiredFlags(t *testing.T) { func TestEvalMeasureCmd_OutputDirIgnoresNestedTelemetry(t *testing.T) { fsDir := t.TempDir() outBase := t.TempDir() - runDir := filepath.Join(outBase, "agent-triage-1") + runDir := filepath.Join(outBase, "agent-triage-1-1") nested := filepath.Join(runDir, "iteration-1", "output") require.NoError(t, os.MkdirAll(nested, 0o755)) @@ -129,7 +136,7 @@ func TestRunEvalMeasure_ErrorIncludesPartialResults(t *testing.T) { }) require.Error(t, err) assert.False(t, skipped) - require.GreaterOrEqual(t, len(results), 1) + require.Len(t, results, 1) assert.Equal(t, "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", results[0].TraceID) printMeasurementResults(ui.New(&buf), results, false) @@ -215,7 +222,42 @@ func TestEvalMeasureCmd_WarnsOnCorruptTelemetryLine(t *testing.T) { "--out-dir", out, }) require.NoError(t, cmd.Execute()) - assert.Contains(t, buf.String(), "unreadable telemetry line") + assert.Contains(t, buf.String(), "skipped 1 unreadable of 2 telemetry line") +} + +func TestEvalMeasureCmd_WarnsOnIncompleteParse(t *testing.T) { + out := t.TempDir() + good, err := os.ReadFile(filepath.Join("..", "evalmeasure", "testdata", "complete.jsonl")) + require.NoError(t, err) + telem := filepath.Join(out, "run-telemetry.jsonl") + f, err := os.Create(telem) + require.NoError(t, err) + _, err = f.Write(append(bytes.TrimSpace(good), '\n')) + require.NoError(t, err) + huge := make([]byte, 11*1024*1024) + for i := range huge { + huge[i] = 'a' + } + _, err = f.Write(huge) + require.NoError(t, err) + _, err = f.Write([]byte("\n")) + require.NoError(t, err) + require.NoError(t, f.Close()) + + cmd := newRootCmd() + buf := &bytes.Buffer{} + cmd.SetOut(buf) + cmd.SetErr(buf) + cmd.SetArgs([]string{ + "eval-measure", + "--telemetry", telem, + "--registry", filepath.Join("..", "evalmeasure", "testdata", "sample-registry.yaml"), + "--out-dir", out, + }) + require.NoError(t, cmd.Execute()) + assert.Contains(t, buf.String(), "telemetry parse incomplete") + assert.Contains(t, buf.String(), "scored available traces") + assert.Contains(t, buf.String(), "Wrote") } func TestActionYML_EvalMeasureNoFloatingV0Curl(t *testing.T) { @@ -246,17 +288,19 @@ func TestPlatformTelemetryFileMatchesRecorder(t *testing.T) { func TestEvalMeasureFetchContext(t *testing.T) { printer := ui.New(io.Discard) - opts, client := evalMeasureFetchContext("", printer) + opts, client := evalMeasureFetchContext("", false, printer) require.NotNil(t, client) assert.NotEmpty(t, opts.OrgAllowlist) assert.NotEmpty(t, opts.WorkspaceRoot) + assert.False(t, opts.FetchPolicy.Offline) dir := t.TempDir() - opts2, _ := evalMeasureFetchContext(dir, printer) + opts2, _ := evalMeasureFetchContext(dir, true, printer) assert.Contains(t, opts2.AuditLogPath, ".fullsend-cache") abs, err := filepath.Abs(dir) require.NoError(t, err) assert.Equal(t, abs, opts2.WorkspaceRoot) + assert.True(t, opts2.FetchPolicy.Offline) } func TestResolveEvalMeasureRegistry_LocalOverride(t *testing.T) { diff --git a/internal/cli/prescript_run_test.go b/internal/cli/prescript_run_test.go index 0284970a81..27ba9a3ee0 100644 --- a/internal/cli/prescript_run_test.go +++ b/internal/cli/prescript_run_test.go @@ -2,6 +2,7 @@ package cli import ( "context" + "fmt" "io" "os" "path/filepath" @@ -200,6 +201,160 @@ func TestRunPreScript_CleansUpOutputFile(t *testing.T) { assert.Empty(t, entries) } +// --- Exit code 78 (neutral skip) tests (issue #582) --- + +func TestRunPreScript_Exit78_SkipsWithStdoutReason(t *testing.T) { + printer := ui.New(io.Discard) + h := &harness.Harness{PreScript: writePreScript(t, + "echo \"No issues need scoring\"\nexit 78\n")} + + res, err := runPreScript(h, t.TempDir(), "", printer) + require.NoError(t, err) + assert.True(t, res.Skipped) + assert.Equal(t, "No issues need scoring", res.Reason) +} + +func TestRunPreScript_Exit78_SkipsWithOutputFileReason(t *testing.T) { + printer := ui.New(io.Discard) + // Script writes a reason to the output file, then exits 78. The file + // reason should take precedence over stdout. + h := &harness.Harness{PreScript: writePreScript(t, + `echo "reason=all scores are fresh" >> "${FULLSEND_PRESCRIPT_OUTPUT}"`+"\n"+ + "echo \"stdout line\"\n"+ + "exit 78\n")} + + res, err := runPreScript(h, t.TempDir(), "", printer) + require.NoError(t, err) + assert.True(t, res.Skipped) + assert.Equal(t, "all scores are fresh", res.Reason) +} + +func TestRunPreScript_Exit78_OverridesSkippedFalseInFile(t *testing.T) { + printer := ui.New(io.Discard) + // Script explicitly writes skipped=false but exits 78. Exit code wins. + h := &harness.Harness{PreScript: writePreScript(t, + `echo "skipped=false" >> "${FULLSEND_PRESCRIPT_OUTPUT}"`+"\n"+ + "exit 78\n")} + + res, err := runPreScript(h, t.TempDir(), "", printer) + require.NoError(t, err) + assert.True(t, res.Skipped, "exit 78 must override skipped=false in output file") +} + +func TestRunPreScript_Exit78_NoReasonDefaultsEmpty(t *testing.T) { + printer := ui.New(io.Discard) + // Script exits 78 with no stdout and no output file content. + h := &harness.Harness{PreScript: writePreScript(t, "exit 78\n")} + + res, err := runPreScript(h, t.TempDir(), "", printer) + require.NoError(t, err) + assert.True(t, res.Skipped) + assert.Empty(t, res.Reason) +} + +func TestRunPreScript_Exit78_DeletedOutputFileStillSkips(t *testing.T) { + printer := ui.New(io.Discard) + // Script deletes the output file then exits 78. Exit 0 treats a missing + // file as a hard error; exit 78 must still skip — the exit code is + // authoritative. + h := &harness.Harness{PreScript: writePreScript(t, + `rm -f "${FULLSEND_PRESCRIPT_OUTPUT}"`+"\n"+ + "echo \"No work today\"\n"+ + "exit 78\n")} + + res, err := runPreScript(h, t.TempDir(), "", printer) + require.NoError(t, err) + assert.True(t, res.Skipped) + assert.Equal(t, "No work today", res.Reason) +} + +func TestRunPreScript_Exit78_MalformedOutputFileStillSkips(t *testing.T) { + printer := ui.New(io.Discard) + // Script writes malformed content to the output file but exits 78. + // The exit code is authoritative — a parse error must not block the skip. + h := &harness.Harness{PreScript: writePreScript(t, + `echo "this is not key=value format but has no equals" >> "${FULLSEND_PRESCRIPT_OUTPUT}"`+"\n"+ + "echo \"Skipping: no work\"\n"+ + "exit 78\n")} + + res, err := runPreScript(h, t.TempDir(), "", printer) + require.NoError(t, err) + assert.True(t, res.Skipped) + assert.Equal(t, "Skipping: no work", res.Reason) +} + +func TestRunPreScript_Exit78_PreservesOtherOutputs(t *testing.T) { + printer := ui.New(io.Discard) + h := &harness.Harness{PreScript: writePreScript(t, + `echo "reason=stale scores refreshed" >> "${FULLSEND_PRESCRIPT_OUTPUT}"`+"\n"+ + `echo "checked_count=42" >> "${FULLSEND_PRESCRIPT_OUTPUT}"`+"\n"+ + "exit 78\n")} + + res, err := runPreScript(h, t.TempDir(), "", printer) + require.NoError(t, err) + assert.True(t, res.Skipped) + assert.Equal(t, "stale scores refreshed", res.Reason) + assert.Equal(t, "42", res.Outputs["checked_count"]) +} + +func TestRunPreScript_Exit78_UsesLastNonEmptyStdoutLine(t *testing.T) { + printer := ui.New(io.Discard) + h := &harness.Harness{PreScript: writePreScript(t, + "echo \"Checking issues...\"\n"+ + "echo \"Checked 5 issues\"\n"+ + "echo \"All scores are current\"\n"+ + "echo \"\"\n"+ + "exit 78\n")} + + res, err := runPreScript(h, t.TempDir(), "", printer) + require.NoError(t, err) + assert.True(t, res.Skipped) + assert.Equal(t, "All scores are current", res.Reason) +} + +func TestRunPreScript_Exit78_StdoutReasonSanitized(t *testing.T) { + printer := ui.New(io.Discard) + h := &harness.Harness{PreScript: writePreScript(t, + "printf 'Has\\ttab and \\x01control'\nexit 78\n")} + + res, err := runPreScript(h, t.TempDir(), "", printer) + require.NoError(t, err) + assert.True(t, res.Skipped) + assert.Equal(t, "Hastab and control", res.Reason) +} + +// Exit code 78 from a pre-script must still relay as skipped=true so +// workflow-level gating works correctly. +func TestRunAgent_PreScriptExit78_RelaysSkippedTrue(t *testing.T) { + usePreScriptStub(t) + out := filepath.Join(t.TempDir(), "github-output") + t.Setenv("GITHUB_ACTIONS", "true") + t.Setenv("GITHUB_OUTPUT", out) + dir := newSkipHarnessDir(t, "echo \"Nothing to do\"\nexit 78\n") + + rFlags := resolveFlags{maxDepth: 10, maxResources: 50} + require.NoError(t, runAgent(context.Background(), "code", dir, "", t.TempDir(), "", nil, false, "", "", + rFlags, statusOpts{}, ui.New(io.Discard), false)) + + data, err := os.ReadFile(out) + require.NoError(t, err) + assert.Contains(t, string(data), "skipped=true") + assert.Contains(t, string(data), "reason=Nothing to do") +} + +// Other non-zero exit codes must remain hard failures — only 78 is neutral. +func TestRunPreScript_OtherNonZeroExitIsStillHardError(t *testing.T) { + printer := ui.New(io.Discard) + for _, code := range []int{1, 2, 77, 79, 127} { + t.Run(fmt.Sprintf("exit_%d", code), func(t *testing.T) { + h := &harness.Harness{PreScript: writePreScript(t, + fmt.Sprintf("exit %d\n", code))} + _, err := runPreScript(h, t.TempDir(), "", printer) + require.ErrorContains(t, err, "running pre-script") + }) + } +} + // usePreScriptStub puts an openshell stub on PATH that passes the gateway // check but refuses sandbox creation, so a run that gets that far fails // recognizably. It also replaces sandbox.RetrySleepFn with a no-op so diff --git a/internal/cli/run.go b/internal/cli/run.go index 20f226a0ba..c7e2f747ab 100644 --- a/internal/cli/run.go +++ b/internal/cli/run.go @@ -1,6 +1,7 @@ package cli import ( + "bytes" "context" "encoding/json" "errors" @@ -897,7 +898,9 @@ func runAgent(ctx context.Context, agentName, fullsendDir, outputBase, targetRep workItemID := resolveWorkItemID() // 3. Create run directory and initialise tracer. - sandboxName := fmt.Sprintf("agent-%s-%d-%d", agentName, os.Getpid(), time.Now().Unix()) + // Lowercase the agent segment so eval-measure --agent (also lowercased) + // matches the host runDir even when the CLI arg was mixed-case. + sandboxName := fmt.Sprintf("agent-%s-%d-%d", strings.ToLower(agentName), os.Getpid(), time.Now().Unix()) if outputBase == "" { outputBase = filepath.Join(os.TempDir(), "fullsend") } @@ -2405,6 +2408,14 @@ func resolveWorkItemID() string { if prNum := strings.TrimSpace(os.Getenv("PR_NUMBER")); prNum != "" { return prNum } + // GitHub retro: reusable-retro.yml sets ORIGINATING_URL (PR/issue HTML URL). + // GitLab agent jobs export GITLAB_ISSUE_URL (issue or MR) when IID is known. + if v := strings.TrimSpace(os.Getenv("ORIGINATING_URL")); v != "" { + return v + } + if v := strings.TrimSpace(os.Getenv("GITLAB_ISSUE_URL")); v != "" { + return v + } return evalmeasure.UnknownSentinel } @@ -2647,12 +2658,40 @@ func runPreScript(h *harness.Harness, runDir, traceparent string, printer *ui.Pr defer cleanup() preCmd := exec.Command(h.PreScript) preCmd.Env = append(childScriptEnv(h.RunnerEnv, traceparent), prescript.EnvVar+"="+outPath) - preCmd.Stdout = os.Stdout + + // Tee stdout so we can use it as a fallback skip reason when the + // script exits 78 without writing a reason to the output file. + var stdoutBuf bytes.Buffer + preCmd.Stdout = io.MultiWriter(os.Stdout, &stdoutBuf) preCmd.Stderr = os.Stderr - if err := preCmd.Run(); err != nil { - printer.StepFail("Pre-script failed") - return prescript.Result{}, fmt.Errorf("running pre-script: %w", err) + + runErr := preCmd.Run() + if runErr != nil { + // Check for exit code 78 (neutral/skip). + var exitErr *exec.ExitError + if !errors.As(runErr, &exitErr) || exitErr.ExitCode() != prescript.ExitCodeNeutral { + printer.StepFail("Pre-script failed") + return prescript.Result{}, fmt.Errorf("running pre-script: %w", runErr) + } + + // Exit 78: parse the output file best-effort for reason and other + // outputs. If parsing fails the exit code is still authoritative. + result, parseErr := prescript.ParseFile(outPath) + if parseErr != nil { + result = prescript.Result{Outputs: map[string]string{}} + } + result.Skipped = true + result.Outputs["skipped"] = "true" + if result.Reason == "" { + result.Reason = lastNonEmptyLine(stdoutBuf.String()) + } + if result.Reason != "" { + result.Outputs["reason"] = result.Reason + } + printer.StepDone(fmt.Sprintf("Pre-script exited 78 (neutral skip) (%.1fs)", time.Since(preStart).Seconds())) + return result, nil } + result, err := prescript.ParseFile(outPath) if err != nil { printer.StepFail("Pre-script output invalid") @@ -2662,6 +2701,36 @@ func runPreScript(h *harness.Harness, runDir, traceparent string, printer *ui.Pr return result, nil } +// lastNonEmptyLine returns the last non-blank line from s, trimmed and +// sanitized. Scripts that exit 78 often print a human-readable reason to +// stdout without writing the output file; this captures that text for +// status reporting. +func lastNonEmptyLine(s string) string { + lines := strings.Split(strings.TrimRight(s, "\n"), "\n") + for i := len(lines) - 1; i >= 0; i-- { + if l := strings.TrimSpace(lines[i]); l != "" { + l = stripControlChars(l) + if len(l) > 1024 { + l = l[:1024] + for len(l) > 0 && !utf8.Valid([]byte(l)) { + l = l[:len(l)-1] + } + } + return l + } + } + return "" +} + +func stripControlChars(s string) string { + return strings.Map(func(r rune) rune { + if r < 0x20 || r == 0x7f { + return -1 + } + return r + }, s) +} + // childScriptEnv builds the environment for a host-side child script (pre- or // post-script): the harness RunnerEnv layered over the process environment, // plus the W3C TRACEPARENT for trace propagation (ADR 0050 Level 1). Any diff --git a/internal/cli/telemetry_run_test.go b/internal/cli/telemetry_run_test.go index a9694a182f..501a2ec0b0 100644 --- a/internal/cli/telemetry_run_test.go +++ b/internal/cli/telemetry_run_test.go @@ -40,14 +40,16 @@ func TestSecurityTraceID_ShellSafe(t *testing.T) { func TestResolveWorkItemID(t *testing.T) { cases := []struct { - name string - issueKey string - repoFull string - issueNumber string - issueURL string - prURL string - prNumber string - want string + name string + issueKey string + repoFull string + issueNumber string + issueURL string + prURL string + prNumber string + originatingURL string + gitlabIssueURL string + want string }{ { name: "ISSUE_KEY wins over everything", @@ -105,6 +107,22 @@ func TestResolveWorkItemID(t *testing.T) { name: "unknown when nothing is set", want: evalmeasure.UnknownSentinel, }, + { + name: "ORIGINATING_URL for retro", + originatingURL: "https://github.com/octo/repo/issues/99", + want: "https://github.com/octo/repo/issues/99", + }, + { + name: "GITLAB_ISSUE_URL for GitLab agent jobs", + gitlabIssueURL: "https://gitlab.example/group/proj/-/issues/12", + want: "https://gitlab.example/group/proj/-/issues/12", + }, + { + name: "ORIGINATING_URL beats GITLAB_ISSUE_URL when both set", + originatingURL: "https://github.com/octo/repo/issues/99", + gitlabIssueURL: "https://gitlab.example/group/proj/-/issues/12", + want: "https://github.com/octo/repo/issues/99", + }, } for _, tc := range cases { t.Run(tc.name, func(t *testing.T) { @@ -114,6 +132,8 @@ func TestResolveWorkItemID(t *testing.T) { t.Setenv("GITHUB_ISSUE_URL", tc.issueURL) t.Setenv("GITHUB_PR_URL", tc.prURL) t.Setenv("PR_NUMBER", tc.prNumber) + t.Setenv("ORIGINATING_URL", tc.originatingURL) + t.Setenv("GITLAB_ISSUE_URL", tc.gitlabIssueURL) assert.Equal(t, tc.want, resolveWorkItemID()) }) } diff --git a/internal/evalmeasure/find.go b/internal/evalmeasure/find.go index 25889d71d6..fa74649dd5 100644 --- a/internal/evalmeasure/find.go +++ b/internal/evalmeasure/find.go @@ -3,6 +3,7 @@ package evalmeasure import ( "os" "path/filepath" + "regexp" "strings" "time" ) @@ -12,14 +13,20 @@ import ( // iteration-N/output/ are agent-writable and must not be scored. const PlatformTelemetryFile = "run-telemetry.jsonl" +// hostRunDirPattern matches agent--- from fullsend run. +// name is lowercased when the sandbox is created; charset matches run +// (ToLower only), so pid/unix must be the trailing numeric pair. +var hostRunDirPattern = regexp.MustCompile(`^agent-(.+)-([0-9]+)-([0-9]+)$`) + // FindPlatformTelemetry returns run-telemetry.jsonl files that sit at the // top of outputDir itself (when outputDir is a runDir) or at the top of // a host-created child runDir (when outputDir is the CI output base). // // Host runDirs are named agent--- (see fullsend run). -// When agent is non-empty, only children with that prefix are considered. -// If several match, only the newest platform file is scored — leftover -// sibling directories from a previous job are ignored. +// When agent is non-empty, only children matching that exact shape for +// the lowercased agent name are considered (so agent-code does not match +// agent-code-review-…). If several match, only the newest platform file +// is scored — leftover sibling directories from a previous job are ignored. // It never walks deeper, so an agent-planted // iteration-N/output/run-telemetry.jsonl is ignored. func FindPlatformTelemetry(outputDir, agent string) ([]string, error) { @@ -36,10 +43,7 @@ func FindPlatformTelemetry(outputDir, agent string) ([]string, error) { } return nil, err } - prefix := "" - if agent != "" { - prefix = "agent-" + strings.ToLower(agent) + "-" - } + wantAgent := strings.ToLower(agent) var bestPath string var bestMod time.Time found := false @@ -47,7 +51,11 @@ func FindPlatformTelemetry(outputDir, agent string) ([]string, error) { if !e.IsDir() { continue } - if prefix != "" && !strings.HasPrefix(e.Name(), prefix) { + m := hostRunDirPattern.FindStringSubmatch(e.Name()) + if m == nil { + continue + } + if wantAgent != "" && m[1] != wantAgent { continue } p := filepath.Join(outputDir, e.Name(), PlatformTelemetryFile) diff --git a/internal/evalmeasure/find_test.go b/internal/evalmeasure/find_test.go index 00a54f4d7b..aa7121741e 100644 --- a/internal/evalmeasure/find_test.go +++ b/internal/evalmeasure/find_test.go @@ -93,3 +93,33 @@ func TestFindPlatformTelemetry_IgnoresSiblingLeftoverRunDir(t *testing.T) { require.NoError(t, err) require.Len(t, gotAllNewest, 1) } + +func TestFindPlatformTelemetry_DoesNotMatchLongerAgentPrefix(t *testing.T) { + t.Parallel() + outputDir := t.TempDir() + codeReview := filepath.Join(outputDir, "agent-code-review-1-1") + require.NoError(t, os.MkdirAll(codeReview, 0o755)) + f := filepath.Join(codeReview, PlatformTelemetryFile) + require.NoError(t, os.WriteFile(f, []byte("x\n"), 0o644)) + + got, err := FindPlatformTelemetry(outputDir, "code") + require.NoError(t, err) + assert.Empty(t, got, "agent-code must not match agent-code-review-*") + + gotReview, err := FindPlatformTelemetry(outputDir, "code-review") + require.NoError(t, err) + require.Equal(t, []string{f}, gotReview) +} + +func TestFindPlatformTelemetry_MatchesUnderscorePrefixedAgent(t *testing.T) { + t.Parallel() + outputDir := t.TempDir() + runDir := filepath.Join(outputDir, "agent-_helper-7-9") + require.NoError(t, os.MkdirAll(runDir, 0o755)) + f := filepath.Join(runDir, PlatformTelemetryFile) + require.NoError(t, os.WriteFile(f, []byte("x\n"), 0o644)) + + got, err := FindPlatformTelemetry(outputDir, "_helper") + require.NoError(t, err) + require.Equal(t, []string{f}, got) +} diff --git a/internal/evalmeasure/fitness.go b/internal/evalmeasure/fitness.go index 9f55f99297..7962c8d47b 100644 --- a/internal/evalmeasure/fitness.go +++ b/internal/evalmeasure/fitness.go @@ -51,6 +51,27 @@ func ScoreFitnessNamed(tr Trace, evalName, version string) EvaluationResult { } } agents := tr.SpansByName("agent") + // No agent span: run never reached an iteration (sandbox/provider/image + // failure). Exclude from pass/(pass+fail) so EM-001 trends measure the + // telemetry contract, not runner health. + if len(agents) == 0 { + spanID := "" + workItem := "" + if hasRun { + spanID = run.SpanID + workItem, _ = run.AttrString(AttrFullsendWorkItemID) + } + return EvaluationResult{ + Name: evalName, + Label: LabelSkip, + Explanation: "no agent span; run never reached an iteration; excluded from trace_fitness", + TraceID: tr.TraceID, + SpanID: spanID, + WorkItemID: workItem, + Agent: tr.AgentName(), + Version: version, + } + } _, hasSandbox := tr.SpanByName("sandbox_create") costOK, costMissing := costToolsTurnsDetail(run, agents) @@ -59,7 +80,8 @@ func ScoreFitnessNamed(tr Trace, evalName, version string) EvaluationResult { pass bool } checks := []check{ - {"span_tree", hasRun && hasSandbox && len(agents) >= 1}, + // len(agents) >= 1 is guaranteed by the early-return guard above. + {"span_tree", hasRun && hasSandbox}, {"identity", identityOK(run, agents)}, // UnknownSentinel is the CLI fallback when no issue- or PR-shaped env is set. // Review jobs that have PR_NUMBER / GITHUB_PR_URL should not hit this @@ -158,7 +180,7 @@ func modelOK(run Span, agents []Span) bool { } hasSystem := false for _, a := range agents { - if attrNonEmpty(a, AttrGenAISystem) { + if attrNonEmpty(a, AttrGenAIProviderName) || attrNonEmpty(a, AttrGenAISystem) { hasSystem = true break } diff --git a/internal/evalmeasure/parse.go b/internal/evalmeasure/parse.go index 8afbe1e57f..4ec713f079 100644 --- a/internal/evalmeasure/parse.go +++ b/internal/evalmeasure/parse.go @@ -40,16 +40,24 @@ type otlpKeyValue struct { Value map[string]any `json:"value"` } -// ParseStats counts telemetry JSONL lines so operators can tell "no traces" -// from "file present but unreadable". +// ParseStats counts telemetry JSONL lines and unusable spans so operators +// can tell "no traces" from "file present but partially unreadable". type ParseStats struct { NonEmptyLines int - SkippedLines int + SkippedLines int // whole JSONL lines that failed to unmarshal + SkippedSpans int // spans inside a valid line that failed convertSpan + // Incomplete is set when the scanner stopped early (e.g. bufio.ErrTooLong) + // but some traces were still recovered. Callers should warn; MeasureAndExport + // still scores those traces and returns a nil error. + Incomplete string } // ParseTelemetryFile reads OTLP JSON TracesData lines from run-telemetry.jsonl -// and merges spans by trace id. Truncated or corrupt lines are skipped -// (fail-open); SkippedLines is the count of those lines. +// and merges spans by trace id. Truncated or corrupt lines, and spans that +// fail conversion, are skipped (fail-open). SkippedLines counts whole-line +// failures; SkippedSpans counts per-span convert failures. An oversized-line +// scanner error returns traces gathered so far alongside the error so the +// caller can still score them. func ParseTelemetryFile(path string) ([]Trace, ParseStats, error) { var stats ParseStats f, err := os.Open(path) @@ -66,9 +74,7 @@ func ParseTelemetryFile(path string) ([]Trace, ParseStats, error) { buf := make([]byte, 0, 64*1024) sc.Buffer(buf, 10*1024*1024) - lineNo := 0 for sc.Scan() { - lineNo++ line := sc.Bytes() if len(line) == 0 { continue @@ -84,7 +90,8 @@ func ParseTelemetryFile(path string) ([]Trace, ParseStats, error) { for _, raw := range ss.Spans { sp, err := convertSpan(raw) if err != nil { - return nil, stats, fmt.Errorf("line %d span %s: %w", lineNo, raw.SpanID, err) + stats.SkippedSpans++ + continue } tr, ok := byID[sp.TraceID] if !ok { @@ -98,7 +105,13 @@ func ParseTelemetryFile(path string) ([]Trace, ParseStats, error) { } } if err := sc.Err(); err != nil { - return nil, stats, err + // Oversized line (bufio.ErrTooLong) or other scanner error: keep + // traces already parsed and surface the error to the caller. + out := make([]Trace, 0, len(order)) + for _, id := range order { + out = append(out, *byID[id]) + } + return out, stats, err } out := make([]Trace, 0, len(order)) diff --git a/internal/evalmeasure/parse_test.go b/internal/evalmeasure/parse_test.go index 8f37462028..5ec316d68a 100644 --- a/internal/evalmeasure/parse_test.go +++ b/internal/evalmeasure/parse_test.go @@ -48,6 +48,7 @@ func TestParseTelemetryFile_InvalidLineSkipped(t *testing.T) { assert.Empty(t, traces) assert.Equal(t, 1, stats.NonEmptyLines) assert.Equal(t, 1, stats.SkippedLines) + assert.Equal(t, 0, stats.SkippedSpans) } func TestParseTelemetryFile_TruncatedLineDoesNotDiscardFile(t *testing.T) { @@ -75,3 +76,20 @@ func TestParseTelemetryFile_MissingFile(t *testing.T) { _, _, err := ParseTelemetryFile(filepath.Join(t.TempDir(), "missing.jsonl")) require.Error(t, err) } + +func TestParseTelemetryFile_BadSpanSkippedKeepsGood(t *testing.T) { + t.Parallel() + good, err := os.ReadFile(filepath.Join("testdata", "complete.jsonl")) + require.NoError(t, err) + dir := t.TempDir() + path := filepath.Join(dir, "mixed.jsonl") + // Valid JSON with a non-numeric startTimeUnixNano on one span. + bad := `{"resourceSpans":[{"scopeSpans":[{"spans":[{"traceId":"cccccccccccccccccccccccccccccccc","spanId":"9999999999999999","name":"run","startTimeUnixNano":"not-a-number","endTimeUnixNano":"2","attributes":[]}]}]}]}` + "\n" + require.NoError(t, os.WriteFile(path, append(append([]byte{}, good...), []byte(bad)...), 0o644)) + traces, stats, err := ParseTelemetryFile(path) + require.NoError(t, err) + require.Len(t, traces, 1) + assert.Equal(t, "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", traces[0].TraceID) + assert.Equal(t, 0, stats.SkippedLines) + assert.GreaterOrEqual(t, stats.SkippedSpans, 1) +} diff --git a/internal/evalmeasure/registry.go b/internal/evalmeasure/registry.go index 781b3ce025..3d34f02f12 100644 --- a/internal/evalmeasure/registry.go +++ b/internal/evalmeasure/registry.go @@ -71,7 +71,7 @@ func (m MeasurementSpec) evalName() string { // manifest can name a scorer this binary does not implement yet). func ScoreTrace(tr Trace, reg Registry) []EvaluationResult { name := tr.AgentName() - if name != "" && name != UnknownSentinel && name != reg.Agent { + if name != "" && name != UnknownSentinel && !strings.EqualFold(name, reg.Agent) { return nil } var out []EvaluationResult diff --git a/internal/evalmeasure/run.go b/internal/evalmeasure/run.go index ace2973786..490c8213a8 100644 --- a/internal/evalmeasure/run.go +++ b/internal/evalmeasure/run.go @@ -38,9 +38,14 @@ func MeasureAndExport(ctx context.Context, telemetryPath, registryPath, outDir s if err != nil { return nil, stats, fmt.Errorf("load registry: %w", err) } - traces, stats, err := ParseTelemetryFile(telemetryPath) - if err != nil { - return nil, stats, fmt.Errorf("parse telemetry: %w", err) + traces, stats, parseErr := ParseTelemetryFile(telemetryPath) + if parseErr != nil && len(traces) == 0 { + return nil, stats, fmt.Errorf("parse telemetry: %w", parseErr) + } + if parseErr != nil { + // Oversized/corrupt tail after good lines: score what we have and + // surface the damage via stats (CLI warns; still exit 0). + stats.Incomplete = parseErr.Error() } ledgerPath := filepath.Join(outDir, LedgerFile) @@ -58,10 +63,12 @@ func MeasureAndExport(ctx context.Context, telemetryPath, registryPath, outDir s if done { continue } - all = append(all, r) if err := AppendMeasurements(measPath, []EvaluationResult{r}); err != nil { return all, stats, fmt.Errorf("append measurements: %w", err) } + // Only count rows that landed in eval-measurements.jsonl so + // CLI stdout matches disk on a later ledger/write error. + all = append(all, r) if err := RecordScored(ledgerPath, r.TraceID, r.Name, r.Version); err != nil { return all, stats, fmt.Errorf("record scored: %w", err) } @@ -70,6 +77,7 @@ func MeasureAndExport(ctx context.Context, telemetryPath, registryPath, outDir s } } } - + // Partial parse with traces already scored is success: scores are data. + // stats.Incomplete (if set) lets the CLI warn without failing the job. return all, stats, nil } diff --git a/internal/evalmeasure/run_test.go b/internal/evalmeasure/run_test.go index 165e69b86a..7e3566aa86 100644 --- a/internal/evalmeasure/run_test.go +++ b/internal/evalmeasure/run_test.go @@ -123,11 +123,42 @@ func TestMeasureAndExport_KeepsFirstWhenSecondPersistFails(t *testing.T) { }) results, _, err := MeasureAndExport(ctx, telem, filepath.Join("testdata", "sample-registry.yaml"), out) require.Error(t, err) - require.GreaterOrEqual(t, len(results), 1) + require.Len(t, results, 1) assert.Equal(t, "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", results[0].TraceID) assert.Contains(t, err.Error(), "append measurements") } +func TestMeasureAndExport_ScoresPartialFileDespiteParseError(t *testing.T) { + // Oversized line after a good line: ParseTelemetryFile keeps the good + // trace and returns sc.Err(); MeasureAndExport still scores it and + // treats the partial parse as success (scores are data). + good, err := os.ReadFile(filepath.Join("testdata", "complete.jsonl")) + require.NoError(t, err) + dir := t.TempDir() + telem := filepath.Join(dir, "run-telemetry.jsonl") + f, err := os.Create(telem) + require.NoError(t, err) + _, err = f.Write(append(bytes.TrimSpace(good), '\n')) + require.NoError(t, err) + huge := make([]byte, 11*1024*1024) + for i := range huge { + huge[i] = 'a' + } + _, err = f.Write(huge) + require.NoError(t, err) + _, err = f.Write([]byte("\n")) + require.NoError(t, err) + require.NoError(t, f.Close()) + + out := t.TempDir() + results, stats, err := MeasureAndExport(context.Background(), telem, filepath.Join("testdata", "sample-registry.yaml"), out) + require.NoError(t, err) + require.Len(t, results, 1) + assert.Equal(t, LabelPass, results[0].Label) + assert.Greater(t, stats.NonEmptyLines, 0) + assert.NotEmpty(t, stats.Incomplete, "oversized-line parse must set Incomplete for CLI warn") +} + func TestMeasureFile_PrescriptSkippedRecordsSkip(t *testing.T) { out := t.TempDir() results, err := MeasureFile( diff --git a/internal/evalmeasure/score_test.go b/internal/evalmeasure/score_test.go index 005bc295a3..c37fb13f10 100644 --- a/internal/evalmeasure/score_test.go +++ b/internal/evalmeasure/score_test.go @@ -210,3 +210,51 @@ func TestScoreFitness_EmptyModelStringFails(t *testing.T) { assert.Equal(t, LabelFail, r.Label) assert.Contains(t, r.Explanation, "model=fail") } + +func TestScoreTrace_AgentNameEqualFold(t *testing.T) { + t.Parallel() + traces, _, err := ParseTelemetryFile(filepath.Join("testdata", "complete.jsonl")) + require.NoError(t, err) + reg := Registry{Agent: "Triage", Measurements: []MeasurementSpec{{ID: "em-001", Scorer: ScorerFitness, Version: 1}}} + results := ScoreTrace(traces[0], reg) + require.Len(t, results, 1) + assert.Equal(t, LabelPass, results[0].Label) +} + +func TestScoreFitness_NoAgentSpanSkipped(t *testing.T) { + t.Parallel() + tr := Trace{ + TraceID: "eeeeeeeeeeeeeeeeeeeeeeeeeeeeeeee", + Spans: []Span{ + { + Name: "run", + Attrs: map[string]any{ + "fullsend.agent": "triage", + "fullsend.work_item_id": "acme/demo#1", + "exit_code": int64(1), + "gen_ai.operation.name": "invoke_agent", + }, + }, + {Name: "sandbox_create"}, + }, + } + r := ScoreFitness(tr) + assert.Equal(t, LabelSkip, r.Label) + assert.Contains(t, r.Explanation, "no agent span") +} + +func TestScoreFitness_ProviderNameAccepted(t *testing.T) { + t.Parallel() + traces, _, err := ParseTelemetryFile(filepath.Join("testdata", "complete.jsonl")) + require.NoError(t, err) + tr := traces[0] + for i := range tr.Spans { + if tr.Spans[i].Name != "agent" { + continue + } + delete(tr.Spans[i].Attrs, "gen_ai.system") + tr.Spans[i].Attrs["gen_ai.provider.name"] = "anthropic" + } + r := ScoreFitness(tr) + assert.Equal(t, LabelPass, r.Label) +} diff --git a/internal/evalmeasure/types.go b/internal/evalmeasure/types.go index 1a633e37c3..0512df4aec 100644 --- a/internal/evalmeasure/types.go +++ b/internal/evalmeasure/types.go @@ -6,14 +6,17 @@ import ( "strconv" ) -// Attribute names used by EM-001. gen_ai.* keys follow experimental -// OpenTelemetry GenAI semantic conventions; an upstream rename is an -// em-001 version bump, not a silent fleet fail. +// Attribute names used by EM-001. Most gen_ai.* keys follow OpenTelemetry +// GenAI semantic conventions. gen_ai.system was renamed to +// gen_ai.provider.name in semconv v1.37.0; modelOK accepts either so +// em-001@1 stays green across the emitter migration. Other upstream +// renames remain an em-001 version bump. const ( AttrFullsendAgent = "fullsend.agent" AttrFullsendWorkItemID = "fullsend.work_item_id" AttrGenAIAgentName = "gen_ai.agent.name" - AttrGenAISystem = "gen_ai.system" + AttrGenAISystem = "gen_ai.system" // deprecated; prefer AttrGenAIProviderName + AttrGenAIProviderName = "gen_ai.provider.name" AttrGenAIRequestModel = "gen_ai.request.model" AttrGenAIUsageInputTokens = "gen_ai.usage.input_tokens" AttrGenAIUsageOutputTokens = "gen_ai.usage.output_tokens" diff --git a/internal/prescript/prescript.go b/internal/prescript/prescript.go index 5be459fffe..0607dd6922 100644 --- a/internal/prescript/prescript.go +++ b/internal/prescript/prescript.go @@ -58,6 +58,17 @@ import ( // (older CLI versions do not export it). const EnvVar = "FULLSEND_PRESCRIPT_OUTPUT" +// ExitCodeNeutral is the exit code a pre-script uses to signal "nothing to +// do, skip cleanly" (issue #582). When fullsend run sees this exit code it +// treats the run as skipped/neutral — no sandbox is created, no LLM is +// invoked, and the run reports a skipped status. The code follows the +// CI convention for "neutral" (used by GitHub Actions and others). +// +// Exit 78 is complementary to the file-based skip protocol (skipped=true). +// Either mechanism alone is sufficient to request a skip. When both are +// present, exit 78 wins — even if the output file says skipped=false. +const ExitCodeNeutral = 78 + const ( skippedKey = "skipped" reasonKey = "reason" diff --git a/internal/scaffold/fullsend-repo-gitlab/.gitlab/ci/fullsend-agent.yml b/internal/scaffold/fullsend-repo-gitlab/.gitlab/ci/fullsend-agent.yml index f2dcf71688..bce69b9323 100644 --- a/internal/scaffold/fullsend-repo-gitlab/.gitlab/ci/fullsend-agent.yml +++ b/internal/scaffold/fullsend-repo-gitlab/.gitlab/ci/fullsend-agent.yml @@ -265,12 +265,21 @@ stages: # Construct the entity URL for the harness. EVENT_TYPE values from # the poller use prefixed forms: issue_note, issue_label (→ issue # URL) and mr_note, mr_event (→ merge request URL). + # Only export a work-item URL when the IID is real. Defaulting to 0 + # invents …/issues/0 (or …/merge_requests/0), which passes EM-001's + # work_item check and greenwashes missing identity. case "${EVENT_TYPE:-}" in issue_*) - export GITLAB_ISSUE_URL="${CI_SERVER_URL}/${CI_PROJECT_PATH}/-/issues/${STATUS_IID:-0}" + if [[ -n "${STATUS_IID:-}" && "${STATUS_IID}" != "0" ]]; then + export GITLAB_ISSUE_URL="${CI_SERVER_URL}/${CI_PROJECT_PATH}/-/issues/${STATUS_IID}" + fi ;; *) - export GITLAB_ISSUE_URL="${CI_SERVER_URL}/${CI_PROJECT_PATH}/-/merge_requests/${CI_MERGE_REQUEST_IID:-${STATUS_IID:-0}}" + _fs_mr_iid="${CI_MERGE_REQUEST_IID:-${STATUS_IID:-}}" + if [[ -n "${_fs_mr_iid}" && "${_fs_mr_iid}" != "0" ]]; then + export GITLAB_ISSUE_URL="${CI_SERVER_URL}/${CI_PROJECT_PATH}/-/merge_requests/${_fs_mr_iid}" + fi + unset _fs_mr_iid ;; esac @@ -278,11 +287,19 @@ stages: # the image field, and creates the sandbox container via Podman. # Capture the run status so eval-measure still runs after a failed # agent (failed runs still write run-telemetry.jsonl). + # Eval measurements (fail-open): same CLI as GitHub Actions. Never + # fail the agent job. Manifest: local .fullsend override, else + # SHA-pinned fetch from public fullsend-ai/agents (GitHub GetRef). + # agents is public, so GetRef works without GH_TOKEN, but unauthenticated + # calls share GitHub's ~60 req/hr per-IP limit — export GH_TOKEN / + # GITHUB_TOKEN on busy shared runners. Local override skips the fetch. + # Write under $CI_PROJECT_DIR so GitLab can retain artifacts (not /tmp). + mkdir -p "${CI_PROJECT_DIR}/output" set +e fullsend run "${STAGE}" \ --fullsend-dir .fullsend \ --target-repo . \ - --output-dir /tmp/fullsend-output \ + --output-dir "${CI_PROJECT_DIR}/output" \ --forge gitlab \ --run-url "${CI_PIPELINE_URL}" \ --status-repo "${CI_PROJECT_PATH}" \ @@ -290,18 +307,18 @@ stages: RUN_STATUS=$? set -e - # Eval measurements (fail-open): same CLI as GitHub Actions. Never - # fail the agent job. Manifest: local .fullsend override, else - # SHA-pinned fetch from public fullsend-ai/agents (GitHub GetRef). - # This job has no GH_TOKEN; stock-agent manifests skip unless an - # operator exports GH_TOKEN or GITHUB_TOKEN. Local override still works. fullsend eval-measure \ --agent "${STAGE}" \ --fullsend-dir .fullsend \ - --output-dir /tmp/fullsend-output \ + --output-dir "${CI_PROJECT_DIR}/output" \ || true exit "${RUN_STATUS}" + artifacts: + when: always + paths: + - output/ + expire_in: 1 week resource_group: "fullsend-${STAGE}-${RESOURCE_KEY}" rules: - if: $STAGE diff --git a/internal/scaffold/scaffold_gitlab_test.go b/internal/scaffold/scaffold_gitlab_test.go index d872a292bf..f9f9457552 100644 --- a/internal/scaffold/scaffold_gitlab_test.go +++ b/internal/scaffold/scaffold_gitlab_test.go @@ -159,7 +159,14 @@ func TestGitLabAgentTemplateContent(t *testing.T) { assert.Contains(t, s, "RUN_STATUS=$?") assert.Contains(t, s, `exit "${RUN_STATUS}"`) assert.Contains(t, s, "|| true") - assert.Contains(t, s, "stock-agent manifests skip") + assert.Contains(t, s, "60 req/hr") + assert.Contains(t, s, "artifacts:") + assert.Contains(t, s, "when: always") + assert.Contains(t, s, `${CI_PROJECT_DIR}/output`) + assert.NotContains(t, s, "/tmp/fullsend-output") + // work_item URL must not invent …/issues/0 when IID is missing + assert.NotContains(t, s, `/-/issues/${STATUS_IID:-0}`) + assert.Contains(t, s, `"${STATUS_IID}" != "0"`) assert.Contains(t, s, "--fullsend-dir") assert.Contains(t, s, "--target-repo") assert.Contains(t, s, "--output-dir") From 8029af1d4bceeec6b028443dcce65a1d0a5a9670 Mon Sep 17 00:00:00 2001 From: Adam Scerra Date: Wed, 19 Aug 2026 14:58:40 -0400 Subject: [PATCH 11/14] fix(eval): address Wayne Aug-19 measurement review Export empty GITLAB_ISSUE_URL on every GitLab path, keep host output/ out of the sandbox tarball and git exclude, prefer runDir telemetry over planted roots, skip EM-001 when the run span never flushed, and document two-step agents@v0 activation (#6384) plus planned Level 3 content scorers. Signed-off-by: Adam Scerra Co-authored-by: Cursor --- ...-eval-measurements-online-trace-scoring.md | 63 ++++++++++++------- .../infrastructure/distributed-tracing.md | 10 ++- .../infrastructure/eval-measurements.md | 44 ++++++++----- internal/cli/evalmeasure_test.go | 37 +++++++++++ internal/cli/run.go | 11 ++-- internal/cli/run_test.go | 2 +- internal/evalmeasure/find.go | 44 +++++++++---- internal/evalmeasure/find_test.go | 32 ++++++++++ internal/evalmeasure/fitness.go | 14 +++++ internal/evalmeasure/score_test.go | 25 ++++++++ internal/sandbox/sandbox.go | 14 ++++- internal/sandbox/sandbox_test.go | 21 +++++++ .../scaffold/fullsend-repo-gitlab/.gitignore | 6 ++ .../.gitlab/ci/fullsend-agent.yml | 12 ++-- internal/scaffold/installfiles.go | 7 ++- internal/scaffold/scaffold_gitlab_test.go | 23 ++++++- 16 files changed, 299 insertions(+), 66 deletions(-) create mode 100644 internal/scaffold/fullsend-repo-gitlab/.gitignore diff --git a/docs/ADRs/0087-eval-measurements-online-trace-scoring.md b/docs/ADRs/0087-eval-measurements-online-trace-scoring.md index bc69e268fc..ceff7a86e6 100644 --- a/docs/ADRs/0087-eval-measurements-online-trace-scoring.md +++ b/docs/ADRs/0087-eval-measurements-online-trace-scoring.md @@ -34,24 +34,29 @@ already choose Phoenix, MLflow, Jaeger, or another OTLP collector for traces. Baking a single product’s Assessments/Quality API into the core CLI or managed workflows would force a tool decision on every install. -Adjacent telemetry proposals (not competing with this score path): - -- **Level 3 content capture** ([#5947](https://github.com/fullsend-ai/fullsend/pull/5947) - — proposed ADRs 0084/0085): richer span content enables later scorers; - first ship reads Level 1/2 metadata in `run-telemetry.jsonl`. Measure CLI is - host-side after sandbox exit. -- **Span status from run outcome** ([#5944](https://github.com/fullsend-ai/fullsend/pull/5944)): +Adjacent telemetry work (not competing with this score path): + +- **Level 3 content capture** ([ADR 0050](0050-distributed-tracing-instrumentation.md); + activation draft closed without merge in + [#5947](https://github.com/fullsend-ai/fullsend/pull/5947)): first ship + reads Level 1/2 metadata in `run-telemetry.jsonl` (fitness foundation). + Content-aware scorers on prompt/completion bodies are the intended next + layer once Level 3 is implemented. Measure CLI is host-side after sandbox + exit. +- **Span status from run outcome** + ([#5944](https://github.com/fullsend-ai/fullsend/pull/5944), merged): OTLP Status (and `fullsend.transcript_error`) become the reliable success/failure signal. EM-001 only checks that `exit_code` is **present** (fitness). Outcome scorers must key on Status, not `exit_code == 0`. -- **Observer / lessons → fixtures** ([#2423](https://github.com/fullsend-ai/fullsend/pull/2423)): - narrative analysis and golden-set promotion. This ADR is same-job - deterministic scoring on traces. -- **Harness snapshot / forge join keys** ([#5524](https://github.com/fullsend-ai/fullsend/pull/5524)): - sibling artifact for harness fingerprint and - forge/CI pointers beside telemetry. Complementary join/identity layer; - primary run facts belong on the OTEL trace (Level 1), while measurements - stay a derived sibling file. +- **Observer / lessons → fixtures** (draft closed without merge in + [#2423](https://github.com/fullsend-ai/fullsend/pull/2423)): narrative + analysis and golden-set promotion remain a sibling idea. This ADR is + same-job deterministic scoring on traces. +- **Harness snapshot / forge join keys** + ([#5524](https://github.com/fullsend-ai/fullsend/pull/5524), open): + sibling artifact for harness fingerprint and forge/CI pointers beside + telemetry. Complementary join/identity layer; primary run facts belong on + the OTEL trace (Level 1), while measurements stay a derived sibling file. ## Options @@ -86,7 +91,13 @@ vendor-specific score adapters in core. `fullsend` owns the parser, scorers, CLI, and GHA step; `fullsend-ai/agents` owns per-agent measurement manifests (`eval/measurements/.yaml`) that declare which scorers to enable. Stock-agent defaults resolve from `agents@v0` at runtime; local files are for -override, opt-out, or custom agents only. +override, opt-out, or custom agents only. Activation is **two-step**: merge +measurement manifests into `fullsend-ai/agents` **and** cut a `v0.x.y` release +that re-points the floating `v0` tag. Merging alone does not activate managed +jobs. Tracking: [#6384](https://github.com/fullsend-ai/fullsend/issues/6384). +Until that release lands, GHA/GitLab `eval-measure` wiring is provisional +(clean skip when the remote manifest is missing). Local `FULLSEND_DIR` +manifests are exercised in unit tests today. The first scorer is `trace_fitness` (catalog id `em-001`) — span-tree and attribute fitness so later scorers can trust the trace. EM-001 reads @@ -94,8 +105,10 @@ OpenTelemetry GenAI attribute names (`gen_ai.*` constants in `internal/evalmeasure`). `gen_ai.system` was renamed to `gen_ai.provider.name` in semconv v1.37.0; `modelOK` accepts either so `em-001@1` survives the emitter migration. Other upstream renames remain an `em-001` version bump. -Pre-script-skipped runs and runs with no `agent` span (never reached an -iteration) record `label: skip` and are excluded from pass/(pass+fail). +Pre-script-skipped runs, runs with no `agent` span (never reached an +iteration), and runs where agent spans flushed but the root `run` span never +ended (hard kill / timeout) record `label: skip` and are excluded from +pass/(pass+fail). ### Versioning (per measurement, not platform “v1”) @@ -126,11 +139,13 @@ Entirely new signal → new `em-NNN` (and usually a new `scorer` string). workflows; remote scores follow OTEL when that path lands. - Functional scenarios (gate) and eval measurements (trend) stay separate; retro can recommend either a manifest scorer or a scenario fixture. -- Richer telemetry (Level 3 / Status fixes) expands what scorers *can* assert; - it does not replace this same-job path. +- Level 1/2 metadata scorers (EM-001) are the foundation; Level 3 content + capture expands what scorers *can* assert (quality / LLM-judge style) once + implemented — it does not replace this same-job path. - Per-measurement versioning (`id@version`) lets pass/fail semantics evolve without mixing trend eras. -- Pre-script skipped runs (`fullsend.prescript.skipped=true` on the root span) - and runs with no `agent` span (never reached an iteration) are excluded from - EM-001: the scorer writes `label: skip` instead of failing a run that never - produced a full telemetry contract. +- Pre-script skipped runs (`fullsend.prescript.skipped=true` on the root span), + runs with no `agent` span (never reached an iteration), and runs where agent + spans flushed but the root `run` span never ended (hard kill / timeout) are + excluded from EM-001: the scorer writes `label: skip` instead of failing a run + that never produced a full telemetry contract. diff --git a/docs/guides/infrastructure/distributed-tracing.md b/docs/guides/infrastructure/distributed-tracing.md index c38db4c1c4..7e050b7bb1 100644 --- a/docs/guides/infrastructure/distributed-tracing.md +++ b/docs/guides/infrastructure/distributed-tracing.md @@ -251,8 +251,14 @@ After each managed agent run, `fullsend eval-measure` scores `eval-measurements.jsonl` beside telemetry when at least one new score is produced (tool-agnostic artifact). Portable remote export will reuse the same `OTEL_EXPORTER_OTLP_*` configuration as -agent traces when implemented. Measurements read the Level 1/2 metadata -contract of `run-telemetry.jsonl` (not Level 3 content). See +agent traces when implemented. + +Today's scorers (starting with EM-001) read the Level 1/2 **metadata** +contract of `run-telemetry.jsonl` — span tree and attributes, not prompt or +completion bodies. That foundation is intentional: fitness scores must trust +the trace before quality scores can. **Planned:** content-aware scorers that +consume Level 3 prompt/completion capture once Level 3 is implemented — +that is where the real quality signal lives. See [Eval Measurements](./eval-measurements.md) and [ADR 0087](../../ADRs/0087-eval-measurements-online-trace-scoring.md). diff --git a/docs/guides/infrastructure/eval-measurements.md b/docs/guides/infrastructure/eval-measurements.md index e445205368..55600a1f4f 100644 --- a/docs/guides/infrastructure/eval-measurements.md +++ b/docs/guides/infrastructure/eval-measurements.md @@ -81,6 +81,14 @@ Managed jobs fetch them from `agents@v0` when no local file exists — users do `${FULLSEND_DIR}/eval/measurements/` only to change policy or score a custom agent. +**Activation is two-step:** merge manifests (e.g. +[fullsend-ai/agents#722](https://github.com/fullsend-ai/agents/pull/722)) +**and** cut a `v0.x.y` release that re-points the floating `v0` tag +([#6384](https://github.com/fullsend-ai/fullsend/issues/6384)). Until that +release lands, managed GHA/GitLab `eval-measure` steps stay provisional +(missing remote manifest → clean skip). Local `FULLSEND_DIR` overrides work +today and are covered by CLI tests. + Scorer *code* stays in fullsend: the measure CLI is the released engine that understands `run-telemetry.jsonl`. Agents ships **policy** (manifests), not Go. EM-001 (`trace_fitness`) evaluates fullsend’s telemetry contract across agents; @@ -110,7 +118,9 @@ measurements: version: 1 ``` -Illustrative **logic-as-config** (future declarative engine — not wired yet): +Illustrative **logic-as-config** (future declarative engine — not wired yet; +attribute names below match what fullsend emits today and are **not** a +contract for the declarative surface): ```yaml agent: code @@ -126,8 +136,8 @@ measurements: checks: - name: turn_token_ratio assert: ratio_lte - numerator: gen_ai.usage.total_tokens - denominator: fullsend.turns + numerator: gen_ai.usage.output_tokens + denominator: fullsend.num_turns max: 8000 ``` @@ -149,20 +159,22 @@ success/failure signal for outcome scorers. Pre-script **skipped** runs set `fullsend.prescript.skipped=true` on the root span and never create a sandbox. EM-001 records `label: skip` for those traces instead of failing the span-tree / model / usage checks. Runs with no `agent` -span (never reached an iteration — e.g. sandbox/provider failure) are also -`label: skip`, so pass/(pass+fail) measures the telemetry contract rather than -runner health. An unknown `scorer:` string (for example a newer `agents@v0` -manifest this binary does not implement yet) also writes `label: skip`, not -`fail`. Trend pass-rate as `pass / (pass + fail)` and drop `skip`. +span (never reached an iteration — e.g. sandbox/provider failure) and runs +where agent spans flushed but the root `run` span never ended (hard kill / +timeout) are also `label: skip`, so pass/(pass+fail) measures the telemetry +contract rather than runner health. An unknown `scorer:` string (for example a +newer `agents@v0` manifest this binary does not implement yet) also writes +`label: skip`, not `fail`. Trend pass-rate as `pass / (pass + fail)` and drop +`skip`. ## Adjacent telemetry work -| Proposal | Relationship to measurements | +| Topic | Relationship to measurements | |---|---| -| [#5947](https://github.com/fullsend-ai/fullsend/pull/5947) Level 3 activation + sandbox OTEL denylist | Richer traces fuel later scorers. First ship reads Level 1/2 local JSONL; content-aware scorers need OTLP/backend (or a widened file contract) under the proposed L3 rules. Measure CLI is host-side after the sandbox exits. | -| [#5944](https://github.com/fullsend-ai/fullsend/pull/5944) Span status from run outcome | Unblocks outcome scorers keyed on Status, not raw exit alone. | -| [#2423](https://github.com/fullsend-ai/fullsend/pull/2423) Semantic observability / observer / lessons | Observer + lessons → fixtures; measurements are the online score path. | -| [#5524](https://github.com/fullsend-ai/fullsend/pull/5524) Harness snapshot / forge join keys | Complementary join/identity proposal beside telemetry; measurements are derived scores, not primary run facts. | +| Level 3 content capture ([ADR 0050](../../ADRs/0050-distributed-tracing-instrumentation.md); activation draft closed without merge in [#5947](https://github.com/fullsend-ai/fullsend/pull/5947)) | First ship scores Level 1/2 metadata fitness. **Planned:** content-aware scorers on Level 3 prompt/completion bodies once L3 is implemented — that is the real quality signal. Measure CLI is host-side after the sandbox exits. | +| [#5944](https://github.com/fullsend-ai/fullsend/pull/5944) Span status from run outcome *(merged)* | Unblocks outcome scorers keyed on Status, not raw exit alone. | +| Semantic observability / observer / lessons (draft closed without merge in [#2423](https://github.com/fullsend-ai/fullsend/pull/2423)) | Observer + lessons → fixtures remains a sibling idea; measurements are the online score path. | +| [#5524](https://github.com/fullsend-ai/fullsend/pull/5524) Harness snapshot / forge join keys *(open)* | Complementary join/identity proposal beside telemetry; measurements are derived scores, not primary run facts. | ## Same-job timing @@ -179,7 +191,11 @@ GitLab CI agent job ``` Add `output/` to the consuming repo's `.gitignore` so local GitLab-checkout -runs do not stage telemetry accidentally. +runs do not stage telemetry accidentally. The GitLab per-repo scaffold embeds +a recommended `.gitignore` fragment (asserted in tests) but does **not** +install it as a root file — that would overwrite an existing consumer ignore +list. `fullsend run` excludes `output/` from the sandbox tarball and from +`.git/info/exclude`. ## Manifest resolution in CI diff --git a/internal/cli/evalmeasure_test.go b/internal/cli/evalmeasure_test.go index 577d5edb8c..2271dcb30a 100644 --- a/internal/cli/evalmeasure_test.go +++ b/internal/cli/evalmeasure_test.go @@ -108,6 +108,43 @@ func TestEvalMeasureCmd_OutputDirIgnoresNestedTelemetry(t *testing.T) { assert.NotContains(t, string(b), "ffffffffffffffffffffffffffffffff") } +// TestEvalMeasureCmd_LocalFullsendDirManifestProducesJSONL locks the +// resolved-manifest path used before agents@v0 carries stock YAML: a local +// FULLSEND_DIR eval/measurements/.yaml must produce eval-measurements.jsonl. +func TestEvalMeasureCmd_LocalFullsendDirManifestProducesJSONL(t *testing.T) { + fsDir := t.TempDir() + outBase := t.TempDir() + runDir := filepath.Join(outBase, "agent-triage-2-2") + require.NoError(t, os.MkdirAll(runDir, 0o755)) + + good, err := os.ReadFile(filepath.Join("..", "evalmeasure", "testdata", "complete.jsonl")) + require.NoError(t, err) + require.NoError(t, os.WriteFile(filepath.Join(runDir, telemetry.TelemetryFile), good, 0o644)) + + reg, err := os.ReadFile(filepath.Join("..", "evalmeasure", "testdata", "sample-registry.yaml")) + require.NoError(t, err) + require.NoError(t, os.MkdirAll(filepath.Join(fsDir, "eval", "measurements"), 0o755)) + require.NoError(t, os.WriteFile(filepath.Join(fsDir, "eval", "measurements", "triage.yaml"), reg, 0o644)) + + cmd := newRootCmd() + buf := &bytes.Buffer{} + cmd.SetOut(buf) + cmd.SetErr(buf) + cmd.SetArgs([]string{ + "eval-measure", + "--agent", "triage", + "--fullsend-dir", fsDir, + "--output-dir", outBase, + "--offline", + }) + require.NoError(t, cmd.Execute()) + + b, err := os.ReadFile(filepath.Join(runDir, evalmeasure.MeasurementsFile)) + require.NoError(t, err) + assert.Contains(t, string(b), `"name":"trace_fitness"`) + assert.Contains(t, buf.String(), "Wrote") +} + func writeTwoTraceTelemetry(t *testing.T) string { t.Helper() src, err := os.ReadFile(filepath.Join("..", "evalmeasure", "testdata", "complete.jsonl")) diff --git a/internal/cli/run.go b/internal/cli/run.go index bd0734e972..b7fe5bf2f3 100644 --- a/internal/cli/run.go +++ b/internal/cli/run.go @@ -102,12 +102,15 @@ var defaultAgentsRepoKnownAgents = map[string]bool{ // runtime tokens). Tests that override it affect both paths. var statusMintToken = mintclient.MintToken -// agentWorkingDirExcludes lists directory patterns that agents may create -// during execution but must never commit. These are added to -// .git/info/exclude before the agent runs so git ignores them entirely. +// agentWorkingDirExcludes lists directory patterns that must stay out of +// commits. They are appended to .git/info/exclude before the agent runs. +// "output/" is the host runDir base on GitLab (under --target-repo); the +// same pattern is also omitted from the UploadDir tarball (see the +// UploadDir call site) so flushed telemetry is not handed to the agent. var agentWorkingDirExcludes = []string{ ".agentready/", ".fullsend-workspace/", + "output/", } // resolveFlags groups CLI flags that control remote resource resolution. @@ -1257,7 +1260,7 @@ func runAgent(ctx context.Context, agentName, fullsendDir, outputBase, targetRep // 8. Make project code available (copy repo root into a named subdirectory). copyStart := time.Now() printer.StepStart("Copying project code into sandbox") - if err := sandbox.UploadDir(sandboxName, hostRepositoryDir, remoteRepositoryDir); err != nil { + if err := sandbox.UploadDir(sandboxName, hostRepositoryDir, remoteRepositoryDir, "output/"); err != nil { printer.StepFail("Failed to copy project code") return fmt.Errorf("copying project code: %w", err) } diff --git a/internal/cli/run_test.go b/internal/cli/run_test.go index b2b6cd9e8f..0880d3a79d 100644 --- a/internal/cli/run_test.go +++ b/internal/cli/run_test.go @@ -2198,7 +2198,7 @@ func TestValidateLinuxBinary_AcceptsHostBinary(t *testing.T) { func TestAgentWorkingDirExcludes_ContainsKnownPatterns(t *testing.T) { // Verify the exclusion list contains the known agent working directories. - expected := []string{".agentready/", ".fullsend-workspace/"} + expected := []string{".agentready/", ".fullsend-workspace/", "output/"} for _, pattern := range expected { found := false for _, exclude := range agentWorkingDirExcludes { diff --git a/internal/evalmeasure/find.go b/internal/evalmeasure/find.go index fa74649dd5..2ff007f9c9 100644 --- a/internal/evalmeasure/find.go +++ b/internal/evalmeasure/find.go @@ -27,26 +27,45 @@ var hostRunDirPattern = regexp.MustCompile(`^agent-(.+)-([0-9]+)-([0-9]+)$`) // the lowercased agent name are considered (so agent-code does not match // agent-code-review-…). If several match, only the newest platform file // is scored — leftover sibling directories from a previous job are ignored. -// It never walks deeper, so an agent-planted -// iteration-N/output/run-telemetry.jsonl is ignored. +// +// Matching child runDirs outrank a root-level run-telemetry.jsonl under +// outputDir so an agent-planted file at the CI output base cannot displace +// the real host recorder. If any matching child runDir exists (even +// without a platform file yet), the root file is ignored. Nested +// iteration-N/output/ copies are never walked. func FindPlatformTelemetry(outputDir, agent string) ([]string, error) { + wantAgent := strings.ToLower(agent) + paths, sawMatch, err := findChildPlatformTelemetry(outputDir, wantAgent) + if err != nil { + return nil, err + } + if sawMatch { + return paths, nil + } + direct := filepath.Join(outputDir, PlatformTelemetryFile) if st, err := os.Stat(direct); err == nil && !st.IsDir() { - // outputDir is a runDir: score only the platform file at the top. + // outputDir is a runDir (or has only a root-level file and no + // matching child): score only the platform file at the top. return []string{direct}, nil } + return nil, nil +} +// findChildPlatformTelemetry looks for agent--- children. +// sawMatch is true when at least one directory matched the pattern (and +// agent filter), whether or not PlatformTelemetryFile was present. +func findChildPlatformTelemetry(outputDir, wantAgent string) (paths []string, sawMatch bool, err error) { entries, err := os.ReadDir(outputDir) if err != nil { if os.IsNotExist(err) { - return nil, nil + return nil, false, nil } - return nil, err + return nil, false, err } - wantAgent := strings.ToLower(agent) var bestPath string var bestMod time.Time - found := false + foundFile := false for _, e := range entries { if !e.IsDir() { continue @@ -58,19 +77,20 @@ func FindPlatformTelemetry(outputDir, agent string) ([]string, error) { if wantAgent != "" && m[1] != wantAgent { continue } + sawMatch = true p := filepath.Join(outputDir, e.Name(), PlatformTelemetryFile) st, err := os.Stat(p) if err != nil || st.IsDir() { continue } - if !found || st.ModTime().After(bestMod) { + if !foundFile || st.ModTime().After(bestMod) { bestPath = p bestMod = st.ModTime() - found = true + foundFile = true } } - if !found { - return nil, nil + if !foundFile { + return nil, sawMatch, nil } - return []string{bestPath}, nil + return []string{bestPath}, sawMatch, nil } diff --git a/internal/evalmeasure/find_test.go b/internal/evalmeasure/find_test.go index aa7121741e..d9ceeb9bfb 100644 --- a/internal/evalmeasure/find_test.go +++ b/internal/evalmeasure/find_test.go @@ -123,3 +123,35 @@ func TestFindPlatformTelemetry_MatchesUnderscorePrefixedAgent(t *testing.T) { require.NoError(t, err) require.Equal(t, []string{f}, got) } + +func TestFindPlatformTelemetry_PrefersRunDirOverPlantedRoot(t *testing.T) { + t.Parallel() + outputDir := t.TempDir() + planted := filepath.Join(outputDir, PlatformTelemetryFile) + require.NoError(t, os.WriteFile(planted, []byte("planted\n"), 0o644)) + + runDir := filepath.Join(outputDir, "agent-review-3-4") + require.NoError(t, os.MkdirAll(runDir, 0o755)) + platform := filepath.Join(runDir, PlatformTelemetryFile) + require.NoError(t, os.WriteFile(platform, []byte("platform\n"), 0o644)) + + got, err := FindPlatformTelemetry(outputDir, "review") + require.NoError(t, err) + require.Equal(t, []string{platform}, got) + + gotAll, err := FindPlatformTelemetry(outputDir, "") + require.NoError(t, err) + require.Equal(t, []string{platform}, gotAll) +} + +func TestFindPlatformTelemetry_EmptyMatchingRunDirIgnoresPlantedRoot(t *testing.T) { + t.Parallel() + outputDir := t.TempDir() + planted := filepath.Join(outputDir, PlatformTelemetryFile) + require.NoError(t, os.WriteFile(planted, []byte("planted\n"), 0o644)) + require.NoError(t, os.MkdirAll(filepath.Join(outputDir, "agent-review-5-6"), 0o755)) + + got, err := FindPlatformTelemetry(outputDir, "review") + require.NoError(t, err) + assert.Empty(t, got, "matching empty runDir must not fall back to planted root") +} diff --git a/internal/evalmeasure/fitness.go b/internal/evalmeasure/fitness.go index 7962c8d47b..cad9393d22 100644 --- a/internal/evalmeasure/fitness.go +++ b/internal/evalmeasure/fitness.go @@ -72,6 +72,20 @@ func ScoreFitnessNamed(tr Trace, evalName, version string) EvaluationResult { Version: version, } } + // Agent spans flushed but root run never ended (SIGKILL / OOM / job + // timeout). Same exclusion: do not score incomplete telemetry as fail. + if !hasRun { + return EvaluationResult{ + Name: evalName, + Label: LabelSkip, + Explanation: "root run span missing; run terminated before flush; excluded from trace_fitness", + TraceID: tr.TraceID, + SpanID: "", + WorkItemID: "", + Agent: tr.AgentName(), + Version: version, + } + } _, hasSandbox := tr.SpanByName("sandbox_create") costOK, costMissing := costToolsTurnsDetail(run, agents) diff --git a/internal/evalmeasure/score_test.go b/internal/evalmeasure/score_test.go index c37fb13f10..54cae036f3 100644 --- a/internal/evalmeasure/score_test.go +++ b/internal/evalmeasure/score_test.go @@ -243,6 +243,31 @@ func TestScoreFitness_NoAgentSpanSkipped(t *testing.T) { assert.Contains(t, r.Explanation, "no agent span") } +func TestScoreFitness_AgentSpansWithoutRunSkipped(t *testing.T) { + t.Parallel() + tr := Trace{ + TraceID: "dddddddddddddddddddddddddddddddd", + Spans: []Span{ + { + Name: "agent", + Attrs: map[string]any{ + "gen_ai.system": "anthropic", + "gen_ai.agent.name": "triage", + "gen_ai.request.model": "claude", + "gen_ai.usage.input_tokens": int64(1), + "gen_ai.usage.output_tokens": int64(1), + "fullsend.cost_usd": 0.01, + "fullsend.tool_calls": int64(0), + "fullsend.num_turns": int64(1), + }, + }, + }, + } + r := ScoreFitness(tr) + assert.Equal(t, LabelSkip, r.Label) + assert.Contains(t, r.Explanation, "root run span missing") +} + func TestScoreFitness_ProviderNameAccepted(t *testing.T) { t.Parallel() traces, _, err := ParseTelemetryFile(filepath.Join("testdata", "complete.jsonl")) diff --git a/internal/sandbox/sandbox.go b/internal/sandbox/sandbox.go index d4aaf51a18..bab56088d4 100644 --- a/internal/sandbox/sandbox.go +++ b/internal/sandbox/sandbox.go @@ -875,7 +875,7 @@ func UploadFile(sandboxName, localPath, remotePath string) error { // same remotePath overwrite deterministically rather than merging files // from both. Do not point two different, unrelated sources at the same // remotePath expecting their content to coexist. -func UploadDir(sandboxName, localPath, remotePath string) error { +func UploadDir(sandboxName, localPath, remotePath string, excludes ...string) error { tmp, err := os.CreateTemp("", "openshell-upload-*.tar.gz") if err != nil { return fmt.Errorf("creating temp tarball: %w", err) @@ -884,7 +884,17 @@ func UploadDir(sandboxName, localPath, remotePath string) error { tmp.Close() defer os.Remove(tmpPath) - tarCmd := exec.Command("tar", "-czf", tmpPath, "-C", localPath, ".") + tarArgs := []string{"-czf", tmpPath, "-C", localPath} + for _, ex := range excludes { + pattern := strings.Trim(ex, "/") + if pattern == "" { + continue + } + // Exclude the directory itself and its contents at the archive root. + tarArgs = append(tarArgs, "--exclude=./"+pattern, "--exclude=./"+pattern+"/*") + } + tarArgs = append(tarArgs, ".") + tarCmd := exec.Command("tar", tarArgs...) // Suppress macOS AppleDouble (._*) files in the tarball. On macOS, // bsdtar generates ._* companion files for any file with extended // attributes. These corrupt .git after a sandbox round-trip. diff --git a/internal/sandbox/sandbox_test.go b/internal/sandbox/sandbox_test.go index ba4f70cdd4..b6116d5711 100644 --- a/internal/sandbox/sandbox_test.go +++ b/internal/sandbox/sandbox_test.go @@ -673,6 +673,27 @@ func TestUploadDir_TarIncludesCopyfileDisable(t *testing.T) { assert.Equal(t, "1", strings.TrimSpace(string(data)), "COPYFILE_DISABLE should be set to 1") } +func TestUploadDir_ExcludesPatternsFromTarball(t *testing.T) { + dir := t.TempDir() + require.NoError(t, os.WriteFile(filepath.Join(dir, "keep.txt"), []byte("keep"), 0o644)) + require.NoError(t, os.MkdirAll(filepath.Join(dir, "output", "agent-x-1-1"), 0o755)) + require.NoError(t, os.WriteFile(filepath.Join(dir, "output", "agent-x-1-1", "run-telemetry.jsonl"), []byte("telem\n"), 0o644)) + + binDir := t.TempDir() + logPath := filepath.Join(binDir, "openshell.log") + sentinelPath := filepath.Join(binDir, "uploaded.tar.gz") + fakeOpenshell(t, binDir, logPath, sentinelPath) + t.Setenv("PATH", binDir+string(os.PathListSeparator)+os.Getenv("PATH")) + + require.NoError(t, UploadDir("test-sandbox", dir, "/sandbox/workspace/repo", "output/")) + + extracted := extractTar(t, sentinelPath) + _, err := os.Stat(filepath.Join(extracted, "keep.txt")) + require.NoError(t, err, "keep.txt should be in the tarball") + _, err = os.Stat(filepath.Join(extracted, "output")) + assert.True(t, os.IsNotExist(err), "output/ must be excluded from the sandbox tarball") +} + // fakeOpenshell writes a script that logs every invocation's full argv to // logPath, and — when invoked as "sandbox upload " — // copies to sentinelPath so the test can inspect exactly what bytes diff --git a/internal/scaffold/fullsend-repo-gitlab/.gitignore b/internal/scaffold/fullsend-repo-gitlab/.gitignore new file mode 100644 index 0000000000..734b5fe12f --- /dev/null +++ b/internal/scaffold/fullsend-repo-gitlab/.gitignore @@ -0,0 +1,6 @@ +# Recommended ignore for host runDir / eval-measure artifacts +# (GitLab CI artifacts: output/). Not installed as a root .gitignore — +# CollectGitLabPerRepoInstallFiles skips this file so it cannot overwrite +# a consumer's existing ignore list. Copy the line below into your repo +# .gitignore (or rely on fullsend run's UploadDir + .git/info/exclude). +output/ diff --git a/internal/scaffold/fullsend-repo-gitlab/.gitlab/ci/fullsend-agent.yml b/internal/scaffold/fullsend-repo-gitlab/.gitlab/ci/fullsend-agent.yml index 447ddb852b..71f49e96b4 100644 --- a/internal/scaffold/fullsend-repo-gitlab/.gitlab/ci/fullsend-agent.yml +++ b/internal/scaffold/fullsend-repo-gitlab/.gitlab/ci/fullsend-agent.yml @@ -265,24 +265,26 @@ stages: # Construct the entity URL for the harness. EVENT_TYPE values from # the poller use prefixed forms: issue_note, issue_label (→ issue # URL) and mr_note, mr_event (→ merge request URL). - # Only export a work-item URL when the IID is real. Defaulting to 0 - # invents …/issues/0 (or …/merge_requests/0), which passes EM-001's - # work_item check and greenwashes missing identity. + # Always export GITLAB_ISSUE_URL (empty is OK for harness env + # validation). Only set a real URL when the IID is non-empty and + # not "0" — inventing …/issues/0 passes EM-001's work_item check. + GITLAB_ISSUE_URL="" case "${EVENT_TYPE:-}" in issue_*) if [[ -n "${STATUS_IID:-}" && "${STATUS_IID}" != "0" ]]; then - export GITLAB_ISSUE_URL="${CI_SERVER_URL}/${CI_PROJECT_PATH}/-/issues/${STATUS_IID}" + GITLAB_ISSUE_URL="${CI_SERVER_URL}/${CI_PROJECT_PATH}/-/issues/${STATUS_IID}" fi ;; *) _fs_mr_iid="${CI_MERGE_REQUEST_IID:-${STATUS_IID:-}}" if [[ -n "${_fs_mr_iid}" && "${_fs_mr_iid}" != "0" ]]; then - export GITLAB_ISSUE_URL="${CI_SERVER_URL}/${CI_PROJECT_PATH}/-/merge_requests/${_fs_mr_iid}" + GITLAB_ISSUE_URL="${CI_SERVER_URL}/${CI_PROJECT_PATH}/-/merge_requests/${_fs_mr_iid}" fi unset _fs_mr_iid export FULLSEND_NOTE_TARGET="merge_requests" ;; esac + export GITLAB_ISSUE_URL # Pre-fetch prior review for the review agent — equivalent to # pre-fetch-prior-review.sh in the GitHub scaffold. Queries the diff --git a/internal/scaffold/installfiles.go b/internal/scaffold/installfiles.go index 391e7e735f..900283ce16 100644 --- a/internal/scaffold/installfiles.go +++ b/internal/scaffold/installfiles.go @@ -68,6 +68,11 @@ func CollectPerRepoInstallFiles(vendored bool, upstreamRef, upstreamTag string) // CollectGitLabPerRepoInstallFiles gathers CI template files for GitLab // per-repo installation. The embedded .fullsend/config.yaml is excluded — // callers generate a config with roles and forge field instead. +// The embedded root .gitignore is also excluded: installing it would +// overwrite a consumer's existing ignore file with a one-line fragment. +// The embed still ships for docs/tests; adopters (and the guide) add +// output/ themselves. Host telemetry is kept out of the sandbox via +// agentWorkingDirExcludes regardless. // runnerTags specifies GitLab runner tags to inject into CI job definitions. // upstreamRef and upstreamTag control the version marker embedded in the // dispatch file for upgrade/status drift detection. @@ -76,7 +81,7 @@ func CollectGitLabPerRepoInstallFiles(runnerTags []string, upstreamRef, upstream versionMarker := FormatVersionMarker(upstreamRef, upstreamTag) var files InstallFiles err := WalkGitLabPerRepo(func(path string, content []byte) error { - if path == ".fullsend/config.yaml" { + if path == ".fullsend/config.yaml" || path == ".gitignore" { return nil } rendered := strings.ReplaceAll(string(content), "__RUNNER_TAGS__", tagYAML) diff --git a/internal/scaffold/scaffold_gitlab_test.go b/internal/scaffold/scaffold_gitlab_test.go index f731baac1a..ac3ad3e792 100644 --- a/internal/scaffold/scaffold_gitlab_test.go +++ b/internal/scaffold/scaffold_gitlab_test.go @@ -10,6 +10,7 @@ import ( func TestGitLabPerRepoFilesExist(t *testing.T) { expected := []string{ + ".gitignore", ".gitlab-ci.yml", ".fullsend/config.yaml", ".gitlab/ci/fullsend-dispatch.yml", @@ -31,6 +32,22 @@ func TestGitLabConfigContent(t *testing.T) { assert.Contains(t, s, "forge: gitlab") } +func TestGitLabGitignoreExcludesOutput(t *testing.T) { + content, err := GitLabPerRepoFile(".gitignore") + require.NoError(t, err) + s := string(content) + assert.Contains(t, s, "output/") +} + +func TestCollectGitLabPerRepoInstallFiles_SkipsGitignore(t *testing.T) { + files, err := CollectGitLabPerRepoInstallFiles(nil, "", "") + require.NoError(t, err) + for _, f := range files { + assert.NotEqual(t, ".gitignore", f.Path, + "install must not overwrite consumer .gitignore with the scaffold fragment") + } +} + func TestGitLabPerRepoFileNotFound(t *testing.T) { _, err := GitLabPerRepoFile("nonexistent-file.yml") assert.Error(t, err) @@ -164,9 +181,13 @@ func TestGitLabAgentTemplateContent(t *testing.T) { assert.Contains(t, s, "when: always") assert.Contains(t, s, `${CI_PROJECT_DIR}/output`) assert.NotContains(t, s, "/tmp/fullsend-output") - // work_item URL must not invent …/issues/0 when IID is missing + // work_item URL must not invent …/issues/0 when IID is missing, but + // GITLAB_ISSUE_URL must still be exported (empty OK) so harness env + // validation does not reject a truly unset variable. assert.NotContains(t, s, `/-/issues/${STATUS_IID:-0}`) assert.Contains(t, s, `"${STATUS_IID}" != "0"`) + assert.Contains(t, s, `GITLAB_ISSUE_URL=""`) + assert.Contains(t, s, "export GITLAB_ISSUE_URL") assert.Contains(t, s, "--fullsend-dir") assert.Contains(t, s, "--target-repo") assert.Contains(t, s, "--output-dir") From 8596989250bd21256b1a6f053553cf914550c90a Mon Sep 17 00:00:00 2001 From: Adam Scerra Date: Wed, 19 Aug 2026 21:16:36 -0400 Subject: [PATCH 12/14] fix(eval): tighten output exclude and evening review follow-ups Enumerate UploadDir members instead of tar --exclude so nested output/ survives on bsdtar, scope host output exclusion to in-repo layouts, fix architecture/declarative docs, use UserCacheDir for eval-measure fetch cache, and round-trip the file exporter through ParseTelemetryFile. Signed-off-by: Adam Scerra Co-authored-by: Cursor --- docs/architecture.md | 2 +- .../infrastructure/eval-measurements.md | 13 ++-- internal/cli/evalmeasure.go | 12 +++- internal/cli/evalmeasure_test.go | 9 +++ internal/cli/run.go | 56 ++++++++++++--- internal/cli/run_test.go | 26 ++++++- internal/evalmeasure/parse_roundtrip_test.go | 69 +++++++++++++++++++ internal/sandbox/sandbox.go | 53 +++++++++++--- internal/sandbox/sandbox_test.go | 15 +++- internal/scaffold/installfiles.go | 5 +- 10 files changed, 231 insertions(+), 29 deletions(-) create mode 100644 internal/evalmeasure/parse_roundtrip_test.go diff --git a/docs/architecture.md b/docs/architecture.md index 9b23913405..31ccd37457 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -311,7 +311,7 @@ Observability is a cross-cutting concern that touches every other component. Eac - What signals matter most — cost, latency, token usage, action logs, decision traces, or something else? - ~~How do we balance detailed tracing (useful for debugging) with the volume of data agents will produce?~~ Decided in [ADR 0050](ADRs/0050-distributed-tracing-instrumentation.md): instrument all lifecycle steps comprehensively; volume is managed by backends not by suppressing data at the source. -- ~~How do we score wild agent traces for trends without a second export stack?~~ Decided in [ADR 0087](ADRs/0087-eval-measurements-online-trace-scoring.md): eval measurements write local JSONL beside telemetry; portable remote export uses the same OTLP config as traces (planned); local eval-measurements.jsonl is always written. +- ~~How do we score wild agent traces for trends without a second export stack?~~ Decided in [ADR 0087](ADRs/0087-eval-measurements-online-trace-scoring.md): eval measurements write local JSONL beside telemetry when at least one new score row is produced (including `label: skip`); portable remote export uses the same OTLP config as traces (planned). The JSONL is absent (not empty) when telemetry/manifest is missing, no traces match, or every candidate is already in the ledger. - What is the retention and access model for agent logs? Who can see what? (JSONL trace access model decided in [ADR 0021](ADRs/0021-jsonl-reasoning-trace-exposure.md); retention policy and broader log access remain open.) - How does observability interact with the security requirement that "every action is logged, attributable, and reviewable"? (See [security-threat-model.md](problems/security-threat-model.md).) - Is there a real-time monitoring requirement (agent is stuck, agent is behaving anomalously), or is observability primarily forensic? diff --git a/docs/guides/infrastructure/eval-measurements.md b/docs/guides/infrastructure/eval-measurements.md index 55600a1f4f..7d8c64e264 100644 --- a/docs/guides/infrastructure/eval-measurements.md +++ b/docs/guides/infrastructure/eval-measurements.md @@ -118,9 +118,9 @@ measurements: version: 1 ``` -Illustrative **logic-as-config** (future declarative engine — not wired yet; -attribute names below match what fullsend emits today and are **not** a -contract for the declarative surface): +Illustrative **logic-as-config** (future declarative engine — not wired yet). +Attribute names in the example match attrs fullsend emits on the **`run`** +span today; they are **not** a contract for the declarative surface: ```yaml agent: code @@ -132,7 +132,7 @@ measurements: scorer: declarative version: 1 where: - span: agent + span: run checks: - name: turn_token_ratio assert: ratio_lte @@ -194,8 +194,9 @@ Add `output/` to the consuming repo's `.gitignore` so local GitLab-checkout runs do not stage telemetry accidentally. The GitLab per-repo scaffold embeds a recommended `.gitignore` fragment (asserted in tests) but does **not** install it as a root file — that would overwrite an existing consumer ignore -list. `fullsend run` excludes `output/` from the sandbox tarball and from -`.git/info/exclude`. +list. When `--output-dir` sits inside `--target-repo` (GitLab layout), +`fullsend run` omits that top-level directory from the sandbox tarball and +`.git/info/exclude`; sibling layouts (GitHub Actions) are unchanged. ## Manifest resolution in CI diff --git a/internal/cli/evalmeasure.go b/internal/cli/evalmeasure.go index 8afa33aaeb..519da87953 100644 --- a/internal/cli/evalmeasure.go +++ b/internal/cli/evalmeasure.go @@ -197,7 +197,17 @@ func localMeasurementManifest(fullsendDir, agent string) (string, error) { func evalMeasureFetchContext(fullsendDir string, offline bool, printer *ui.Printer) (harness.ComposeOpts, forge.Client) { workspace := fullsendDir if workspace == "" { - workspace = os.TempDir() + // Prefer a per-user cache dir over shared os.TempDir() (sticky, + // multi-user /tmp races and predictable paths). Fall back to a + // process-private temp dir if UserCacheDir is unavailable. + if cache, err := os.UserCacheDir(); err == nil && cache != "" { + workspace = filepath.Join(cache, "fullsend", "eval-measure") + } else { + workspace = filepath.Join(os.TempDir(), fmt.Sprintf("fullsend-evalmeasure-%d", os.Getpid())) + } + if err := os.MkdirAll(workspace, 0o700); err != nil && printer != nil { + printer.StepWarn("Could not create eval-measure cache dir: " + err.Error()) + } } abs, err := filepath.Abs(workspace) if err != nil { diff --git a/internal/cli/evalmeasure_test.go b/internal/cli/evalmeasure_test.go index 2271dcb30a..64700def6c 100644 --- a/internal/cli/evalmeasure_test.go +++ b/internal/cli/evalmeasure_test.go @@ -325,11 +325,20 @@ func TestPlatformTelemetryFileMatchesRecorder(t *testing.T) { func TestEvalMeasureFetchContext(t *testing.T) { printer := ui.New(io.Discard) + home := t.TempDir() + t.Setenv("HOME", home) + t.Setenv("XDG_CACHE_HOME", filepath.Join(home, ".cache")) + opts, client := evalMeasureFetchContext("", false, printer) require.NotNil(t, client) assert.NotEmpty(t, opts.OrgAllowlist) assert.NotEmpty(t, opts.WorkspaceRoot) assert.False(t, opts.FetchPolicy.Offline) + assert.NotEqual(t, filepath.Clean(os.TempDir()), filepath.Clean(opts.WorkspaceRoot), + "empty --fullsend-dir must not use shared os.TempDir() as WorkspaceRoot") + if cache, err := os.UserCacheDir(); err == nil && cache != "" { + assert.Equal(t, filepath.Join(cache, "fullsend", "eval-measure"), opts.WorkspaceRoot) + } dir := t.TempDir() opts2, _ := evalMeasureFetchContext(dir, true, printer) diff --git a/internal/cli/run.go b/internal/cli/run.go index b7fe5bf2f3..18f2b2a08e 100644 --- a/internal/cli/run.go +++ b/internal/cli/run.go @@ -102,15 +102,13 @@ var defaultAgentsRepoKnownAgents = map[string]bool{ // runtime tokens). Tests that override it affect both paths. var statusMintToken = mintclient.MintToken -// agentWorkingDirExcludes lists directory patterns that must stay out of -// commits. They are appended to .git/info/exclude before the agent runs. -// "output/" is the host runDir base on GitLab (under --target-repo); the -// same pattern is also omitted from the UploadDir tarball (see the -// UploadDir call site) so flushed telemetry is not handed to the agent. +// agentWorkingDirExcludes lists fullsend-reserved directory patterns that +// must stay out of commits. They are appended to .git/info/exclude before +// the agent runs. Host run output (often named output/) is excluded only +// when it actually sits inside --target-repo — see outputDirExcludeRel. var agentWorkingDirExcludes = []string{ ".agentready/", ".fullsend-workspace/", - "output/", } // resolveFlags groups CLI flags that control remote resource resolution. @@ -1258,9 +1256,19 @@ func runAgent(ctx context.Context, agentName, fullsendDir, outputBase, targetRep printer.StepDone(fmt.Sprintf("Sandbox bootstrapped (%.1fs)", time.Since(bootstrapStart).Seconds())) // 8. Make project code available (copy repo root into a named subdirectory). + // When --output-dir sits inside --target-repo (GitLab CI layout), omit that + // top-level directory from the tarball and git exclude so host telemetry + // is not uploaded or committed. GHA keeps output as a sibling, so Rel + // fails IsLocal and nothing is excluded. copyStart := time.Now() printer.StepStart("Copying project code into sandbox") - if err := sandbox.UploadDir(sandboxName, hostRepositoryDir, remoteRepositoryDir, "output/"); err != nil { + var uploadExcludes []string + var gitExtraExcludes []string + if rel, ok := outputDirExcludeRel(hostRepositoryDir, outputBase); ok { + uploadExcludes = append(uploadExcludes, rel) + gitExtraExcludes = append(gitExtraExcludes, rel+"/") + } + if err := sandbox.UploadDir(sandboxName, hostRepositoryDir, remoteRepositoryDir, uploadExcludes...); err != nil { printer.StepFail("Failed to copy project code") return fmt.Errorf("copying project code: %w", err) } @@ -1302,7 +1310,7 @@ func runAgent(ctx context.Context, agentName, fullsendDir, outputBase, targetRep // Agents may create working directories (e.g. .agentready/) during // execution. These must never appear in commits. Adding them to // .git/info/exclude ensures git status/add ignores them entirely. - if err := excludeAgentWorkingDirs(sandboxName, remoteRepositoryDir, printer); err != nil { + if err := excludeAgentWorkingDirs(sandboxName, remoteRepositoryDir, gitExtraExcludes, printer); err != nil { printer.StepWarn("Could not exclude agent working dirs: " + err.Error()) } @@ -3046,13 +3054,43 @@ func relOrAbs(base, path string) string { return rel } +// outputDirExcludeRel returns the top-level directory name to omit from the +// sandbox upload when outputBase is inside hostRepositoryDir. Only single- +// segment relative paths are returned so nested names like build/output are +// not handled by dropping an entire parent tree (GitLab uses top-level +// output/). Returns ok=false when output is a sibling of the checkout +// (GitHub Actions layout), multi-segment, or otherwise outside the repo. +func outputDirExcludeRel(hostRepositoryDir, outputBase string) (string, bool) { + if hostRepositoryDir == "" || outputBase == "" { + return "", false + } + absRepo, err := filepath.Abs(hostRepositoryDir) + if err != nil { + return "", false + } + absOut, err := filepath.Abs(outputBase) + if err != nil { + return "", false + } + rel, err := filepath.Rel(absRepo, absOut) + if err != nil || !filepath.IsLocal(rel) || rel == "." { + return "", false + } + if strings.ContainsRune(rel, os.PathSeparator) { + return "", false + } + return rel, true +} + // excludeAgentWorkingDirs adds agent working directory patterns to // .git/info/exclude so they are invisible to git status and git add. -func excludeAgentWorkingDirs(sandboxName, repoDir string, printer *ui.Printer) error { +// extra holds layout-specific patterns (e.g. host output/ when nested). +func excludeAgentWorkingDirs(sandboxName, repoDir string, extra []string, printer *ui.Printer) error { var lines []string for _, pattern := range agentWorkingDirExcludes { lines = append(lines, pattern) } + lines = append(lines, extra...) if len(lines) == 0 { return nil } diff --git a/internal/cli/run_test.go b/internal/cli/run_test.go index 0880d3a79d..fba9cf0a60 100644 --- a/internal/cli/run_test.go +++ b/internal/cli/run_test.go @@ -2198,7 +2198,8 @@ func TestValidateLinuxBinary_AcceptsHostBinary(t *testing.T) { func TestAgentWorkingDirExcludes_ContainsKnownPatterns(t *testing.T) { // Verify the exclusion list contains the known agent working directories. - expected := []string{".agentready/", ".fullsend-workspace/", "output/"} + // Host output/ is layout-scoped via outputDirExcludeRel, not this list. + expected := []string{".agentready/", ".fullsend-workspace/"} for _, pattern := range expected { found := false for _, exclude := range agentWorkingDirExcludes { @@ -2209,6 +2210,29 @@ func TestAgentWorkingDirExcludes_ContainsKnownPatterns(t *testing.T) { } assert.True(t, found, "agentWorkingDirExcludes should contain %q", pattern) } + for _, exclude := range agentWorkingDirExcludes { + assert.NotEqual(t, "output/", exclude, "output/ must not be hardcoded in agentWorkingDirExcludes") + } +} + +func TestOutputDirExcludeRel(t *testing.T) { + t.Parallel() + repo := t.TempDir() + nested := filepath.Join(repo, "output") + require.NoError(t, os.MkdirAll(nested, 0o755)) + + rel, ok := outputDirExcludeRel(repo, nested) + assert.True(t, ok) + assert.Equal(t, "output", rel) + + sibling := filepath.Join(filepath.Dir(repo), "output-sibling") + _, ok = outputDirExcludeRel(repo, sibling) + assert.False(t, ok, "sibling output must not be excluded") + + deep := filepath.Join(repo, "build", "output") + require.NoError(t, os.MkdirAll(deep, 0o755)) + _, ok = outputDirExcludeRel(repo, deep) + assert.False(t, ok, "multi-segment Rel must not exclude a whole parent tree") } func TestAgentWorkingDirExcludes_NotEmpty(t *testing.T) { diff --git a/internal/evalmeasure/parse_roundtrip_test.go b/internal/evalmeasure/parse_roundtrip_test.go new file mode 100644 index 0000000000..fa3cd81710 --- /dev/null +++ b/internal/evalmeasure/parse_roundtrip_test.go @@ -0,0 +1,69 @@ +package evalmeasure + +import ( + "context" + "os" + "path/filepath" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "go.opentelemetry.io/otel/attribute" + + "github.com/fullsend-ai/fullsend/internal/telemetry" +) + +// TestParseTelemetryFile_RoundTripFromExporter binds the evalmeasure reader +// to the real file exporter: if the writer ever switches encoding (protojson +// enums, base64 ids, etc.), this fails instead of silently scoring nothing. +func TestParseTelemetryFile_RoundTripFromExporter(t *testing.T) { + t.Setenv("OTEL_EXPORTER_OTLP_ENDPOINT", "") + t.Setenv("OTEL_EXPORTER_OTLP_TRACES_ENDPOINT", "") + + dir := t.TempDir() + tracer, cleanup := telemetry.Setup(dir, "roundtrip-test") + + ctx := context.Background() + ctx, run := tracer.Start(ctx, "run") + run.SetAttributes( + attribute.String("fullsend.agent", "triage"), + attribute.String("fullsend.work_item_id", "acme/demo#1"), + attribute.String("gen_ai.operation.name", "invoke_agent"), + attribute.Int64("exit_code", 0), + attribute.Int64("fullsend.num_turns", 1), + attribute.Float64("fullsend.cost_usd", 0.01), + attribute.Int64("fullsend.tool_calls", 0), + attribute.Int64("fullsend.iterations", 1), + attribute.String("gen_ai.request.model", "test-model"), + attribute.Int64("gen_ai.usage.input_tokens", 1), + attribute.Int64("gen_ai.usage.output_tokens", 1), + ) + _, sandbox := tracer.Start(ctx, "sandbox_create") + sandbox.End() + _, agent := tracer.Start(ctx, "agent") + agent.SetAttributes( + attribute.String("gen_ai.agent.name", "triage"), + attribute.String("gen_ai.system", "anthropic"), + attribute.String("gen_ai.request.model", "test-model"), + attribute.Int64("gen_ai.usage.input_tokens", 1), + attribute.Int64("gen_ai.usage.output_tokens", 1), + attribute.Float64("fullsend.cost_usd", 0.01), + attribute.Int64("fullsend.tool_calls", 0), + ) + agent.End() + run.End() + cleanup(ctx) + + path := filepath.Join(dir, telemetry.TelemetryFile) + _, err := os.Stat(path) + require.NoError(t, err) + + traces, stats, err := ParseTelemetryFile(path) + require.NoError(t, err) + assert.Equal(t, 0, stats.SkippedLines, "exporter JSONL must unmarshal") + assert.Equal(t, 0, stats.SkippedSpans, "exporter spans must convert") + require.NotEmpty(t, traces) + + r := ScoreFitness(traces[0]) + assert.Equal(t, LabelPass, r.Label, "round-tripped EM-001 fixture should pass: %s", r.Explanation) +} diff --git a/internal/sandbox/sandbox.go b/internal/sandbox/sandbox.go index 01d077fd9f..f3d7acdba3 100644 --- a/internal/sandbox/sandbox.go +++ b/internal/sandbox/sandbox.go @@ -894,16 +894,20 @@ func UploadDir(sandboxName, localPath, remotePath string, excludes ...string) er tmp.Close() defer os.Remove(tmpPath) + members, err := tarRootMembers(localPath, excludes...) + if err != nil { + return fmt.Errorf("listing %q for upload: %w", localPath, err) + } tarArgs := []string{"-czf", tmpPath, "-C", localPath} - for _, ex := range excludes { - pattern := strings.Trim(ex, "/") - if pattern == "" { - continue - } - // Exclude the directory itself and its contents at the archive root. - tarArgs = append(tarArgs, "--exclude=./"+pattern, "--exclude=./"+pattern+"/*") + if len(members) == 0 { + // Everything at the root was excluded (or the dir is empty). An + // empty --files-from list yields an empty archive on GNU tar and + // bsdtar alike; do not fall back to "." (that would re-include + // excluded names). + tarArgs = append(tarArgs, "--files-from", os.DevNull) + } else { + tarArgs = append(tarArgs, members...) } - tarArgs = append(tarArgs, ".") tarCmd := exec.Command("tar", tarArgs...) // Suppress macOS AppleDouble (._*) files in the tarball. On macOS, // bsdtar generates ._* companion files for any file with extended @@ -941,6 +945,39 @@ func UploadDir(sandboxName, localPath, remotePath string, excludes ...string) er return nil } +// tarRootMembers lists top-level archive members under localPath, omitting +// excludes. Matching is by top-level entry name only (nested path components +// in an exclude are ignored beyond the first segment), so nested directories +// like sub/output/ are never dropped — unlike tar --exclude on bsdtar, +// which matches the basename at any depth. +func tarRootMembers(localPath string, excludes ...string) ([]string, error) { + skip := make(map[string]struct{}, len(excludes)) + for _, ex := range excludes { + name := strings.Trim(ex, `/\`) + if name == "" { + continue + } + // Top-level basenames only. Rejecting nested paths avoids silently + // truncating "build/output" to "build" and dropping an entire tree. + if strings.ContainsAny(name, `/\`) { + return nil, fmt.Errorf("upload exclude %q must be a top-level name", ex) + } + skip[name] = struct{}{} + } + entries, err := os.ReadDir(localPath) + if err != nil { + return nil, err + } + members := make([]string, 0, len(entries)) + for _, e := range entries { + if _, ok := skip[e.Name()]; ok { + continue + } + members = append(members, "./"+e.Name()) + } + return members, nil +} + // Download copies a file or directory from a sandbox to the local machine. // The localPath is always treated as a directory by openshell — for single-file // downloads use DownloadFile instead. diff --git a/internal/sandbox/sandbox_test.go b/internal/sandbox/sandbox_test.go index ee28723763..5b64528b20 100644 --- a/internal/sandbox/sandbox_test.go +++ b/internal/sandbox/sandbox_test.go @@ -678,6 +678,8 @@ func TestUploadDir_ExcludesPatternsFromTarball(t *testing.T) { require.NoError(t, os.WriteFile(filepath.Join(dir, "keep.txt"), []byte("keep"), 0o644)) require.NoError(t, os.MkdirAll(filepath.Join(dir, "output", "agent-x-1-1"), 0o755)) require.NoError(t, os.WriteFile(filepath.Join(dir, "output", "agent-x-1-1", "run-telemetry.jsonl"), []byte("telem\n"), 0o644)) + require.NoError(t, os.MkdirAll(filepath.Join(dir, "sub", "output"), 0o755)) + require.NoError(t, os.WriteFile(filepath.Join(dir, "sub", "output", "keep.txt"), []byte("nested\n"), 0o644)) binDir := t.TempDir() logPath := filepath.Join(binDir, "openshell.log") @@ -691,7 +693,18 @@ func TestUploadDir_ExcludesPatternsFromTarball(t *testing.T) { _, err := os.Stat(filepath.Join(extracted, "keep.txt")) require.NoError(t, err, "keep.txt should be in the tarball") _, err = os.Stat(filepath.Join(extracted, "output")) - assert.True(t, os.IsNotExist(err), "output/ must be excluded from the sandbox tarball") + assert.True(t, os.IsNotExist(err), "root output/ must be excluded from the sandbox tarball") + got, err := os.ReadFile(filepath.Join(extracted, "sub", "output", "keep.txt")) + require.NoError(t, err, "nested sub/output/ must survive (bsdtar --exclude would drop it)") + assert.Equal(t, "nested\n", string(got)) +} + +func TestTarRootMembers_RejectsNestedExclude(t *testing.T) { + dir := t.TempDir() + require.NoError(t, os.WriteFile(filepath.Join(dir, "keep.txt"), []byte("x"), 0o644)) + _, err := tarRootMembers(dir, "build/output") + require.Error(t, err) + assert.Contains(t, err.Error(), "top-level") } // fakeOpenshell writes a script that logs every invocation's full argv to diff --git a/internal/scaffold/installfiles.go b/internal/scaffold/installfiles.go index 900283ce16..5e069d71d5 100644 --- a/internal/scaffold/installfiles.go +++ b/internal/scaffold/installfiles.go @@ -71,8 +71,9 @@ func CollectPerRepoInstallFiles(vendored bool, upstreamRef, upstreamTag string) // The embedded root .gitignore is also excluded: installing it would // overwrite a consumer's existing ignore file with a one-line fragment. // The embed still ships for docs/tests; adopters (and the guide) add -// output/ themselves. Host telemetry is kept out of the sandbox via -// agentWorkingDirExcludes regardless. +// output/ themselves. When --output-dir sits inside --target-repo +// (GitLab layout), fullsend run omits that top-level directory from the +// sandbox tarball and .git/info/exclude via outputDirExcludeRel. // runnerTags specifies GitLab runner tags to inject into CI job definitions. // upstreamRef and upstreamTag control the version marker embedded in the // dispatch file for upgrade/status drift detection. From 8f3564d6845a6fb5cb0e3f2195af945044f873fe Mon Sep 17 00:00:00 2001 From: Adam Scerra Date: Thu, 20 Aug 2026 10:35:57 -0400 Subject: [PATCH 13/14] fix(eval)!: trust default-branch measurement manifests in CI Managed GHA/GitLab eval-measure no longer prefers MR/PR-tree .fullsend/eval overrides; use default/base tip via --registry or SHA-pinned agents@v0. Label the GitLab output-dir default move as breaking per COMMITS.md. BREAKING CHANGE: GitLab fullsend-agent scaffold default --output-dir now writes under ${CI_PROJECT_DIR}/output (was an ephemeral tmp path) so eval-measurements.jsonl can be retained as a CI artifact. Re-sync adds a top-level output/ directory, excludes it from the sandbox upload/git when nested under --target-repo, and keeps telemetry as a 1-week artifact. Jobs that depended on the old tmp layout must adopt the new path or pin an older scaffold. Signed-off-by: Adam Scerra Co-authored-by: Cursor --- action.yml | 29 ++++++++++---- .../infrastructure/eval-measurements.md | 21 +++++++--- internal/cli/evalmeasure.go | 3 ++ internal/cli/evalmeasure_test.go | 3 ++ .../.gitlab/ci/fullsend-agent.yml | 38 +++++++++++++------ internal/scaffold/scaffold_gitlab_test.go | 9 +++++ 6 files changed, 78 insertions(+), 25 deletions(-) diff --git a/action.yml b/action.yml index 461086a8b7..d4ff2d2fb8 100644 --- a/action.yml +++ b/action.yml @@ -413,24 +413,37 @@ runs: # measurement manifest. Same job as fullsend run; never fails the agent. # Writes eval-measurements.jsonl when at least one new score row is produced # (tool-agnostic artifact); missing telemetry/manifest skips with no file. + # Manifest trust: prefer a local override from the PR base SHA (same + # trusted tip reusable-* workflows check out for kill-switch config). + # Never prefer a PR-head .fullsend/eval tree for which scorers run. + # When no base override exists, fetch SHA-pinned agents@v0 (no + # --fullsend-dir) so a hostile checkout cannot swap measurement policy. - name: Eval measurements if: always() && inputs.agent != '__install_only__' continue-on-error: true shell: bash env: AGENT: ${{ inputs.agent }} - FULLSEND_DIR: ${{ inputs.fullsend-dir }} # For GetRef of agents@v0 (SHA pin). Not sent to raw.githubusercontent.com. GH_TOKEN: ${{ inputs.github_token }} + PR_BASE_SHA: ${{ github.event.pull_request.base.sha }} run: | set -euo pipefail - FULLSEND_DIR="${FULLSEND_DIR:-${GITHUB_WORKSPACE}}" - # Binary resolves the SHA-pinned agents manifest (allowlist/hash/audit) - # and scores only platform run-telemetry.jsonl at the top of each runDir. - fullsend eval-measure \ - --agent "${AGENT}" \ - --fullsend-dir "${FULLSEND_DIR}" \ - --output-dir "${GITHUB_WORKSPACE}/output" + MEASURE_ARGS=(--agent "${AGENT}" --output-dir "${GITHUB_WORKSPACE}/output") + TRUSTED_REF="${PR_BASE_SHA:-}" + if [[ -z "${TRUSTED_REF}" ]]; then + TRUSTED_REF="${GITHUB_SHA:-}" + fi + if [[ -n "${TRUSTED_REF}" ]] && git cat-file -e "${TRUSTED_REF}:.fullsend/eval/measurements/${AGENT}.yaml" 2>/dev/null; then + MEASURE_FILE="${GITHUB_WORKSPACE}/output/.fullsend-measure-${AGENT}.yaml" + mkdir -p "${GITHUB_WORKSPACE}/output" + git show "${TRUSTED_REF}:.fullsend/eval/measurements/${AGENT}.yaml" > "${MEASURE_FILE}" + MEASURE_ARGS+=(--registry "${MEASURE_FILE}") + fi + # Binary resolves SHA-pinned agents@v0 when --registry is unset + # (allowlist/hash/audit) and scores only platform run-telemetry.jsonl + # at the top of each runDir. + fullsend eval-measure "${MEASURE_ARGS[@]}" - name: Upload fullsend artifacts if: always() && inputs.agent != '__install_only__' diff --git a/docs/guides/infrastructure/eval-measurements.md b/docs/guides/infrastructure/eval-measurements.md index 7d8c64e264..1efd751386 100644 --- a/docs/guides/infrastructure/eval-measurements.md +++ b/docs/guides/infrastructure/eval-measurements.md @@ -202,18 +202,27 @@ list. When `--output-dir` sits inside `--target-repo` (GitLab layout), `fullsend eval-measure` resolves the measurement manifest for the agent: -1. `${FULLSEND_DIR}/eval/measurements/${AGENT}.yaml` if present, else +1. Explicit `--registry ` when the managed job materializes a **trusted** + local override (GitLab: `git show ${DEFAULT_BRANCH_SHA}:.fullsend/eval/measurements/…`; + GHA: `git show` of `pull_request.base.sha` or `GITHUB_SHA`), else 2. SHA-pinned `eval/measurements/${AGENT}.yaml` from public `fullsend-ai/agents` (same `v0` → commit SHA, allowlist, hash, and fetch audit as harness fallback — not a floating `raw.githubusercontent.com/.../v0/...` curl). GitHub Actions injects `GH_TOKEN` for that `GetRef`. GitLab CI has no GitHub token by default; because `agents` is public, `GetRef` still runs unauthenticated (~60 req/hr per IP). On busy shared runners, export - `GH_TOKEN` / `GITHUB_TOKEN` to avoid rate-limit skips. A local - FULLSEND_DIR override skips the remote fetch entirely. - -Step 2 is how stock-agent defaults reach every install. Step 1 is override / -BYOA only. + `GH_TOKEN` / `GITHUB_TOKEN` to avoid rate-limit skips. + +Managed GHA/GitLab scaffolds deliberately **do not** pass `--fullsend-dir` into +`eval-measure`: that flag would prefer `${FULLSEND_DIR}/eval/measurements/` +from the checked-out MR/PR working tree and let an author change which +already-shipped scorers run (or their `id@version`) for that job's trend — +unlike kill-switch/role config, which already reads the default/base tip. +Local/dev invocations may still pass `--fullsend-dir` when you intentionally +want the working-tree override. + +Step 2 is how stock-agent defaults reach every install. Step 1 is org override +on the **default/base branch** only (or an explicit `--registry` path). Platform telemetry is `run-telemetry.jsonl` at the top of the host run directory (`agent---` under the CI output base). Nested diff --git a/internal/cli/evalmeasure.go b/internal/cli/evalmeasure.go index 519da87953..e3db77ed4f 100644 --- a/internal/cli/evalmeasure.go +++ b/internal/cli/evalmeasure.go @@ -159,6 +159,9 @@ func resolveEvalMeasureRegistry(ctx context.Context, printer *ui.Printer, opts e if err != nil { return "", err } + // --fullsend-dir prefers a working-tree override. Managed CI scaffolds + // must not pass the MR/PR checkout here (trend-poisoning); they pass + // --registry from the default/base tip or omit both and fetch agents@v0. if opts.fullsendDir != "" { local, err := localMeasurementManifest(opts.fullsendDir, agent) if err != nil { diff --git a/internal/cli/evalmeasure_test.go b/internal/cli/evalmeasure_test.go index 64700def6c..ed90ff9ae7 100644 --- a/internal/cli/evalmeasure_test.go +++ b/internal/cli/evalmeasure_test.go @@ -315,6 +315,9 @@ func TestActionYML_EvalMeasureNoFloatingV0Curl(t *testing.T) { assert.Contains(t, step, "continue-on-error: true") assert.Contains(t, step, "if: always()") assert.Contains(t, step, "GH_TOKEN:") + assert.Contains(t, step, "PR_BASE_SHA:") + assert.Contains(t, step, "--registry") + assert.NotContains(t, step, "--fullsend-dir") assert.NotContains(t, step, "curl ") assert.NotContains(t, step, "Authorization: Bearer") } diff --git a/internal/scaffold/fullsend-repo-gitlab/.gitlab/ci/fullsend-agent.yml b/internal/scaffold/fullsend-repo-gitlab/.gitlab/ci/fullsend-agent.yml index 71f49e96b4..318ca85de4 100644 --- a/internal/scaffold/fullsend-repo-gitlab/.gitlab/ci/fullsend-agent.yml +++ b/internal/scaffold/fullsend-repo-gitlab/.gitlab/ci/fullsend-agent.yml @@ -174,14 +174,18 @@ stages: # Read config from default branch (trusted), not MR source branch. # GitHub reads config from pull_request.base.sha; this is the GitLab # equivalent. git fetch is needed because CI shallow clones only - # include the pipeline commit. + # include the pipeline commit. Save the SHA so later steps (eval + # measurement manifests) can reuse the same trusted tip even if + # FETCH_HEAD moves. CONFIG_YAML="" + DEFAULT_BRANCH_SHA="" if [ -n "${CI_DEFAULT_BRANCH:-}" ]; then if ! git fetch origin "${CI_DEFAULT_BRANCH}" --depth=1; then echo "ERROR: cannot fetch default branch — refusing to run without trusted config" exit 1 fi - CONFIG_YAML=$(git show "FETCH_HEAD:.fullsend/config.yaml" 2>/dev/null || echo "") + DEFAULT_BRANCH_SHA=$(git rev-parse FETCH_HEAD) + CONFIG_YAML=$(git show "${DEFAULT_BRANCH_SHA}:.fullsend/config.yaml" 2>/dev/null || echo "") fi # Kill switch — halt all agent dispatch when active @@ -379,12 +383,20 @@ stages: # Capture the run status so eval-measure still runs after a failed # agent (failed runs still write run-telemetry.jsonl). # Eval measurements (fail-open): same CLI as GitHub Actions. Never - # fail the agent job. Manifest: local .fullsend override, else - # SHA-pinned fetch from public fullsend-ai/agents (GitHub GetRef). + # fail the agent job. + # Manifest trust (mirrors kill-switch config): prefer a local override + # from the default branch tip (DEFAULT_BRANCH_SHA), else SHA-pinned + # fetch from public fullsend-ai/agents (GitHub GetRef). Never read + # .fullsend/eval/measurements/ from the MR source tree — that would + # let an MR author change which scorers run or their id@version for + # the job's trend. Do not pass --fullsend-dir to eval-measure here. # agents is public, so GetRef works without GH_TOKEN, but unauthenticated # calls share GitHub's ~60 req/hr per-IP limit — export GH_TOKEN / - # GITHUB_TOKEN on busy shared runners. Local override skips the fetch. - # Write under $CI_PROJECT_DIR so GitLab can retain artifacts (not /tmp). + # GITHUB_TOKEN on busy shared runners. + # Write under $CI_PROJECT_DIR so GitLab can retain artifacts (not an + # ephemeral tmp path). BREAKING CHANGE vs older scaffolds: default + # --output-dir now lives inside the project dir (artifact retention + # + top-level output/ sandbox exclude). Re-sync adopts the new layout. mkdir -p "${CI_PROJECT_DIR}/output" set +e fullsend run "${STAGE}" \ @@ -398,11 +410,15 @@ stages: RUN_STATUS=$? set -e - fullsend eval-measure \ - --agent "${STAGE}" \ - --fullsend-dir .fullsend \ - --output-dir "${CI_PROJECT_DIR}/output" \ - || true + MEASURE_ARGS=(--agent "${STAGE}" --output-dir "${CI_PROJECT_DIR}/output") + if [ -n "${DEFAULT_BRANCH_SHA}" ]; then + if MEASURE_YAML=$(git show "${DEFAULT_BRANCH_SHA}:.fullsend/eval/measurements/${STAGE}.yaml" 2>/dev/null); then + MEASURE_FILE="${CI_PROJECT_DIR}/output/.fullsend-measure-${STAGE}.yaml" + printf '%s\n' "${MEASURE_YAML}" > "${MEASURE_FILE}" + MEASURE_ARGS+=(--registry "${MEASURE_FILE}") + fi + fi + fullsend eval-measure "${MEASURE_ARGS[@]}" || true exit "${RUN_STATUS}" artifacts: diff --git a/internal/scaffold/scaffold_gitlab_test.go b/internal/scaffold/scaffold_gitlab_test.go index ac3ad3e792..4762e7a1c2 100644 --- a/internal/scaffold/scaffold_gitlab_test.go +++ b/internal/scaffold/scaffold_gitlab_test.go @@ -181,6 +181,14 @@ func TestGitLabAgentTemplateContent(t *testing.T) { assert.Contains(t, s, "when: always") assert.Contains(t, s, `${CI_PROJECT_DIR}/output`) assert.NotContains(t, s, "/tmp/fullsend-output") + // Measurement override from default branch tip — never MR-tree --fullsend-dir. + assert.Contains(t, s, "DEFAULT_BRANCH_SHA") + assert.Contains(t, s, `git show "${DEFAULT_BRANCH_SHA}:.fullsend/eval/measurements/${STAGE}.yaml"`) + assert.Contains(t, s, `MEASURE_ARGS+=(--registry "${MEASURE_FILE}")`) + assert.Contains(t, s, `fullsend eval-measure "${MEASURE_ARGS[@]}"`) + assert.NotContains(t, s, `eval-measure \ + --agent "${STAGE}" \ + --fullsend-dir .fullsend`) // work_item URL must not invent …/issues/0 when IID is missing, but // GITLAB_ISSUE_URL must still be exported (empty OK) so harness env // validation does not reject a truly unset variable. @@ -262,6 +270,7 @@ func TestGitLabAgentTemplateKillSwitch(t *testing.T) { assert.Contains(t, s, "kill_switch: false") // Config read from default branch (trusted), not MR source assert.Contains(t, s, "CI_DEFAULT_BRANCH") + assert.Contains(t, s, "DEFAULT_BRANCH_SHA") assert.Contains(t, s, "FETCH_HEAD") assert.Contains(t, s, "CONFIG_YAML") // Fetch failure fails the job (not silently permissive) From 903c0a428a27fb258165cb748db143802c4c1d60 Mon Sep 17 00:00:00 2001 From: Adam Scerra Date: Thu, 20 Aug 2026 10:39:15 -0400 Subject: [PATCH 14/14] fix(eval): resolve trusted measure path from fullsend-dir layout MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Per-org installs use workspace-root eval/measurements/, not .fullsend/eval/… — derive the git-show path from inputs.fullsend-dir while still reading bytes from the PR base / GITHUB_SHA tip. Signed-off-by: Adam Scerra Co-authored-by: Cursor --- action.yml | 28 +++++++++++++++++++++++----- internal/cli/evalmeasure_test.go | 2 ++ 2 files changed, 25 insertions(+), 5 deletions(-) diff --git a/action.yml b/action.yml index d4ff2d2fb8..83639d7e6f 100644 --- a/action.yml +++ b/action.yml @@ -415,15 +415,17 @@ runs: # (tool-agnostic artifact); missing telemetry/manifest skips with no file. # Manifest trust: prefer a local override from the PR base SHA (same # trusted tip reusable-* workflows check out for kill-switch config). - # Never prefer a PR-head .fullsend/eval tree for which scorers run. - # When no base override exists, fetch SHA-pinned agents@v0 (no - # --fullsend-dir) so a hostile checkout cannot swap measurement policy. + # Path prefix comes from inputs.fullsend-dir (install layout: per-repo + # `.fullsend/` or per-org workspace root); file bytes come from the + # trusted ref — never from a PR-head working tree. When no trusted + # override exists, fetch SHA-pinned agents@v0 (no --fullsend-dir). - name: Eval measurements if: always() && inputs.agent != '__install_only__' continue-on-error: true shell: bash env: AGENT: ${{ inputs.agent }} + FULLSEND_DIR: ${{ inputs.fullsend-dir }} # For GetRef of agents@v0 (SHA pin). Not sent to raw.githubusercontent.com. GH_TOKEN: ${{ inputs.github_token }} PR_BASE_SHA: ${{ github.event.pull_request.base.sha }} @@ -434,10 +436,26 @@ runs: if [[ -z "${TRUSTED_REF}" ]]; then TRUSTED_REF="${GITHUB_SHA:-}" fi - if [[ -n "${TRUSTED_REF}" ]] && git cat-file -e "${TRUSTED_REF}:.fullsend/eval/measurements/${AGENT}.yaml" 2>/dev/null; then + FULLSEND_DIR="${FULLSEND_DIR:-.fullsend}" + MEASURE_REL="" + case "${FULLSEND_DIR}" in + "${GITHUB_WORKSPACE}"/*) + MEASURE_REL="${FULLSEND_DIR#"${GITHUB_WORKSPACE}"/}" + ;; + /*) + MEASURE_REL="" + ;; + *) + MEASURE_REL="${FULLSEND_DIR}" + ;; + esac + MEASURE_REL="${MEASURE_REL#./}" + MEASURE_REL="${MEASURE_REL%/}" + if [[ -n "${TRUSTED_REF}" && -n "${MEASURE_REL}" ]] \ + && git cat-file -e "${TRUSTED_REF}:${MEASURE_REL}/eval/measurements/${AGENT}.yaml" 2>/dev/null; then MEASURE_FILE="${GITHUB_WORKSPACE}/output/.fullsend-measure-${AGENT}.yaml" mkdir -p "${GITHUB_WORKSPACE}/output" - git show "${TRUSTED_REF}:.fullsend/eval/measurements/${AGENT}.yaml" > "${MEASURE_FILE}" + git show "${TRUSTED_REF}:${MEASURE_REL}/eval/measurements/${AGENT}.yaml" > "${MEASURE_FILE}" MEASURE_ARGS+=(--registry "${MEASURE_FILE}") fi # Binary resolves SHA-pinned agents@v0 when --registry is unset diff --git a/internal/cli/evalmeasure_test.go b/internal/cli/evalmeasure_test.go index ed90ff9ae7..b3e1321a9e 100644 --- a/internal/cli/evalmeasure_test.go +++ b/internal/cli/evalmeasure_test.go @@ -317,6 +317,8 @@ func TestActionYML_EvalMeasureNoFloatingV0Curl(t *testing.T) { assert.Contains(t, step, "GH_TOKEN:") assert.Contains(t, step, "PR_BASE_SHA:") assert.Contains(t, step, "--registry") + assert.Contains(t, step, "FULLSEND_DIR:") + assert.Contains(t, step, "MEASURE_REL") assert.NotContains(t, step, "--fullsend-dir") assert.NotContains(t, step, "curl ") assert.NotContains(t, step, "Authorization: Bearer")