diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 134d4b5..7fa004d 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -26,6 +26,7 @@ jobs: - run: python -m pip install -e ".[test]" build - run: python tools/check_versions.py - run: python tools/check_schemas.py + - run: python tools/check_otel_compatibility.py - run: python conformance/runner/validate.py - run: python -m unittest discover -s tests -v - run: python -m compileall -q src diff --git a/MANIFEST.in b/MANIFEST.in index 0f2aace..712de52 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -11,6 +11,7 @@ include ROADMAP.md include SECURITY.md include SUPPORT.md include VERSIONING.md +recursive-include compatibility *.json recursive-include conformance *.json *.py *.txt recursive-include docs *.md recursive-include examples *.py diff --git a/README.md b/README.md index 35037e9..dbbaa0f 100644 --- a/README.md +++ b/README.md @@ -103,6 +103,7 @@ python examples/governed_workflow.py - [Architecture](docs/architecture.md) - [OpenTelemetry projection](docs/otel-projection.md) +- [OpenTelemetry GenAI compatibility](docs/otel-genai-compatibility.md) - [Action execution events](docs/action-events.md) - [Data-flow classification](docs/data-flow-classification.md) - [Usage and cost attribution](docs/usage-attribution.md) diff --git a/ROADMAP.md b/ROADMAP.md index 7a5369a..5eb04f8 100644 --- a/ROADMAP.md +++ b/ROADMAP.md @@ -8,6 +8,7 @@ - Structured-log projection contract. - Reproducible package and CI gates. - Per-call usage facts and coverage-labelled per-agent/workflow cost rollups. +- Executable OpenTelemetry GenAI compatibility matrix pinned to upstream source. ## Next diff --git a/compatibility/otel-genai.json b/compatibility/otel-genai.json new file mode 100644 index 0000000..954538b --- /dev/null +++ b/compatibility/otel-genai.json @@ -0,0 +1,58 @@ +{ + "profile": "agentrust-otel-genai-compatibility-v1", + "captured_at": "2026-08-18", + "upstream": { + "repository": "https://github.com/open-telemetry/semantic-conventions-genai", + "commit": "a685613a207a580163353b8e48a7ad88967e7b42", + "status": "development", + "schema_url": null + }, + "relations": ["exact", "extension", "complementary", "non_equivalent", "deferred"], + "span_attributes": [ + {"agentrust": "gen_ai.agent.id", "otel": "gen_ai.agent.id", "relation": "exact", "note": "Exact only when agent_id is a stable agent resource identifier; transient process instance IDs must not use this field."}, + {"agentrust": "agentrust.telemetry.spec_version", "otel": null, "relation": "extension", "note": "AgentTrust contract version."}, + {"agentrust": "agentrust.event.id", "otel": null, "relation": "extension", "note": "Durable event identifier."}, + {"agentrust": "agentrust.run.id", "otel": null, "relation": "extension", "note": "Durable governed execution correlation; not a conversation or trace ID."}, + {"agentrust": "agentrust.workflow.id", "otel": "gen_ai.workflow.name", "relation": "non_equivalent", "note": "AgentTrust stores a high-cardinality instance ID; OTel defines a low-cardinality workflow name."}, + {"agentrust": "agentrust.agent.parent.id", "otel": null, "relation": "extension", "note": "Durable logical parent across process boundaries; OTel span parentage and links remain authoritative for trace causality."}, + {"agentrust": "agentrust.task.id", "otel": null, "relation": "extension", "note": "Governed task instance identifier."}, + {"agentrust": "agentrust.policy.decision", "otel": null, "relation": "extension", "note": "Policy outcome, not a GenAI response outcome."}, + {"agentrust": "agentrust.approval.id", "otel": null, "relation": "extension", "note": "Action-bound approval lifecycle identifier."}, + {"agentrust": "agentrust.usage.scope", "otel": "gen_ai.operation.name", "relation": "non_equivalent", "note": "Attribution boundary, not operation type."}, + {"agentrust": "agentrust.data_flow.direction", "otel": null, "relation": "extension", "note": "Classified metadata flow direction."}, + {"agentrust": "agentrust.data_flow.policy_decision", "otel": null, "relation": "extension", "note": "Policy result attached to a classified flow."}, + {"agentrust": "agentrust.evidence.capture_profile", "otel": null, "relation": "extension", "note": "Evidence privacy profile."}, + {"agentrust": "agentrust.evidence.completeness", "otel": null, "relation": "extension", "note": "Durable evidence completeness claim."}, + {"agentrust": "agentrust.action.kind", "otel": "gen_ai.operation.name", "relation": "non_equivalent", "note": "Broad governed action category; only tool actions may coexist with execute_tool spans."}, + {"agentrust": "agentrust.action.outcome", "otel": "error.type", "relation": "non_equivalent", "note": "Lifecycle outcome; error.type is an error class and only applies on failure."} + ], + "metrics": [ + {"agentrust": "agentrust.policy.decisions", "instrument": "counter", "otel": null, "relation": "extension"}, + {"agentrust": "agentrust.policy.evaluation.duration", "instrument": "histogram", "otel": null, "relation": "extension"}, + {"agentrust": "agentrust.approval.events", "instrument": "counter", "otel": null, "relation": "extension"}, + {"agentrust": "agentrust.action.executions", "instrument": "counter", "otel": null, "relation": "extension"}, + {"agentrust": "agentrust.action.duration", "instrument": "histogram", "otel": "gen_ai.execute_tool.duration", "relation": "non_equivalent", "note": "AgentTrust includes non-tool and denied actions and does not claim the GenAI tool metric."}, + {"agentrust": "agentrust.data_flow.events", "instrument": "counter", "otel": null, "relation": "extension"}, + {"agentrust": "agentrust.usage.tokens", "instrument": "counter", "otel": "gen_ai.client.token.usage", "relation": "non_equivalent", "note": "OTel specifies a histogram per client operation with input/output token type; AgentTrust records additive reported facts across attribution scopes and additional token categories."}, + {"agentrust": "agentrust.usage.cost", "instrument": "counter", "otel": null, "relation": "extension", "note": "Includes explicit currency and provenance; the pinned OTel GenAI model has no general cost metric."} + ], + "event_families": [ + {"agentrust": "action.executed", "otel": "gen_ai.execute_tool", "relation": "complementary", "note": "A governed tool action may correlate with an OTel tool span, but AgentTrust also represents denied attempts and non-tool actions."}, + {"agentrust": "policy.decision", "otel": null, "relation": "extension"}, + {"agentrust": "usage.recorded", "otel": "gen_ai.client.token.usage", "relation": "complementary", "note": "Structured attribution fact; does not replace model client metrics or span usage attributes."}, + {"agentrust": "data_flow.observed", "otel": null, "relation": "extension"}, + {"agentrust": "approval.*", "otel": null, "relation": "extension"}, + {"agentrust": "evidence.*", "otel": null, "relation": "extension"} + ], + "native_context": [ + {"agentrust": "trace_id/span_id", "otel": "W3C Trace Context", "relation": "exact", "note": "Never duplicated as attributes; active OTel context is authoritative."} + ], + "deliberate_omissions": [ + {"otel": "gen_ai.input.messages", "relation": "deferred", "note": "Content-bearing and outside the metadata-only profile."}, + {"otel": "gen_ai.output.messages", "relation": "deferred", "note": "Content-bearing and outside the metadata-only profile."}, + {"otel": "gen_ai.operation.name", "relation": "deferred", "note": "Owned by the GenAI span instrumentation; AgentTrust does not create or mutate those spans."}, + {"otel": "gen_ai.provider.name", "relation": "deferred", "note": "Provider identity remains on model/provider telemetry; usage records retain caller-observed provider metadata."}, + {"otel": "gen_ai.request.model", "relation": "deferred", "note": "AgentTrust structured usage retains the requested model but does not duplicate it onto arbitrary governance span events."}, + {"otel": "gen_ai.response.model", "relation": "deferred", "note": "AgentTrust structured usage retains the response model but does not duplicate it onto arbitrary governance span events."} + ] +} diff --git a/docs/otel-genai-compatibility.md b/docs/otel-genai-compatibility.md new file mode 100644 index 0000000..bbfdbab --- /dev/null +++ b/docs/otel-genai-compatibility.md @@ -0,0 +1,48 @@ +# OpenTelemetry GenAI compatibility + +Status: pinned compatibility assessment for AgentTrust contract `0.1.0-dev`. + +The machine-readable matrix is [`compatibility/otel-genai.json`](../compatibility/otel-genai.json). +It is pinned to OpenTelemetry's dedicated GenAI semantic-conventions repository +at commit `a685613a207a580163353b8e48a7ad88967e7b42` (2026-08-15). +Those conventions are in Development status. At the pinned revision the upstream +schema URL is still unpublished, so AgentTrust does not advertise one. + +Primary upstream references: + +- [Pinned GenAI repository revision](https://github.com/open-telemetry/semantic-conventions-genai/tree/a685613a207a580163353b8e48a7ad88967e7b42) +- [GenAI agent spans](https://github.com/open-telemetry/semantic-conventions-genai/blob/a685613a207a580163353b8e48a7ad88967e7b42/docs/gen-ai/gen-ai-agent-spans.md) +- [GenAI model spans](https://github.com/open-telemetry/semantic-conventions-genai/blob/a685613a207a580163353b8e48a7ad88967e7b42/docs/gen-ai/gen-ai-spans.md) +- [GenAI metrics](https://github.com/open-telemetry/semantic-conventions-genai/blob/a685613a207a580163353b8e48a7ad88967e7b42/docs/gen-ai/gen-ai-metrics.md) +- [GenAI events](https://github.com/open-telemetry/semantic-conventions-genai/blob/a685613a207a580163353b8e48a7ad88967e7b42/docs/gen-ai/gen-ai-events.md) + +## Compatibility meanings + +- `exact`: the same field has the same semantics, subject to its stated precondition. +- `extension`: AgentTrust adds a governance or evidence concept OTel does not define. +- `complementary`: both signals may describe the same operation from different layers. +- `non_equivalent`: names or subject matter overlap, but substitution would be incorrect. +- `deferred`: OTel owns the signal or it conflicts with the metadata-only profile. + +AgentTrust currently has one exact GenAI attribute mapping: a stable `agent_id` may +project to `gen_ai.agent.id`. It does not use that attribute for a transient process +instance. W3C trace and span context is also native rather than an AgentTrust extension. + +The `agentrust.usage.tokens` Counter is deliberately **not** represented as +`gen_ai.client.token.usage`. The pinned OTel convention defines a per-operation +Histogram with `input` and `output` token types. AgentTrust records additive, +attribution-scoped facts and also distinguishes cache and reasoning categories. +Applications should retain their existing GenAI instrumentation and use AgentTrust +events for governance attribution. + +Likewise, `action.executed` does not replace `gen_ai.execute_tool` spans: it includes +denied attempts and MCP, A2A, file, HTTP, and database actions. A successful tool action +can correlate with the application's tool span through the active trace context. + +## Drift gate + +`python tools/check_otel_compatibility.py` parses the shipped projection and metric +instrument definitions and fails if the matrix omits or misstates one. It also checks +the complete normalized event-family set and the pinned upstream metadata. Updating +the upstream commit is a reviewed compatibility decision, not an automatic network +operation in CI. diff --git a/tests/test_repository_gates.py b/tests/test_repository_gates.py index d3465f4..50a246a 100644 --- a/tests/test_repository_gates.py +++ b/tests/test_repository_gates.py @@ -17,6 +17,7 @@ def load_tool(name): check_schemas = load_tool("check_schemas") +check_otel_compatibility = load_tool("check_otel_compatibility") check_versions = load_tool("check_versions") @@ -50,6 +51,36 @@ def test_schema_gate_detects_missing_and_extra_files(self): def test_declared_versions_are_consistent(self): self.assertEqual(check_versions.main(), 0) + def test_otel_matrix_matches_shipped_projection(self): + self.assertEqual(check_otel_compatibility.main(), 0) + + def test_source_manifest_includes_otel_matrix(self): + manifest = (ROOT / "MANIFEST.in").read_text(encoding="utf-8") + self.assertIn("recursive-include compatibility *.json", manifest.splitlines()) + + def test_otel_matrix_gate_detects_attribute_metric_and_family_drift(self): + document = json.loads( + (ROOT / "compatibility" / "otel-genai.json").read_text(encoding="utf-8") + ) + errors = check_otel_compatibility.validate_matrix( + document, + {"gen_ai.agent.id", "agentrust.unmapped"}, + {"agentrust.unmapped.metric": "counter"}, + ) + self.assertTrue(any("span attribute drift" in error for error in errors)) + self.assertTrue(any("metric drift" in error for error in errors)) + document["event_families"].pop() + family_errors = check_otel_compatibility.validate_matrix( + document, + set(check_otel_compatibility._literal_assignment( + ROOT / "src" / "agentrust_telemetry" / "projection.py", "DIRECT_ATTRIBUTES" + ).values()), + check_otel_compatibility._instruments( + ROOT / "src" / "agentrust_telemetry" / "otel.py" + ), + ) + self.assertTrue(any("event family drift" in error for error in family_errors)) + def test_issue_templates_are_valid_yaml_when_parser_available(self): try: import yaml diff --git a/tools/check_otel_compatibility.py b/tools/check_otel_compatibility.py new file mode 100644 index 0000000..8c66029 --- /dev/null +++ b/tools/check_otel_compatibility.py @@ -0,0 +1,101 @@ +"""Fail when shipped OTel projections drift from the compatibility matrix.""" + +from __future__ import annotations + +import ast +import json +import re +import sys +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[1] +MATRIX = ROOT / "compatibility" / "otel-genai.json" +PROJECTION = ROOT / "src" / "agentrust_telemetry" / "projection.py" +OTEL = ROOT / "src" / "agentrust_telemetry" / "otel.py" + + +def _literal_assignment(path: Path, name: str): + tree = ast.parse(path.read_text(encoding="utf-8")) + for node in tree.body: + if isinstance(node, ast.Assign) and any( + isinstance(target, ast.Name) and target.id == name for target in node.targets + ): + return ast.literal_eval(node.value) + raise ValueError(f"{name} assignment not found in {path.name}") + + +def _instruments(path: Path) -> dict[str, str]: + tree = ast.parse(path.read_text(encoding="utf-8")) + found: dict[str, str] = {} + for node in ast.walk(tree): + if not isinstance(node, ast.Call) or not isinstance(node.func, ast.Attribute): + continue + kind = {"create_counter": "counter", "create_histogram": "histogram"}.get( + node.func.attr + ) + if kind and node.args and isinstance(node.args[0], ast.Constant): + found[str(node.args[0].value)] = kind + return found + + +def validate_matrix(document: dict, attributes: set[str], instruments: dict[str, str]) -> list[str]: + errors: list[str] = [] + upstream = document.get("upstream", {}) + if not re.fullmatch(r"[0-9a-f]{40}", str(upstream.get("commit", ""))): + errors.append("upstream commit must be a full lowercase Git SHA") + if upstream.get("status") != "development": + errors.append("pinned OTel GenAI status must be development") + if upstream.get("schema_url") is not None: + errors.append("schema_url must remain null until upstream publishes one") + + allowed = set(document.get("relations", [])) + for section in ("span_attributes", "metrics", "event_families", "native_context", "deliberate_omissions"): + for entry in document.get(section, []): + if entry.get("relation") not in allowed: + errors.append(f"{section} has unsupported relation: {entry.get('relation')!r}") + + matrix_attributes = [entry["agentrust"] for entry in document.get("span_attributes", [])] + if len(matrix_attributes) != len(set(matrix_attributes)): + errors.append("span_attributes contains duplicates") + missing = sorted(attributes - set(matrix_attributes)) + extra = sorted(set(matrix_attributes) - attributes) + if missing or extra: + errors.append(f"span attribute drift missing={missing} extra={extra}") + + matrix_metrics = { + entry["agentrust"]: entry["instrument"] for entry in document.get("metrics", []) + } + if matrix_metrics != instruments: + errors.append(f"metric drift expected={instruments} matrix={matrix_metrics}") + + expected_families = { + "action.executed", "policy.decision", "usage.recorded", "data_flow.observed", + "approval.*", "evidence.*", + } + actual_families = {entry["agentrust"] for entry in document.get("event_families", [])} + if actual_families != expected_families: + errors.append( + f"event family drift missing={sorted(expected_families - actual_families)} " + f"extra={sorted(actual_families - expected_families)}" + ) + return errors + + +def main() -> int: + document = json.loads(MATRIX.read_text(encoding="utf-8")) + attributes = set(_literal_assignment(PROJECTION, "DIRECT_ATTRIBUTES").values()) + instruments = _instruments(OTEL) + errors = validate_matrix(document, attributes, instruments) + if errors: + for error in errors: + print(f"FAIL {error}", file=sys.stderr) + return 1 + print( + f"PASS OTel GenAI matrix covers {len(attributes)} span attributes, " + f"{len(instruments)} metrics, and {len(document['event_families'])} event families" + ) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main())