Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 2 additions & 2 deletions integrations/ramen-ai-cmcp/integration.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -21,5 +21,5 @@ trace_conformance_level: 0
trace_roles:
- record-producer
tested_against:
agentrust-trace: "0.10.0"
agentrust-trace-tests: "0.5.1"
agentrust-trace: "0.11.0"
agentrust-trace-tests: "0.6.1"
2 changes: 1 addition & 1 deletion integrations/ramen-ai-cmcp/pyproject.toml
Original file line number Diff line number Diff line change
Expand Up @@ -15,7 +15,7 @@ dependencies = [

[project.optional-dependencies]
test = [
"agentrust-trace-tests==0.5.1",
"agentrust-trace-tests==0.6.1",
"pytest==9.1.1",
]

Expand Down
27 changes: 21 additions & 6 deletions integrations/ramen-ai-cmcp/tests/test_mapping.py
Original file line number Diff line number Diff line change
Expand Up @@ -251,19 +251,30 @@ def test_signed_software_record_passes_level_0(self, trace_key):
record = _build(_load("vector1_allowed.json"), iat=int(time.time()))
results = run_trace_tests(record, "trace", level=0)
findings = [finding for group in results.values() for finding in group]
assert len(findings) == 8
assert all(finding.status is Status.PASS for finding in findings)
assert len(findings) == 15
# agentrust-trace-tests 0.6 adds three checks that do not apply to this
# record at Level 0 and report SKIP; everything else passes.
assert sorted(
finding.code for finding in findings if finding.status is not Status.PASS
) == ["TR-APR-003", "TR-APR-005", "TR-POL-003"]
assert all(
finding.status is Status.SKIP
for finding in findings
if finding.status is not Status.PASS
)
assert [
finding.status for finding in findings if finding.code == "TR-SIG-005"
] == [Status.PASS]

def test_signed_software_record_fails_level_1_runtime_rules(self, trace_key):
"""A software-only record fails Level 1 on runtime rules only.

Two of them as of agentrust-trace-tests 0.5.1. TR-RTE-001 is the platform
rule and has always fired here. TR-RTE-004 was added in 0.5.1 and fires
because no verifier nonce is supplied: this harness does not issue one, and
the record carries no ``runtime.nonce`` to match it if it did. The assertion
Three of them as of agentrust-trace-tests 0.6.1. TR-APR-005 was added in
0.6 and fires because the record's appraisal.status is 'none', not
'affirming'. TR-RTE-001 is the platform rule and has always fired here.
TR-RTE-004 was added in 0.5.1 and fires because no verifier nonce is
supplied: this harness does not issue one, and the record carries no
``runtime.nonce`` to match it if it did. The assertion
stays an exact comparison on purpose, so that a future change to the Level 1
finding set fails this test rather than passing silently.
"""
Expand All @@ -276,6 +287,10 @@ def test_signed_software_record_fails_level_1_runtime_rules(self, trace_key):
if finding.status is Status.FAIL
]
assert [(finding.code, finding.message) for finding in failures] == [
(
"TR-APR-005",
"TR-APR-005: Level 1 requires appraisal.status 'affirming', got 'none'",
),
(
"TR-RTE-001",
"TR-RTE-001: runtime.platform 'software-only' is development-mode "
Expand Down
Loading