From 0e13cef74f68a19c2d91a99de829fab0076275c9 Mon Sep 17 00:00:00 2001 From: Jeff Otterson Date: Sat, 11 Jul 2026 22:56:15 -0600 Subject: [PATCH 1/2] crucible: scope config for rag_guard/guard.py --- pyproject.toml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/pyproject.toml b/pyproject.toml index f25bc7a..b87ff0c 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -43,3 +43,8 @@ testpaths = ["tests"] source = ["rag_guard"] # relative_files lets the coverage-comment action resolve paths from a checkout relative_files = true + +[tool.mutmut] +source_paths = ["rag_guard/guard.py"] +also_copy = ["rag_guard"] +pytest_add_cli_args_test_selection = ["--ignore=tests/test_hook.py"] From 408c14e9f5bc80eff82b8a193abd55fcd5710ef6 Mon Sep 17 00:00:00 2001 From: Jeff Otterson Date: Sat, 11 Jul 2026 23:13:07 -0600 Subject: [PATCH 2/2] =?UTF-8?q?tests:=20crucible=20harden=20of=20rag=5Fgua?= =?UTF-8?q?rd/guard.py=20=E2=80=94=2025/25=20baseline=20survivors=20killed?= =?UTF-8?q?=20(tester=2023=20+=20critic=202);=20receipt=20~/.crucible-runs?= =?UTF-8?q?/rag-guard/20260712T050833Z-rag-guard-harden?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- tests/crucible_r0_harden_test.py | 183 +++++++++++++++++++++++++++++++ tests/crucible_r1_harden_test.py | 44 ++++++++ 2 files changed, 227 insertions(+) create mode 100644 tests/crucible_r0_harden_test.py create mode 100644 tests/crucible_r1_harden_test.py diff --git a/tests/crucible_r0_harden_test.py b/tests/crucible_r0_harden_test.py new file mode 100644 index 0000000..3e5ac3e --- /dev/null +++ b/tests/crucible_r0_harden_test.py @@ -0,0 +1,183 @@ +"""Tests for rag_guard.guard: should_refuse, groundedness, redact_pii.""" +import pytest + +from rag_guard.guard import groundedness, redact_pii, should_refuse + + +# --------------------------------------------------------------------------- +# should_refuse +# --------------------------------------------------------------------------- + +def test_should_refuse_empty_hits_refuses(): + assert should_refuse([]) is True + + +def test_should_refuse_score_above_default_threshold_does_not_refuse(): + assert should_refuse([{"score": 0.9}]) is False + + +def test_should_refuse_score_below_default_threshold_refuses(): + assert should_refuse([{"score": 0.01}]) is True + + +def test_should_refuse_score_exactly_at_default_threshold_does_not_refuse(): + # condition is strictly `max < min_score`, so equality does not refuse + assert should_refuse([{"score": 0.05}]) is False + + +def test_should_refuse_missing_score_key_defaults_to_zero_and_refuses(): + assert should_refuse([{}]) is True + + +def test_should_refuse_uses_max_score_across_multiple_hits(): + hits = [{"score": 0.01}, {"score": 0.2}, {"score": 0.0}] + assert should_refuse(hits) is False + + +def test_should_refuse_all_low_scores_refuses(): + hits = [{"score": 0.01}, {"score": 0.02}, {"score": 0.03}] + assert should_refuse(hits) is True + + +def test_should_refuse_custom_min_score_below_threshold_refuses(): + assert should_refuse([{"score": 0.5}], min_score=0.6) is True + + +def test_should_refuse_custom_min_score_equal_does_not_refuse(): + assert should_refuse([{"score": 0.5}], min_score=0.5) is False + + +# --------------------------------------------------------------------------- +# groundedness +# --------------------------------------------------------------------------- + +def test_groundedness_empty_answer_is_not_grounded_and_zero_support(): + result = groundedness("", ["anything here"]) + assert result["grounded"] is False + assert result["support"] == pytest.approx(0.0) + + +def test_groundedness_answer_of_only_stopwords_is_not_grounded(): + result = groundedness("the and for", ["anything here"]) + assert result["grounded"] is False + assert result["support"] == pytest.approx(0.0) + + +def test_groundedness_half_support_is_grounded_at_default_threshold(): + # answer content tokens: {cats, dogs}; context tokens: {cats, great, pets} + # ("are" is a stopword) -> intersection {cats} -> support 1/2 = 0.5 + result = groundedness("cats dogs", ["cats are great pets"]) + assert result["support"] == pytest.approx(0.5) + assert result["grounded"] is True + + +def test_groundedness_quarter_support_is_not_grounded_at_default_threshold(): + # answer tokens: {cats, dogs, birds, fish}; context tokens: {cats, everywhere} + # intersection {cats} -> support 1/4 = 0.25 + result = groundedness("cats dogs birds fish", ["cats everywhere"]) + assert result["support"] == pytest.approx(0.25) + assert result["grounded"] is False + + +def test_groundedness_empty_contexts_list_gives_zero_support(): + result = groundedness("cats dogs", []) + assert result["support"] == pytest.approx(0.0) + assert result["grounded"] is False + + +def test_groundedness_rounds_support_to_four_decimal_places(): + # answer tokens: {cats, dogs, birds}; context tokens: {cats, only, here} + # intersection {cats} -> support 1/3 = 0.3333... + result = groundedness("cats dogs birds", ["cats only here"], threshold=0.3) + assert result["support"] == pytest.approx(0.3333, abs=1e-4) + assert result["grounded"] is True + + +def test_groundedness_merges_tokens_across_multiple_contexts(): + # answer tokens: {cats, dogs, birds}; contexts merge to {cats, dogs} + # intersection {cats, dogs} -> support 2/3 = 0.6667 + result = groundedness("cats dogs birds", ["cats", "dogs"]) + assert result["support"] == pytest.approx(0.6667, abs=1e-4) + assert result["grounded"] is True + + +def test_groundedness_full_support_is_grounded(): + result = groundedness("cats dogs", ["cats dogs everywhere"]) + assert result["support"] == pytest.approx(1.0) + assert result["grounded"] is True + + +def test_groundedness_zero_support_is_not_grounded(): + result = groundedness("cats dogs", ["nothing relevant whatsoever"]) + assert result["support"] == pytest.approx(0.0) + assert result["grounded"] is False + + +# --------------------------------------------------------------------------- +# redact_pii +# --------------------------------------------------------------------------- + +def test_redact_pii_redacts_email(): + text = "Contact me at foo.bar@example.com please" + assert redact_pii(text) == "Contact me at [redacted-email] please" + + +def test_redact_pii_redacts_ssn(): + text = "SSN: 123-45-6789" + assert redact_pii(text) == "SSN: [redacted-ssn]" + + +def test_redact_pii_redacts_phone_with_dashes(): + text = "Call 555-123-4567 now" + assert redact_pii(text) == "Call [redacted-phone] now" + + +def test_redact_pii_redacts_phone_with_dots(): + text = "Phone: 555.123.4567" + assert redact_pii(text) == "Phone: [redacted-phone]" + + +def test_redact_pii_redacts_16_digit_card_number(): + text = "Card 4111111111111111 expires" + assert redact_pii(text) == "Card [redacted-card]expires" + + +def test_redact_pii_redacts_13_digit_card_number_lower_bound(): + text = "Card 1234567890123 test" + assert redact_pii(text) == "Card [redacted-card]test" + + +def test_redact_pii_does_not_redact_12_digit_number_below_card_lower_bound(): + text = "Number 123456789012 test" + assert redact_pii(text) == "Number 123456789012 test" + + +def test_redact_pii_does_not_redact_unseparated_10_digit_number(): + text = "Number 5551234567 here" + assert redact_pii(text) == "Number 5551234567 here" + + +def test_redact_pii_does_not_redact_space_separated_ssn_like_number(): + # SSN pattern only matches dash-separated groups + text = "SSN 123 45 6789" + assert redact_pii(text) == "SSN 123 45 6789" + + +def test_redact_pii_redacts_multiple_emails(): + text = "a@b.com and c@d.com" + assert redact_pii(text) == "[redacted-email] and [redacted-email]" + + +def test_redact_pii_redacts_mixed_pii_types_in_one_string(): + text = "Email me at a@b.com or call 555-123-4567, SSN 123-45-6789" + expected = "Email me at [redacted-email] or call [redacted-phone], SSN [redacted-ssn]" + assert redact_pii(text) == expected + + +def test_redact_pii_leaves_text_without_pii_unchanged(): + text = "Hello world, nothing sensitive here." + assert redact_pii(text) == text + + +def test_redact_pii_empty_string_returns_empty_string(): + assert redact_pii("") == "" diff --git a/tests/crucible_r1_harden_test.py b/tests/crucible_r1_harden_test.py new file mode 100644 index 0000000..9b913f3 --- /dev/null +++ b/tests/crucible_r1_harden_test.py @@ -0,0 +1,44 @@ +import pytest + +from rag_guard.guard import _content_tokens, groundedness + + +def test_content_tokens_none_input_yields_empty_set(): + # `_content_tokens(None)` must fall back to "" (not a placeholder word), + # so no tokens are extracted from missing text. + assert _content_tokens(None) == set() + + +def test_content_tokens_empty_string_yields_empty_set(): + # Same fallback path as None: empty string has no content tokens. + assert _content_tokens("") == set() + + +def test_content_tokens_falsy_text_does_not_leak_placeholder_token(): + # Guards against a mutant that substitutes a non-empty placeholder (e.g. "XXXX") + # for falsy text in `(text or "").lower()` — that would inject a spurious + # "xxxx" token into the result. + assert "xxxx" not in _content_tokens(None) + assert "xxxx" not in _content_tokens("") + + +def test_groundedness_support_rounded_to_four_decimal_places(): + # answer content tokens: {"apple", "banana", "cherry"} (3 tokens) + # context content tokens: {"apple", "info"} + # overlap = {"apple"} -> support = 1/3 = 0.333333... -> round(x, 4) == 0.3333 + # A mutant that rounds to 5 decimals instead would yield 0.33333, which is + # NOT approx-equal to 0.3333 at the tight tolerance used below. + result = groundedness("apple banana cherry", ["apple info"]) + expected_support = round(1 / 3, 4) + assert expected_support == pytest.approx(0.3333, rel=1e-6) + assert result["support"] == pytest.approx(expected_support, abs=1e-9) + assert result["grounded"] is False + + +def test_groundedness_support_exact_value_not_five_decimals(): + # Independent computation: 1/3 rounded to 4 places is 0.3333, which differs + # from the 5-decimal rounding (0.33333) by 3e-5 -- well outside a 1e-9 tolerance, + # so this fails against the mutant that rounds to 5 decimal places. + result = groundedness("apple banana cherry", ["apple info"]) + assert result["support"] != pytest.approx(0.33333, abs=1e-9) + assert result["support"] == pytest.approx(0.3333, abs=1e-9)