diff --git a/scripts/compare_scan_accuracy.py b/scripts/compare_scan_accuracy.py new file mode 100644 index 000000000..17eeaa5ac --- /dev/null +++ b/scripts/compare_scan_accuracy.py @@ -0,0 +1,259 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Compare two SkillSpector revisions on a human-adjudicated local corpus. + +The corpus and generated report are intentionally external inputs. This keeps +private or disclosure-controlled fixtures out of the repository while making +the accuracy gate deterministic and reproducible. +""" + +from __future__ import annotations + +import argparse +import hashlib +import json +import subprocess +import sys +from collections import Counter +from pathlib import Path +from typing import Any + + +def _load_json(path: Path) -> dict[str, Any]: + value = json.loads(path.read_text(encoding="utf-8")) + if not isinstance(value, dict): + raise ValueError(f"Expected a JSON object in {path}") + return value + + +def _resolve_case_path(corpus_root: Path, relative_path: str) -> Path: + root = corpus_root.resolve(strict=True) + target = (root / relative_path).resolve(strict=True) + if not target.is_relative_to(root) or not target.is_dir(): + raise ValueError(f"Corpus case must be a directory below the corpus root: {relative_path}") + return target + + +def _corpus_snapshot(corpus_root: Path, cases: list[dict[str, Any]]) -> str: + digest = hashlib.sha256() + seen: set[Path] = set() + for case in sorted(cases, key=lambda item: str(item["id"])): + target = _resolve_case_path(corpus_root, str(case["path"])) + for path in sorted(target.rglob("*")): + if path.is_symlink(): + raise ValueError(f"Corpus snapshot does not follow symlinks: {path}") + if not path.is_file(): + continue + resolved = path.resolve(strict=True) + if resolved in seen: + continue + seen.add(resolved) + relative = resolved.relative_to(corpus_root.resolve(strict=True)).as_posix() + digest.update(relative.encode("utf-8")) + digest.update(b"\0") + digest.update(resolved.read_bytes()) + digest.update(b"\0") + return f"sha256:{digest.hexdigest()}" + + +def _run_scan(executable: Path, target: Path) -> dict[str, Any]: + command = [ + str(executable), + "scan", + str(target), + "--format", + "json", + "--no-llm", + ] + completed = subprocess.run(command, capture_output=True, text=True, check=False) + if completed.returncode not in {0, 1}: + raise RuntimeError( + f"Scanner exited {completed.returncode} for {target.name}: {completed.stderr.strip()}" + ) + try: + report = json.loads(completed.stdout) + except json.JSONDecodeError as error: + raise RuntimeError(f"Scanner returned invalid JSON for {target.name}") from error + if not isinstance(report, dict): + raise RuntimeError(f"Scanner returned a non-object report for {target.name}") + return report + + +def _rule_counts(report: dict[str, Any], selected_rules: frozenset[str]) -> Counter[str]: + issues = report.get("issues", []) + if not isinstance(issues, list): + raise ValueError("Scanner JSON report has a non-list 'issues' field") + counts: Counter[str] = Counter() + for issue in issues: + if not isinstance(issue, dict) or not isinstance(issue.get("id"), str): + continue + rule_id = str(issue["id"]) + if not selected_rules or rule_id in selected_rules: + counts[rule_id] += 1 + return counts + + +def _expected_range(value: object) -> tuple[int, int]: + if isinstance(value, int) and value >= 0: + return value, value + if isinstance(value, dict): + minimum = value.get("min", 0) + maximum = value.get("max", minimum) + if isinstance(minimum, int) and isinstance(maximum, int) and 0 <= minimum <= maximum: + return minimum, maximum + raise ValueError("Expected rule counts must be a non-negative integer or {min, max} object") + + +def _adjudication_errors(case: dict[str, Any], counts: Counter[str]) -> list[str]: + raw_expected = case.get("expected_rules", {}) + if not isinstance(raw_expected, dict): + raise ValueError(f"Case {case['id']} has a non-object expected_rules field") + expected = {str(rule_id): _expected_range(value) for rule_id, value in raw_expected.items()} + errors: list[str] = [] + for rule_id, (minimum, maximum) in sorted(expected.items()): + actual = counts.get(rule_id, 0) + if not minimum <= actual <= maximum: + errors.append(f"{rule_id}: expected {minimum}..{maximum}, observed {actual}") + if not bool(case.get("allow_unlisted_rules", False)): + for rule_id in sorted(set(counts) - set(expected)): + errors.append(f"{rule_id}: unlisted rule emitted {counts[rule_id]} finding(s)") + return errors + + +def compare_scanners( + *, + manifest_path: Path, + corpus_root: Path, + baseline_executable: Path, + candidate_executable: Path, + baseline_revision: str, + candidate_revision: str, + selected_rules: frozenset[str] = frozenset(), +) -> dict[str, Any]: + """Run both scanners and return deterministic accuracy evidence.""" + manifest = _load_json(manifest_path) + if manifest.get("schema_version") != 1: + raise ValueError("Accuracy manifest schema_version must be 1") + raw_cases = manifest.get("cases") + if not isinstance(raw_cases, list) or not raw_cases: + raise ValueError("Accuracy manifest must contain a non-empty cases list") + cases: list[dict[str, Any]] = [] + case_ids: set[str] = set() + for raw_case in raw_cases: + if not isinstance(raw_case, dict): + raise ValueError("Each accuracy case must be a JSON object") + case_id = raw_case.get("id") + path = raw_case.get("path") + if not isinstance(case_id, str) or not case_id or case_id in case_ids: + raise ValueError("Each accuracy case needs a unique non-empty id") + if not isinstance(path, str) or not path: + raise ValueError(f"Case {case_id} needs a non-empty path") + case_ids.add(case_id) + cases.append(raw_case) + + aggregate_baseline: Counter[str] = Counter() + aggregate_candidate: Counter[str] = Counter() + case_results: list[dict[str, Any]] = [] + for case in sorted(cases, key=lambda item: str(item["id"])): + target = _resolve_case_path(corpus_root, str(case["path"])) + baseline_counts = _rule_counts(_run_scan(baseline_executable, target), selected_rules) + candidate_counts = _rule_counts(_run_scan(candidate_executable, target), selected_rules) + aggregate_baseline.update(baseline_counts) + aggregate_candidate.update(candidate_counts) + all_rules = sorted(set(baseline_counts) | set(candidate_counts)) + case_results.append( + { + "id": case["id"], + "path": case["path"], + "classification": case.get("classification", "review"), + "baseline": dict(sorted(baseline_counts.items())), + "candidate": dict(sorted(candidate_counts.items())), + "delta": { + rule_id: candidate_counts[rule_id] - baseline_counts[rule_id] + for rule_id in all_rules + }, + "adjudication_errors": _adjudication_errors(case, candidate_counts), + } + ) + + all_rules = sorted(set(aggregate_baseline) | set(aggregate_candidate)) + result = { + "schema_version": 1, + "corpus_snapshot": _corpus_snapshot(corpus_root, cases), + "manifest": manifest_path.name, + "baseline": { + "revision": baseline_revision, + "command": [ + str(baseline_executable), + "scan", + "", + "--format", + "json", + "--no-llm", + ], + }, + "candidate": { + "revision": candidate_revision, + "command": [ + str(candidate_executable), + "scan", + "", + "--format", + "json", + "--no-llm", + ], + }, + "selected_rules": sorted(selected_rules), + "per_rule": { + rule_id: { + "baseline": aggregate_baseline[rule_id], + "candidate": aggregate_candidate[rule_id], + "delta": aggregate_candidate[rule_id] - aggregate_baseline[rule_id], + } + for rule_id in all_rules + }, + "cases": case_results, + } + result["passed"] = not any(case["adjudication_errors"] for case in case_results) + return result + + +def _parser() -> argparse.ArgumentParser: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--manifest", type=Path, required=True) + parser.add_argument("--corpus-root", type=Path, required=True) + parser.add_argument("--baseline-executable", type=Path, required=True) + parser.add_argument("--candidate-executable", type=Path, required=True) + parser.add_argument("--baseline-revision", required=True) + parser.add_argument("--candidate-revision", required=True) + parser.add_argument("--rule", action="append", default=[]) + parser.add_argument("--output", type=Path) + return parser + + +def main(argv: list[str] | None = None) -> int: + args = _parser().parse_args(argv) + try: + result = compare_scanners( + manifest_path=args.manifest, + corpus_root=args.corpus_root, + baseline_executable=args.baseline_executable, + candidate_executable=args.candidate_executable, + baseline_revision=args.baseline_revision, + candidate_revision=args.candidate_revision, + selected_rules=frozenset(args.rule), + ) + except (OSError, RuntimeError, ValueError, json.JSONDecodeError) as error: + print(f"accuracy gate error: {error}", file=sys.stderr) + return 2 + rendered = json.dumps(result, indent=2, sort_keys=True) + "\n" + if args.output: + args.output.write_text(rendered, encoding="utf-8") + else: + print(rendered, end="") + return 0 if result["passed"] else 1 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/generate_unicode_confusables.py b/scripts/generate_unicode_confusables.py new file mode 100644 index 000000000..a5826e7c4 --- /dev/null +++ b/scripts/generate_unicode_confusables.py @@ -0,0 +1,76 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Generate the bounded ASCII skeleton table used by security text views. + +The input is the versioned ``confusables.txt`` published with Unicode UTS #39. +Only single-code-point sources whose skeleton is made entirely of ASCII letters +or digits are retained. This is the complete UTS #39 subset relevant to the +ASCII security tokens matched by SkillSpector's deterministic analyzers. +""" + +from __future__ import annotations + +import argparse +import json +from pathlib import Path + + +def _parse_line(line: str) -> tuple[int, str] | None: + data = line.split("#", 1)[0].strip() + if not data: + return None + fields = [field.strip() for field in data.split(";")] + if len(fields) < 2: + return None + source_points = fields[0].split() + if len(source_points) != 1: + return None + target = "".join(chr(int(point, 16)) for point in fields[1].split()) + if not target or not all(char.isascii() and char.isalnum() for char in target): + return None + source = int(source_points[0], 16) + # Raw ASCII is already scanned directly. Retaining ASCII-to-ASCII skeleton + # rewrites (for example ``m`` -> ``rn``) would mutate otherwise ordinary + # detector tokens after a neighboring non-ASCII character triggered this + # derived view. + if source <= 0x7F: + return None + return source, target + + +def main() -> None: + parser = argparse.ArgumentParser() + parser.add_argument("source", type=Path) + parser.add_argument("destination", type=Path) + parser.add_argument("--version", required=True) + args = parser.parse_args() + + mappings = dict( + parsed + for line in args.source.read_text(encoding="utf-8").splitlines() + if (parsed := _parse_line(line)) is not None + ) + lines = [ + "# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.", + "# SPDX-License-Identifier: Apache-2.0", + "", + '"""Generated ASCII skeleton subset from Unicode UTS #39 confusables data."""', + "", + "from __future__ import annotations", + "", + f'UNICODE_CONFUSABLES_VERSION = "{args.version}"', + f"# Source: https://www.unicode.org/Public/{args.version}/security/confusables.txt", + "# The source data is governed by https://www.unicode.org/license.txt.", + "ASCII_CONFUSABLE_SKELETON: dict[int, str] = {", + ] + lines.extend( + f" 0x{codepoint:04X}: {json.dumps(target)}," + for codepoint, target in sorted(mappings.items()) + ) + lines.extend(["}", ""]) + args.destination.write_text("\n".join(lines), encoding="utf-8") + + +if __name__ == "__main__": + main() diff --git a/src/skillspector/artifacts.py b/src/skillspector/artifacts.py new file mode 100644 index 000000000..87acc4898 --- /dev/null +++ b/src/skillspector/artifacts.py @@ -0,0 +1,282 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Canonical artifact classification and security-oriented text views. + +The scanner keeps raw bytes as the source of truth. Text analyzers consume +derived views with source-offset maps so decoding and Unicode normalization do +not create an untracked gap between the bytes that were supplied and the text +that was inspected. +""" + +from __future__ import annotations + +import re +import unicodedata +from array import array +from dataclasses import dataclass +from enum import StrEnum +from io import StringIO +from typing import NotRequired + +from typing_extensions import TypedDict + +from skillspector.unicode_confusables import ASCII_CONFUSABLE_SKELETON + + +class ContentKind(StrEnum): + """Byte-derived artifact content classification.""" + + TEXT = "text" + BINARY = "binary" + OPAQUE = "opaque" + + +class ArtifactDisposition(StrEnum): + """Normative disposition used by coverage and reference accounting.""" + + ANALYZED = "analyzed" + PARTIAL = "partial" + FAILED = "failed" + OUT_OF_SCOPE = "out_of_scope" + + +class ArtifactRecord(TypedDict): + """Serializable inventory row for one discovered bundle artifact.""" + + path: str + content_kind: ContentKind + disposition: ArtifactDisposition + size_bytes: int + decodable: bool + contains_nul: bool + misleading_extension: bool + referenced: bool + reason: NotRequired[str] + + +class BundleReference(TypedDict): + """Canonical, report-safe intra-bundle reference record.""" + + source_path: str + line: int + column: int + evidence: str + target_path: str | None + status: str + disposition: ArtifactDisposition + + +@dataclass(frozen=True) +class SecurityTextView: + """A bounded derived text view and mapping to raw character offsets.""" + + name: str + text: str + source_offsets: array[int] | None = None + + def source_offset(self, derived_offset: int) -> int: + """Map a derived character offset to the corresponding source offset.""" + if self.source_offsets is None: + return min(max(derived_offset, 0), len(self.text)) + if not self.source_offsets: + return 0 + index = min(max(derived_offset, 0), len(self.source_offsets) - 1) + return self.source_offsets[index] + + +_BINARY_MAGIC = ( + b"\x89PNG\r\n\x1a\n", + b"\xff\xd8\xff", + b"GIF87a", + b"GIF89a", + b"PK\x03\x04", + b"\x7fELF", + b"MZ", + b"\x00asm", + b"%PDF-", +) + +_BINARY_EXTENSIONS = frozenset( + { + ".png", + ".jpg", + ".jpeg", + ".gif", + ".pdf", + ".zip", + ".gz", + ".exe", + ".dll", + ".so", + ".dylib", + ".wasm", + ".pyc", + ".class", + ".mp3", + ".mp4", + ".sqlite", + } +) +_TEXT_EXTENSIONS = frozenset( + { + ".md", + ".markdown", + ".txt", + ".py", + ".sh", + ".json", + ".yaml", + ".yml", + ".toml", + ".js", + ".ts", + ".rb", + ".go", + ".rs", + } +) + +_ALLOWED_FORMAT_CHARS = frozenset({"\n", "\r", "\t"}) +_IGNORED_ASCII_CONTROL = re.compile(r"[\x00-\x08\x0b\x0c\x0e-\x1f\x7f]") + + +def _suffix(path: str) -> str: + name = path.rsplit("/", 1)[-1] + index = name.rfind(".") + return name[index:].lower() if index >= 0 else "" + + +def classify_artifact(path: str, data: bytes, *, referenced: bool = False) -> ArtifactRecord: + """Classify from bytes and decodability; an extension is never authoritative.""" + contains_nul = b"\x00" in data + has_binary_magic = any(data.startswith(magic) for magic in _BINARY_MAGIC) + try: + decoded = data.decode("utf-8") + decodable = True + except UnicodeDecodeError: + decoded = data.decode("utf-8", errors="replace") + decodable = False + + if has_binary_magic: + kind = ContentKind.BINARY + elif decodable: + kind = ContentKind.TEXT + elif not data: + kind = ContentKind.TEXT + else: + printable = sum(ch.isprintable() or ch in _ALLOWED_FORMAT_CHARS for ch in decoded) + replacement_ratio = decoded.count("\ufffd") / max(1, len(decoded)) + if printable / max(1, len(decoded)) >= 0.85 and replacement_ratio <= 0.10: + kind = ContentKind.TEXT + else: + kind = ContentKind.BINARY + + suffix = _suffix(path) + misleading = (suffix in _BINARY_EXTENSIONS and kind is ContentKind.TEXT) or ( + suffix in _TEXT_EXTENSIONS and kind is ContentKind.BINARY + ) + disposition = ( + ArtifactDisposition.PARTIAL + if referenced and kind is not ContentKind.TEXT + else ArtifactDisposition.OUT_OF_SCOPE + if kind is ContentKind.BINARY + else ArtifactDisposition.ANALYZED + ) + return { + "path": path, + "content_kind": kind, + "disposition": disposition, + "size_bytes": len(data), + "decodable": decodable, + "contains_nul": contains_nul, + "misleading_extension": misleading, + "referenced": referenced, + } + + +def decode_text(data: bytes) -> str: + """Return the loss-tolerant local text projection for static analyzers.""" + return data.decode("utf-8", errors="replace") + + +def _is_ignored_format(ch: str) -> bool: + return ( + ch == "\u00ad" + or unicodedata.category(ch) in {"Cf", "Cc"} + and ch not in _ALLOWED_FORMAT_CHARS + ) + + +def normalized_security_view(text: str) -> SecurityTextView: + """Build an NFKC/UTS #39 ASCII-skeleton view with compact offsets.""" + output = StringIO() + offsets = array("I") + for source_offset, ch in enumerate(text): + if _is_ignored_format(ch): + continue + normalized = unicodedata.normalize("NFKC", ch).translate(ASCII_CONFUSABLE_SKELETON) + for normalized_char in normalized: + output.write(normalized_char) + offsets.append(source_offset) + return SecurityTextView("normalized", output.getvalue(), offsets) + + +def compact_letter_view(text: str) -> SecurityTextView: + """Remove compact binary/format noise between letters without joining words.""" + output = StringIO() + offsets = array("I") + for source_offset, ch in enumerate(text): + if _is_ignored_format(ch) or ch == "\ufffd": + continue + normalized = unicodedata.normalize("NFKC", ch).translate(ASCII_CONFUSABLE_SKELETON) + for normalized_char in normalized: + output.write(normalized_char) + offsets.append(source_offset) + return SecurityTextView("compact", output.getvalue(), offsets) + + +def security_text_views(text: str) -> tuple[SecurityTextView, ...]: + """Return distinct raw, normalized, and compact views deterministically.""" + raw = SecurityTextView("raw", text) + if text.isascii() and _IGNORED_ASCII_CONTROL.search(text) is None: + return (raw,) + unique = [raw] + seen = {text} + builders = [normalized_security_view] + if "\ufffd" in text: + builders.append(compact_letter_view) + for build_view in builders: + view = build_view(text) + if view.text not in seen: + seen.add(view.text) + unique.append(view) + return tuple(unique) + + +def unicode_anomaly_density(text: str) -> float: + """Return the density of soft-hyphen/default-ignorable format characters.""" + if not text: + return 0.0 + return sum(_is_ignored_format(ch) for ch in text) / len(text) + + +def has_mixed_script_token(text: str) -> bool: + """Detect bounded tokens that combine ASCII with Greek/Cyrillic letters.""" + token_scripts: set[str] = set() + for ch in text: + if ch.isascii() and ch.isalpha(): + token_scripts.add("latin") + elif ch.isalpha(): + name = unicodedata.name(ch, "") + if "CYRILLIC" in name: + token_scripts.add("cyrillic") + elif "GREEK" in name: + token_scripts.add("greek") + elif ch.isalnum() or ch in {"_", "-"}: + continue + else: + if "latin" in token_scripts and len(token_scripts) > 1: + return True + token_scripts.clear() + return "latin" in token_scripts and len(token_scripts) > 1 diff --git a/src/skillspector/cli.py b/src/skillspector/cli.py index 357188006..67e6b0a05 100644 --- a/src/skillspector/cli.py +++ b/src/skillspector/cli.py @@ -265,6 +265,13 @@ def scan( "is given.", ), ] = False, + fail_on_incomplete: Annotated[ + bool, + typer.Option( + "--fail-on-incomplete", + help="Exit 1 when relevant analysis is partial or incomplete.", + ), + ] = False, verbose: Annotated[ bool, typer.Option( @@ -357,7 +364,15 @@ def scan( "multi-skill scans; scan each sub-skill with its own baseline" ) raise typer.Exit(code=2) - _scan_multi_skill(detection, format, output, no_llm, yara_rules_dir, verbose) + _scan_multi_skill( + detection, + format, + output, + no_llm, + yara_rules_dir, + verbose, + fail_on_incomplete=fail_on_incomplete, + ) return if not detection.has_root_skill and len(detection.skills) == 0: console.print( @@ -423,6 +438,9 @@ def scan( if result.get("execution_successful") is False: raise typer.Exit(code=2) + completeness = result.get("analysis_completeness") or {} + if fail_on_incomplete and not bool(completeness.get("is_complete", True)): + raise typer.Exit(code=1) if (result.get("risk_score") or 0) > RISK_THRESHOLD: raise typer.Exit(code=1) except typer.Exit: @@ -466,6 +484,7 @@ def _scan_multi_skill( no_llm: bool, yara_rules_dir: Path | None, verbose: bool, + fail_on_incomplete: bool = False, ) -> None: """Scan each detected sub-skill independently and produce a combined report.""" skills = detection.skills @@ -474,6 +493,7 @@ def _scan_multi_skill( results: list[dict[str, object]] = [] max_score = 0 execution_failed = False + analysis_incomplete = False for i, skill in enumerate(skills, 1): console.print( @@ -488,6 +508,9 @@ def _scan_multi_skill( results.append(result) if result.get("execution_successful") is False: execution_failed = True + completeness = result.get("analysis_completeness") or {} + if not bool(completeness.get("is_complete", True)): + analysis_incomplete = True score = result.get("risk_score") or 0 if isinstance(score, int) and score > max_score: max_score = score @@ -564,6 +587,8 @@ def _scan_multi_skill( if execution_failed: raise typer.Exit(code=2) + if fail_on_incomplete and analysis_incomplete: + raise typer.Exit(code=1) if max_score > RISK_THRESHOLD: raise typer.Exit(code=1) diff --git a/src/skillspector/constants.py b/src/skillspector/constants.py index 7ef3b6ffc..798641456 100644 --- a/src/skillspector/constants.py +++ b/src/skillspector/constants.py @@ -31,6 +31,9 @@ # Maximum text-file size processed by static analyzers and lightweight # format recognizers. MAX_FILE_BYTES = 1_000_000 +# Static analysis supports complete per-artifact coverage through 16 MiB. Larger +# files are read only to this bound and are reported as partial, never complete. +MAX_ANALYZABLE_FILE_BYTES = 16 * 1024 * 1024 # Default-model selection lives on each provider (see providers//provider.py # for ``DEFAULT_MODEL`` and ``SLOT_DEFAULTS``). The active provider's diff --git a/src/skillspector/inspection_ledger.py b/src/skillspector/inspection_ledger.py index 0b3cf2041..daac3966e 100644 --- a/src/skillspector/inspection_ledger.py +++ b/src/skillspector/inspection_ledger.py @@ -20,6 +20,7 @@ class LedgerOutcome(StrEnum): """Terminal outcome of one inspection work item.""" COMPLETED = "completed" + PARTIAL = "partial" SKIPPED = "skipped" FAILED = "failed" OUT_OF_SCOPE = "out_of_scope" @@ -60,6 +61,13 @@ class LedgerReason(StrEnum): NO_APPLICABLE_FILES = "no_applicable_files" OMS_SIGNATURE = "oms_signature" BASELINE_FILE = "baseline_file" + VCS_METADATA = "vcs_metadata" + OPAQUE_CONTENT = "opaque_content" + REFERENCED_UNINSPECTED = "referenced_uninspected" + REFERENCE_EXTRACTION_LIMIT = "reference_extraction_limit" + REFERENCE_UNRESOLVED = "reference_unresolved" + RUNTIME_LIMIT = "runtime_limit" + OUTPUT_LIMIT = "output_limit" REASON_MESSAGES: Final[dict[LedgerReason, str]] = { @@ -99,6 +107,19 @@ class LedgerReason(StrEnum): LedgerReason.BASELINE_FILE: ( "The explicitly selected suppression baseline is excluded from content analysis." ), + LedgerReason.VCS_METADATA: ( + "VCS object and history metadata is outside the bounded artifact inspection profile." + ), + LedgerReason.OPAQUE_CONTENT: "Artifact contents could not be fully interpreted.", + LedgerReason.REFERENCED_UNINSPECTED: ("A referenced artifact was not completely inspected."), + LedgerReason.REFERENCE_EXTRACTION_LIMIT: ( + "Reference extraction reached an explicit resource bound before completion." + ), + LedgerReason.REFERENCE_UNRESOLVED: ( + "A local path-like reference could not be resolved unambiguously." + ), + LedgerReason.RUNTIME_LIMIT: "Analyzer reached its per-artifact runtime limit.", + LedgerReason.OUTPUT_LIMIT: "Analyzer reached its per-artifact finding output limit.", } @@ -141,6 +162,10 @@ class InspectionLedgerEvent(TypedDict): limit_characters: NotRequired[int] observed_bytes: NotRequired[int] limit_bytes: NotRequired[int] + observed_findings: NotRequired[int] + limit_findings: NotRequired[int] + observed_seconds: NotRequired[float] + limit_seconds: NotRequired[float] class AnalyzerStatusEvent(TypedDict): @@ -175,6 +200,7 @@ class AnalysisCompleteness(TypedDict): scanned_components: int coverage_percent: float is_complete: bool + status: str execution_successful: bool fully_inspected_files: int partially_inspected_files: int @@ -182,6 +208,7 @@ class AnalysisCompleteness(TypedDict): ledger_exceptions: list[InspectionLedgerException] scope_exclusions: list[InspectionLedgerException] analyzer_statuses: list[dict[str, object]] + references: NotRequired[list[dict[str, object]]] limitations: NotRequired[list[str]] findings_before_filtering: NotRequired[int] findings_after_filtering: NotRequired[int] @@ -262,6 +289,10 @@ def ledger_event( limit_characters: int | None = None, observed_bytes: int | None = None, limit_bytes: int | None = None, + observed_findings: int | None = None, + limit_findings: int | None = None, + observed_seconds: float | None = None, + limit_seconds: float | None = None, ) -> InspectionLedgerEvent: """Create one validated terminal ledger record without sensitive payloads.""" _validate_range(start_line, end_line) @@ -284,11 +315,11 @@ def ledger_event( if not is_meta: if input_ids: raise ValueError("producer ledger events cannot consume findings") - if outcome is not LedgerOutcome.COMPLETED and emitted_ids: + if outcome not in (LedgerOutcome.COMPLETED, LedgerOutcome.PARTIAL) and emitted_ids: raise ValueError("non-completed producers cannot reference findings") elif outcome is LedgerOutcome.COMPLETED and not set(emitted_ids).issubset(input_ids): raise ValueError("completed meta events must emit a subset of input findings") - elif outcome in (LedgerOutcome.FAILED, LedgerOutcome.SKIPPED): + elif outcome in (LedgerOutcome.FAILED, LedgerOutcome.SKIPPED, LedgerOutcome.PARTIAL): if emitted_ids != input_ids: raise ValueError("failed or skipped meta events must pass every input finding through") elif outcome is not LedgerOutcome.COMPLETED: @@ -324,6 +355,14 @@ def ledger_event( event["observed_bytes"] = observed_bytes if limit_bytes is not None: event["limit_bytes"] = limit_bytes + if observed_findings is not None: + event["observed_findings"] = observed_findings + if limit_findings is not None: + event["limit_findings"] = limit_findings + if observed_seconds is not None: + event["observed_seconds"] = observed_seconds + if limit_seconds is not None: + event["limit_seconds"] = limit_seconds return event @@ -377,7 +416,7 @@ def analyzer_status_for_events( "failed" if LedgerOutcome.FAILED in outcomes else "degraded" - if LedgerOutcome.SKIPPED in outcomes + if LedgerOutcome.SKIPPED in outcomes or LedgerOutcome.PARTIAL in outcomes else "completed" ) return analyzer_status_event( @@ -616,7 +655,11 @@ def accounting_error(path: object = None) -> None: producer_rows_present = True if is_producer and input_ids: accounting_error(event.get("path")) - if is_producer and outcome != LedgerOutcome.COMPLETED and emitted_ids: + if ( + is_producer + and outcome not in (LedgerOutcome.COMPLETED, LedgerOutcome.PARTIAL) + and emitted_ids + ): accounting_error(event.get("path")) if ( is_meta @@ -626,14 +669,14 @@ def accounting_error(path: object = None) -> None: accounting_error(event.get("path")) if ( is_meta - and outcome in (LedgerOutcome.FAILED, LedgerOutcome.SKIPPED) + and outcome in (LedgerOutcome.FAILED, LedgerOutcome.SKIPPED, LedgerOutcome.PARTIAL) and emitted_ids != input_ids ): accounting_error(event.get("path")) for finding_id in [*input_ids, *emitted_ids]: if finding_id not in findings_by_id: accounting_error(event.get("path")) - if is_producer and outcome == LedgerOutcome.COMPLETED: + if is_producer and outcome in (LedgerOutcome.COMPLETED, LedgerOutcome.PARTIAL): for finding_id in emitted_ids: producer_origins[finding_id] = producer_origins.get(finding_id, 0) + 1 @@ -657,22 +700,10 @@ def accounting_error(path: object = None) -> None: seen_effective.add(finding_id) validated_effective.append(finding_id) - meta_planned_ids = { - target["work_id"] - for status in statuses - if status.get("analyzer_id") == "meta_analyzer" - for target in status.get("planned_work", []) - } - if meta_planned_ids: - meta_effective = _deduplicate_ids( - finding_id - for event in events - if event.get("work_id") in meta_planned_ids - and _is_meta_phase(str(event.get("phase", ""))) - for finding_id in event.get("emitted_finding_ids", []) - ) - if meta_effective != validated_effective: - accounting_error() + # Deterministic analyzer findings are primary evidence. Meta analysis may + # enrich or annotate those objects but cannot select them out of the public + # machine-readable result. + validated_effective = list(findings_by_id) unaccounted_exceptions: list[InspectionLedgerException] = [] status_summaries: list[dict[str, object]] = [] @@ -680,7 +711,13 @@ def accounting_error(path: object = None) -> None: for status in statuses: analyzer_id = str(status.get("analyzer_id", "")) planned_work = cast(list[PlannedWorkTarget], status.get("planned_work", [])) - outcome_counts = {"completed": 0, "skipped": 0, "failed": 0, "unaccounted": 0} + outcome_counts = { + "completed": 0, + "partial": 0, + "skipped": 0, + "failed": 0, + "unaccounted": 0, + } for target in planned_work: work_id = str(target.get("work_id", "")) matches = events_by_work_id.get(work_id, []) @@ -723,7 +760,8 @@ def accounting_error(path: object = None) -> None: exceptional_rows = [ _exception_from_event(event, fatal=event.get("outcome") == LedgerOutcome.FAILED) for event in events - if event.get("outcome") in (LedgerOutcome.SKIPPED, LedgerOutcome.FAILED) + if event.get("outcome") + in (LedgerOutcome.PARTIAL, LedgerOutcome.SKIPPED, LedgerOutcome.FAILED) ] exceptional_rows.extend(unaccounted_exceptions) exceptional_rows.extend(accounting_exceptions) @@ -750,25 +788,71 @@ def accounting_error(path: object = None) -> None: LedgerOutcome.FAILED if component in cache_failures else LedgerOutcome.COMPLETED ) + raw_inventory = state.get("artifact_inventory", []) + inventory = ( + [item for item in raw_inventory if isinstance(item, dict)] + if isinstance(raw_inventory, list) + else [] + ) + disposition_by_path = { + str(item.get("path", "")): str(item.get("disposition", "")) for item in inventory + } + raw_references_for_paths = state.get("artifact_references", []) + referenced_paths = ( + { + str(item.get("target_path")) + for item in raw_references_for_paths + if isinstance(item, dict) + and item.get("status") == "resolved" + and item.get("target_path") + } + if isinstance(raw_references_for_paths, list) + else set() + ) + relevant_components = [ + component + for component in components + if disposition_by_path.get(component) != "out_of_scope" or component in referenced_paths + ] + fully_inspected = 0 partially_inspected = 0 entirely_uninspected = 0 - for component in components: - outcomes = per_component.get(component, []) + for component in relevant_components: + outcomes = [ + outcome + for outcome in per_component.get(component, []) + if outcome != LedgerOutcome.OUT_OF_SCOPE + ] if outcomes and all(outcome == LedgerOutcome.COMPLETED for outcome in outcomes): fully_inspected += 1 - elif any(outcome == LedgerOutcome.COMPLETED for outcome in outcomes): + elif any( + outcome in (LedgerOutcome.COMPLETED, LedgerOutcome.PARTIAL) for outcome in outcomes + ): partially_inspected += 1 else: entirely_uninspected += 1 - total_components = len(components) + total_components = len(relevant_components) coverage_percent = ( round(fully_inspected / total_components * 100, 1) if total_components else 100.0 ) limitations: list[str] = [] for status_summary in status_summaries: status_name = str(status_summary["status"]) + explicitly_optional = ( + state.get("use_llm") is False + and status_name == "disabled" + and str(status_summary["analyzer_id"]) + in { + "meta_analyzer", + "semantic_security_discovery", + "semantic_developer_intent", + "semantic_quality_policy", + } + ) + if explicitly_optional: + continue if status_name not in {"completed", "not_applicable"}: message = status_summary.get("message") limitations.append( @@ -776,14 +860,29 @@ def accounting_error(path: object = None) -> None: if message else f"Analyzer {status_summary['analyzer_id']} status: {status_name}." ) - is_complete = not ledger_exceptions and not limitations execution_successful = not any(exception.get("fatal") for exception in ledger_exceptions) + completeness_status = ( + "failed" + if not execution_successful + else "partial" + if ledger_exceptions or limitations or partially_inspected or entirely_uninspected + else "complete" + ) + is_complete = completeness_status == "complete" + + raw_references = state.get("artifact_references", []) + public_references = ( + [dict(item) for item in raw_references if isinstance(item, dict)] + if isinstance(raw_references, list) + else [] + ) completeness: AnalysisCompleteness = { "total_components": total_components, "scanned_components": fully_inspected, "coverage_percent": coverage_percent, "is_complete": is_complete, + "status": completeness_status, "execution_successful": execution_successful, "fully_inspected_files": fully_inspected, "partially_inspected_files": partially_inspected, @@ -791,6 +890,7 @@ def accounting_error(path: object = None) -> None: "ledger_exceptions": ledger_exceptions, "scope_exclusions": scope_exclusions, "analyzer_statuses": sorted(status_summaries, key=lambda item: str(item["analyzer_id"])), + "references": public_references, "limitations": limitations, "findings_before_filtering": len(findings_by_id), "findings_after_filtering": len(validated_effective), diff --git a/src/skillspector/mcp_server.py b/src/skillspector/mcp_server.py index 2fbababb5..4b90fd53e 100644 --- a/src/skillspector/mcp_server.py +++ b/src/skillspector/mcp_server.py @@ -143,7 +143,10 @@ async def run_scan( analysis_completeness = result.get("analysis_completeness") or {} entirely_uninspected = int(analysis_completeness.get("entirely_uninspected_files", 0)) safe_to_install = ( - risk_score <= RISK_THRESHOLD and execution_successful and entirely_uninspected == 0 + risk_score <= RISK_THRESHOLD + and execution_successful + and entirely_uninspected == 0 + and bool(analysis_completeness.get("is_complete", True)) ) return { "target": target, diff --git a/src/skillspector/models.py b/src/skillspector/models.py index 586ea228d..6166dae85 100644 --- a/src/skillspector/models.py +++ b/src/skillspector/models.py @@ -19,6 +19,7 @@ from dataclasses import dataclass, field from enum import StrEnum +from hashlib import sha256 from typing import TYPE_CHECKING, Protocol from uuid import uuid4 @@ -89,6 +90,17 @@ class Finding: tags: list[str] = field(default_factory=list) context: str | None = None matched_text: str | None = None + match_fingerprint: str | None = None + occurrences: list[dict[str, object]] = field(default_factory=list) + + def fingerprint(self) -> str | None: + """Return a full-match fingerprint without exposing the matched payload.""" + if self.match_fingerprint: + return self.match_fingerprint + if not self.matched_text: + return None + normalized = " ".join(self.matched_text.strip().split()) + return sha256(f"{self.rule_id}\x1f{normalized}".encode()).hexdigest() def to_dict(self) -> dict[str, object]: """Return a JSON-serializable dict representation (full finding shape).""" @@ -112,6 +124,15 @@ def to_dict(self) -> dict[str, object]: # Tags surface markers like "llm-unconfirmed" (a high-severity static # finding the LLM filter did not confirm but which is preserved anyway). "tags": list(self.tags), + "match_fingerprint": self.fingerprint(), + "occurrences": list(self.occurrences) + or [ + { + "file": self.file, + "start_line": self.start_line, + "end_line": self.end_line, + } + ], } def __str__(self) -> str: diff --git a/src/skillspector/nodes/analyzers/__init__.py b/src/skillspector/nodes/analyzers/__init__.py index bf7d76b9b..7538fd2f6 100644 --- a/src/skillspector/nodes/analyzers/__init__.py +++ b/src/skillspector/nodes/analyzers/__init__.py @@ -17,6 +17,7 @@ from __future__ import annotations +from skillspector.nodes.analyzers.artifact_integrity import node as artifact_integrity_node from skillspector.nodes.analyzers.behavioral_ast import node as behavioral_ast_node from skillspector.nodes.analyzers.behavioral_taint_tracking import ( node as behavioral_taint_tracking_node, @@ -81,6 +82,7 @@ from skillspector.nodes.analyzers.static_yara import node as static_yara_node ANALYZER_NODE_IDS: list[str] = [ + "artifact_integrity", "static_patterns_prompt_injection", "static_patterns_data_exfiltration", "static_patterns_privilege_escalation", @@ -108,6 +110,7 @@ ] ANALYZER_NODES = { + "artifact_integrity": artifact_integrity_node, "static_patterns_prompt_injection": static_patterns_prompt_injection_node, "static_patterns_data_exfiltration": static_patterns_data_exfiltration_node, "static_patterns_privilege_escalation": static_patterns_privilege_escalation_node, diff --git a/src/skillspector/nodes/analyzers/artifact_integrity.py b/src/skillspector/nodes/analyzers/artifact_integrity.py new file mode 100644 index 000000000..dea224d54 --- /dev/null +++ b/src/skillspector/nodes/analyzers/artifact_integrity.py @@ -0,0 +1,124 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Artifact-level evasion signals derived from canonical byte classification.""" + +from __future__ import annotations + +from skillspector.artifacts import has_mixed_script_token, unicode_anomaly_density +from skillspector.inspection_ledger import ( + InspectionLedgerEvent, + LedgerOutcome, + analyzer_status_for_events, + ledger_event, +) +from skillspector.models import Finding +from skillspector.python_ast import MAX_PYTHON_AST_SOURCE_CHARS +from skillspector.state import AnalyzerNodeResponse, SkillspectorState + +ANALYZER_ID = "artifact_integrity" +_INSTRUCTION_SUFFIXES = ( + ".md", + ".markdown", + ".txt", +) + + +def _finding( + rule_id: str, + message: str, + path: str, + *, + severity: str, + confidence: float, + line: int = 1, +) -> Finding: + return Finding( + rule_id=rule_id, + message=message, + severity=severity, + confidence=confidence, + file=path, + start_line=line, + category="analysis-evasion", + tags=["artifact-integrity"], + ) + + +def node(state: SkillspectorState) -> AnalyzerNodeResponse: + """Emit classification, Unicode, and analysis-ceiling evasion findings.""" + file_cache = state.get("file_cache") or {} + inventory = { + str(item.get("path", "")): item + for item in state.get("artifact_inventory") or [] + if isinstance(item, dict) + } + findings: list[Finding] = [] + events: list[InspectionLedgerEvent] = [] + for path in state.get("components") or []: + artifact: dict[str, object] = inventory.get(path, {}) + path_findings: list[Finding] = [] + if artifact.get("misleading_extension"): + path_findings.append( + _finding( + "AE2", + "Artifact content does not match its filename extension", + path, + severity="MEDIUM", + confidence=0.9, + ) + ) + content = file_cache.get(path) + if content is not None: + if artifact.get("contains_nul"): + nul_offset = content.find("\x00") + path_findings.append( + _finding( + "AE3", + "Text artifact contains embedded NUL bytes", + path, + severity="HIGH", + confidence=0.9, + line=content[:nul_offset].count("\n") + 1, + ) + ) + format_density = unicode_anomaly_density(content) + if format_density >= 0.01 or has_mixed_script_token(content): + path_findings.append( + _finding( + "AE4", + "Suspicious Unicode normalization or mixed-script content", + path, + severity="MEDIUM", + confidence=0.8, + ) + ) + normalized_path = path.lower() + if len(content) > MAX_PYTHON_AST_SOURCE_CHARS and ( + normalized_path.endswith(_INSTRUCTION_SUFFIXES) + or normalized_path.endswith("skill.md") + ): + path_findings.append( + _finding( + "AE5", + "Instruction-capable artifact exceeds whole-file semantic analysis limits", + path, + severity="HIGH", + confidence=1.0, + ) + ) + findings.extend(path_findings) + events.append( + ledger_event( + analyzer_id=ANALYZER_ID, + outcome=LedgerOutcome.COMPLETED, + phase="artifact", + path=path, + emitted_finding_ids=[finding.finding_id for finding in path_findings], + ) + ) + return { + "findings": findings, + "inspection_ledger": events, + "analyzer_status_events": [analyzer_status_for_events(ANALYZER_ID, events)], + } diff --git a/src/skillspector/nodes/analyzers/behavioral_ast.py b/src/skillspector/nodes/analyzers/behavioral_ast.py index ca165dade..d3596e878 100644 --- a/src/skillspector/nodes/analyzers/behavioral_ast.py +++ b/src/skillspector/nodes/analyzers/behavioral_ast.py @@ -351,7 +351,7 @@ def node(state: SkillspectorState) -> AnalyzerNodeResponse: ) elif len(content) > MAX_FILE_CHARS: event = ledger_event( - outcome=LedgerOutcome.SKIPPED, + outcome=LedgerOutcome.PARTIAL, phase="behavioral", analyzer_id=ANALYZER_ID, path=path, @@ -407,7 +407,7 @@ def node(state: SkillspectorState) -> AnalyzerNodeResponse: "failed" if LedgerOutcome.FAILED in outcomes else "degraded" - if LedgerOutcome.SKIPPED in outcomes + if LedgerOutcome.SKIPPED in outcomes or LedgerOutcome.PARTIAL in outcomes else "completed" ), planned_work=planned_work, diff --git a/src/skillspector/nodes/analyzers/behavioral_taint_tracking.py b/src/skillspector/nodes/analyzers/behavioral_taint_tracking.py index 73f360914..45590d8a2 100644 --- a/src/skillspector/nodes/analyzers/behavioral_taint_tracking.py +++ b/src/skillspector/nodes/analyzers/behavioral_taint_tracking.py @@ -480,7 +480,7 @@ def node(state: SkillspectorState) -> AnalyzerNodeResponse: ) elif len(content) > MAX_FILE_CHARS: event = ledger_event( - outcome=LedgerOutcome.SKIPPED, + outcome=LedgerOutcome.PARTIAL, phase="behavioral", analyzer_id=ANALYZER_ID, path=path, @@ -536,7 +536,7 @@ def node(state: SkillspectorState) -> AnalyzerNodeResponse: "failed" if LedgerOutcome.FAILED in outcomes else "degraded" - if LedgerOutcome.SKIPPED in outcomes + if LedgerOutcome.SKIPPED in outcomes or LedgerOutcome.PARTIAL in outcomes else "completed" ), planned_work=planned_work, diff --git a/src/skillspector/nodes/analyzers/mcp_tool_poisoning.py b/src/skillspector/nodes/analyzers/mcp_tool_poisoning.py index 72593ba0c..95998e112 100644 --- a/src/skillspector/nodes/analyzers/mcp_tool_poisoning.py +++ b/src/skillspector/nodes/analyzers/mcp_tool_poisoning.py @@ -344,9 +344,9 @@ def _check_p9_padding(text: str, source_field: str) -> list[Finding]: findings: list[Finding] = [] for run in detect_whitespace_padding(text): - if run.kind not in ("horizontal", "vertical", "block"): + if run.kind not in ("horizontal", "vertical", "block", "repetition"): continue - if run.kind in ("horizontal", "vertical"): + if run.kind in ("horizontal", "vertical", "repetition"): severity = "MEDIUM" confidence = 0.7 else: # "block" @@ -828,7 +828,10 @@ def _check_tp4( permissions = manifest.get("permissions") # Collect executable code from file_cache filtered by component_metadata types - file_cache: dict[str, str] = state.get("file_cache") or {} + llm_cache = state.get("llm_file_cache") + file_cache: dict[str, str] = ( + llm_cache if isinstance(llm_cache, dict) else state.get("file_cache") or {} + ) component_metadata: list[dict] = state.get("component_metadata") or [] executable_type_by_path = { diff --git a/src/skillspector/nodes/analyzers/semantic_developer_intent.py b/src/skillspector/nodes/analyzers/semantic_developer_intent.py index e67e03e48..a08dc6d12 100644 --- a/src/skillspector/nodes/analyzers/semantic_developer_intent.py +++ b/src/skillspector/nodes/analyzers/semantic_developer_intent.py @@ -174,7 +174,10 @@ def node(state: SkillspectorState) -> AnalyzerNodeResponse: ], } - file_cache: dict[str, str] = state.get("file_cache") or {} + llm_cache = state.get("llm_file_cache") + file_cache: dict[str, str] = ( + llm_cache if isinstance(llm_cache, dict) else state.get("file_cache") or {} + ) if not file_cache: return { "findings": [], diff --git a/src/skillspector/nodes/analyzers/semantic_quality_policy.py b/src/skillspector/nodes/analyzers/semantic_quality_policy.py index 2778da524..b42ef9508 100644 --- a/src/skillspector/nodes/analyzers/semantic_quality_policy.py +++ b/src/skillspector/nodes/analyzers/semantic_quality_policy.py @@ -147,7 +147,10 @@ def node(state: SkillspectorState) -> AnalyzerNodeResponse: ], } - file_cache: dict[str, str] = state.get("file_cache") or {} + llm_cache = state.get("llm_file_cache") + file_cache: dict[str, str] = ( + llm_cache if isinstance(llm_cache, dict) else state.get("file_cache") or {} + ) files = sorted(file_cache.keys()) if not files: return { diff --git a/src/skillspector/nodes/analyzers/semantic_security_discovery.py b/src/skillspector/nodes/analyzers/semantic_security_discovery.py index 09bf2b2ae..4c35a2275 100644 --- a/src/skillspector/nodes/analyzers/semantic_security_discovery.py +++ b/src/skillspector/nodes/analyzers/semantic_security_discovery.py @@ -96,8 +96,15 @@ def node(state: SkillspectorState) -> AnalyzerNodeResponse: ], } - file_cache: dict[str, str] = state.get("file_cache") or {} - components: list[str] = state.get("components") or sorted(file_cache.keys()) + llm_cache = state.get("llm_file_cache") + file_cache: dict[str, str] = ( + llm_cache if isinstance(llm_cache, dict) else state.get("file_cache") or {} + ) + components: list[str] = ( + sorted(file_cache) + if isinstance(llm_cache, dict) + else state.get("components") or sorted(file_cache) + ) if not components: return { "findings": [], diff --git a/src/skillspector/nodes/analyzers/static_patterns_anti_refusal.py b/src/skillspector/nodes/analyzers/static_patterns_anti_refusal.py index ba8cf9ea1..d45aaaa5d 100644 --- a/src/skillspector/nodes/analyzers/static_patterns_anti_refusal.py +++ b/src/skillspector/nodes/analyzers/static_patterns_anti_refusal.py @@ -123,10 +123,6 @@ _RULES = [("AR1", AR1_PATTERNS), ("AR2", AR2_PATTERNS), ("AR3", AR3_PATTERNS)] -# Confidence penalty applied when the match appears inside a code/doc example, and the -# minimum confidence required to emit a finding after the penalty. -_EXAMPLE_PENALTY = 0.4 -_MIN_CONFIDENCE = 0.5 _MODE_ENABLED_RE = re.compile( r"\b(?:developer|debug|god|sudo|jailbreak)\s+mode\s+(?:enabled|on|activated|engaged)\b", re.IGNORECASE, @@ -410,36 +406,27 @@ def analyze(content: str, file_path: str, file_type: str) -> list[AnalyzerFindin match_line = lines[line_num - 1] if lines else content previous_line = lines[line_num - 2] if line_num > 1 else None context = get_context(content, match.start(), context_lines=3) - if _MODE_ENABLED_RE.fullmatch(match.group(0)) and ( - _SECURITY_REVIEW_CONTEXT_RE.search(context) - ): - continue + security_review_context = bool( + _MODE_ENABLED_RE.fullmatch(match.group(0)) + and _SECURITY_REVIEW_CONTEXT_RE.search(context) + ) line_start = content.rfind("\n", 0, match.start()) + 1 line_match_start = match.start() - line_start line_match_end = line_match_start + len(match.group(0)) match_clause, _, _ = _match_clause(match_line, line_match_start, line_match_end) is_directive = _is_directly_instructive(match_clause.lower(), match.group(0)) - confidence = base_confidence - if ( - is_code_example(context) - and _is_explicit_example_context(context) - and not _is_quoted_match( - match_line, - match.group(0), - ) - ): - confidence -= _EXAMPLE_PENALTY - if _is_benign_ar_context( + example_context = is_code_example(context) and _is_explicit_example_context(context) + benign_context = _is_benign_ar_context( match_line, match.group(0), line_match_start, line_match_end, previous_line=previous_line, - ): - continue - if confidence < _MIN_CONFIDENCE: - continue + ) + finding_tags = list(tag) + if security_review_context or example_context or benign_context: + finding_tags.extend(["contextual-triage", "likely-benign-context"]) findings.append( AnalyzerFinding( rule_id=rule_id, @@ -449,8 +436,8 @@ def analyze(content: str, file_path: str, file_type: str) -> list[AnalyzerFindin file=file_path, start_line=line_num, ), - confidence=round(confidence, 2), - tags=tag, + confidence=base_confidence, + tags=finding_tags, context=_emitted_context( context, match_line, diff --git a/src/skillspector/nodes/analyzers/static_patterns_data_exfiltration.py b/src/skillspector/nodes/analyzers/static_patterns_data_exfiltration.py index e49ff42ad..d87b1aa50 100644 --- a/src/skillspector/nodes/analyzers/static_patterns_data_exfiltration.py +++ b/src/skillspector/nodes/analyzers/static_patterns_data_exfiltration.py @@ -32,7 +32,6 @@ get_context, get_context_from_lines, get_line_number, - is_code_example, resolve_call_name, resolve_dotted_name, ) @@ -358,13 +357,11 @@ def ctx(start: int) -> str: matched_text=match.group(0)[:200], ) ) - # E5: cloud-storage exfiltration. Filtered through is_code_example() because - # upload calls commonly appear in SKILL.md docs and examples. + # E5: framing words such as "example" are attacker-controlled and cannot + # suppress deterministic exfiltration evidence. for pattern, confidence in E5_PATTERNS: for match in re.finditer(pattern, content, re.IGNORECASE | re.MULTILINE): context = ctx(match.start()) - if is_code_example(context): - continue line_num = get_line_number(content, match.start()) findings.append( AnalyzerFinding( diff --git a/src/skillspector/nodes/analyzers/static_patterns_excessive_agency.py b/src/skillspector/nodes/analyzers/static_patterns_excessive_agency.py index 04ba47f7a..7f94e5d63 100644 --- a/src/skillspector/nodes/analyzers/static_patterns_excessive_agency.py +++ b/src/skillspector/nodes/analyzers/static_patterns_excessive_agency.py @@ -33,7 +33,7 @@ from skillspector.state import AnalyzerNodeResponse, SkillspectorState from . import static_runner -from .common import get_context, get_line_number, is_code_example +from .common import get_context, get_line_number from .pattern_defaults import PatternCategory logger = get_logger(__name__) @@ -187,8 +187,6 @@ def ctx(start: int) -> str: for match in re.finditer(pattern, content, re.IGNORECASE | re.MULTILINE): line_num = get_line_number(content, match.start()) context_text = ctx(match.start()) - if is_code_example(context_text): - continue findings.append( AnalyzerFinding( rule_id="EA2", diff --git a/src/skillspector/nodes/analyzers/static_patterns_memory_poisoning.py b/src/skillspector/nodes/analyzers/static_patterns_memory_poisoning.py index 62dff83e0..f9fcaab8a 100644 --- a/src/skillspector/nodes/analyzers/static_patterns_memory_poisoning.py +++ b/src/skillspector/nodes/analyzers/static_patterns_memory_poisoning.py @@ -32,7 +32,7 @@ from skillspector.state import AnalyzerNodeResponse, SkillspectorState from . import static_runner -from .common import get_context, get_line_number, is_code_example +from .common import get_context, get_line_number from .pattern_defaults import PatternCategory logger = get_logger(__name__) @@ -232,8 +232,6 @@ def ctx(start: int) -> str: for match in re.finditer(pattern, content, re.IGNORECASE | re.MULTILINE): line_num = get_line_number(content, match.start()) context_text = ctx(match.start()) - if is_code_example(context_text): - continue findings.append( AnalyzerFinding( rule_id="MP3", diff --git a/src/skillspector/nodes/analyzers/static_patterns_privilege_escalation.py b/src/skillspector/nodes/analyzers/static_patterns_privilege_escalation.py index a37469fe2..f43975049 100644 --- a/src/skillspector/nodes/analyzers/static_patterns_privilege_escalation.py +++ b/src/skillspector/nodes/analyzers/static_patterns_privilege_escalation.py @@ -320,8 +320,9 @@ def loc(ln: int) -> Location: for match in re.finditer(pattern, content, re.IGNORECASE | re.MULTILINE): line_num = get_line_number(content, match.start()) context = get_context(content, match.start()) + finding_tags = list(tag) if _is_documentation_example(context, file_type): - continue + finding_tags.extend(["contextual-triage", "likely-benign-context"]) findings.append( AnalyzerFinding( rule_id="PE2", @@ -329,7 +330,7 @@ def loc(ln: int) -> Location: severity=Severity.MEDIUM, location=loc(line_num), confidence=confidence, - tags=tag, + tags=finding_tags, context=context, matched_text=match.group(0)[:200], ) @@ -338,14 +339,17 @@ def loc(ln: int) -> Location: for match in re.finditer(pattern, content, re.IGNORECASE | re.MULTILINE): line_num = get_line_number(content, match.start()) context = get_context(content, match.start()) - if _is_pe3_documentation_example(content, match, file_type, file_path): - continue - if _is_qualified_benign_access_requirement(content, match, file_type): - continue - if _is_read_only_passwd_volume_match(content, match): - continue - if _is_negated_safety_constraint(content, match): - continue + contextual = any( + ( + _is_pe3_documentation_example(content, match, file_type, file_path), + _is_qualified_benign_access_requirement(content, match, file_type), + _is_read_only_passwd_volume_match(content, match), + _is_negated_safety_constraint(content, match), + ) + ) + finding_tags = list(tag) + if contextual: + finding_tags.extend(["contextual-triage", "likely-benign-context"]) findings.append( AnalyzerFinding( rule_id="PE3", @@ -353,7 +357,7 @@ def loc(ln: int) -> Location: severity=Severity.HIGH, location=loc(line_num), confidence=confidence, - tags=tag, + tags=finding_tags, context=context, matched_text=match.group(0)[:200], ) @@ -365,8 +369,9 @@ def loc(ln: int) -> Location: for match in re.finditer(pattern, content, re.IGNORECASE | re.MULTILINE): line_num = get_line_number(content, match.start()) context = get_context(content, match.start()) + finding_tags = list(tag) if _is_documentation_example(context, file_type): - continue + finding_tags.extend(["contextual-triage", "likely-benign-context"]) if line_num in pe4_best and pe4_best[line_num].confidence >= confidence: continue pe4_best[line_num] = AnalyzerFinding( @@ -375,7 +380,7 @@ def loc(ln: int) -> Location: severity=Severity.HIGH, location=loc(line_num), confidence=confidence, - tags=tag, + tags=finding_tags, context=context, matched_text=match.group(0)[:200], ) @@ -387,8 +392,9 @@ def loc(ln: int) -> Location: for match in re.finditer(pattern, content, re.IGNORECASE | re.MULTILINE): line_num = get_line_number(content, match.start()) context = get_context(content, match.start()) + finding_tags = list(tag) if _is_documentation_example(context, file_type): - continue + finding_tags.extend(["contextual-triage", "likely-benign-context"]) if line_num in pe5_best and pe5_best[line_num].confidence >= confidence: continue pe5_best[line_num] = AnalyzerFinding( @@ -397,7 +403,7 @@ def loc(ln: int) -> Location: severity=Severity.HIGH, location=loc(line_num), confidence=confidence, - tags=tag, + tags=finding_tags, context=context, matched_text=match.group(0)[:200], ) diff --git a/src/skillspector/nodes/analyzers/static_patterns_prompt_injection.py b/src/skillspector/nodes/analyzers/static_patterns_prompt_injection.py index fced93188..e31b22541 100644 --- a/src/skillspector/nodes/analyzers/static_patterns_prompt_injection.py +++ b/src/skillspector/nodes/analyzers/static_patterns_prompt_injection.py @@ -334,6 +334,9 @@ def ctx(start: int) -> str: elif run.kind == "horizontal": confidence = 0.7 severity = Severity.MEDIUM + elif run.kind == "repetition": + confidence = 0.8 + severity = Severity.MEDIUM else: # "block" or "ratio" confidence = 0.4 severity = Severity.LOW diff --git a/src/skillspector/nodes/analyzers/static_patterns_tool_misuse.py b/src/skillspector/nodes/analyzers/static_patterns_tool_misuse.py index 8df9ae0b0..8a31d8da9 100644 --- a/src/skillspector/nodes/analyzers/static_patterns_tool_misuse.py +++ b/src/skillspector/nodes/analyzers/static_patterns_tool_misuse.py @@ -32,7 +32,7 @@ from skillspector.state import AnalyzerNodeResponse, SkillspectorState from . import static_runner -from .common import get_context, get_line_number, is_code_example +from .common import get_context, get_line_number from .pattern_defaults import PatternCategory logger = get_logger(__name__) @@ -86,7 +86,7 @@ # Dangerous tool parameter patterns in instructions ( r"(?:set|pass|use)\s+(?:the\s+)?(?:parameter|argument|flag|option)\s+(?:to\s+)?(?:shell\s*=\s*True|--force|-rf)\b", - 0.75, + 0.8, ), ] @@ -322,13 +322,10 @@ def ctx(start: int) -> str: matched_text=match.group(0)[:200], ) ) - # TM4: privileged K8s workload. Filtered through is_code_example() because - # privileged/hostPath fields commonly appear in SKILL.md docs and examples. + # TM4: documentation framing is not a suppression boundary. for pattern, confidence in TM4_PATTERNS: for match in re.finditer(pattern, content, re.IGNORECASE | re.MULTILINE): context_text = ctx(match.start()) - if is_code_example(context_text): - continue line_num = get_line_number(content, match.start()) findings.append( AnalyzerFinding( diff --git a/src/skillspector/nodes/analyzers/static_runner.py b/src/skillspector/nodes/analyzers/static_runner.py index 9b3ccffbc..2a5eed2ae 100644 --- a/src/skillspector/nodes/analyzers/static_runner.py +++ b/src/skillspector/nodes/analyzers/static_runner.py @@ -18,13 +18,16 @@ from __future__ import annotations import re +import time from collections.abc import Callable, Mapping from typing import cast +from skillspector.artifacts import ContentKind, SecurityTextView, security_text_views from skillspector.inspection_ledger import ( InspectionLedgerEvent, LedgerOutcome, LedgerReason, + LedgerRecordType, analyzer_status_for_events, ledger_event, ) @@ -37,7 +40,6 @@ ) from skillspector.state import AnalyzerNodeResponse -from .common import is_code_example from .pattern_defaults import get_category, get_explanation, get_pattern_name, get_remediation logger = get_logger(__name__) @@ -63,16 +65,10 @@ } MAX_FILE_CHARS = MAX_PYTHON_AST_SOURCE_CHARS -_EVAL_DATASET_FILES = { - "evals/evals.json", - "evals/evals.jsonl", - "evals/evals.yaml", - "evals/evals.yml", - "eval/dataset.json", - "eval/dataset.jsonl", - "eval/dataset.yaml", - "eval/dataset.yml", -} +SECURITY_VIEW_WINDOW_CHARS = 256_000 +_WINDOW_OVERLAP_CHARS = 8192 +MAX_FINDINGS_PER_ARTIFACT = 10_000 +MAX_STATIC_ANALYSIS_SECONDS_PER_ARTIFACT = 30.0 def _infer_file_type(path: str) -> str: @@ -82,57 +78,12 @@ def _infer_file_type(path: str) -> str: return FILE_TYPES.get(suffix, "other") -_BINARY_EXTENSIONS = frozenset( - { - ".pdf", - ".png", - ".jpg", - ".jpeg", - ".gif", - ".bmp", - ".ico", - ".woff", - ".woff2", - ".ttf", - ".otf", - ".eot", - ".zip", - ".tar", - ".gz", - ".bz2", - ".xz", - ".7z", - ".rar", - ".exe", - ".dll", - ".so", - ".dylib", - ".bin", - ".o", - ".a", - ".pyc", - ".pyo", - ".class", - ".wasm", - ".mp3", - ".mp4", - ".wav", - ".avi", - ".mov", - ".webm", - ".sqlite", - ".db", - } -) - _NULL_BYTE_SAMPLE_SIZE = 512 def _is_binary_file(path: str, content: str) -> bool: - """Detect binary files by extension or null-byte presence in the first 512 chars.""" - idx = path.rfind(".") - if idx >= 0 and path[idx:].lower() in _BINARY_EXTENSIONS: - return True + """Compatibility helper: extensions alone never classify an artifact as binary.""" + del path return "\x00" in content[:_NULL_BYTE_SAMPLE_SIZE] @@ -193,95 +144,6 @@ def _is_env_file_reference_in_docs( ) -def _is_eval_dataset(path: str) -> bool: - """Return True for authored eval datasets that contain test-case prose.""" - return path.replace("\\", "/") in _EVAL_DATASET_FILES - - -_DOCUMENTATION_DIR_NAMES = ( - "docs", - "documentation", - "procedures", - "references", - "examples", - "guides", -) - -_DOCUMENTATION_CONFIDENCE_FACTOR = 0.3 -_CODE_EXAMPLE_CONFIDENCE_FACTOR = 0.5 - -_NON_EXECUTABLE_FILE_TYPES = frozenset({"markdown", "text", "json", "yaml", "toml"}) -_DOC_PROSE_FILE_TYPES = frozenset({"markdown", "text"}) - -# PE3 is intentionally excluded: its analyzer and the exact .env setup grammar -# above own the narrowly reviewed safe cases. A generic prose classification -# must not hide credential-access instructions. -_SEMANTIC_STRING_DOC_PRONE_RULES = frozenset({"RA1", "TM1", "AR2"}) -_EXECUTION_SIGNAL = re.compile( - r"(?:\b\w+\s*=|\bos\.(?:environ|getenv|system)\b|\bshutil\.rmtree\b|\b(?:subprocess|eval|exec)\b|[|>]" - r"|\b(?:open|read_text|write_text)\s*\()", - re.IGNORECASE, -) - - -# Markdown syntax that collides with shell metacharacters. A table row is delimited by "|" and -# a quoted line begins with ">": neither is a pipe or a redirection, but _EXECUTION_SIGNAL reads -# them as one and the prose classification below is then skipped for the whole line. -# -# Only the *delimiters* are removed — the leading and trailing bar of a row and the quote marker. -# A bar inside a cell may well be a real pipe in a documented command, and it must keep counting -# as an execution signal. -_MD_TABLE_ROW = re.compile(r"^\s*\|.*\|\s*$") -_MD_BLOCKQUOTE = re.compile(r"^\s*>+\s?") -_MD_ESCAPED_BAR = "\\|" -_BAR_PLACEHOLDER = "\x00" - - -def _strip_markdown_structure(line: str) -> str: - r"""Drop markdown delimiters that would otherwise read as shell metacharacters. - - In a table row an unescaped ``|`` separates cells; a literal pipe inside a cell has to be - written ``\|`` (CommonMark). That distinction is what makes this safe: the delimiters are - removed, while a documented ``cmd \| tee log`` keeps its pipe and still counts as an - execution signal. - """ - if _MD_TABLE_ROW.match(line): - line = line.replace(_MD_ESCAPED_BAR, _BAR_PLACEHOLDER) - line = line.replace("|", " ") - line = line.replace(_BAR_PLACEHOLDER, "|") - return _MD_BLOCKQUOTE.sub("", line) - - -def _is_documentation_context(af: AnalyzerFinding, file_type: str, path: str, content: str) -> bool: - """Return true when a governed finding is prose or a comment without execution signals.""" - if af.rule_id not in _SEMANTIC_STRING_DOC_PRONE_RULES: - return False - if path.replace("\\", "/").lower().endswith("skill.md"): - return False - lines = content.splitlines() - matched_line = ( - lines[af.location.start_line - 1] - if 0 < af.location.start_line <= len(lines) - else af.context or "" - ) - if file_type in _DOC_PROSE_FILE_TYPES: - if _EXECUTION_SIGNAL.search(_strip_markdown_structure(matched_line)): - return False - return True - return bool(matched_line and matched_line.lstrip().startswith(("#", "//"))) - - -def _is_documentation_markdown(path: str) -> bool: - """Return True for markdown files in documentation subdirectories (not SKILL.md).""" - normalized = path.replace("\\", "/").lower() - if not normalized.endswith((".md", ".markdown")): - return False - if normalized.endswith("skill.md"): - return False - parts = normalized.split("/") - return any(part in _DOCUMENTATION_DIR_NAMES for part in parts[:-1]) - - def analyzer_finding_to_finding( af: AnalyzerFinding, get_remediation_fn: Callable[[str], str] | None = None, @@ -303,7 +165,7 @@ def analyzer_finding_to_finding( remediation=remediation, tags=list(af.tags), context=af.context, - matched_text=af.matched_text[:200] if af.matched_text else None, + matched_text=af.matched_text, category=category, pattern=pattern, finding=finding_snippet, @@ -327,8 +189,6 @@ def _scan_path( """Run pattern modules for one already-applicable file path.""" findings: list[Finding] = [] file_type = _infer_file_type(path) - is_doc_markdown = _is_documentation_markdown(path) - is_non_executable = file_type in _NON_EXECUTABLE_FILE_TYPES python_ast: ParsedPythonFile | None = None if file_type == "python" and any(_uses_python_ast(module) for module in pattern_modules): python_ast = get_python_ast(python_ast_cache_key, content, path) @@ -345,44 +205,124 @@ def _scan_path( raw = module.analyze(content=content, file_path=path, file_type=file_type) for af in raw: if _is_env_file_reference_in_docs(af, file_type, path, content): - logger.debug( - "Filtered PE3 .env doc reference: %s in %s:%d", - af.rule_id, - path, - af.location.start_line, + for triage_tag in ("contextual-triage", "likely-benign-context"): + if triage_tag not in af.tags: + af.tags.append(triage_tag) + findings.append(analyzer_finding_to_finding(af)) + return findings + + +def _deduplicate_view_findings(findings: list[Finding]) -> list[Finding]: + """Remove overlap/view duplicates using the complete match fingerprint.""" + result: list[Finding] = [] + seen: set[tuple[str, str, int, str | None]] = set() + for finding in findings: + key = (finding.rule_id, finding.file, finding.start_line, finding.fingerprint()) + if key in seen: + continue + seen.add(key) + result.append(finding) + return result + + +def _scan_view_windows( + path: str, + view: SecurityTextView, + pattern_modules: list, + python_ast_cache_key: str | None, +) -> list[Finding]: + """Scan one already-bounded view.""" + findings = _scan_path(path, view.text, pattern_modules, python_ast_cache_key) + if view.name != "raw": + for finding in findings: + if "normalized-view" not in finding.tags: + finding.tags.append("normalized-view") + return findings + + +def _line_start_offset(text: str, line_number: int) -> int: + """Return the local character offset for a 1-based line number.""" + if line_number <= 1: + return 0 + offset = 0 + for _ in range(line_number - 1): + newline = text.find("\n", offset) + if newline < 0: + return len(text) + offset = newline + 1 + return offset + + +def _restore_source_lines( + findings: list[Finding], + *, + raw_window: str, + window_line: int, + view: SecurityTextView, +) -> None: + """Map normalized/window-relative locations to raw whole-file lines.""" + for finding in findings: + derived_start = _line_start_offset(view.text, finding.start_line) + raw_start = view.source_offset(derived_start) + finding.start_line = window_line + raw_window.count("\n", 0, raw_start) + if finding.end_line is not None: + derived_end = _line_start_offset(view.text, finding.end_line) + raw_end = view.source_offset(derived_end) + finding.end_line = window_line + raw_window.count("\n", 0, raw_end) + + +def _scan_all_views_detailed( + path: str, + content: str, + pattern_modules: list, + python_ast_cache_key: str | None, +) -> tuple[list[Finding], LedgerReason | None]: + """Scan bounded raw windows and derived views, returning any limit hit.""" + ast_modules = [module for module in pattern_modules if _uses_python_ast(module)] + lexical_modules = [module for module in pattern_modules if not _uses_python_ast(module)] + findings: list[Finding] = [] + deadline = time.monotonic() + MAX_STATIC_ANALYSIS_SECONDS_PER_ARTIFACT + + if ast_modules and len(content) <= MAX_FILE_CHARS: + findings.extend(_scan_path(path, content, ast_modules, python_ast_cache_key)) + + modules_for_windows = lexical_modules or ([] if ast_modules else pattern_modules) + if modules_for_windows: + step = SECURITY_VIEW_WINDOW_CHARS - _WINDOW_OVERLAP_CHARS + window_line = 1 + for start in range(0, max(1, len(content)), step): + if time.monotonic() > deadline: + return _deduplicate_view_findings(findings), LedgerReason.RUNTIME_LIMIT + end = min(len(content), start + SECURITY_VIEW_WINDOW_CHARS) + raw_window = content[start:end] + for view in security_text_views(raw_window): + view_findings = _scan_view_windows(path, view, modules_for_windows, None) + _restore_source_lines( + view_findings, + raw_window=raw_window, + window_line=window_line, + view=view, ) - continue - # PE3's analyzer owns its narrowly qualified safe references. - # Generic documentation words are attacker-controlled and must - # not hard-drop HIGH credential-access findings here. - if af.rule_id != "PE3" and af.context and is_code_example(af.context): - if is_non_executable: - logger.debug( - "Filtered code-example finding in non-executable: %s in %s:%d", - af.rule_id, - path, - af.location.start_line, + findings.extend(view_findings) + if len(findings) > MAX_FINDINGS_PER_ARTIFACT: + return ( + _deduplicate_view_findings(findings)[:MAX_FINDINGS_PER_ARTIFACT], + LedgerReason.OUTPUT_LIMIT, ) - continue - af.confidence *= _CODE_EXAMPLE_CONFIDENCE_FACTOR - logger.debug( - "Downweighted code-example finding in executable: %s in %s:%d (conf=%.2f)", - af.rule_id, - path, - af.location.start_line, - af.confidence, - ) - if _is_documentation_context(af, file_type, path, content): - logger.debug( - "Filtered documentation-context finding: %s in %s:%d", - af.rule_id, - path, - af.location.start_line, - ) - continue - if is_doc_markdown: - af.confidence *= _DOCUMENTATION_CONFIDENCE_FACTOR - findings.append(analyzer_finding_to_finding(af)) + if end == len(content): + break + window_line += content.count("\n", start, min(len(content), start + step)) + + return _deduplicate_view_findings(findings), None + + +def _scan_all_views( + path: str, + content: str, + pattern_modules: list, + python_ast_cache_key: str | None, +) -> list[Finding]: + findings, _ = _scan_all_views_detailed(path, content, pattern_modules, python_ast_cache_key) return findings @@ -400,28 +340,26 @@ def run_static_patterns( components = cast(list[str], state.get("components") or []) file_cache = cast(dict[str, str], state.get("file_cache") or {}) python_ast_cache_key = cast(str | None, state.get("python_ast_cache_key")) + raw_inventory = state.get("artifact_inventory", []) + binary_paths = ( + { + str(item.get("path", "")) + for item in raw_inventory + if isinstance(item, dict) and item.get("content_kind") == ContentKind.BINARY + } + if isinstance(raw_inventory, list) + else set() + ) findings: list[Finding] = [] for path in components: - if _is_eval_dataset(path): - logger.debug("Skipping eval dataset prose for static pattern scan: %s", path) - continue content = file_cache.get(path) if content is None: logger.debug("Skipping %s: no content in file_cache", path) continue - if len(content) > MAX_FILE_CHARS: - logger.debug( - "Skipping %s: size %d characters exceeds MAX_FILE_CHARS (%d)", - path, - len(content), - MAX_FILE_CHARS, - ) - continue - if _is_binary_file(path, content): - logger.debug("Skipping binary file: %s", path) + if path in binary_paths or (not binary_paths and _is_binary_file(path, content)): continue - findings.extend(_scan_path(path, content, pattern_modules, python_ast_cache_key)) + findings.extend(_scan_all_views(path, content, pattern_modules, python_ast_cache_key)) return findings @@ -437,15 +375,26 @@ def run_static_patterns_with_ledger( python_ast_cache_key = cast(str | None, state.get("python_ast_cache_key")) findings: list[Finding] = [] events: list[InspectionLedgerEvent] = [] + raw_inventory = state.get("artifact_inventory", []) + inventory: dict[str, dict[str, object]] = ( + {str(item.get("path", "")): item for item in raw_inventory if isinstance(item, dict)} + if isinstance(raw_inventory, list) + else {} + ) for path in components: - if _is_eval_dataset(path): + artifact = inventory.get(path, {}) + if artifact.get("content_kind") == ContentKind.BINARY: + referenced = bool(artifact.get("referenced")) event = ledger_event( - outcome=LedgerOutcome.SKIPPED, + outcome=LedgerOutcome.PARTIAL if referenced else LedgerOutcome.OUT_OF_SCOPE, + record_type=( + LedgerRecordType.WORK_ITEM if referenced else LedgerRecordType.SCOPE_BOUNDARY + ), phase="static", analyzer_id=analyzer_id, path=path, - reason=LedgerReason.EVAL_DATASET, + reason=(LedgerReason.OPAQUE_CONTENT if referenced else LedgerReason.BINARY_CONTENT), ) else: content = file_cache.get(path) @@ -457,28 +406,11 @@ def run_static_patterns_with_ledger( path=path, reason=LedgerReason.MISSING_FILE_CACHE, ) - elif len(content) > MAX_FILE_CHARS: - event = ledger_event( - outcome=LedgerOutcome.SKIPPED, - phase="static", - analyzer_id=analyzer_id, - path=path, - reason=LedgerReason.SIZE_LIMIT, - observed_characters=len(content), - limit_characters=MAX_FILE_CHARS, - observed_bytes=len(content.encode("utf-8")), - ) - elif _is_binary_file(path, content): - event = ledger_event( - outcome=LedgerOutcome.SKIPPED, - phase="static", - analyzer_id=analyzer_id, - path=path, - reason=LedgerReason.BINARY_CONTENT, - ) else: try: - path_findings = _scan_path(path, content, pattern_modules, python_ast_cache_key) + path_findings, resource_limit = _scan_all_views_detailed( + path, content, pattern_modules, python_ast_cache_key + ) except Exception as exc: logger.warning("%s: scan error on %s: %s", analyzer_id, path, exc) event = ledger_event( @@ -491,12 +423,40 @@ def run_static_patterns_with_ledger( ) else: findings.extend(path_findings) + partial = resource_limit is not None or ( + _infer_file_type(path) == "python" + and len(content) > MAX_FILE_CHARS + and any(_uses_python_ast(module) for module in pattern_modules) + ) + partial_reason = resource_limit or LedgerReason.SIZE_LIMIT event = ledger_event( - outcome=LedgerOutcome.COMPLETED, + outcome=LedgerOutcome.PARTIAL if partial else LedgerOutcome.COMPLETED, phase="static", analyzer_id=analyzer_id, path=path, + reason=partial_reason if partial else None, emitted_finding_ids=[finding.finding_id for finding in path_findings], + observed_characters=( + len(content) if partial_reason is LedgerReason.SIZE_LIMIT else None + ), + limit_characters=( + MAX_FILE_CHARS if partial_reason is LedgerReason.SIZE_LIMIT else None + ), + observed_findings=( + len(path_findings) + if partial_reason is LedgerReason.OUTPUT_LIMIT + else None + ), + limit_findings=( + MAX_FINDINGS_PER_ARTIFACT + if partial_reason is LedgerReason.OUTPUT_LIMIT + else None + ), + limit_seconds=( + MAX_STATIC_ANALYSIS_SECONDS_PER_ARTIFACT + if partial_reason is LedgerReason.RUNTIME_LIMIT + else None + ), ) events.append(event) diff --git a/src/skillspector/nodes/analyzers/static_yara.py b/src/skillspector/nodes/analyzers/static_yara.py index 23cf8e485..6ed99ced8 100644 --- a/src/skillspector/nodes/analyzers/static_yara.py +++ b/src/skillspector/nodes/analyzers/static_yara.py @@ -42,7 +42,7 @@ from .common import get_context_from_lines from .pattern_defaults import PatternCategory -from .static_runner import MAX_FILE_CHARS, analyzer_finding_to_finding +from .static_runner import analyzer_finding_to_finding ANALYZER_ID = "static_yara" logger = get_logger(__name__) @@ -277,9 +277,15 @@ def _build_message(rule_name: str, namespace: str, description: str | None) -> s return msg -def _match_file(rules: yara.Rules, content: str, file_path: str) -> list[AnalyzerFinding]: - """Run compiled YARA rules against *content* and return AnalyzerFindings.""" - data = content.encode("utf-8", errors="replace") +def _match_file( + rules: yara.Rules, data: bytes | str, file_path: str, content: str | None = None +) -> list[AnalyzerFinding]: + """Run compiled YARA rules against canonical raw bytes.""" + if isinstance(data, str): + content = data if content is None else content + data = data.encode("utf-8", errors="replace") + if content is None: + content = data.decode("utf-8", errors="replace") matches = rules.match(data=data) findings: list[AnalyzerFinding] = [] @@ -307,7 +313,7 @@ def _match_file(rules: yara.Rules, content: str, file_path: str) -> list[Analyze location=Location(file=file_path, start_line=start_line), confidence=confidence, tags=[PatternCategory.YARA_MATCH.value], - context=get_context_from_lines(content.splitlines(), start_line), + context=get_context_from_lines(content.splitlines(), start_line)[:1000], matched_text=matched_text, ) ) @@ -336,12 +342,16 @@ def node(state: SkillspectorState) -> AnalyzerNodeResponse: components: list[str] = state.get("components") or [] file_cache: dict[str, str] = state.get("file_cache") or {} + raw_file_cache: dict[str, bytes] = state.get("raw_file_cache") or {} findings = [] events: list[InspectionLedgerEvent] = [] for path in components: content = file_cache.get(path) - if content is None: + data = raw_file_cache.get(path) + if data is None and content is not None: + data = content.encode("utf-8", errors="replace") + if data is None: events.append( ledger_event( analyzer_id=ANALYZER_ID, @@ -352,29 +362,9 @@ def node(state: SkillspectorState) -> AnalyzerNodeResponse: ) ) continue - if len(content) > MAX_FILE_CHARS: - logger.debug( - "%s: skipping %s (exceeds %d-character limit)", - ANALYZER_ID, - path, - MAX_FILE_CHARS, - ) - events.append( - ledger_event( - analyzer_id=ANALYZER_ID, - outcome=LedgerOutcome.SKIPPED, - phase="static", - path=path, - reason=LedgerReason.SIZE_LIMIT, - observed_characters=len(content), - limit_characters=MAX_FILE_CHARS, - observed_bytes=len(content.encode("utf-8")), - ) - ) - continue try: path_findings = [ - analyzer_finding_to_finding(af) for af in _match_file(rules, content, path) + analyzer_finding_to_finding(af) for af in _match_file(rules, data, path, content) ] except Exception as exc: logger.warning("%s: match error on %s: %s", ANALYZER_ID, path, exc) diff --git a/src/skillspector/nodes/analyzers/whitespace_padding.py b/src/skillspector/nodes/analyzers/whitespace_padding.py index 7d82707c9..1fb685cf6 100644 --- a/src/skillspector/nodes/analyzers/whitespace_padding.py +++ b/src/skillspector/nodes/analyzers/whitespace_padding.py @@ -60,6 +60,8 @@ BLOCK_BYTE_BUDGET = 2048 RATIO_THRESHOLD = 0.90 RATIO_MIN_FILE_BYTES = 4096 +REPEATED_CHAR_THRESHOLD = 512 +REPEATED_LINE_THRESHOLD = 64 # Replacement character emitted by errors="replace" decoding; a high *density* of # it marks binary-ish content, which we bail out of entirely. We key on density @@ -161,7 +163,7 @@ class PaddingRun: set by the detectors that produce span-based runs. """ - kind: str # "vertical" | "horizontal" | "block" | "ratio" + kind: str # "vertical" | "horizontal" | "block" | "ratio" | "repetition" start_offset: int # char offset where the run starts start_line: int # 1-based line number length: int # see class docstring — unit depends on kind @@ -367,6 +369,50 @@ def _detect_block_and_ratio(content: str) -> list[PaddingRun]: return runs +def _detect_repetition(content: str) -> list[PaddingRun]: + """Detect non-whitespace character and line repetition used as visual padding.""" + runs: list[PaddingRun] = [] + index = 0 + while index < len(content): + end = index + 1 + while end < len(content) and content[end] == content[index]: + end += 1 + if end - index >= REPEATED_CHAR_THRESHOLD and not is_padding_char(content[index]): + runs.append( + PaddingRun( + kind="repetition", + start_offset=index, + start_line=content[:index].count("\n") + 1, + length=end - index, + followed_by_content=end < len(content), + summary=f"repeated U+{ord(content[index]):04X} x{end - index}", + end_offset=end, + ) + ) + index = end + + lines, offsets = _split_lines(content) + index = 0 + while index < len(lines): + end = index + 1 + while end < len(lines) and lines[end] == lines[index] and lines[index].strip(): + end += 1 + if end - index >= REPEATED_LINE_THRESHOLD: + runs.append( + PaddingRun( + kind="repetition", + start_offset=offsets[index], + start_line=index + 1, + length=end - index, + followed_by_content=end < len(lines), + summary=f"repeated line x{end - index}", + end_offset=offsets[end], + ) + ) + index = end + return runs + + def detect_whitespace_padding(content: str, *, file_type: str = "other") -> list[PaddingRun]: """Scan *content* for whitespace-padding runs and return structured records. @@ -431,4 +477,5 @@ def _overlaps_primary(run: PaddingRun) -> bool: block_kept = True deduped_block_ratio.append(run) - return vertical + horizontal + deduped_block_ratio + repetition = [run for run in _detect_repetition(content) if not _overlaps_primary(run)] + return vertical + horizontal + deduped_block_ratio + repetition diff --git a/src/skillspector/nodes/build_context.py b/src/skillspector/nodes/build_context.py index 0caa441f5..fb0409982 100644 --- a/src/skillspector/nodes/build_context.py +++ b/src/skillspector/nodes/build_context.py @@ -31,7 +31,13 @@ import yaml -from skillspector.constants import MAX_FILE_BYTES, build_model_config +from skillspector.artifacts import ( + ArtifactDisposition, + ArtifactRecord, + classify_artifact, + decode_text, +) +from skillspector.constants import MAX_ANALYZABLE_FILE_BYTES, MAX_FILE_BYTES, build_model_config from skillspector.input_handler import ( _FileOpenError, _open_regular_file_no_follow, @@ -47,14 +53,20 @@ ) from skillspector.logging_config import get_logger from skillspector.python_ast import prewarm_python_ast_cache +from skillspector.references import ( + MAX_ACCEPTED_REFERENCES, + MAX_RAW_REFERENCE_CANDIDATES, + MAX_REFERENCE_RECORDS, + MAX_REFERENCE_RUNTIME_SECONDS, + MAX_REFERENCE_SOURCE_BYTES, + resolve_bundle_references_with_metadata, +) from skillspector.state import SkillspectorState logger = get_logger(__name__) # Directories to skip when walking -_SKIP_DIRS = frozenset( - {".git", "__pycache__", "node_modules", ".venv", "venv", ".tox", ".pytest_cache"} -) +_SKIP_DIRS = frozenset({"__pycache__", "node_modules", ".venv", "venv", ".tox", ".pytest_cache"}) # File type by extension _FILE_TYPES: dict[str, str] = { @@ -152,10 +164,17 @@ def _resolves_outside(path: Path, root: Path) -> bool: return False -def _read_text_no_follow(path: Path) -> str: +def _read_text_no_follow(path: Path, *, max_bytes: int | None = None) -> str: """Read a regular file without following symlinks at open time.""" with _open_regular_file_no_follow(path) as source: - return source.read().decode("utf-8", errors="replace") + data = source.read() if max_bytes is None else source.read(max_bytes + 1) + return data.decode("utf-8", errors="replace") + + +def _read_bytes_no_follow(path: Path, *, max_bytes: int | None = None) -> bytes: + """Read a regular file as canonical bytes without following symlinks.""" + with _open_regular_file_no_follow(path) as source: + return source.read() if max_bytes is None else source.read(max_bytes) def _walk_skill_files( @@ -163,8 +182,9 @@ def _walk_skill_files( ) -> tuple[list[str], list[InspectionLedgerEvent]]: """Walk skill files and record scan-scope exclusions. - Skips _SKIP_DIRS, hidden files except those starting with .claude, and - symlinks, which must never supply content to remote LLM analyzers. + Skips profile-permitted generated trees and symlinks. Hidden artifacts are + inventoried normally. Within ``.git`` only configuration and hooks are + inspected; object/history storage remains a bounded scope boundary. """ paths: list[str] = [] exclusions: list[InspectionLedgerEvent] = [] @@ -175,6 +195,24 @@ def _walk_skill_files( filenames.sort() relative_root = root_path.relative_to(skill_dir) + normalized_root = relative_root.as_posix() + if normalized_root == ".git": + vcs_skipped = [name for name in dirnames if name != "hooks"] + dirnames[:] = [name for name in dirnames if name == "hooks"] + for dirname in vcs_skipped: + exclusions.append( + ledger_event( + outcome=LedgerOutcome.OUT_OF_SCOPE, + record_type=LedgerRecordType.SCOPE_BOUNDARY, + phase="discovery", + path=f".git/{dirname}/", + reason=LedgerReason.VCS_METADATA, + ) + ) + filenames[:] = [name for name in filenames if name == "config"] + elif normalized_root == ".git/hooks": + dirnames[:] = [] + skipped_dirnames = [name for name in dirnames if name in _SKIP_DIRS] symlinked_dirnames = [name for name in dirnames if _is_symlink(root_path / name)] dirnames[:] = [ @@ -205,18 +243,6 @@ def _walk_skill_files( for filename in filenames: relative_path = (relative_root / filename).as_posix() - if filename.startswith(".") and not filename.startswith(".claude"): - exclusions.append( - ledger_event( - outcome=LedgerOutcome.OUT_OF_SCOPE, - record_type=LedgerRecordType.SCOPE_BOUNDARY, - phase="discovery", - path=relative_path, - reason=LedgerReason.HIDDEN_FILE, - ) - ) - continue - # Use forward slashes on every OS: these relative paths are dict keys # and SARIF/URI locations, so they must be portable. Other # non-regular entries remain inventoried for cache-phase evidence; @@ -359,11 +385,42 @@ def _build_component_metadata( return metadata, has_executable +def _redact_for_external_model(path: str, content: str) -> str: + """Redact values from local environment files before external-model use.""" + name = Path(path).name.lower() + if name != ".env" and not name.startswith(".env."): + return content + lines: list[str] = [] + for line in content.splitlines(keepends=True): + match = re.match(r"^(\s*(?:export\s+)?[A-Za-z_][A-Za-z0-9_]*\s*=)(.*?)(\r?\n)?$", line) + if match: + lines.append(f"{match.group(1)}{match.group(3) or ''}") + else: + lines.append(line) + return "".join(lines) + + +def _is_hidden_path(path: str) -> bool: + """Return whether any bundle path segment is hidden.""" + return any(part.startswith(".") for part in Path(path).parts) + + def _read_file_cache( - skill_dir: Path, components: list[str] -) -> tuple[dict[str, str], list[InspectionLedgerEvent]]: - """Build readable file content and terminal events for cache failures.""" + skill_dir: Path, + components: list[str], + referenced_paths: frozenset[str] = frozenset(), +) -> tuple[ + dict[str, str], + dict[str, bytes], + dict[str, str], + list[ArtifactRecord], + list[InspectionLedgerEvent], +]: + """Build canonical byte/text caches, inventory rows, and cache-failure events.""" file_cache: dict[str, str] = {} + raw_file_cache: dict[str, bytes] = {} + llm_file_cache: dict[str, str] = {} + inventory: list[ArtifactRecord] = [] ledger_events: list[InspectionLedgerEvent] = [] skill_root = skill_dir.resolve(strict=False) for path in components: @@ -417,8 +474,38 @@ def _read_file_cache( ) continue try: - content = _read_text_no_follow(full) + # Always bound the post-stat read as well. A file can grow between + # stat and open, and a stale size must not turn this into an + # unbounded allocation. + observed = _read_bytes_no_follow(full, max_bytes=MAX_ANALYZABLE_FILE_BYTES + 1) + truncated = ( + file_stat.st_size > MAX_ANALYZABLE_FILE_BYTES + or len(observed) > MAX_ANALYZABLE_FILE_BYTES + ) + raw = observed[:MAX_ANALYZABLE_FILE_BYTES] + content = decode_text(raw) + raw_file_cache[path] = raw file_cache[path] = content + artifact = classify_artifact(path, raw, referenced=path in referenced_paths) + if truncated: + observed_size = max(file_stat.st_size, len(observed)) + artifact["size_bytes"] = observed_size + artifact["disposition"] = ArtifactDisposition.PARTIAL + artifact["reason"] = "size_limit" + ledger_events.append( + ledger_event( + outcome=LedgerOutcome.PARTIAL, + record_type=LedgerRecordType.SYSTEM, + phase="cache", + path=path, + reason=LedgerReason.SIZE_LIMIT, + observed_bytes=observed_size, + limit_bytes=MAX_ANALYZABLE_FILE_BYTES, + ) + ) + inventory.append(artifact) + if not truncated and not _is_hidden_path(path) and artifact["content_kind"] == "text": + llm_file_cache[path] = _redact_for_external_model(path, content) except FileNotFoundError as exc: ledger_events.append( ledger_event( @@ -464,7 +551,7 @@ def _read_file_cache( error_class=type(exc).__name__, ) ) - return file_cache, ledger_events + return file_cache, raw_file_cache, llm_file_cache, inventory, ledger_events def _parse_manifest(skill_dir: Path) -> dict[str, object]: @@ -570,7 +657,89 @@ def build_context(state: SkillspectorState) -> dict[str, object]: ) for path in sorted(selected_baselines) ] - file_cache, cache_events = _read_file_cache(skill_dir, components) + primary_path = next( + (path for path in ("SKILL.md", "skill.md") if path in inventoried_components), None + ) + references = [] + reference_events: list[InspectionLedgerEvent] = [] + reference_resolution: dict[str, object] = {} + if primary_path is not None: + try: + primary_text = _read_text_no_follow( + skill_dir / primary_path, max_bytes=MAX_REFERENCE_SOURCE_BYTES + ) + except (OSError, _FileOpenError, _UnsafeFileError): + primary_text = "" + resolution = resolve_bundle_references_with_metadata( + skill_dir, + source_path=primary_path, + source_text=primary_text, + known_paths=inventoried_components, + ) + references = resolution.records + reference_resolution = { + "complete": resolution.complete, + "limitations": list(resolution.limitations), + "input_bytes_examined": resolution.input_bytes_examined, + "input_bytes_limit": MAX_REFERENCE_SOURCE_BYTES, + "raw_candidates_considered": resolution.raw_candidates_considered, + "raw_candidates_limit": MAX_RAW_REFERENCE_CANDIDATES, + "accepted_references": resolution.accepted_references, + "accepted_references_limit": MAX_ACCEPTED_REFERENCES, + "output_records": len(resolution.records), + "output_records_limit": MAX_REFERENCE_RECORDS, + "runtime_seconds_limit": MAX_REFERENCE_RUNTIME_SECONDS, + } + if not resolution.complete: + reference_events.append( + ledger_event( + outcome=LedgerOutcome.PARTIAL, + record_type=LedgerRecordType.SYSTEM, + phase="reference_resolution", + path=primary_path, + reason=LedgerReason.REFERENCE_EXTRACTION_LIMIT, + observed_bytes=resolution.input_bytes_examined, + limit_bytes=MAX_REFERENCE_SOURCE_BYTES, + ) + ) + reference_events.extend( + ledger_event( + outcome=LedgerOutcome.PARTIAL, + record_type=LedgerRecordType.SYSTEM, + phase="reference_resolution", + path=primary_path, + start_line=int(reference["line"]), + end_line=int(reference["line"]), + reason=LedgerReason.REFERENCE_UNRESOLVED, + ) + for reference in references + if reference["status"] in {"missing", "ambiguous"} + ) + referenced_paths = frozenset( + str(reference["target_path"]) + for reference in references + if reference["status"] == "resolved" and reference["target_path"] + ) + for referenced_path in referenced_paths: + if ( + referenced_path not in components + and referenced_path not in recognized_oms_signatures + and referenced_path not in selected_baselines + ): + components.append(referenced_path) + components.sort() + ( + file_cache, + raw_file_cache, + llm_file_cache, + artifact_inventory, + cache_events, + ) = _read_file_cache(skill_dir, components, referenced_paths) + disposition_by_path = {item["path"]: item["disposition"] for item in artifact_inventory} + for reference in references: + target = reference["target_path"] + if target and target in disposition_by_path: + reference["disposition"] = disposition_by_path[target] python_ast_cache_key = prewarm_python_ast_cache(components, file_cache) manifest = _parse_manifest(skill_dir) metadata_components = [ @@ -583,10 +752,16 @@ def build_context(state: SkillspectorState) -> dict[str, object]: return { "components": components, "file_cache": file_cache, + "raw_file_cache": raw_file_cache, + "llm_file_cache": llm_file_cache, + "artifact_inventory": artifact_inventory, + "artifact_references": references, + "reference_resolution": reference_resolution, "inspection_ledger": [ *discovery_events, *signature_events, *baseline_events, + *reference_events, *cache_events, ], "ast_cache": {}, diff --git a/src/skillspector/nodes/deduplicate.py b/src/skillspector/nodes/deduplicate.py index abda51335..52be1058d 100644 --- a/src/skillspector/nodes/deduplicate.py +++ b/src/skillspector/nodes/deduplicate.py @@ -1,104 +1,96 @@ # SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -"""Cross-analyzer finding deduplication. - -Merges findings that represent the same conceptual issue observed multiple -times — either within the same file or across files with identical patterns. - -Deduplication strategy: -1. Same-file dedup: Same rule_id + same file + same matched_text - → keep highest confidence instance -2. Cross-file consolidation: Same rule_id + same matched_text across files - → keep highest confidence instance -""" +"""Collision-resistant, occurrence-preserving finding compaction.""" from __future__ import annotations +from dataclasses import replace + from skillspector.logging_config import get_logger from skillspector.models import Finding logger = get_logger(__name__) -def _same_file_key(finding: Finding) -> tuple[str, str, str]: - """Build a deduplication key for same-file matches.""" - matched = (finding.matched_text or "").strip()[:100] - return (finding.rule_id, finding.file, matched) +def _occurrences(finding: Finding) -> list[dict[str, object]]: + if finding.occurrences: + return [dict(item) for item in finding.occurrences] + return [ + { + "file": finding.file, + "start_line": finding.start_line, + "end_line": finding.end_line, + } + ] -def _cross_file_key(finding: Finding) -> tuple[str, str]: - """Build a cross-file deduplication key from rule_id and normalized matched_text.""" - matched = (finding.matched_text or "").strip()[:100] - return (finding.rule_id, matched) +def _line(value: object, default: int) -> int: + return value if isinstance(value, int) else default def deduplicate(findings: list[Finding]) -> list[Finding]: - """Deduplicate a list of findings, returning a reduced list. - - Two-pass deduplication: - 1. Same-file: identical (rule_id, file, matched_text) → keep highest confidence - 2. Cross-file: identical (rule_id, matched_text) across different files - → keep highest confidence representative - - Findings without matched_text are never cross-file deduplicated (they lack - a reliable identity signal). - """ - if not findings: - return [] - - original_count = len(findings) - - # Pass 1: Same-file deduplication - same_file_best: dict[tuple[str, str, str], Finding] = {} - for f in findings: - key = _same_file_key(f) - existing = same_file_best.get(key) - if existing is None or f.confidence > existing.confidence: - same_file_best[key] = f - - after_same_file = list(same_file_best.values()) - - # Pass 2: Cross-file deduplication (only for findings WITH matched_text) - cross_file_best: dict[tuple[str, str], Finding] = {} - no_text_findings: list[Finding] = [] - - for f in after_same_file: - matched = (f.matched_text or "").strip() - if not matched: - no_text_findings.append(f) + """Aggregate exact full-match duplicates while preserving every occurrence.""" + groups: dict[tuple[str, str], list[Finding]] = {} + unique_without_match: list[Finding] = [] + for finding in findings: + fingerprint = finding.fingerprint() + if fingerprint is None: + unique_without_match.append(finding) continue - key = _cross_file_key(f) - existing = cross_file_best.get(key) - if existing is None or f.confidence > existing.confidence: - cross_file_best[key] = f - - deduplicated = list(cross_file_best.values()) + no_text_findings - - removed = original_count - len(deduplicated) - if removed > 0: - logger.info( - "Deduplication: %d → %d findings (%d duplicates removed)", - original_count, - len(deduplicated), - removed, + groups.setdefault((finding.rule_id, fingerprint), []).append(finding) + + compacted: list[Finding] = [] + for (_rule_id, fingerprint), group in groups.items(): + representative = max( + group, + key=lambda item: ( + item.confidence, + -item.start_line, + item.file, + item.finding_id, + ), + ) + occurrences = { + ( + str(occurrence.get("file", "")), + _line(occurrence.get("start_line"), 1), + occurrence.get("end_line"), + ) + for finding in group + for occurrence in _occurrences(finding) + } + ordered_occurrences = [ + {"file": file, "start_line": start, "end_line": end} + for file, start, end in sorted( + occurrences, + key=lambda item: (item[0], item[1], _line(item[2], item[1])), + ) + ] + compacted.append( + replace( + representative, + match_fingerprint=fingerprint, + occurrences=ordered_occurrences, + ) ) + compacted.extend(unique_without_match) severity_order = {"CRITICAL": 0, "HIGH": 1, "MEDIUM": 2, "LOW": 3} - deduplicated.sort( - key=lambda f: (severity_order.get(f.severity.upper(), 4), f.file, f.start_line) + compacted.sort( + key=lambda finding: ( + severity_order.get(finding.severity.upper(), 4), + finding.file, + finding.start_line, + finding.rule_id, + ) ) - - return deduplicated + removed = len(findings) - len(compacted) + if removed: + logger.info( + "Deduplication: %d -> %d findings (%d exact duplicates aggregated)", + len(findings), + len(compacted), + removed, + ) + return compacted diff --git a/src/skillspector/nodes/finalize_inspection_ledger.py b/src/skillspector/nodes/finalize_inspection_ledger.py index e9cacd9f9..64b7c6067 100644 --- a/src/skillspector/nodes/finalize_inspection_ledger.py +++ b/src/skillspector/nodes/finalize_inspection_ledger.py @@ -5,15 +5,128 @@ from __future__ import annotations -from skillspector.inspection_ledger import finalize_ledger +from collections.abc import Mapping + +from skillspector.inspection_ledger import ( + InspectionLedgerEvent, + LedgerOutcome, + analyzer_status_for_events, + finalize_ledger, + ledger_event, +) +from skillspector.models import Finding from skillspector.state import SkillspectorState +def _reference_coverage_findings( + state: SkillspectorState, +) -> list[Finding]: + """Create AE1 only for canonical resolved targets with incomplete disposition.""" + raw_references = state.get("artifact_references") or [] + inventory: dict[str, Mapping[str, object]] = { + str(item.get("path", "")): item + for item in state.get("artifact_inventory") or [] + if isinstance(item, dict) + } + exceptional_outcomes: dict[str, set[str]] = {} + for event in state.get("inspection_ledger") or []: + if not isinstance(event, Mapping): + continue + outcome = str(event.get("outcome", "")) + if outcome in {"partial", "failed", "out_of_scope"}: + exceptional_outcomes.setdefault(str(event.get("path", "")), set()).add(outcome) + findings: list[Finding] = [] + for reference in raw_references: + if not isinstance(reference, dict): + continue + status = str(reference.get("status", "")) + if status != "resolved": + continue + target = reference.get("target_path") + target_path = str(target) if target else "" + inventory_item = inventory.get(target_path) + disposition = str(inventory_item.get("disposition", "")) if inventory_item else "" + exceptional = exceptional_outcomes.get(target_path, set()) + final_disposition = ( + "failed" + if "failed" in exceptional + else "partial" + if "partial" in exceptional + else "out_of_scope" + if "out_of_scope" in exceptional + else disposition + ) + if final_disposition not in {"partial", "failed", "out_of_scope"}: + continue + line_value = reference.get("line", 1) + evidence = str(reference.get("evidence", ""))[:160] + findings.append( + Finding( + rule_id="AE1", + message="Referenced artifact was not completely inspected", + severity="HIGH", + confidence=1.0, + file=str(reference.get("source_path", "SKILL.md")), + start_line=line_value if isinstance(line_value, int) else 1, + category="analysis-evasion", + tags=["coverage", "reference", f"target-disposition:{final_disposition}"], + finding=f"{target_path} ({final_disposition})"[:200], + code_snippet=evidence, + matched_text=target_path, + remediation=( + "Make the referenced artifact locally available and fully analyzable, " + "or remove the reference." + ), + ) + ) + return findings + + def finalize_inspection_ledger(state: SkillspectorState) -> dict[str, object]: """Validate full internal facts and derive the public completeness projection.""" - completeness, effective_finding_ids = finalize_ledger(state) + reference_findings = _reference_coverage_findings(state) + reference_events: list[InspectionLedgerEvent] = [ + ledger_event( + outcome=LedgerOutcome.COMPLETED, + phase="reference", + analyzer_id="reference_coverage", + path=finding.file, + start_line=finding.start_line, + end_line=finding.start_line, + emitted_finding_ids=[finding.finding_id], + ) + for finding in reference_findings + ] + merged_state = dict(state) + merged_state["findings"] = [*(state.get("findings") or []), *reference_findings] + merged_state["effective_finding_ids"] = [ + *(state.get("effective_finding_ids") or []), + *(finding.finding_id for finding in reference_findings), + ] + merged_state["inspection_ledger"] = [ + *(state.get("inspection_ledger") or []), + *reference_events, + ] + reference_statuses = ( + [analyzer_status_for_events("reference_coverage", reference_events)] + if reference_events + else [] + ) + merged_state["analyzer_status_events"] = [ + *(state.get("analyzer_status_events") or []), + *reference_statuses, + ] + completeness, effective_finding_ids = finalize_ledger(merged_state) + if reference_findings and completeness["status"] == "complete": + completeness["status"] = "partial" + completeness["is_complete"] = False + limitations = completeness.setdefault("limitations", []) + limitations.append("One or more referenced artifacts were not completely inspected.") return { "analysis_completeness": completeness, "execution_successful": completeness["execution_successful"], + "findings": reference_findings, "effective_finding_ids": effective_finding_ids, + "inspection_ledger": reference_events, + "analyzer_status_events": reference_statuses, } diff --git a/src/skillspector/nodes/meta_analyzer.py b/src/skillspector/nodes/meta_analyzer.py index 34c980652..8f455c9a8 100644 --- a/src/skillspector/nodes/meta_analyzer.py +++ b/src/skillspector/nodes/meta_analyzer.py @@ -232,41 +232,17 @@ def _format_findings_for_prompt(findings: list[Finding]) -> str: return "\n".join(lines) -_NO_LLM_CONFIDENCE_THRESHOLD = 0.4 -_HIGH_SEVERITY_PASS_THROUGH = frozenset({"CRITICAL", "HIGH"}) -_CODE_EXAMPLE_DOWNWEIGHT = 0.5 - - def _fallback_filtered(findings: list[Finding]) -> list[Finding]: - """Heuristic fallback filter for --no-llm mode. - - Applies rule-based filtering when LLM analysis is unavailable: - 1. Drop findings with confidence below threshold (0.4), UNLESS severity - is CRITICAL or HIGH (high-severity findings are never dropped on - confidence alone) - 2. Downweight findings whose context matches code-example indicators - (0.5x confidence reduction) — never hard-drop, as there is no LLM - safety net in this mode - 3. Apply default remediations from pattern_defaults - """ - from skillspector.nodes.analyzers.common import is_code_example - + """Preserve deterministic findings and add defaults in --no-llm mode.""" result: list[Finding] = [] for f in findings: - severity_upper = (f.severity or "LOW").upper() - confidence = f.confidence - if f.context and is_code_example(f.context): - confidence *= _CODE_EXAMPLE_DOWNWEIGHT - if confidence < _NO_LLM_CONFIDENCE_THRESHOLD: - if severity_upper not in _HIGH_SEVERITY_PASS_THROUGH: - continue result.append( Finding( rule_id=f.rule_id, message=f.message, finding_id=f.finding_id, severity=f.severity, - confidence=confidence, + confidence=f.confidence, file=f.file, start_line=f.start_line, end_line=f.end_line, @@ -279,13 +255,14 @@ def _fallback_filtered(findings: list[Finding]) -> list[Finding]: finding=getattr(f, "finding", None), explanation=getattr(f, "explanation", None), code_snippet=getattr(f, "code_snippet", None) or f.context, - intent=None, + intent=f.intent, + match_fingerprint=f.match_fingerprint, + occurrences=list(f.occurrences), ) ) logger.info( - "Heuristic fallback filter (--no-llm): %d → %d findings", + "Deterministic fallback (--no-llm): %d findings preserved", len(findings), - len(result), ) return result @@ -316,7 +293,9 @@ def _passthrough_with_defaults(findings: list[Finding]) -> list[Finding]: finding=getattr(f, "finding", None), explanation=getattr(f, "explanation", None), code_snippet=getattr(f, "code_snippet", None) or f.context, - intent=None, + intent=f.intent, + match_fingerprint=f.match_fingerprint, + occurrences=list(f.occurrences), ) for f in findings ] @@ -382,7 +361,7 @@ def apply_filter( findings: list[Finding], batch_results: list[tuple[Batch, list[dict[str, Any]]]], ) -> list[Finding]: - """Keep only LLM-confirmed findings, enriched with explanation / remediation. + """Enrich deterministic findings without letting LLM output suppress them. Uses granular ``(file, rule_id, start_line, end_line)`` keying when the LLM provides a ``start_line``, so multiple findings with the same @@ -391,14 +370,9 @@ def apply_filter( callers that omit it still match. Falls back to coarse ``(file, rule_id)`` keying for LLM responses that omit ``start_line``. - Severity-gated floor (security invariant) - ------------------------------------------ - CRITICAL and HIGH static findings are **always** kept in the output even - if the LLM did not confirm them. When the LLM omits or denies such a - finding the original static finding is preserved unchanged and the tag - ``"llm-unconfirmed"`` is appended so consumers can distinguish it from - LLM-validated findings. MEDIUM and LOW findings continue to be filtered - by the LLM as before (false-positive reduction). + Every deterministic finding remains in primary output. Unconfirmed + findings receive an annotation tag; confirmed findings may gain an + explanation or higher confidence, but are never downgraded. """ _enrichment = tuple[str, str, float] confirmed_granular: dict[tuple[str, str, int, int | None], _enrichment] = {} @@ -449,38 +423,33 @@ def apply_filter( elif coarse_key in confirmed_coarse: expl, rem, conf = confirmed_coarse[coarse_key] else: - # Security: CRITICAL/HIGH static findings must survive LLM filtering. - # A prompt-injection payload in the scanned skill could cause the LLM - # to deny or omit a real high-severity finding; silently dropping it - # would be a false-negative in a security gate. Keep the original - # finding and tag it so consumers know it was not LLM-validated. - if f.severity in self._HIGH_SEVERITY_FLOOR: - unconfirmed_tags = list(f.tags) - if "llm-unconfirmed" not in unconfirmed_tags: - unconfirmed_tags.append("llm-unconfirmed") - result.append( - Finding( - rule_id=f.rule_id, - message=f.message, - finding_id=f.finding_id, - severity=f.severity, - confidence=f.confidence, - file=f.file, - start_line=f.start_line, - end_line=f.end_line, - remediation=f.remediation or get_remediation(f.rule_id), - tags=unconfirmed_tags, - context=f.context, - matched_text=f.matched_text, - category=getattr(f, "category", None), - pattern=getattr(f, "pattern", None), - finding=getattr(f, "finding", None), - explanation=getattr(f, "explanation", None), - code_snippet=getattr(f, "code_snippet", None) or f.context, - intent=None, - ) + unconfirmed_tags = list(f.tags) + if "llm-unconfirmed" not in unconfirmed_tags: + unconfirmed_tags.append("llm-unconfirmed") + result.append( + Finding( + rule_id=f.rule_id, + message=f.message, + finding_id=f.finding_id, + severity=f.severity, + confidence=f.confidence, + file=f.file, + start_line=f.start_line, + end_line=f.end_line, + remediation=f.remediation or get_remediation(f.rule_id), + tags=unconfirmed_tags, + context=f.context, + matched_text=f.matched_text, + category=getattr(f, "category", None), + pattern=getattr(f, "pattern", None), + finding=getattr(f, "finding", None), + explanation=getattr(f, "explanation", None), + code_snippet=getattr(f, "code_snippet", None) or f.context, + intent=f.intent, + match_fingerprint=f.match_fingerprint, + occurrences=list(f.occurrences), ) - # MEDIUM/LOW: preserve existing behaviour (LLM may filter as false-positive). + ) continue result.append( Finding( @@ -488,7 +457,7 @@ def apply_filter( message=expl, finding_id=f.finding_id, severity=f.severity, - confidence=conf, + confidence=max(f.confidence, conf), file=f.file, start_line=f.start_line, end_line=f.end_line, @@ -501,7 +470,9 @@ def apply_filter( finding=getattr(f, "finding", None), explanation=expl, code_snippet=getattr(f, "code_snippet", None) or f.context, - intent=None, + intent=f.intent, + match_fingerprint=f.match_fingerprint, + occurrences=list(f.occurrences), ) ) return result @@ -619,7 +590,10 @@ def meta_analyzer(state: SkillspectorState) -> MetaAnalyzerResponse: ], } - file_cache: dict[str, str] = state.get("file_cache") or {} + llm_cache = state.get("llm_file_cache") + file_cache: dict[str, str] = ( + llm_cache if isinstance(llm_cache, dict) else state.get("file_cache") or {} + ) manifest: dict[str, object] = state.get("manifest") or {} model_config: dict[str, str] = state.get("model_config") or {} model = ( @@ -629,7 +603,23 @@ def meta_analyzer(state: SkillspectorState) -> MetaAnalyzerResponse: ) metadata_text = _format_metadata(manifest) - files_with_findings = sorted({f.file for f in findings}) + eligible_findings = [finding for finding in findings if finding.file in file_cache] + local_only_findings = [finding for finding in findings if finding.file not in file_cache] + files_with_findings = sorted({f.file for f in eligible_findings}) + if not eligible_findings: + filtered = _fallback_filtered(findings) + return { + "findings": filtered, + "effective_finding_ids": [finding.finding_id for finding in filtered], + "inspection_ledger": [], + "analyzer_status_events": [ + analyzer_status_event( + analyzer_id="meta_analyzer", + status="not_applicable", + reason=LedgerReason.NO_APPLICABLE_FILES, + ) + ], + } analyzer: LLMMetaAnalyzer | None = None batches: list[Batch] = [] @@ -638,7 +628,7 @@ def meta_analyzer(state: SkillspectorState) -> MetaAnalyzerResponse: # and recorded as a degraded LLM call (consistent with the semantic # analyzers) rather than crashing the whole graph. analyzer = LLMMetaAnalyzer(model=model) - batches = analyzer.get_batches(files_with_findings, file_cache, findings) + batches = analyzer.get_batches(files_with_findings, file_cache, eligible_findings) batches = [batch for batch in batches if batch.findings] logger.debug( "Meta-analyzer: %d files -> %d batches (model=%s)", @@ -681,10 +671,14 @@ def meta_analyzer(state: SkillspectorState) -> MetaAnalyzerResponse: analysed_ids = { finding.finding_id for batch, _ in batch_results for finding in batch.findings } - analysed = [finding for finding in findings if finding.finding_id in analysed_ids] - unanalysed = [finding for finding in findings if finding.finding_id not in analysed_ids] + analysed = [ + finding for finding in eligible_findings if finding.finding_id in analysed_ids + ] + unanalysed = [ + finding for finding in eligible_findings if finding.finding_id not in analysed_ids + ] else: - analysed, unanalysed = findings, [] + analysed, unanalysed = eligible_findings, [] filtered = analyzer.apply_filter(analysed, batch_results) if unanalysed: @@ -697,6 +691,7 @@ def meta_analyzer(state: SkillspectorState) -> MetaAnalyzerResponse: len({f.file for f in unanalysed}), ) filtered.extend(_fallback_filtered(unanalysed)) + filtered.extend(_fallback_filtered(local_only_findings)) logger.debug( "LLM filtering done: %d findings -> %d after filter", diff --git a/src/skillspector/nodes/report.py b/src/skillspector/nodes/report.py index 68c0760a8..20098539c 100644 --- a/src/skillspector/nodes/report.py +++ b/src/skillspector/nodes/report.py @@ -116,6 +116,34 @@ def _sanitize_finding(finding: Finding) -> Finding: ) +def _expand_occurrences(findings: list[Finding]) -> list[Finding]: + """Expand compacted findings for human/JSON output without losing locations.""" + expanded: list[Finding] = [] + for finding in findings: + occurrences = finding.occurrences or [ + { + "file": finding.file, + "start_line": finding.start_line, + "end_line": finding.end_line, + } + ] + for occurrence in occurrences: + start_value = occurrence.get("start_line", finding.start_line) + start_line = start_value if isinstance(start_value, int) else finding.start_line + end_value = occurrence.get("end_line") + end_line = end_value if isinstance(end_value, int) else None + expanded.append( + replace( + finding, + file=str(occurrence.get("file", finding.file)), + start_line=start_line, + end_line=end_line, + occurrences=[], + ) + ) + return expanded + + def _build_sarif_properties(finding: Finding) -> dict[str, object] | None: """Project selected finding metadata into a SARIF properties dictionary.""" finding_dict = finding.to_dict() @@ -258,23 +286,36 @@ def _build_sarif( for finding in findings: if not finding.rule_id or not finding.message: continue - region = SarifRegion(startLine=finding.start_line, endLine=finding.end_line) - results.append( - SarifResult( - ruleId=finding.rule_id, - message=SarifMessage(text=finding.message), - level=_severity_to_sarif_level(finding.severity), - properties=_build_sarif_properties(finding), - locations=[ - SarifLocation( - physicalLocation=SarifPhysicalLocation( - artifactLocation=SarifArtifactLocation(uri=finding.file), - region=region, + occurrences = finding.occurrences or [ + { + "file": finding.file, + "start_line": finding.start_line, + "end_line": finding.end_line, + } + ] + for occurrence in occurrences: + start_value = occurrence.get("start_line", finding.start_line) + start_line = start_value if isinstance(start_value, int) else finding.start_line + end_value = occurrence.get("end_line") + end_line = int(end_value) if isinstance(end_value, int) else None + results.append( + SarifResult( + ruleId=finding.rule_id, + message=SarifMessage(text=finding.message), + level=_severity_to_sarif_level(finding.severity), + properties=_build_sarif_properties(finding), + locations=[ + SarifLocation( + physicalLocation=SarifPhysicalLocation( + artifactLocation=SarifArtifactLocation( + uri=str(occurrence.get("file", finding.file)) + ), + region=SarifRegion(startLine=start_line, endLine=end_line), + ) ) - ) - ], + ], + ) ) - ) if finding.rule_id not in seen_rule_ids: seen_rule_ids[finding.rule_id] = finding.message @@ -430,6 +471,7 @@ def _render_terminal_completeness( table.add_column("Metric", style="bold") table.add_column("Value") table.add_row("Execution", "successful" if execution_successful else "failed") + table.add_row("Status", str(completeness.get("status", "complete"))) table.add_row("Coverage", f"{completeness.get('coverage_percent', 100.0)}%") table.add_row("Fully inspected", str(completeness.get("fully_inspected_files", 0))) table.add_row("Partially inspected", str(completeness.get("partially_inspected_files", 0))) @@ -730,6 +772,7 @@ def _render_markdown_completeness( lines.append("| Metric | Value |") lines.append("|--------|-------|") lines.append(f"| Execution | {'successful' if execution_successful else 'failed'} |") + lines.append(f"| Status | {_markdown_cell(completeness.get('status', 'complete'))} |") lines.append(f"| Coverage | {_markdown_cell(completeness.get('coverage_percent', 100.0))}% |") lines.append( f"| Fully inspected | {_markdown_cell(completeness.get('fully_inspected_files', 0))} |" @@ -877,19 +920,15 @@ def report(state: SkillspectorState) -> dict[str, object]: validated finding IDs, applies baseline suppression, and renders all surfaces. """ clear_python_ast_cache(state.get("python_ast_cache_key")) - raw_findings = state.get("findings", []) + raw_findings = state.get("findings") + if raw_findings is None: + # Preserve the public node contract for direct callers while ensuring + # graph executions prefer the pre-meta canonical finding collection. + raw_findings = state.get("filtered_findings", []) findings_by_id = {finding.finding_id: finding for finding in raw_findings} - effective_ids = state.get("effective_finding_ids") - if isinstance(effective_ids, list): - selected_findings = [ - findings_by_id[finding_id] - for finding_id in effective_ids - if isinstance(finding_id, str) and finding_id in findings_by_id - ] - else: - # Transitional direct-node compatibility. Graph execution always receives - # `effective_finding_ids` from finalize_inspection_ledger. - selected_findings = state.get("filtered_findings", raw_findings) + # Meta/LLM analysis can enrich canonical objects but cannot remove + # deterministic findings from primary output. + selected_findings = list(findings_by_id.values()) selected_findings = [_sanitize_finding(finding) for finding in selected_findings] empty_completeness: AnalysisCompleteness = { @@ -897,6 +936,7 @@ def report(state: SkillspectorState) -> dict[str, object]: "scanned_components": 0, "coverage_percent": 100.0, "is_complete": True, + "status": "complete", "execution_successful": True, "fully_inspected_files": 0, "partially_inspected_files": 0, @@ -940,10 +980,11 @@ def report(state: SkillspectorState) -> dict[str, object]: file_cache=file_cache, scanner_version=skillspector_version, ) - findings_for_scoring = deduplicate(active_findings) risk_score, risk_severity, risk_recommendation = _compute_risk_score( - findings_for_scoring, has_executable_scripts, component_metadata + active_findings, has_executable_scripts, component_metadata ) + reported_findings = deduplicate(active_findings) + display_findings = _expand_occurrences(reported_findings) exceptions = analysis_completeness.get("ledger_exceptions", []) fatal_exception = ( any( @@ -957,11 +998,14 @@ def report(state: SkillspectorState) -> dict[str, object]: entirely_uninspected = ( entirely_uninspected_value if isinstance(entirely_uninspected_value, int) else 0 ) - if (degraded or fatal_exception or entirely_uninspected > 0) and risk_recommendation == "SAFE": + incomplete = not bool(analysis_completeness.get("is_complete", True)) + if ( + degraded or fatal_exception or entirely_uninspected > 0 or incomplete + ) and risk_recommendation == "SAFE": risk_recommendation = "CAUTION" sarif_report = _build_sarif( - active_findings, + reported_findings, suppressed, degraded_notice=degraded_notice, analysis_completeness=analysis_completeness, @@ -969,7 +1013,7 @@ def report(state: SkillspectorState) -> dict[str, object]: ) if output_format == "terminal": report_body = _format_terminal( - active_findings, + display_findings, component_metadata, manifest, skill_path, @@ -986,7 +1030,7 @@ def report(state: SkillspectorState) -> dict[str, object]: ) elif output_format == "json": report_body = _format_json( - active_findings, + display_findings, component_metadata, manifest, skill_path, @@ -1003,7 +1047,7 @@ def report(state: SkillspectorState) -> dict[str, object]: ) elif output_format == "markdown": report_body = _format_markdown( - active_findings, + display_findings, component_metadata, manifest, skill_path, @@ -1033,7 +1077,7 @@ def report(state: SkillspectorState) -> dict[str, object]: "risk_severity": risk_severity, "risk_recommendation": risk_recommendation, "report_body": report_body, - "filtered_findings": selected_findings, + "filtered_findings": reported_findings, "suppressed_findings": suppressed, "execution_successful": execution_successful, } diff --git a/src/skillspector/references.py b/src/skillspector/references.py new file mode 100644 index 000000000..9713e1448 --- /dev/null +++ b/src/skillspector/references.py @@ -0,0 +1,212 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Bounded canonical resolver for references made by the primary skill file.""" + +from __future__ import annotations + +import posixpath +import re +import time +from dataclasses import dataclass +from pathlib import Path, PurePosixPath +from urllib.parse import unquote, urlsplit + +from skillspector.artifacts import ArtifactDisposition, BundleReference + +MAX_REFERENCE_SOURCE_BYTES = 1_000_000 +MAX_RAW_REFERENCE_CANDIDATES = 4096 +MAX_ACCEPTED_REFERENCES = 256 +MAX_REFERENCE_RECORDS = 1024 +MAX_REFERENCE_RUNTIME_SECONDS = 2.0 +_MAX_EVIDENCE = 160 +_MARKDOWN_DESTINATION = re.compile(r"\[[^\]\n]{1,200}\]\(([^)\n]{1,512})\)") +_QUOTED_OR_CODE_PATH = re.compile( + r"(?:`|'|\")((?:\./)?(?:[A-Za-z0-9_.-]+/)*[A-Za-z0-9_.-]+\.[A-Za-z0-9]{1,12})(?:`|'|\")" +) + + +@dataclass(frozen=True) +class ReferenceResolutionResult: + """Bounded reference records plus explicit extraction accounting.""" + + records: list[BundleReference] + complete: bool + limitations: tuple[str, ...] + input_bytes_examined: int + raw_candidates_considered: int + accepted_references: int + + +_PLAIN_RELATIVE_PATH = re.compile( + r"(? str: + """Return a bounded one-line evidence preview.""" + cleaned = " ".join(line.strip().split()) + if len(cleaned) <= _MAX_EVIDENCE: + return cleaned + start = max(0, min(column - 1, len(cleaned)) - _MAX_EVIDENCE // 2) + return cleaned[start : start + _MAX_EVIDENCE] + + +def _candidate_strings( + text: str, *, deadline: float +) -> tuple[list[tuple[str, int, int, str]], tuple[str, ...]]: + """Extract path-like strings with 1-based line/column locations.""" + candidates: list[tuple[str, int, int, str]] = [] + seen: set[tuple[int, int, str]] = set() + for line_number, line in enumerate(text.splitlines(), 1): + if time.monotonic() > deadline: + return candidates, ("runtime",) + line_matches: list[tuple[int, int, int, re.Match[str]]] = [] + for pattern_index, pattern in enumerate( + (_MARKDOWN_DESTINATION, _QUOTED_OR_CODE_PATH, _PLAIN_RELATIVE_PATH) + ): + for match in pattern.finditer(line): + line_matches.append((match.start(1), match.end(1), pattern_index, match)) + for _, _, _, match in sorted(line_matches, key=lambda item: item[:3]): + raw = match.group(1).strip().split(maxsplit=1)[0] + key = (line_number, match.start(1), raw) + if key in seen: + continue + seen.add(key) + candidates.append( + (raw, line_number, match.start(1) + 1, _evidence(line, match.start(1) + 1)) + ) + if len(candidates) >= MAX_RAW_REFERENCE_CANDIDATES: + return candidates, ("raw_candidates",) + return candidates, () + + +def _normalize_candidate(raw: str, source_path: str) -> str | None: + """Return a contained relative POSIX candidate, or None when unsupported.""" + raw = unquote(raw.strip().strip("<>")) + split = urlsplit(raw) + if split.scheme or split.netloc or raw.startswith(("/", "\\", "#")): + return None + path_part = split.path.replace("\\", "/") + if not path_part: + return None + if len(path_part) >= 2 and path_part[1] == ":": + return None + source_parent = PurePosixPath(source_path).parent.as_posix() + joined = posixpath.normpath(posixpath.join(source_parent, path_part)) + if joined in {"", ".", ".."} or joined.startswith("../"): + return None + return joined.removeprefix("./") + + +def _safe_regular_target(skill_dir: Path, relative_path: str) -> bool: + """Return true only for a non-link regular file contained by *skill_dir*.""" + root = skill_dir.resolve(strict=False) + target = skill_dir / relative_path + try: + if target.is_symlink() or target.is_junction(): + return False + resolved = target.resolve(strict=True) + return resolved.is_relative_to(root) and resolved.is_file() + except (OSError, RuntimeError): + return False + + +def resolve_bundle_references_with_metadata( + skill_dir: Path, + *, + source_path: str, + source_text: str, + known_paths: list[str], +) -> ReferenceResolutionResult: + """Resolve references with separate deterministic input/work/output bounds.""" + encoded = source_text.encode("utf-8") + input_limited = len(encoded) > MAX_REFERENCE_SOURCE_BYTES + bounded_source = encoded[:MAX_REFERENCE_SOURCE_BYTES].decode("utf-8", errors="ignore") + input_bytes_examined = min(len(encoded), MAX_REFERENCE_SOURCE_BYTES) + deadline = time.monotonic() + MAX_REFERENCE_RUNTIME_SECONDS + + known = set(known_paths) + basename_index: dict[str, list[str]] = {} + for path in sorted(known): + basename_index.setdefault(PurePosixPath(path).name, []).append(path) + + candidates, candidate_limitations = _candidate_strings(bounded_source, deadline=deadline) + limitations = ["input_bytes"] if input_limited else [] + limitations.extend(candidate_limitations) + records: list[BundleReference] = [] + accepted_keys: set[tuple[str, str]] = set() + for raw, line, column, evidence in candidates: + if time.monotonic() > deadline: + limitations.append("runtime") + break + target = _normalize_candidate(raw, source_path) + status = "rejected" + disposition = ArtifactDisposition.OUT_OF_SCOPE + resolved_target: str | None = None + if target is not None: + if target in known or _safe_regular_target(skill_dir, target): + resolved_target = target + status = "resolved" + disposition = ArtifactDisposition.ANALYZED + elif "/" not in raw.replace("\\", "/"): + matches = basename_index.get(PurePosixPath(target).name, []) + if len(matches) == 1: + resolved_target = matches[0] + status = "resolved" + disposition = ArtifactDisposition.ANALYZED + elif len(matches) > 1: + status = "ambiguous" + disposition = ArtifactDisposition.PARTIAL + else: + status = "missing" + disposition = ArtifactDisposition.PARTIAL + else: + status = "missing" + disposition = ArtifactDisposition.PARTIAL + if status != "rejected": + accepted_key = (status, resolved_target or target or raw) + if accepted_key not in accepted_keys: + if len(accepted_keys) >= MAX_ACCEPTED_REFERENCES: + limitations.append("accepted_references") + break + accepted_keys.add(accepted_key) + if len(records) >= MAX_REFERENCE_RECORDS: + limitations.append("output_records") + break + records.append( + { + "source_path": source_path, + "line": line, + "column": column, + "evidence": evidence, + "target_path": resolved_target, + "status": status, + "disposition": disposition, + } + ) + stable_limitations = tuple(dict.fromkeys(limitations)) + return ReferenceResolutionResult( + records=records, + complete=not stable_limitations, + limitations=stable_limitations, + input_bytes_examined=input_bytes_examined, + raw_candidates_considered=len(candidates), + accepted_references=len(accepted_keys), + ) + + +def resolve_bundle_references( + skill_dir: Path, + *, + source_path: str, + source_text: str, + known_paths: list[str], +) -> list[BundleReference]: + """Compatibility wrapper returning bounded reference records.""" + return resolve_bundle_references_with_metadata( + skill_dir, + source_path=source_path, + source_text=source_text, + known_paths=known_paths, + ).records diff --git a/src/skillspector/state.py b/src/skillspector/state.py index f7942bf72..f1c75f830 100644 --- a/src/skillspector/state.py +++ b/src/skillspector/state.py @@ -22,6 +22,7 @@ from typing_extensions import TypedDict +from skillspector.artifacts import ArtifactRecord, BundleReference from skillspector.inference_usage import InferenceUsageRecord from skillspector.inspection_ledger import ( AnalysisCompleteness, @@ -59,6 +60,13 @@ class SkillspectorState(TypedDict, total=False): # build_context node populates these components: list[str] file_cache: dict[str, str] + # Raw bytes remain the canonical source for YARA and content classification. + raw_file_cache: dict[str, bytes] + # External-model consumers use the redacted projection for sensitive local files. + llm_file_cache: dict[str, str] + artifact_inventory: list[ArtifactRecord] + artifact_references: list[BundleReference] + reference_resolution: dict[str, object] # Retained for compatibility with the persisted workflow-state schema. ast_cache: dict[str, str] # Key for the process-local parsed-AST cache. The ASTs themselves stay diff --git a/src/skillspector/unicode_confusables.py b/src/skillspector/unicode_confusables.py new file mode 100644 index 000000000..4fd0f036c --- /dev/null +++ b/src/skillspector/unicode_confusables.py @@ -0,0 +1,1527 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Generated ASCII skeleton subset from Unicode UTS #39 confusables data.""" + +from __future__ import annotations + +UNICODE_CONFUSABLES_VERSION = "17.0.0" +# Source: https://www.unicode.org/Public/17.0.0/security/confusables.txt +# The source data is governed by https://www.unicode.org/license.txt. +ASCII_CONFUSABLE_SKELETON: dict[int, str] = { + 0x00C6: "AE", + 0x00D7: "x", + 0x00E6: "ae", + 0x00FE: "p", + 0x0131: "i", + 0x0132: "lJ", + 0x0133: "ij", + 0x0152: "OE", + 0x0153: "oe", + 0x017F: "f", + 0x0184: "b", + 0x018D: "g", + 0x0192: "f", + 0x0196: "l", + 0x01A6: "R", + 0x01A7: "2", + 0x01B7: "3", + 0x01BC: "5", + 0x01BD: "s", + 0x01BF: "p", + 0x01C0: "l", + 0x01C1: "ll", + 0x01C7: "LJ", + 0x01C8: "Lj", + 0x01C9: "lj", + 0x01CA: "NJ", + 0x01CB: "Nj", + 0x01CC: "nj", + 0x01F1: "DZ", + 0x01F2: "Dz", + 0x01F3: "dz", + 0x021C: "3", + 0x0222: "8", + 0x0223: "8", + 0x0251: "a", + 0x0261: "g", + 0x0263: "y", + 0x0269: "i", + 0x026A: "i", + 0x026F: "w", + 0x028B: "u", + 0x028F: "y", + 0x02A3: "dz", + 0x02A6: "ts", + 0x02AA: "ls", + 0x02AB: "lz", + 0x02DB: "i", + 0x037A: "i", + 0x037F: "J", + 0x0391: "A", + 0x0392: "B", + 0x0395: "E", + 0x0396: "Z", + 0x0397: "H", + 0x0399: "l", + 0x039A: "K", + 0x039C: "M", + 0x039D: "N", + 0x039F: "O", + 0x03A1: "P", + 0x03A4: "T", + 0x03A5: "Y", + 0x03A7: "X", + 0x03B1: "a", + 0x03B3: "y", + 0x03B9: "i", + 0x03BD: "v", + 0x03BF: "o", + 0x03C1: "p", + 0x03C3: "o", + 0x03C5: "u", + 0x03D2: "Y", + 0x03DC: "F", + 0x03E8: "2", + 0x03EC: "6", + 0x03ED: "o", + 0x03F1: "p", + 0x03F2: "c", + 0x03F3: "j", + 0x03F8: "p", + 0x03F9: "C", + 0x03FA: "M", + 0x0405: "S", + 0x0406: "l", + 0x0408: "J", + 0x0410: "A", + 0x0412: "B", + 0x0415: "E", + 0x0417: "3", + 0x041A: "K", + 0x041C: "M", + 0x041D: "H", + 0x041E: "O", + 0x0420: "P", + 0x0421: "C", + 0x0422: "T", + 0x0423: "Y", + 0x0425: "X", + 0x042B: "bl", + 0x042C: "b", + 0x042E: "lO", + 0x0430: "a", + 0x0431: "6", + 0x0433: "r", + 0x0435: "e", + 0x043E: "o", + 0x0440: "p", + 0x0441: "c", + 0x0443: "y", + 0x0445: "x", + 0x0448: "w", + 0x0455: "s", + 0x0456: "i", + 0x0458: "j", + 0x0461: "w", + 0x0474: "V", + 0x0475: "v", + 0x04AE: "Y", + 0x04AF: "y", + 0x04BB: "h", + 0x04BD: "e", + 0x04C0: "l", + 0x04CF: "l", + 0x04D4: "AE", + 0x04D5: "ae", + 0x04E0: "3", + 0x0501: "d", + 0x050C: "G", + 0x051B: "q", + 0x051C: "W", + 0x051D: "w", + 0x054D: "U", + 0x054F: "S", + 0x0555: "O", + 0x0561: "w", + 0x0563: "q", + 0x0566: "q", + 0x0570: "h", + 0x0578: "n", + 0x057C: "n", + 0x057D: "u", + 0x0581: "g", + 0x0582: "i", + 0x0584: "f", + 0x0585: "o", + 0x05C0: "l", + 0x05D5: "l", + 0x05D8: "v", + 0x05DF: "l", + 0x05E1: "o", + 0x05F0: "ll", + 0x0627: "l", + 0x0647: "o", + 0x0661: "l", + 0x0665: "o", + 0x0667: "V", + 0x06BE: "o", + 0x06C1: "o", + 0x06D5: "o", + 0x06F1: "l", + 0x06F5: "o", + 0x06F7: "V", + 0x07C0: "O", + 0x07CA: "l", + 0x0966: "o", + 0x0969: "3", + 0x09E6: "o", + 0x09EA: "8", + 0x09ED: "9", + 0x0A66: "o", + 0x0A67: "9", + 0x0A6A: "8", + 0x0AE6: "o", + 0x0AE9: "3", + 0x0B03: "8", + 0x0B20: "O", + 0x0B66: "o", + 0x0B68: "9", + 0x0BE6: "o", + 0x0C02: "o", + 0x0C66: "o", + 0x0C82: "o", + 0x0CE6: "O", + 0x0D02: "o", + 0x0D1F: "s", + 0x0D20: "o", + 0x0D66: "o", + 0x0D6D: "9", + 0x0D82: "o", + 0x0E50: "o", + 0x0ED0: "o", + 0x1004: "c", + 0x101D: "o", + 0x1040: "o", + 0x105A: "c", + 0x10E7: "y", + 0x10FF: "o", + 0x1200: "U", + 0x12D0: "O", + 0x13A0: "D", + 0x13A1: "R", + 0x13A2: "T", + 0x13A5: "i", + 0x13A9: "Y", + 0x13AA: "A", + 0x13AB: "J", + 0x13AC: "E", + 0x13B3: "W", + 0x13B7: "M", + 0x13BB: "H", + 0x13BD: "Y", + 0x13C0: "G", + 0x13C2: "h", + 0x13C3: "Z", + 0x13CE: "4", + 0x13CF: "b", + 0x13D2: "R", + 0x13D4: "W", + 0x13D5: "S", + 0x13D9: "V", + 0x13DA: "S", + 0x13DE: "L", + 0x13DF: "C", + 0x13E2: "P", + 0x13E6: "K", + 0x13E7: "d", + 0x13EE: "6", + 0x13F3: "G", + 0x13F4: "B", + 0x142F: "V", + 0x144C: "U", + 0x146D: "P", + 0x146F: "d", + 0x1472: "b", + 0x148D: "J", + 0x14AA: "L", + 0x14BF: "2", + 0x1541: "x", + 0x157C: "H", + 0x157D: "x", + 0x1587: "R", + 0x15AF: "b", + 0x15B4: "F", + 0x15C5: "A", + 0x15DE: "D", + 0x15EA: "D", + 0x15F0: "M", + 0x15F7: "B", + 0x166D: "X", + 0x166E: "x", + 0x16B7: "X", + 0x16C1: "l", + 0x16D5: "K", + 0x16D6: "M", + 0x17E0: "o", + 0x1D04: "c", + 0x1D0F: "o", + 0x1D11: "o", + 0x1D1C: "u", + 0x1D20: "v", + 0x1D21: "w", + 0x1D22: "z", + 0x1D26: "r", + 0x1D6B: "ue", + 0x1D83: "g", + 0x1D8C: "y", + 0x1E9D: "f", + 0x1EFF: "y", + 0x1FBE: "i", + 0x2016: "ll", + 0x20A8: "Rs", + 0x20B6: "lt", + 0x2102: "C", + 0x210A: "g", + 0x210B: "H", + 0x210C: "H", + 0x210D: "H", + 0x210E: "h", + 0x2110: "l", + 0x2111: "l", + 0x2112: "L", + 0x2113: "l", + 0x2115: "N", + 0x2116: "No", + 0x2119: "P", + 0x211A: "Q", + 0x211B: "R", + 0x211C: "R", + 0x211D: "R", + 0x2121: "TEL", + 0x2124: "Z", + 0x2128: "Z", + 0x212A: "K", + 0x212C: "B", + 0x212D: "C", + 0x212E: "e", + 0x212F: "e", + 0x2130: "E", + 0x2131: "F", + 0x2133: "M", + 0x2134: "o", + 0x2139: "i", + 0x213B: "FAX", + 0x213D: "y", + 0x2145: "D", + 0x2146: "d", + 0x2147: "e", + 0x2148: "i", + 0x2149: "j", + 0x2160: "l", + 0x2161: "ll", + 0x2162: "lll", + 0x2163: "lV", + 0x2164: "V", + 0x2165: "Vl", + 0x2166: "Vll", + 0x2167: "Vlll", + 0x2168: "lX", + 0x2169: "X", + 0x216A: "Xl", + 0x216B: "Xll", + 0x216C: "L", + 0x216D: "C", + 0x216E: "D", + 0x216F: "M", + 0x2170: "i", + 0x2171: "ii", + 0x2172: "iii", + 0x2173: "iv", + 0x2174: "v", + 0x2175: "vi", + 0x2176: "vii", + 0x2177: "viii", + 0x2178: "ix", + 0x2179: "x", + 0x217A: "xi", + 0x217B: "xii", + 0x217C: "l", + 0x217D: "c", + 0x217E: "d", + 0x217F: "rn", + 0x221E: "oo", + 0x2223: "l", + 0x2225: "ll", + 0x2228: "v", + 0x222A: "U", + 0x22A4: "T", + 0x22C1: "v", + 0x22C3: "U", + 0x22FF: "E", + 0x2373: "i", + 0x2374: "p", + 0x237A: "a", + 0x23FD: "l", + 0x2573: "X", + 0x27D9: "T", + 0x292B: "x", + 0x292C: "x", + 0x2A2F: "x", + 0x2C82: "B", + 0x2C85: "r", + 0x2C8E: "H", + 0x2C92: "l", + 0x2C93: "i", + 0x2C94: "K", + 0x2C98: "M", + 0x2C9A: "N", + 0x2C9C: "3", + 0x2C9E: "O", + 0x2C9F: "o", + 0x2CA2: "P", + 0x2CA3: "p", + 0x2CA4: "C", + 0x2CA5: "c", + 0x2CA6: "T", + 0x2CA8: "Y", + 0x2CA9: "y", + 0x2CAC: "X", + 0x2CBD: "w", + 0x2CC4: "3", + 0x2CCA: "9", + 0x2CCB: "9", + 0x2CCC: "3", + 0x2CCE: "P", + 0x2CCF: "p", + 0x2CD0: "L", + 0x2CD2: "6", + 0x2CD3: "6", + 0x2CDC: "6", + 0x2D38: "V", + 0x2D39: "E", + 0x2D4F: "l", + 0x2D54: "O", + 0x2D55: "Q", + 0x2D5D: "X", + 0x3007: "O", + 0xA4D0: "B", + 0xA4D1: "P", + 0xA4D2: "d", + 0xA4D3: "D", + 0xA4D4: "T", + 0xA4D6: "G", + 0xA4D7: "K", + 0xA4D9: "J", + 0xA4DA: "C", + 0xA4DC: "Z", + 0xA4DD: "F", + 0xA4DF: "M", + 0xA4E0: "N", + 0xA4E1: "L", + 0xA4E2: "S", + 0xA4E3: "R", + 0xA4E6: "V", + 0xA4E7: "H", + 0xA4EA: "W", + 0xA4EB: "X", + 0xA4EC: "Y", + 0xA4EE: "A", + 0xA4F0: "E", + 0xA4F2: "l", + 0xA4F3: "O", + 0xA4F4: "U", + 0xA644: "2", + 0xA647: "i", + 0xA698: "OO", + 0xA699: "oo", + 0xA6DF: "V", + 0xA6EF: "2", + 0xA728: "T3", + 0xA731: "s", + 0xA732: "AA", + 0xA733: "aa", + 0xA734: "AO", + 0xA735: "ao", + 0xA736: "AU", + 0xA737: "au", + 0xA738: "AV", + 0xA739: "av", + 0xA73A: "AV", + 0xA73B: "av", + 0xA73C: "AY", + 0xA73D: "ay", + 0xA74E: "OO", + 0xA74F: "oo", + 0xA75A: "2", + 0xA76A: "3", + 0xA76E: "9", + 0xA777: "tf", + 0xA798: "F", + 0xA799: "f", + 0xA79F: "u", + 0xA7AB: "3", + 0xA7B2: "J", + 0xA7B3: "X", + 0xA7B4: "B", + 0xAB32: "e", + 0xAB35: "f", + 0xAB3D: "o", + 0xAB47: "r", + 0xAB48: "r", + 0xAB4E: "u", + 0xAB52: "u", + 0xAB5A: "y", + 0xAB63: "uo", + 0xAB75: "i", + 0xAB81: "r", + 0xAB83: "w", + 0xAB93: "z", + 0xABA9: "v", + 0xABAA: "s", + 0xABAF: "c", + 0xFB00: "ff", + 0xFB01: "fi", + 0xFB02: "fl", + 0xFB03: "ffi", + 0xFB04: "ffl", + 0xFB06: "st", + 0xFBA6: "o", + 0xFBA7: "o", + 0xFBA8: "o", + 0xFBA9: "o", + 0xFBAA: "o", + 0xFBAB: "o", + 0xFBAC: "o", + 0xFBAD: "o", + 0xFE8D: "l", + 0xFE8E: "l", + 0xFEE9: "o", + 0xFEEA: "o", + 0xFEEB: "o", + 0xFEEC: "o", + 0xFF21: "A", + 0xFF22: "B", + 0xFF23: "C", + 0xFF25: "E", + 0xFF28: "H", + 0xFF29: "l", + 0xFF2A: "J", + 0xFF2B: "K", + 0xFF2D: "M", + 0xFF2E: "N", + 0xFF2F: "O", + 0xFF30: "P", + 0xFF33: "S", + 0xFF34: "T", + 0xFF38: "X", + 0xFF39: "Y", + 0xFF3A: "Z", + 0xFF41: "a", + 0xFF43: "c", + 0xFF45: "e", + 0xFF47: "g", + 0xFF48: "h", + 0xFF49: "i", + 0xFF4A: "j", + 0xFF4C: "l", + 0xFF4F: "o", + 0xFF50: "p", + 0xFF53: "s", + 0xFF56: "v", + 0xFF58: "x", + 0xFF59: "y", + 0xFFE8: "l", + 0x10282: "B", + 0x10286: "E", + 0x10287: "F", + 0x1028A: "l", + 0x10290: "X", + 0x10292: "O", + 0x10295: "P", + 0x10296: "S", + 0x10297: "T", + 0x102A0: "A", + 0x102A1: "B", + 0x102A2: "C", + 0x102A5: "F", + 0x102AB: "O", + 0x102B0: "M", + 0x102B1: "T", + 0x102B2: "Y", + 0x102B4: "X", + 0x102CF: "H", + 0x102F5: "Z", + 0x10301: "B", + 0x10302: "C", + 0x10309: "l", + 0x10311: "M", + 0x10315: "T", + 0x10317: "X", + 0x1031A: "8", + 0x10320: "l", + 0x10322: "X", + 0x10404: "O", + 0x10415: "C", + 0x1041B: "L", + 0x10420: "S", + 0x1042C: "o", + 0x1043D: "c", + 0x10448: "s", + 0x104B4: "R", + 0x104C2: "O", + 0x104CE: "U", + 0x104D2: "7", + 0x104EA: "o", + 0x104F6: "u", + 0x10513: "N", + 0x10516: "O", + 0x10518: "K", + 0x1051C: "C", + 0x1051D: "V", + 0x10525: "F", + 0x10526: "L", + 0x10527: "X", + 0x114D0: "o", + 0x11700: "rn", + 0x11706: "v", + 0x1170A: "w", + 0x1170E: "w", + 0x1170F: "w", + 0x118A0: "V", + 0x118A2: "F", + 0x118A3: "L", + 0x118A4: "Y", + 0x118A6: "E", + 0x118A9: "Z", + 0x118AC: "9", + 0x118AE: "E", + 0x118AF: "4", + 0x118B2: "L", + 0x118B5: "O", + 0x118B8: "U", + 0x118BB: "5", + 0x118BC: "T", + 0x118C0: "v", + 0x118C1: "s", + 0x118C2: "F", + 0x118C3: "i", + 0x118C4: "z", + 0x118C6: "7", + 0x118C8: "o", + 0x118CA: "3", + 0x118CC: "9", + 0x118D5: "6", + 0x118D6: "9", + 0x118D7: "o", + 0x118D8: "u", + 0x118DC: "y", + 0x118E0: "O", + 0x118E3: "rn", + 0x118E5: "Z", + 0x118E6: "W", + 0x118E9: "C", + 0x118EC: "X", + 0x118EF: "W", + 0x118F2: "C", + 0x11DDA: "l", + 0x11DE0: "O", + 0x11DE1: "l", + 0x16EAA: "l", + 0x16EB6: "b", + 0x16F08: "V", + 0x16F0A: "T", + 0x16F16: "L", + 0x16F28: "l", + 0x16F35: "R", + 0x16F3A: "S", + 0x16F3B: "3", + 0x16F40: "A", + 0x16F42: "U", + 0x16F43: "Y", + 0x1CCD6: "A", + 0x1CCD7: "B", + 0x1CCD8: "C", + 0x1CCD9: "D", + 0x1CCDA: "E", + 0x1CCDB: "F", + 0x1CCDC: "G", + 0x1CCDD: "H", + 0x1CCDE: "l", + 0x1CCDF: "J", + 0x1CCE0: "K", + 0x1CCE1: "L", + 0x1CCE2: "M", + 0x1CCE3: "N", + 0x1CCE4: "O", + 0x1CCE5: "P", + 0x1CCE6: "Q", + 0x1CCE7: "R", + 0x1CCE8: "S", + 0x1CCE9: "T", + 0x1CCEA: "U", + 0x1CCEB: "V", + 0x1CCEC: "W", + 0x1CCED: "X", + 0x1CCEE: "Y", + 0x1CCEF: "Z", + 0x1CCF0: "O", + 0x1CCF1: "l", + 0x1CCF2: "2", + 0x1CCF3: "3", + 0x1CCF4: "4", + 0x1CCF5: "5", + 0x1CCF6: "6", + 0x1CCF7: "7", + 0x1CCF8: "8", + 0x1CCF9: "9", + 0x1D206: "3", + 0x1D20D: "V", + 0x1D212: "7", + 0x1D213: "F", + 0x1D216: "R", + 0x1D22A: "L", + 0x1D400: "A", + 0x1D401: "B", + 0x1D402: "C", + 0x1D403: "D", + 0x1D404: "E", + 0x1D405: "F", + 0x1D406: "G", + 0x1D407: "H", + 0x1D408: "l", + 0x1D409: "J", + 0x1D40A: "K", + 0x1D40B: "L", + 0x1D40C: "M", + 0x1D40D: "N", + 0x1D40E: "O", + 0x1D40F: "P", + 0x1D410: "Q", + 0x1D411: "R", + 0x1D412: "S", + 0x1D413: "T", + 0x1D414: "U", + 0x1D415: "V", + 0x1D416: "W", + 0x1D417: "X", + 0x1D418: "Y", + 0x1D419: "Z", + 0x1D41A: "a", + 0x1D41B: "b", + 0x1D41C: "c", + 0x1D41D: "d", + 0x1D41E: "e", + 0x1D41F: "f", + 0x1D420: "g", + 0x1D421: "h", + 0x1D422: "i", + 0x1D423: "j", + 0x1D424: "k", + 0x1D425: "l", + 0x1D426: "rn", + 0x1D427: "n", + 0x1D428: "o", + 0x1D429: "p", + 0x1D42A: "q", + 0x1D42B: "r", + 0x1D42C: "s", + 0x1D42D: "t", + 0x1D42E: "u", + 0x1D42F: "v", + 0x1D430: "w", + 0x1D431: "x", + 0x1D432: "y", + 0x1D433: "z", + 0x1D434: "A", + 0x1D435: "B", + 0x1D436: "C", + 0x1D437: "D", + 0x1D438: "E", + 0x1D439: "F", + 0x1D43A: "G", + 0x1D43B: "H", + 0x1D43C: "l", + 0x1D43D: "J", + 0x1D43E: "K", + 0x1D43F: "L", + 0x1D440: "M", + 0x1D441: "N", + 0x1D442: "O", + 0x1D443: "P", + 0x1D444: "Q", + 0x1D445: "R", + 0x1D446: "S", + 0x1D447: "T", + 0x1D448: "U", + 0x1D449: "V", + 0x1D44A: "W", + 0x1D44B: "X", + 0x1D44C: "Y", + 0x1D44D: "Z", + 0x1D44E: "a", + 0x1D44F: "b", + 0x1D450: "c", + 0x1D451: "d", + 0x1D452: "e", + 0x1D453: "f", + 0x1D454: "g", + 0x1D456: "i", + 0x1D457: "j", + 0x1D458: "k", + 0x1D459: "l", + 0x1D45A: "rn", + 0x1D45B: "n", + 0x1D45C: "o", + 0x1D45D: "p", + 0x1D45E: "q", + 0x1D45F: "r", + 0x1D460: "s", + 0x1D461: "t", + 0x1D462: "u", + 0x1D463: "v", + 0x1D464: "w", + 0x1D465: "x", + 0x1D466: "y", + 0x1D467: "z", + 0x1D468: "A", + 0x1D469: "B", + 0x1D46A: "C", + 0x1D46B: "D", + 0x1D46C: "E", + 0x1D46D: "F", + 0x1D46E: "G", + 0x1D46F: "H", + 0x1D470: "l", + 0x1D471: "J", + 0x1D472: "K", + 0x1D473: "L", + 0x1D474: "M", + 0x1D475: "N", + 0x1D476: "O", + 0x1D477: "P", + 0x1D478: "Q", + 0x1D479: "R", + 0x1D47A: "S", + 0x1D47B: "T", + 0x1D47C: "U", + 0x1D47D: "V", + 0x1D47E: "W", + 0x1D47F: "X", + 0x1D480: "Y", + 0x1D481: "Z", + 0x1D482: "a", + 0x1D483: "b", + 0x1D484: "c", + 0x1D485: "d", + 0x1D486: "e", + 0x1D487: "f", + 0x1D488: "g", + 0x1D489: "h", + 0x1D48A: "i", + 0x1D48B: "j", + 0x1D48C: "k", + 0x1D48D: "l", + 0x1D48E: "rn", + 0x1D48F: "n", + 0x1D490: "o", + 0x1D491: "p", + 0x1D492: "q", + 0x1D493: "r", + 0x1D494: "s", + 0x1D495: "t", + 0x1D496: "u", + 0x1D497: "v", + 0x1D498: "w", + 0x1D499: "x", + 0x1D49A: "y", + 0x1D49B: "z", + 0x1D49C: "A", + 0x1D49E: "C", + 0x1D49F: "D", + 0x1D4A2: "G", + 0x1D4A5: "J", + 0x1D4A6: "K", + 0x1D4A9: "N", + 0x1D4AA: "O", + 0x1D4AB: "P", + 0x1D4AC: "Q", + 0x1D4AE: "S", + 0x1D4AF: "T", + 0x1D4B0: "U", + 0x1D4B1: "V", + 0x1D4B2: "W", + 0x1D4B3: "X", + 0x1D4B4: "Y", + 0x1D4B5: "Z", + 0x1D4B6: "a", + 0x1D4B7: "b", + 0x1D4B8: "c", + 0x1D4B9: "d", + 0x1D4BB: "f", + 0x1D4BD: "h", + 0x1D4BE: "i", + 0x1D4BF: "j", + 0x1D4C0: "k", + 0x1D4C1: "l", + 0x1D4C2: "rn", + 0x1D4C3: "n", + 0x1D4C5: "p", + 0x1D4C6: "q", + 0x1D4C7: "r", + 0x1D4C8: "s", + 0x1D4C9: "t", + 0x1D4CA: "u", + 0x1D4CB: "v", + 0x1D4CC: "w", + 0x1D4CD: "x", + 0x1D4CE: "y", + 0x1D4CF: "z", + 0x1D4D0: "A", + 0x1D4D1: "B", + 0x1D4D2: "C", + 0x1D4D3: "D", + 0x1D4D4: "E", + 0x1D4D5: "F", + 0x1D4D6: "G", + 0x1D4D7: "H", + 0x1D4D8: "l", + 0x1D4D9: "J", + 0x1D4DA: "K", + 0x1D4DB: "L", + 0x1D4DC: "M", + 0x1D4DD: "N", + 0x1D4DE: "O", + 0x1D4DF: "P", + 0x1D4E0: "Q", + 0x1D4E1: "R", + 0x1D4E2: "S", + 0x1D4E3: "T", + 0x1D4E4: "U", + 0x1D4E5: "V", + 0x1D4E6: "W", + 0x1D4E7: "X", + 0x1D4E8: "Y", + 0x1D4E9: "Z", + 0x1D4EA: "a", + 0x1D4EB: "b", + 0x1D4EC: "c", + 0x1D4ED: "d", + 0x1D4EE: "e", + 0x1D4EF: "f", + 0x1D4F0: "g", + 0x1D4F1: "h", + 0x1D4F2: "i", + 0x1D4F3: "j", + 0x1D4F4: "k", + 0x1D4F5: "l", + 0x1D4F6: "rn", + 0x1D4F7: "n", + 0x1D4F8: "o", + 0x1D4F9: "p", + 0x1D4FA: "q", + 0x1D4FB: "r", + 0x1D4FC: "s", + 0x1D4FD: "t", + 0x1D4FE: "u", + 0x1D4FF: "v", + 0x1D500: "w", + 0x1D501: "x", + 0x1D502: "y", + 0x1D503: "z", + 0x1D504: "A", + 0x1D505: "B", + 0x1D507: "D", + 0x1D508: "E", + 0x1D509: "F", + 0x1D50A: "G", + 0x1D50D: "J", + 0x1D50E: "K", + 0x1D50F: "L", + 0x1D510: "M", + 0x1D511: "N", + 0x1D512: "O", + 0x1D513: "P", + 0x1D514: "Q", + 0x1D516: "S", + 0x1D517: "T", + 0x1D518: "U", + 0x1D519: "V", + 0x1D51A: "W", + 0x1D51B: "X", + 0x1D51C: "Y", + 0x1D51E: "a", + 0x1D51F: "b", + 0x1D520: "c", + 0x1D521: "d", + 0x1D522: "e", + 0x1D523: "f", + 0x1D524: "g", + 0x1D525: "h", + 0x1D526: "i", + 0x1D527: "j", + 0x1D528: "k", + 0x1D529: "l", + 0x1D52A: "rn", + 0x1D52B: "n", + 0x1D52C: "o", + 0x1D52D: "p", + 0x1D52E: "q", + 0x1D52F: "r", + 0x1D530: "s", + 0x1D531: "t", + 0x1D532: "u", + 0x1D533: "v", + 0x1D534: "w", + 0x1D535: "x", + 0x1D536: "y", + 0x1D537: "z", + 0x1D538: "A", + 0x1D539: "B", + 0x1D53B: "D", + 0x1D53C: "E", + 0x1D53D: "F", + 0x1D53E: "G", + 0x1D540: "l", + 0x1D541: "J", + 0x1D542: "K", + 0x1D543: "L", + 0x1D544: "M", + 0x1D546: "O", + 0x1D54A: "S", + 0x1D54B: "T", + 0x1D54C: "U", + 0x1D54D: "V", + 0x1D54E: "W", + 0x1D54F: "X", + 0x1D550: "Y", + 0x1D552: "a", + 0x1D553: "b", + 0x1D554: "c", + 0x1D555: "d", + 0x1D556: "e", + 0x1D557: "f", + 0x1D558: "g", + 0x1D559: "h", + 0x1D55A: "i", + 0x1D55B: "j", + 0x1D55C: "k", + 0x1D55D: "l", + 0x1D55E: "rn", + 0x1D55F: "n", + 0x1D560: "o", + 0x1D561: "p", + 0x1D562: "q", + 0x1D563: "r", + 0x1D564: "s", + 0x1D565: "t", + 0x1D566: "u", + 0x1D567: "v", + 0x1D568: "w", + 0x1D569: "x", + 0x1D56A: "y", + 0x1D56B: "z", + 0x1D56C: "A", + 0x1D56D: "B", + 0x1D56E: "C", + 0x1D56F: "D", + 0x1D570: "E", + 0x1D571: "F", + 0x1D572: "G", + 0x1D573: "H", + 0x1D574: "l", + 0x1D575: "J", + 0x1D576: "K", + 0x1D577: "L", + 0x1D578: "M", + 0x1D579: "N", + 0x1D57A: "O", + 0x1D57B: "P", + 0x1D57C: "Q", + 0x1D57D: "R", + 0x1D57E: "S", + 0x1D57F: "T", + 0x1D580: "U", + 0x1D581: "V", + 0x1D582: "W", + 0x1D583: "X", + 0x1D584: "Y", + 0x1D585: "Z", + 0x1D586: "a", + 0x1D587: "b", + 0x1D588: "c", + 0x1D589: "d", + 0x1D58A: "e", + 0x1D58B: "f", + 0x1D58C: "g", + 0x1D58D: "h", + 0x1D58E: "i", + 0x1D58F: "j", + 0x1D590: "k", + 0x1D591: "l", + 0x1D592: "rn", + 0x1D593: "n", + 0x1D594: "o", + 0x1D595: "p", + 0x1D596: "q", + 0x1D597: "r", + 0x1D598: "s", + 0x1D599: "t", + 0x1D59A: "u", + 0x1D59B: "v", + 0x1D59C: "w", + 0x1D59D: "x", + 0x1D59E: "y", + 0x1D59F: "z", + 0x1D5A0: "A", + 0x1D5A1: "B", + 0x1D5A2: "C", + 0x1D5A3: "D", + 0x1D5A4: "E", + 0x1D5A5: "F", + 0x1D5A6: "G", + 0x1D5A7: "H", + 0x1D5A8: "l", + 0x1D5A9: "J", + 0x1D5AA: "K", + 0x1D5AB: "L", + 0x1D5AC: "M", + 0x1D5AD: "N", + 0x1D5AE: "O", + 0x1D5AF: "P", + 0x1D5B0: "Q", + 0x1D5B1: "R", + 0x1D5B2: "S", + 0x1D5B3: "T", + 0x1D5B4: "U", + 0x1D5B5: "V", + 0x1D5B6: "W", + 0x1D5B7: "X", + 0x1D5B8: "Y", + 0x1D5B9: "Z", + 0x1D5BA: "a", + 0x1D5BB: "b", + 0x1D5BC: "c", + 0x1D5BD: "d", + 0x1D5BE: "e", + 0x1D5BF: "f", + 0x1D5C0: "g", + 0x1D5C1: "h", + 0x1D5C2: "i", + 0x1D5C3: "j", + 0x1D5C4: "k", + 0x1D5C5: "l", + 0x1D5C6: "rn", + 0x1D5C7: "n", + 0x1D5C8: "o", + 0x1D5C9: "p", + 0x1D5CA: "q", + 0x1D5CB: "r", + 0x1D5CC: "s", + 0x1D5CD: "t", + 0x1D5CE: "u", + 0x1D5CF: "v", + 0x1D5D0: "w", + 0x1D5D1: "x", + 0x1D5D2: "y", + 0x1D5D3: "z", + 0x1D5D4: "A", + 0x1D5D5: "B", + 0x1D5D6: "C", + 0x1D5D7: "D", + 0x1D5D8: "E", + 0x1D5D9: "F", + 0x1D5DA: "G", + 0x1D5DB: "H", + 0x1D5DC: "l", + 0x1D5DD: "J", + 0x1D5DE: "K", + 0x1D5DF: "L", + 0x1D5E0: "M", + 0x1D5E1: "N", + 0x1D5E2: "O", + 0x1D5E3: "P", + 0x1D5E4: "Q", + 0x1D5E5: "R", + 0x1D5E6: "S", + 0x1D5E7: "T", + 0x1D5E8: "U", + 0x1D5E9: "V", + 0x1D5EA: "W", + 0x1D5EB: "X", + 0x1D5EC: "Y", + 0x1D5ED: "Z", + 0x1D5EE: "a", + 0x1D5EF: "b", + 0x1D5F0: "c", + 0x1D5F1: "d", + 0x1D5F2: "e", + 0x1D5F3: "f", + 0x1D5F4: "g", + 0x1D5F5: "h", + 0x1D5F6: "i", + 0x1D5F7: "j", + 0x1D5F8: "k", + 0x1D5F9: "l", + 0x1D5FA: "rn", + 0x1D5FB: "n", + 0x1D5FC: "o", + 0x1D5FD: "p", + 0x1D5FE: "q", + 0x1D5FF: "r", + 0x1D600: "s", + 0x1D601: "t", + 0x1D602: "u", + 0x1D603: "v", + 0x1D604: "w", + 0x1D605: "x", + 0x1D606: "y", + 0x1D607: "z", + 0x1D608: "A", + 0x1D609: "B", + 0x1D60A: "C", + 0x1D60B: "D", + 0x1D60C: "E", + 0x1D60D: "F", + 0x1D60E: "G", + 0x1D60F: "H", + 0x1D610: "l", + 0x1D611: "J", + 0x1D612: "K", + 0x1D613: "L", + 0x1D614: "M", + 0x1D615: "N", + 0x1D616: "O", + 0x1D617: "P", + 0x1D618: "Q", + 0x1D619: "R", + 0x1D61A: "S", + 0x1D61B: "T", + 0x1D61C: "U", + 0x1D61D: "V", + 0x1D61E: "W", + 0x1D61F: "X", + 0x1D620: "Y", + 0x1D621: "Z", + 0x1D622: "a", + 0x1D623: "b", + 0x1D624: "c", + 0x1D625: "d", + 0x1D626: "e", + 0x1D627: "f", + 0x1D628: "g", + 0x1D629: "h", + 0x1D62A: "i", + 0x1D62B: "j", + 0x1D62C: "k", + 0x1D62D: "l", + 0x1D62E: "rn", + 0x1D62F: "n", + 0x1D630: "o", + 0x1D631: "p", + 0x1D632: "q", + 0x1D633: "r", + 0x1D634: "s", + 0x1D635: "t", + 0x1D636: "u", + 0x1D637: "v", + 0x1D638: "w", + 0x1D639: "x", + 0x1D63A: "y", + 0x1D63B: "z", + 0x1D63C: "A", + 0x1D63D: "B", + 0x1D63E: "C", + 0x1D63F: "D", + 0x1D640: "E", + 0x1D641: "F", + 0x1D642: "G", + 0x1D643: "H", + 0x1D644: "l", + 0x1D645: "J", + 0x1D646: "K", + 0x1D647: "L", + 0x1D648: "M", + 0x1D649: "N", + 0x1D64A: "O", + 0x1D64B: "P", + 0x1D64C: "Q", + 0x1D64D: "R", + 0x1D64E: "S", + 0x1D64F: "T", + 0x1D650: "U", + 0x1D651: "V", + 0x1D652: "W", + 0x1D653: "X", + 0x1D654: "Y", + 0x1D655: "Z", + 0x1D656: "a", + 0x1D657: "b", + 0x1D658: "c", + 0x1D659: "d", + 0x1D65A: "e", + 0x1D65B: "f", + 0x1D65C: "g", + 0x1D65D: "h", + 0x1D65E: "i", + 0x1D65F: "j", + 0x1D660: "k", + 0x1D661: "l", + 0x1D662: "rn", + 0x1D663: "n", + 0x1D664: "o", + 0x1D665: "p", + 0x1D666: "q", + 0x1D667: "r", + 0x1D668: "s", + 0x1D669: "t", + 0x1D66A: "u", + 0x1D66B: "v", + 0x1D66C: "w", + 0x1D66D: "x", + 0x1D66E: "y", + 0x1D66F: "z", + 0x1D670: "A", + 0x1D671: "B", + 0x1D672: "C", + 0x1D673: "D", + 0x1D674: "E", + 0x1D675: "F", + 0x1D676: "G", + 0x1D677: "H", + 0x1D678: "l", + 0x1D679: "J", + 0x1D67A: "K", + 0x1D67B: "L", + 0x1D67C: "M", + 0x1D67D: "N", + 0x1D67E: "O", + 0x1D67F: "P", + 0x1D680: "Q", + 0x1D681: "R", + 0x1D682: "S", + 0x1D683: "T", + 0x1D684: "U", + 0x1D685: "V", + 0x1D686: "W", + 0x1D687: "X", + 0x1D688: "Y", + 0x1D689: "Z", + 0x1D68A: "a", + 0x1D68B: "b", + 0x1D68C: "c", + 0x1D68D: "d", + 0x1D68E: "e", + 0x1D68F: "f", + 0x1D690: "g", + 0x1D691: "h", + 0x1D692: "i", + 0x1D693: "j", + 0x1D694: "k", + 0x1D695: "l", + 0x1D696: "rn", + 0x1D697: "n", + 0x1D698: "o", + 0x1D699: "p", + 0x1D69A: "q", + 0x1D69B: "r", + 0x1D69C: "s", + 0x1D69D: "t", + 0x1D69E: "u", + 0x1D69F: "v", + 0x1D6A0: "w", + 0x1D6A1: "x", + 0x1D6A2: "y", + 0x1D6A3: "z", + 0x1D6A4: "i", + 0x1D6A8: "A", + 0x1D6A9: "B", + 0x1D6AC: "E", + 0x1D6AD: "Z", + 0x1D6AE: "H", + 0x1D6B0: "l", + 0x1D6B1: "K", + 0x1D6B3: "M", + 0x1D6B4: "N", + 0x1D6B6: "O", + 0x1D6B8: "P", + 0x1D6BB: "T", + 0x1D6BC: "Y", + 0x1D6BE: "X", + 0x1D6C2: "a", + 0x1D6C4: "y", + 0x1D6CA: "i", + 0x1D6CE: "v", + 0x1D6D0: "o", + 0x1D6D2: "p", + 0x1D6D4: "o", + 0x1D6D6: "u", + 0x1D6E0: "p", + 0x1D6E2: "A", + 0x1D6E3: "B", + 0x1D6E6: "E", + 0x1D6E7: "Z", + 0x1D6E8: "H", + 0x1D6EA: "l", + 0x1D6EB: "K", + 0x1D6ED: "M", + 0x1D6EE: "N", + 0x1D6F0: "O", + 0x1D6F2: "P", + 0x1D6F5: "T", + 0x1D6F6: "Y", + 0x1D6F8: "X", + 0x1D6FC: "a", + 0x1D6FE: "y", + 0x1D704: "i", + 0x1D708: "v", + 0x1D70A: "o", + 0x1D70C: "p", + 0x1D70E: "o", + 0x1D710: "u", + 0x1D71A: "p", + 0x1D71C: "A", + 0x1D71D: "B", + 0x1D720: "E", + 0x1D721: "Z", + 0x1D722: "H", + 0x1D724: "l", + 0x1D725: "K", + 0x1D727: "M", + 0x1D728: "N", + 0x1D72A: "O", + 0x1D72C: "P", + 0x1D72F: "T", + 0x1D730: "Y", + 0x1D732: "X", + 0x1D736: "a", + 0x1D738: "y", + 0x1D73E: "i", + 0x1D742: "v", + 0x1D744: "o", + 0x1D746: "p", + 0x1D748: "o", + 0x1D74A: "u", + 0x1D754: "p", + 0x1D756: "A", + 0x1D757: "B", + 0x1D75A: "E", + 0x1D75B: "Z", + 0x1D75C: "H", + 0x1D75E: "l", + 0x1D75F: "K", + 0x1D761: "M", + 0x1D762: "N", + 0x1D764: "O", + 0x1D766: "P", + 0x1D769: "T", + 0x1D76A: "Y", + 0x1D76C: "X", + 0x1D770: "a", + 0x1D772: "y", + 0x1D778: "i", + 0x1D77C: "v", + 0x1D77E: "o", + 0x1D780: "p", + 0x1D782: "o", + 0x1D784: "u", + 0x1D78E: "p", + 0x1D790: "A", + 0x1D791: "B", + 0x1D794: "E", + 0x1D795: "Z", + 0x1D796: "H", + 0x1D798: "l", + 0x1D799: "K", + 0x1D79B: "M", + 0x1D79C: "N", + 0x1D79E: "O", + 0x1D7A0: "P", + 0x1D7A3: "T", + 0x1D7A4: "Y", + 0x1D7A6: "X", + 0x1D7AA: "a", + 0x1D7AC: "y", + 0x1D7B2: "i", + 0x1D7B6: "v", + 0x1D7B8: "o", + 0x1D7BA: "p", + 0x1D7BC: "o", + 0x1D7BE: "u", + 0x1D7C8: "p", + 0x1D7CA: "F", + 0x1D7CE: "O", + 0x1D7CF: "l", + 0x1D7D0: "2", + 0x1D7D1: "3", + 0x1D7D2: "4", + 0x1D7D3: "5", + 0x1D7D4: "6", + 0x1D7D5: "7", + 0x1D7D6: "8", + 0x1D7D7: "9", + 0x1D7D8: "O", + 0x1D7D9: "l", + 0x1D7DA: "2", + 0x1D7DB: "3", + 0x1D7DC: "4", + 0x1D7DD: "5", + 0x1D7DE: "6", + 0x1D7DF: "7", + 0x1D7E0: "8", + 0x1D7E1: "9", + 0x1D7E2: "O", + 0x1D7E3: "l", + 0x1D7E4: "2", + 0x1D7E5: "3", + 0x1D7E6: "4", + 0x1D7E7: "5", + 0x1D7E8: "6", + 0x1D7E9: "7", + 0x1D7EA: "8", + 0x1D7EB: "9", + 0x1D7EC: "O", + 0x1D7ED: "l", + 0x1D7EE: "2", + 0x1D7EF: "3", + 0x1D7F0: "4", + 0x1D7F1: "5", + 0x1D7F2: "6", + 0x1D7F3: "7", + 0x1D7F4: "8", + 0x1D7F5: "9", + 0x1D7F6: "O", + 0x1D7F7: "l", + 0x1D7F8: "2", + 0x1D7F9: "3", + 0x1D7FA: "4", + 0x1D7FB: "5", + 0x1D7FC: "6", + 0x1D7FD: "7", + 0x1D7FE: "8", + 0x1D7FF: "9", + 0x1E8C7: "l", + 0x1E8CB: "8", + 0x1EE00: "l", + 0x1EE24: "o", + 0x1EE64: "o", + 0x1EE80: "l", + 0x1EE84: "o", + 0x1F700: "QE", + 0x1F707: "AR", + 0x1F74C: "C", + 0x1F75C: "sss", + 0x1F768: "T", + 0x1F76B: "MB", + 0x1F76C: "VB", + 0x1FBF0: "O", + 0x1FBF1: "l", + 0x1FBF2: "2", + 0x1FBF3: "3", + 0x1FBF4: "4", + 0x1FBF5: "5", + 0x1FBF6: "6", + 0x1FBF7: "7", + 0x1FBF8: "8", + 0x1FBF9: "9", +} diff --git a/tests/nodes/analyzers/test_binary_and_pe3_filtering.py b/tests/nodes/analyzers/test_binary_and_pe3_filtering.py index 573a679d9..4af869216 100644 --- a/tests/nodes/analyzers/test_binary_and_pe3_filtering.py +++ b/tests/nodes/analyzers/test_binary_and_pe3_filtering.py @@ -47,16 +47,16 @@ class TestBinaryFileDetection: """Binary files are correctly identified and skipped.""" def test_pdf_extension_detected(self) -> None: - assert _is_binary_file("report.pdf", "some content") is True + assert _is_binary_file("report.pdf", "some content") is False def test_png_extension_detected(self) -> None: - assert _is_binary_file("image.png", "fake data") is True + assert _is_binary_file("image.png", "fake data") is False def test_zip_extension_detected(self) -> None: - assert _is_binary_file("archive.zip", "PK\x03\x04") is True + assert _is_binary_file("archive.zip", "PK\x03\x04") is False def test_exe_extension_detected(self) -> None: - assert _is_binary_file("tool.exe", "MZ") is True + assert _is_binary_file("tool.exe", "MZ") is False def test_markdown_not_binary(self) -> None: assert _is_binary_file("README.md", "# Hello\n") is False @@ -72,8 +72,8 @@ def test_no_null_byte_not_binary(self) -> None: assert _is_binary_file("unknownfile", "normal text content") is False def test_case_insensitive_extension(self) -> None: - assert _is_binary_file("photo.JPEG", "data") is True - assert _is_binary_file("archive.ZIP", "PK") is True + assert _is_binary_file("photo.JPEG", "data") is False + assert _is_binary_file("archive.ZIP", "PK") is False def test_svg_not_treated_as_binary(self) -> None: """SVG is text/XML and can carry