Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
15 changes: 4 additions & 11 deletions ceres/analyzers/eval/safety_config.py
Original file line number Diff line number Diff line change
Expand Up @@ -134,28 +134,21 @@ def _scan_python_file(path: Path, ctx: AnalyzerContext) -> list[Finding]:
rel = ctx.rel(path)
findings: list[Finding] = []

class Visitor(ast.NodeVisitor):
def visit_Assign(self, node: ast.Assign) -> None:
for node in ast.walk(tree):
if isinstance(node, ast.Assign):
for target in node.targets:
key = _target_name(target)
if key:
_check_key_value(key, _literal(node.value), rel, [key], node.lineno, findings, ctx)
self.generic_visit(node)

def visit_AnnAssign(self, node: ast.AnnAssign) -> None:
elif isinstance(node, ast.AnnAssign):
key = _target_name(node.target)
if key:
_check_key_value(key, _literal(node.value), rel, [key], node.lineno, findings, ctx)
self.generic_visit(node)

def visit_Dict(self, node: ast.Dict) -> None:
elif isinstance(node, ast.Dict):
for key_node, value_node in zip(node.keys, node.values):
key = _literal(key_node)
if isinstance(key, str):
_check_key_value(key, _literal(value_node), rel, [key], getattr(node, "lineno", None), findings, ctx)
self.generic_visit(node)

Visitor().visit(tree)
return findings


Expand Down
10 changes: 5 additions & 5 deletions ceres/analyzers/model/pickle_static.py
Original file line number Diff line number Diff line change
@@ -1,8 +1,8 @@
from __future__ import annotations

import pickle
import pickletools
import zipfile
import pickle
from dataclasses import dataclass
from pathlib import Path

Expand Down Expand Up @@ -62,7 +62,7 @@ def scan_pickle_bytes(data: bytes) -> PickleScanResult:
count = 0
truncated = False
try:
for op, arg, _pos in pickletools.genops(data[:_MAX_BYTES]):
for op, arg, _ in pickletools.genops(data[:_MAX_BYTES]):
count += 1
if count > _MAX_OPS:
truncated = True
Expand All @@ -76,17 +76,17 @@ def scan_pickle_bytes(data: bytes) -> PickleScanResult:
continue
ref = arg if isinstance(arg, str) else " ".join(arg) if arg else ""
ref_norm = ref.replace(" ", ".")
if _matches_any(ref_norm, _SUSPICIOUS_GLOBALS):
if _matches_any(ref_norm):
suspicious.append(ref_norm)
if name in {"INST", "OBJ"}:
if isinstance(arg, str) and _matches_any(arg.replace(" ", "."), _SUSPICIOUS_GLOBALS):
if isinstance(arg, str) and _matches_any(arg.replace(" ", ".")):
suspicious.append(arg)
except Exception as e: # noqa: BLE001
return PickleScanResult(suspicious, has_reduce, count, truncated, error=str(e))
return PickleScanResult(suspicious, has_reduce, count, truncated)


def _matches_any(ref: str, fragments: set[str]) -> bool:
def _matches_any(ref: str) -> bool:
if ref in _SUSPICIOUS_EXACT:
return True
return any(ref.startswith(prefix) for prefix in _SUSPICIOUS_PREFIXES)
Expand Down
17 changes: 17 additions & 0 deletions tests/test_correctness_regressions.py
Original file line number Diff line number Diff line change
Expand Up @@ -139,3 +139,20 @@ def test_root_temperature_without_ai_context_is_not_flagged(tmp_path):
cfg.write_text("temperature: 1.4\n")
findings, _suppressed, _counts, _passed, _inv = run_scan(repo, Policy(), None, None)
assert "ceres.eval.generation_temperature_high" in _rule_ids(findings)


def test_eval_safety_python_assignments_are_scanned(tmp_path):
repo = tmp_path / "repo"
src = repo / "src"
src.mkdir(parents=True)
(src / "eval_config.py").write_text(
"skip_safety_eval = True\n"
"enable_content_filter: bool = False\n"
"generation_config = {'temperature': 1.4}\n"
)

findings, _suppressed, _counts, _passed, _inv = run_scan(repo, Policy(), None, None)
rule_ids = _rule_ids(findings)
assert "ceres.eval.safety_eval_disabled" in rule_ids
assert "ceres.eval.safety_filter_disabled" in rule_ids
assert "ceres.eval.generation_temperature_high" in rule_ids
Loading