diff --git a/sdk/typescript/_bundled_plugin/scripts/workbench_cli.py b/sdk/typescript/_bundled_plugin/scripts/workbench_cli.py index 2fc1c8f8..db48da90 100644 --- a/sdk/typescript/_bundled_plugin/scripts/workbench_cli.py +++ b/sdk/typescript/_bundled_plugin/scripts/workbench_cli.py @@ -163,7 +163,9 @@ def parse_args(description: str) -> argparse.Namespace: save_scan_comparison = subparsers.add_parser("save-scan-comparison") save_scan_comparison.add_argument("--before-scan-id", required=True) save_scan_comparison.add_argument("--after-scan-id", required=True) - save_scan_comparison.add_argument("--matches-json", required=True) + matches_transport = save_scan_comparison.add_mutually_exclusive_group(required=True) + matches_transport.add_argument("--matches-json") + matches_transport.add_argument("--matches-json-stdin", action="store_true") list_global_findings = subparsers.add_parser("list-global-findings") list_global_findings.add_argument("--query") @@ -315,7 +317,13 @@ def parse_args(description: str) -> argparse.Namespace: parser.error("pass exactly one user-context transport") index = arguments.index("--user-context-stdin") arguments[index : index + 1] = ["--user-context", sys.stdin.read()] - return parser.parse_args(arguments) + parsed = parser.parse_args(arguments) + if getattr(parsed, "matches_json_stdin", False): + try: + parsed.matches_json = sys.stdin.buffer.read().decode("utf-8") + except UnicodeDecodeError: + parser.error("--matches-json-stdin requires UTF-8 JSON") + return parsed def non_negative_int(value: str) -> int: diff --git a/sdk/typescript/_bundled_plugin/scripts/workbench_db.py b/sdk/typescript/_bundled_plugin/scripts/workbench_db.py index 5584d3a1..7935deff 100644 --- a/sdk/typescript/_bundled_plugin/scripts/workbench_db.py +++ b/sdk/typescript/_bundled_plugin/scripts/workbench_db.py @@ -3203,9 +3203,15 @@ def list_findings(connection: sqlite3.Connection, args: argparse.Namespace) -> d values, ).fetchone()[0] next_offset = args.offset + len(rows) + relations = scan_history.finding_relations( + connection, scan["id"], (row["id"] for row in rows) + ) return { "findingsPage": { - "findings": [finding_result(connection, scan, row) for row in rows], + "findings": [ + finding_result(connection, scan, row, related=relations.get(row["id"], [])) + for row in rows + ], "limit": limit, "nextOffset": next_offset if next_offset < total else None, "offset": args.offset, @@ -3297,6 +3303,9 @@ def scan_result( "completed": independent_reviews["completed"], "consolidating": independent_reviews["consolidating"], } + relations = scan_history.finding_relations( + connection, scan["id"], (row["id"] for row in occurrence_rows) + ) return { "artifacts": artifacts, "canceledAt": scan["canceled_at"], @@ -3304,7 +3313,10 @@ def scan_result( "contract": scan_contract(scan), "continuationThreadId": scan["continuation_thread_id"], "failureMessage": scan["failure_message"], - "findings": [finding_result(connection, scan, row) for row in occurrence_rows], + "findings": [ + finding_result(connection, scan, row, related=relations.get(row["id"], [])) + for row in occurrence_rows + ], "findingCount": finding_count, "findingsTruncated": finding_count > len(occurrence_rows), "severityCounts": severity_counts, @@ -3449,6 +3461,8 @@ def finding_result( connection: sqlite3.Connection, scan: sqlite3.Row, occurrence: sqlite3.Row, + *, + related: list[dict[str, Any]], ) -> dict[str, Any]: details = bounded_finding_details(read_finding_details(occurrence["details_json"])) confidence = details.get("confidence") @@ -3513,6 +3527,8 @@ def finding_result( result["matches"] = matches result["knownSince"] = known_since result["knownScanIds"] = known_scan_ids + if related: + result["related"] = related result.pop("artifactPaths", None) source_excerpt = finding_source_excerpt(scan, target, locations) if source_excerpt: diff --git a/sdk/typescript/_bundled_plugin/scripts/workbench_scan_history.py b/sdk/typescript/_bundled_plugin/scripts/workbench_scan_history.py index abf18b62..25d2fc4c 100644 --- a/sdk/typescript/_bundled_plugin/scripts/workbench_scan_history.py +++ b/sdk/typescript/_bundled_plugin/scripts/workbench_scan_history.py @@ -6,6 +6,8 @@ import os import sqlite3 import sys +from collections.abc import Iterable, Iterator +from itertools import chain from pathlib import Path, PurePosixPath from typing import Any, Callable from urllib.parse import urlsplit @@ -28,10 +30,12 @@ def _same_repository( *, after_identity: tuple[str | None, tuple[str, str] | None] | None = None, ) -> bool: - if before["target_id"] == after["target_id"]: + if before["target_id"] is not None and before["target_id"] == after["target_id"]: return True before_target = Path(before["target_path"]) after_target = Path(after["target_path"]) + if before_target.resolve() == after_target.resolve(): + return True before_git_dir = git_output( before_target, "rev-parse", "--path-format=absolute", "--git-common-dir" ) @@ -271,6 +275,7 @@ def list_unmatched_scan_pairs( batches = [] skipped = 0 matching_findings: dict[str, list[dict[str, Any]]] = {} + known_links: list[sqlite3.Row] | None = None for index, after in enumerate(available): previous = [ before @@ -280,6 +285,12 @@ def list_unmatched_scan_pairs( skipped += index - len(previous) if not previous: continue + if known_links is None: + known_links = ( + [] + if args.force + else _saved_finding_links(connection, {scan["id"] for scan in selected}) + ) for scan in (*previous, after): if scan["id"] not in matching_findings: backfill_finding_details(connection, scan) @@ -287,6 +298,14 @@ def list_unmatched_scan_pairs( _matching_input(row) for row in _scan_findings(connection, scan["id"]).values() ] + known_groups = _known_finding_groups( + known_links, + { + scan["id"] + for scan in selected + if (scan["started_at"], scan["id"]) <= (after["started_at"], after["id"]) + }, + ) batches.append( { "afterFindings": matching_findings[after["id"]], @@ -298,6 +317,7 @@ def list_unmatched_scan_pairs( } for before in previous ], + **({"knownFindingGroups": known_groups} if known_groups else {}), } ) return { @@ -309,6 +329,58 @@ def list_unmatched_scan_pairs( } +def _saved_finding_links( + connection: sqlite3.Connection, scan_ids: set[str] +) -> list[sqlite3.Row]: + return [ + row + for row in _rows_for_ids( + connection, + """ + SELECT before.scan_id AS before_scan_id, before.finding_id AS before_finding_id, + after.scan_id AS after_scan_id, after.finding_id AS after_finding_id + FROM scan_comparison_matches AS matches + JOIN finding_occurrences AS before ON before.id = matches.before_occurrence_id + JOIN finding_occurrences AS after ON after.id = matches.after_occurrence_id + WHERE matches.before_scan_id IN ({placeholders}) + ORDER BY matches.before_scan_id, after.scan_id, before.finding_id, after.finding_id + """, + sorted(scan_ids), + ) + if row["before_scan_id"] in scan_ids and row["after_scan_id"] in scan_ids + ] + + +def _finding_aliases(links: Iterable[tuple[str, str]]) -> dict[str, str]: + parents: dict[str, str] = {} + + def root(value: str) -> str: + parents.setdefault(value, value) + while parents[value] != value: + parents[value] = parents[parents[value]] + value = parents[value] + return value + + for before_id, after_id in links: + before = root(before_id) + after = root(after_id) + if before != after: + parents[after] = before + return {finding_id: root(finding_id) for finding_id in parents} + + +def _known_finding_groups(links: list[sqlite3.Row], scan_ids: set[str]) -> list[list[str]]: + aliases = _finding_aliases( + (link["before_finding_id"], link["after_finding_id"]) + for link in links + if link["before_scan_id"] in scan_ids and link["after_scan_id"] in scan_ids + ) + groups: dict[str, list[str]] = {} + for finding_id, identity in aliases.items(): + groups.setdefault(identity, []).append(finding_id) + return sorted(sorted(group) for group in groups.values() if len(group) > 1) + + def compare_scans( connection: sqlite3.Connection, args: argparse.Namespace, @@ -352,7 +424,12 @@ def compare_scans( before_findings = _scan_findings(connection, before["id"]) after_findings = _scan_findings(connection, after["id"]) matches = json.loads(cached["result_json"]) if cached is not None else None - groups = _finding_groups(before_findings, after_findings, matches) + saved_matches = matches["matches"] if matches is not None else [] + occurrences = { + row["id"]: row for row in chain(before_findings.values(), after_findings.values()) + } + aliases = _confirmed_finding_aliases(connection, occurrences) + groups = _finding_groups(before_findings, after_findings, saved_matches, aliases) uncertain = ( { (side, match[f"{side}OccurrenceId"]): match["reason"] @@ -366,7 +443,11 @@ def compare_scans( summary = {status: 0 for status in ("new", "persisting", "resolved", "reopened", "unknown")} for previous_rows, current_rows, match_reason in groups: - previous = previous_rows[0] if previous_rows else None + previous = ( + min(previous_rows, key=lambda row: SEVERITY_ORDER[row["severity"]]) + if previous_rows + else None + ) current = ( min(current_rows, key=lambda row: SEVERITY_ORDER[row["severity"]]) if current_rows @@ -381,8 +462,16 @@ def compare_scans( "severity": selected["severity"], "title": selected["title"], } + side = "after" if current_rows else "before" + uncertain_reason = next( + ( + uncertain[(side, row["id"])] + for row in current_rows or previous_rows + if (side, row["id"]) in uncertain + ), + None, + ) if previous is None: - uncertain_reason = uncertain.get(("after", current["id"])) if current else None if uncertain_reason is None: status = "new" else: @@ -400,17 +489,20 @@ def compare_scans( ) if match_reason is not None: item["matchReason"] = match_reason - elif (uncertain_reason := uncertain.get(("before", previous["id"]))) is not None: + elif uncertain_reason is not None: status = "unknown" item["reason"] = uncertain_reason elif not comparable: status = "unknown" item["reason"] = "The later scan has incomplete coverage." - elif not scan_covers_path( - after, - target_id=after["target_id"], - path=previous["relative_path"], - coverage=after_coverage, + elif not all( + scan_covers_path( + after, + target_id=after["target_id"], + path=row["relative_path"], + coverage=after_coverage, + ) + for row in previous_rows ): status = "unknown" item["reason"] = "The affected path was excluded or outside the later scope." @@ -442,11 +534,41 @@ def compare_scans( "repository": before["target_path"], "summary": summary, } + if matches is not None and matches.get("related"): + related = _separate_finding_pairs(matches["related"], occurrences, aliases) + if related: + result["related"] = [ + { + **pair, + "beforeTitle": occurrences[pair["beforeOccurrenceId"]]["title"], + "afterTitle": occurrences[pair["afterOccurrenceId"]]["title"], + } + for pair in related + ] if include_matching_inputs: + known_scan_ids = { + scan["id"] + for scan in connection.execute( + "SELECT * FROM scans WHERE status = 'complete' " + "AND (started_at < ? OR (started_at = ? AND id <= ?))", + (after["started_at"], after["started_at"], after["id"]), + ) + if _same_repository(scan, after) + } + excluded_pairs = {(before["id"], after["id"]), (after["id"], before["id"])} + known_groups = _known_finding_groups( + [ + link + for link in _saved_finding_links(connection, known_scan_ids) + if (link["before_scan_id"], link["after_scan_id"]) not in excluded_pairs + ], + known_scan_ids, + ) result["matchingCached"] = cached is not None result["matchingInputs"] = { "before": [_matching_input(row) for row in before_findings.values()], "after": [_matching_input(row) for row in after_findings.values()], + **({"knownFindingGroups": known_groups} if known_groups else {}), } return result @@ -474,37 +596,71 @@ def save_scan_comparison( payload = json.loads(args.matches_json) except (TypeError, ValueError) as exc: raise SystemExit("Scan comparison matches must be a valid JSON object.") from exc - if not isinstance(payload, dict) or set(payload) != {"matches", "uncertain"}: + if ( + not isinstance(payload, dict) + or not {"matches", "uncertain"}.issubset(payload) + or set(payload) - {"matches", "uncertain", "related"} + ): raise SystemExit("Scan comparison matches must contain matches and uncertain arrays.") - if not isinstance(payload["matches"], list) or not isinstance(payload["uncertain"], list): + if any(not isinstance(payload.get(key, []), list) for key in ("matches", "uncertain", "related")): raise SystemExit("Scan comparison matches must contain matches and uncertain arrays.") allowed = { "before": {row["id"] for row in before_findings.values()}, "after": {row["id"] for row in after_findings.values()}, } - consumed: dict[str, set[str]] = {"before": set(), "after": set()} - for match in payload["matches"]: + consumed: dict[str, dict[str, int]] = {"before": {}, "after": {}} + for group, match in enumerate(payload["matches"]): + if ( + not isinstance(match, dict) + or match.get("confidence") != "high" + or not isinstance(match.get("reason"), str) + or not match["reason"].strip() + ): + raise SystemExit("Scan comparison matches must have high confidence and a reason.") for side in ("before", "after"): - occurrences = match[f"{side}OccurrenceIds"] + occurrences = match.get(f"{side}OccurrenceIds") + if not isinstance(occurrences, list) or any( + not isinstance(value, str) for value in occurrences + ): + raise SystemExit("Scan comparison matches must identify distinct scan findings.") unique = set(occurrences) if ( not occurrences or len(unique) != len(occurrences) or not unique.issubset(allowed[side]) - or not consumed[side].isdisjoint(unique) + or not unique.isdisjoint(consumed[side]) ): raise SystemExit("Scan comparison matches must identify distinct scan findings.") - consumed[side].update(unique) + consumed[side].update((occurrence_id, group) for occurrence_id in unique) uncertain_pairs = set() for match in payload["uncertain"]: + if not _valid_finding_pair(match): + raise SystemExit("Uncertain scan comparison matches must identify distinct findings.") pair = (match["beforeOccurrenceId"], match["afterOccurrenceId"]) if ( - pair[0] not in allowed["before"] - consumed["before"] - or pair[1] not in allowed["after"] - consumed["after"] + pair[0] not in allowed["before"] + or pair[0] in consumed["before"] + or pair[1] not in allowed["after"] + or pair[1] in consumed["after"] or pair in uncertain_pairs ): raise SystemExit("Uncertain scan comparison matches must identify distinct findings.") uncertain_pairs.add(pair) + related_pairs = set() + for match in payload.get("related", []): + if not _valid_finding_pair(match): + raise SystemExit("Related scan comparison findings must identify distinct findings.") + pair = (match["beforeOccurrenceId"], match["afterOccurrenceId"]) + group = consumed["before"].get(pair[0]) + if ( + pair[0] not in allowed["before"] + or pair[1] not in allowed["after"] + or (group is not None and group == consumed["after"].get(pair[1])) + or pair in uncertain_pairs + or pair in related_pairs + ): + raise SystemExit("Related scan comparison findings must identify distinct findings.") + related_pairs.add(pair) timestamp = now() with connection: connection.execute("BEGIN IMMEDIATE") @@ -542,6 +698,143 @@ def save_scan_comparison( return compare_scans(connection, args, require_scan=require_scan, read_coverage=read_coverage) +def _valid_finding_pair(value: Any) -> bool: + return ( + isinstance(value, dict) + and set(value) == {"beforeOccurrenceId", "afterOccurrenceId", "reason"} + and all(isinstance(item, str) and item.strip() for item in value.values()) + ) + + +def _rows_for_ids( + connection: sqlite3.Connection, query: str, ids: Iterable[str] +) -> Iterator[sqlite3.Row]: + values = tuple(dict.fromkeys(ids)) + getlimit = getattr(connection, "getlimit", None) + # Python 3.10 lacks getlimit; 999 is SQLite's older host-parameter limit. + limit = getlimit(sqlite3.SQLITE_LIMIT_VARIABLE_NUMBER) if getlimit else 999 + for start in range(0, len(values), limit): + batch = values[start : start + limit] + yield from connection.execute( + query.format(placeholders=", ".join("?" for _ in batch)), batch + ) + + +def _confirmed_finding_aliases( + connection: sqlite3.Connection, occurrence_ids: Iterable[str] +) -> dict[str, str]: + # Stable finding IDs already include the target identity. Follow their indexed + # occurrences instead of resolving repository paths or scanning every saved link. + neighbors = """ + FROM linked + CROSS JOIN finding_occurrences AS source + ON source.finding_id = linked.finding_id + CROSS JOIN scan_comparison_matches AS matches + ON matches.before_occurrence_id = source.id OR matches.after_occurrence_id = source.id + CROSS JOIN finding_occurrences AS neighbor ON neighbor.id = CASE + WHEN matches.before_occurrence_id = source.id THEN matches.after_occurrence_id + ELSE matches.before_occurrence_id END + """ + # Traverse only the selected findings' components, including recurring stable IDs. + query = f""" + WITH RECURSIVE linked(finding_id) AS ( + SELECT occurrences.finding_id + FROM finding_occurrences AS occurrences + WHERE occurrences.id IN ({{placeholders}}) + UNION + SELECT neighbor.finding_id + {neighbors} + ) + SELECT DISTINCT linked.finding_id AS before_finding_id, + neighbor.finding_id AS after_finding_id + {neighbors} + """ + return _finding_aliases( + (row["before_finding_id"], row["after_finding_id"]) + for row in _rows_for_ids(connection, query, occurrence_ids) + ) + + +def _separate_finding_pairs( + pairs: list[dict[str, Any]], + occurrences: dict[str, sqlite3.Row], + aliases: dict[str, str], +) -> list[dict[str, Any]]: + def identity(occurrence_id: str) -> str: + finding_id = occurrences[occurrence_id]["finding_id"] + return aliases.get(finding_id, finding_id) + + return [ + pair + for pair in pairs + if identity(pair["beforeOccurrenceId"]) != identity(pair["afterOccurrenceId"]) + ] + + +def finding_relations( + connection: sqlite3.Connection, scan_id: str, occurrence_ids: Iterable[str] +) -> dict[str, list[dict[str, Any]]]: + selected = set(occurrence_ids) + if not selected: + return {} + pairs = [] + for comparison in connection.execute( + "SELECT before_scan_id, after_scan_id, result_json FROM scan_comparisons " + "WHERE before_scan_id = ? OR after_scan_id = ? " + "ORDER BY before_scan_id, after_scan_id", + (scan_id, scan_id), + ): + side = "before" if comparison["before_scan_id"] == scan_id else "after" + other = "after" if side == "before" else "before" + for pair in json.loads(comparison["result_json"]).get("related", []): + if pair[f"{side}OccurrenceId"] in selected: + pairs.append( + { + "beforeOccurrenceId": pair[f"{side}OccurrenceId"], + "afterOccurrenceId": pair[f"{other}OccurrenceId"], + "afterScanId": comparison[f"{other}_scan_id"], + "reason": pair["reason"], + } + ) + occurrences = { + row["id"]: row + for row in _rows_for_ids( + connection, + "SELECT id, finding_id, scan_id, title FROM finding_occurrences " + "WHERE id IN ({placeholders})", + ( + pair[key] + for pair in pairs + for key in ("beforeOccurrenceId", "afterOccurrenceId") + ), + ) + } + pairs = [ + pair + for pair in pairs + if pair["beforeOccurrenceId"] in occurrences + and occurrences[pair["beforeOccurrenceId"]]["scan_id"] == scan_id + and pair["afterOccurrenceId"] in occurrences + and occurrences[pair["afterOccurrenceId"]]["scan_id"] == pair["afterScanId"] + ] + result: dict[str, list[dict[str, Any]]] = {} + aliases = _confirmed_finding_aliases( + connection, (pair["beforeOccurrenceId"] for pair in pairs) + ) + for pair in _separate_finding_pairs(pairs, occurrences, aliases): + finding = occurrences[pair["afterOccurrenceId"]] + result.setdefault(pair["beforeOccurrenceId"], []).append( + { + "findingId": finding["finding_id"], + "occurrenceId": finding["id"], + "reason": pair["reason"], + "scanId": pair["afterScanId"], + "title": finding["title"], + } + ) + return result + + def finding_matches( connection: sqlite3.Connection, occurrence_id: str, scan_id: str, started_at: str ) -> tuple[list[dict[str, Any]], str, list[str]]: @@ -612,26 +905,45 @@ def finding_matches( def _finding_groups( before_findings: dict[str, sqlite3.Row], after_findings: dict[str, sqlite3.Row], - matches: dict[str, Any] | None, + matches: list[dict[str, Any]], + aliases: dict[str, str], ) -> list[tuple[list[sqlite3.Row], list[sqlite3.Row], str | None]]: rows = { side: {row["id"]: row for row in findings.values()} for side, findings in (("before", before_findings), ("after", after_findings)) } - consumed: dict[str, set[str]] = {"before": set(), "after": set()} - result = [] - for match in matches["matches"] if matches is not None else []: - previous = [rows["before"][value] for value in match["beforeOccurrenceIds"]] - current = [rows["after"][value] for value in match["afterOccurrenceIds"]] - consumed["before"].update(match["beforeOccurrenceIds"]) - consumed["after"].update(match["afterOccurrenceIds"]) - result.append((previous, current, match["reason"])) - result.extend( - ([row], [], None) for key, row in rows["before"].items() if key not in consumed["before"] - ) - result.extend( - ([], [row], None) for key, row in rows["after"].items() if key not in consumed["after"] - ) + groups: dict[str, tuple[list[sqlite3.Row], list[sqlite3.Row], list[str]]] = {} + + def group(row: sqlite3.Row) -> tuple[list[sqlite3.Row], list[sqlite3.Row], list[str]]: + finding_id = row["finding_id"] + return groups.setdefault(aliases.get(finding_id, finding_id), ([], [], [])) + + for match in matches: + group(rows["before"][match["beforeOccurrenceIds"][0]])[2].append(match["reason"]) + for index, side in enumerate(("before", "after")): + occurrence_ids = dict.fromkeys( + chain( + (value for match in matches for value in match[f"{side}OccurrenceIds"]), + rows[side], + ) + ) + for occurrence_id in occurrence_ids: + row = rows[side][occurrence_id] + group(row)[index].append(row) + result = [ + ( + previous, + current, + ( + " ".join(dict.fromkeys(reasons)) + if reasons + else "The findings share a stable identity or a previously confirmed link." + ) + if previous and current + else None, + ) + for previous, current, reasons in groups.values() + ] return sorted( result, key=lambda group: ( diff --git a/sdk/typescript/_bundled_plugin/scripts/workbench_schema.py b/sdk/typescript/_bundled_plugin/scripts/workbench_schema.py index 7fb414ae..9e9fa62c 100644 --- a/sdk/typescript/_bundled_plugin/scripts/workbench_schema.py +++ b/sdk/typescript/_bundled_plugin/scripts/workbench_schema.py @@ -683,6 +683,17 @@ WHERE project_id IS NULL; """, ), + ( + 31, + "index finding identity and comparison history", + """ + CREATE INDEX finding_occurrences_by_finding + ON finding_occurrences(finding_id, id); + + CREATE INDEX scan_comparisons_by_after_scan + ON scan_comparisons(after_scan_id, before_scan_id); + """, + ), ) diff --git a/sdk/typescript/src/runtime.ts b/sdk/typescript/src/runtime.ts index 3569c11b..76e56e69 100644 --- a/sdk/typescript/src/runtime.ts +++ b/sdk/typescript/src/runtime.ts @@ -1366,30 +1366,30 @@ export async function preparePersistentOutputRoot( return root; } +const workbenchComparisonStdinSupport = new Map(); + export async function runWorkbench( options: WorkbenchCommandOptions, args: readonly string[], input?: string, ): Promise { - let stdout: string; - try { - const environment = Object.fromEntries( - Object.entries(options.environment).filter( - ([name]) => - name.toUpperCase() !== "OPENAI_API_KEY" && - name.toUpperCase() !== "CODEX_API_KEY" && - name.toUpperCase() !== "OPENROUTER_API_KEY" && - name.toUpperCase() !== "FIREWORKS_API_KEY", - ), - ); + const script = join(options.pluginRoot, "scripts", "workbench_db.py"); + const environment = Object.fromEntries( + Object.entries(options.environment).filter( + ([name]) => + name.toUpperCase() !== "OPENAI_API_KEY" && + name.toUpperCase() !== "CODEX_API_KEY" && + name.toUpperCase() !== "OPENROUTER_API_KEY" && + name.toUpperCase() !== "FIREWORKS_API_KEY", + ), + ); + const run = async ( + arguments_: readonly string[], + input?: string, + ): Promise => { const result = await runCodexCommand( { command: options.python }, - [ - "-I", - "-B", - join(options.pluginRoot, "scripts", "workbench_db.py"), - ...args, - ], + ["-I", "-B", script, ...arguments_], environment, input, options.signal, @@ -1401,7 +1401,36 @@ export async function runWorkbench( `Workbench exited with status ${result.exitCode}.`, ); } - stdout = result.stdout; + return result.stdout; + }; + let stdout: string; + try { + const arguments_ = [...args]; + const matchesIndex = arguments_.indexOf("--matches-json"); + const matches = + matchesIndex === -1 ? undefined : arguments_[matchesIndex + 1]; + if (arguments_[0] === "save-scan-comparison" && matches !== undefined) { + const key = JSON.stringify([options.python, script]); + let supportsStdin = workbenchComparisonStdinSupport.get(key); + if (supportsStdin === undefined) { + const help = await run(["save-scan-comparison", "--help"]); + options.signal?.throwIfAborted(); + supportsStdin = help.includes("--matches-json-stdin"); + workbenchComparisonStdinSupport.set(key, supportsStdin); + } + if (supportsStdin) { + input = matches; + arguments_.splice(matchesIndex, 2, "--matches-json-stdin"); + } else { + // Older custom plugins accept only the original comparison format. + const comparison: unknown = JSON.parse(matches); + if (isRecord(comparison) && "related" in comparison) { + delete comparison["related"]; + arguments_[matchesIndex + 1] = JSON.stringify(comparison); + } + } + } + stdout = await run(arguments_, input); } catch (error) { if (options.signal?.aborted) throw error; const detail = processErrorDetail(error); diff --git a/sdk/typescript/tests-ts/publication-store.test.ts b/sdk/typescript/tests-ts/publication-store.test.ts index 194230bf..34d461a4 100644 --- a/sdk/typescript/tests-ts/publication-store.test.ts +++ b/sdk/typescript/tests-ts/publication-store.test.ts @@ -234,9 +234,11 @@ connection.close() test("upgrades existing scan history and verifies every completed finding before publication", async () => { const fixture = await publicationFixture(); databaseRows(fixture, "DROP TABLE finding_publications"); - databaseRows(fixture, "DELETE FROM schema_migrations WHERE version >= ?", [ - 29, - ]); + databaseRows( + fixture, + "DELETE FROM schema_migrations WHERE version BETWEEN ? AND ?", + [29, 30], + ); await expect( preparePublicationStore(fixture.publication, fixture.environment), @@ -245,8 +247,8 @@ connection.close() expect( databaseRows( fixture, - "SELECT version, name FROM schema_migrations WHERE version >= ? ORDER BY version", - [29], + "SELECT version, name FROM schema_migrations WHERE version BETWEEN ? AND ? ORDER BY version", + [29, 30], ), ).toEqual([ { version: 29, name: "persist finding publication associations" }, diff --git a/sdk/typescript/tests-ts/runtime.test.ts b/sdk/typescript/tests-ts/runtime.test.ts index d27a2116..e932ff43 100644 --- a/sdk/typescript/tests-ts/runtime.test.ts +++ b/sdk/typescript/tests-ts/runtime.test.ts @@ -3931,11 +3931,10 @@ describe("runtime directories and plugin Python boundary", () => { "print(json.dumps({'ok': True, 'inputLength': len(payload), 'details': 'x' * (5 * 1024 * 1024)}))", ].join("\n"), ); - const python = Bun.which("python3") ?? Bun.which("python"); - expect(python).not.toBeNull(); + const python = await resolvePluginPython(); const result = await runWorkbench( { - python: python!, + python, pluginRoot, environment: { PATH: process.env["PATH"], @@ -3953,6 +3952,117 @@ describe("runtime directories and plugin Python boundary", () => { expect(result["details"]).toHaveLength(5 * 1024 * 1024); }); + test.each(["legacy", "current"])( + "saves comparisons with a %s custom plugin", + async (version) => { + const supportsStdin = version === "current"; + const root = await temporaryDirectory(); + const pluginRoot = join(root, "custom plugin"); + const scripts = join(pluginRoot, "scripts"); + await mkdir(scripts, { recursive: true }); + await writeFile( + join(scripts, "workbench_db.py"), + [ + "import argparse, json, os, sys", + "from pathlib import Path", + "assert sys.flags.isolated and sys.dont_write_bytecode", + "assert os.environ.get('OPENAI_API_KEY') is None", + "assert os.environ.get('CODEX_API_KEY') is None", + "assert os.environ.get('OPENROUTER_API_KEY') is None", + "assert os.environ.get('FIREWORKS_API_KEY') is None", + "if '--help' in sys.argv:", + " with Path(__file__).with_name('help-calls').open('ab') as calls: calls.write(b'help\\n')", + " if os.environ.get('FAIL_COMPARISON_HELP'): sys.exit('Synthetic help failure')", + "parser = argparse.ArgumentParser()", + "command = parser.add_subparsers(dest='command', required=True).add_parser('save-scan-comparison')", + "command.add_argument('--before-scan-id', required=True)", + "command.add_argument('--after-scan-id', required=True)", + ...(supportsStdin + ? [ + "transport = command.add_mutually_exclusive_group(required=True)", + "transport.add_argument('--matches-json')", + "transport.add_argument('--matches-json-stdin', action='store_true')", + ] + : ["command.add_argument('--matches-json', required=True)"]), + "args = parser.parse_args()", + "uses_stdin = getattr(args, 'matches_json_stdin', False)", + "payload = json.loads(sys.stdin.buffer.read().decode('utf-8') if uses_stdin else args.matches_json)", + ...(!supportsStdin + ? ["assert set(payload) == {'matches', 'uncertain'}"] + : []), + "print(json.dumps({'payload': payload, 'usesStdin': uses_stdin}))", + ].join("\n"), + ); + const python = await resolvePluginPython(); + const options = { + python, + pluginRoot, + environment: { + PATH: process.env["PATH"], + OPENAI_API_KEY: "synthetic-openai-key", + CODEX_API_KEY: "synthetic-codex-key", + OPENROUTER_API_KEY: "synthetic-openrouter-key", + FIREWORKS_API_KEY: "synthetic-fireworks-key", + }, + }; + const original = { + matches: Array.from( + { length: supportsStdin ? 10_000 : 1 }, + (_, index) => ({ + beforeOccurrenceIds: [`before-${index}`], + afterOccurrenceIds: [`after-${index}`], + confidence: "high", + reason: "Same synthetic control.", + }), + ), + uncertain: [ + { + beforeOccurrenceId: "uncertain-before", + afterOccurrenceId: "uncertain-after", + reason: "Needs more evidence.", + }, + ], + related: [ + { + beforeOccurrenceId: "related-before", + afterOccurrenceId: "related-after", + reason: "Separate synthetic controls. 🙂", + }, + ], + }; + const args = [ + "save-scan-comparison", + "--before-scan-id", + "before-scan", + "--after-scan-id", + "after-scan", + "--matches-json", + JSON.stringify(original), + ]; + await expect( + runWorkbench( + { + ...options, + environment: { ...options.environment, FAIL_COMPARISON_HELP: "1" }, + }, + args, + ), + ).rejects.toThrow("Synthetic help failure"); + const expected = { + usesStdin: supportsStdin, + payload: supportsStdin + ? original + : { matches: original.matches, uncertain: original.uncertain }, + }; + expect(await runWorkbench(options, args)).toEqual(expected); + expect(await runWorkbench(options, args)).toEqual(expected); + expect(await readFile(join(scripts, "help-calls"), "utf8")).toBe( + "help\nhelp\n", + ); + expect(JSON.parse(args.at(-1)!)).toEqual(original); + }, + ); + test("upgrades colliding legacy execution-profile and public CLI migrations", async () => { const root = await temporaryDirectory("codex-security-legacy-migrations-"); const repository = join(root, "repository"); diff --git a/sdk/typescript/tests-ts/workbench-scan-history.test.ts b/sdk/typescript/tests-ts/workbench-scan-history.test.ts index 57ad27e7..0d7a0025 100644 --- a/sdk/typescript/tests-ts/workbench-scan-history.test.ts +++ b/sdk/typescript/tests-ts/workbench-scan-history.test.ts @@ -5,9 +5,298 @@ import { expect, test } from "bun:test"; import { resolvePluginPython } from "../src/runtime.js"; import { PLUGIN_ROOT } from "./plugin-root.js"; -test("loads each scan's matching findings once across historical batches", async () => { +async function runPythonProbe( + program: string, + ...args: string[] +): Promise> { const python = await resolvePluginPython(); + const result = spawnSync( + python, + ["-I", "-B", "-", join(PLUGIN_ROOT, "scripts"), ...args], + { input: program, encoding: "utf8", timeout: 10_000, windowsHide: true }, + ); + expect(result.status, result.stderr || result.error?.message).toBe(0); + expect(result.stderr).toBe(""); + return JSON.parse(result.stdout) as Record; +} + +test("keeps inline and stdin comparison transports compatible", async () => { + const python = await resolvePluginPython(); + const probe = [ + "import json, sys", + "sys.path.insert(0, sys.argv.pop(1))", + "from workbench_cli import parse_args", + "args = parse_args('Synthetic comparison transport')", + "print(json.dumps(json.loads(args.matches_json)))", + ].join("\n"); + const args = [ + "-I", + "-B", + "-c", + probe, + join(PLUGIN_ROOT, "scripts"), + "save-scan-comparison", + "--before-scan-id", + "before", + "--after-scan-id", + "after", + ]; + const payload = JSON.stringify({ + matches: [ + { + beforeOccurrenceIds: ["before"], + afterOccurrenceIds: ["after"], + confidence: "high", + reason: "Synthetic comparison 🙂", + }, + ], + uncertain: [], + }); + for (const transport of [ + ["--matches-json", payload], + ["--matches-json-stdin"], + ]) { + const result = spawnSync(python, [...args, ...transport], { + input: payload, + encoding: "utf8", + timeout: 10_000, + windowsHide: true, + }); + expect(result.status, result.stderr).toBe(0); + expect(JSON.parse(result.stdout)).toEqual(JSON.parse(payload)); + } + const conflicting = spawnSync( + python, + [...args, "--matches-json", payload, "--matches-json-stdin"], + { input: payload, encoding: "utf8", timeout: 10_000, windowsHide: true }, + ); + expect(conflicting.status).toBe(2); + expect(conflicting.stderr).toContain("not allowed with argument"); +}); + +test("keeps unrelated legacy repositories out of matching inputs", async () => { + const observed = await runPythonProbe( + ` +import argparse, json, sqlite3, sys +from pathlib import Path +sys.path.insert(0, sys.argv[1]) +import workbench_scan_history as history +connection = sqlite3.connect(':memory:') +connection.row_factory = sqlite3.Row +connection.executescript(''' +CREATE TABLE scans (id TEXT PRIMARY KEY, target_id TEXT, target_path TEXT, status TEXT, started_at TEXT); +CREATE TABLE finding_occurrences (id TEXT PRIMARY KEY, finding_id TEXT, scan_id TEXT); +CREATE TABLE finding_triage (occurrence_id TEXT, status TEXT, close_reason TEXT); +CREATE TABLE finding_locations (occurrence_id TEXT, relative_path TEXT, role TEXT, sort_order INTEGER); +CREATE TABLE scan_comparisons (before_scan_id TEXT, after_scan_id TEXT, result_json TEXT); +CREATE TABLE scan_comparison_matches (before_scan_id TEXT, after_scan_id TEXT, before_occurrence_id TEXT, after_occurrence_id TEXT); +''') +for index, (scan, repository) in enumerate([ + ('unrelated-before', 'unrelated'), ('unrelated-after', 'unrelated'), + ('before', 'selected'), ('after', 'selected') +]): + connection.execute('INSERT INTO scans VALUES (?, NULL, ?, ?, ?)', + (scan, str(Path(sys.argv[2]) / repository), 'complete', str(index))) +connection.executemany('INSERT INTO finding_occurrences VALUES (?, ?, ?)', [ + ('unrelated-first', 'unrelated-identity-a', 'unrelated-before'), + ('unrelated-second', 'unrelated-identity-b', 'unrelated-after') +]) +connection.execute('INSERT INTO scan_comparison_matches VALUES (?, ?, ?, ?)', + ('unrelated-before', 'unrelated-after', 'unrelated-first', 'unrelated-second')) +comparison = history.compare_scans( + connection, argparse.Namespace(before_scan_id='before', after_scan_id='after'), + require_scan=lambda db, scan: db.execute('SELECT * FROM scans WHERE id = ?', (scan,)).fetchone(), + read_coverage=lambda _: {'completeness': 'complete'}, include_matching_inputs=True) +print(json.dumps(comparison['matchingInputs'])) +`, + join(tmpdir(), "codex-security-legacy-repositories"), + ); + expect(observed).toEqual({ before: [], after: [] }); +}); + +test("validates related pairs by confirmed group without replacing saved results", async () => { + const probe = ` +import argparse, json, sqlite3, sys +sys.path.insert(0, sys.argv[1]) +import workbench_scan_history as history +connection = sqlite3.connect(':memory:') +connection.row_factory = sqlite3.Row +connection.executescript(''' +PRAGMA foreign_keys = ON; +CREATE TABLE scans (id TEXT PRIMARY KEY, target_path TEXT, target_id TEXT, status TEXT); +CREATE TABLE finding_occurrences ( + id TEXT PRIMARY KEY, finding_id TEXT, scan_id TEXT, title TEXT, severity TEXT +); +CREATE TABLE finding_triage (occurrence_id TEXT, status TEXT, close_reason TEXT); +CREATE TABLE finding_locations (occurrence_id TEXT, relative_path TEXT, role TEXT, sort_order INTEGER); +CREATE TABLE scan_comparisons ( + before_scan_id TEXT, after_scan_id TEXT, result_json TEXT, created_at TEXT, updated_at TEXT, + PRIMARY KEY(before_scan_id, after_scan_id) +); +CREATE TABLE scan_comparison_matches ( + before_scan_id TEXT, after_scan_id TEXT, before_occurrence_id TEXT, after_occurrence_id TEXT, + reason TEXT, + FOREIGN KEY(before_scan_id, after_scan_id) + REFERENCES scan_comparisons(before_scan_id, after_scan_id) ON DELETE CASCADE +); +CREATE INDEX matches_before ON scan_comparison_matches(before_occurrence_id); +CREATE INDEX matches_after ON scan_comparison_matches(after_occurrence_id); +''') +for scan, names in [('before', ('a1', 'a2', 'b', 'c')), ('after', ('x1', 'x2', 'y', 'z'))]: + connection.execute('INSERT INTO scans VALUES (?, ?, ?, ?)', (scan, sys.argv[2], 'target', 'complete')) + for name in names: + connection.execute('INSERT INTO finding_occurrences VALUES (?, ?, ?, ?, ?)', (name, name, scan, name, 'high')) + connection.execute('INSERT INTO finding_locations VALUES (?, ?, ?, ?)', (name, 'src/example.py', 'root_control', 0)) +connection.commit() +def pair(before, after): + return {'beforeOccurrenceId': before, 'afterOccurrenceId': after, 'reason': 'Separate synthetic controls.'} +def group(before, after): + return {'beforeOccurrenceIds': before, 'afterOccurrenceIds': after, + 'confidence': 'high', 'reason': 'The same synthetic control.'} +payload = {'matches': [group(['a1', 'a2'], ['x1', 'x2']), group(['b'], ['y'])], + 'uncertain': [], 'related': [pair('a2', 'y'), pair('c', 'z')]} +def save(value): + return history.save_scan_comparison( + connection, argparse.Namespace(before_scan_id='before', after_scan_id='after', matches_json=json.dumps(value)), + now=lambda: '2026-01-01T00:00:00Z', + require_scan=lambda db, scan: db.execute('SELECT * FROM scans WHERE id = ?', (scan,)).fetchone(), + read_coverage=lambda _: {'completeness': 'complete', 'includePaths': ['src'], + 'excludePaths': [], 'explicitExclusions': []}) +def snapshot(): + return (connection.execute('SELECT result_json FROM scan_comparisons').fetchone()[0], + [tuple(row) for row in connection.execute('SELECT * FROM scan_comparison_matches ORDER BY before_occurrence_id, after_occurrence_id')]) +accepted = save(payload) +original = snapshot() +invalid = [ + {**payload, 'related': [pair('a2', 'x2')]}, + {**payload, 'related': [pair('b', 'y')]}, + {**payload, 'related': [pair('a2', 'y'), pair('a2', 'y')]}, + {**payload, 'related': [pair('outside', 'z')]}, + {**payload, 'uncertain': [pair('c', 'z')]}, + {**payload, 'uncertain': [pair('a1', 'z')]}, + {**payload, 'uncertain': [pair('c', 'y')]}, + {**payload, 'matches': [payload['matches'][0], group(['b', 'a1'], ['y'])]}, +] +for value in invalid: + try: + save(value) + except SystemExit: + pass + else: + raise AssertionError('Invalid comparison was accepted') + assert snapshot() == original +print(json.dumps({'summary': accepted['summary'], + 'related': [(item['beforeOccurrenceId'], item['afterOccurrenceId']) for item in accepted['related']], + 'savedPairs': len(original[1]), 'rejected': len(invalid)})) +`; + expect( + await runPythonProbe( + probe, + join(tmpdir(), "codex-security-validation-fixture"), + ), + ).toEqual({ + summary: { new: 1, persisting: 2, reopened: 0, resolved: 1, unknown: 0 }, + related: [ + ["a2", "y"], + ["c", "z"], + ], + savedPairs: 5, + rejected: 8, + }); +}); + +test("upgrades existing history with indexed identity and reverse comparison lookups", async () => { + const probe = ` +import json, sqlite3, sys +from pathlib import Path +sys.path.insert(0, sys.argv[1]) +from finalize_scan_contract import _derived_finding_identity_rows +from workbench_schema import MIGRATIONS, apply_migrations +connection = sqlite3.connect(':memory:') +connection.row_factory = sqlite3.Row +connection.execute('PRAGMA foreign_keys = ON') +timestamp = '2026-01-01T00:00:00Z' +def migrate(migrations): + apply_migrations(connection, migrations, lambda: timestamp, lambda _: None) +migrate(tuple(item for item in MIGRATIONS if item[0] < 31)) +connection.execute('INSERT INTO security_targets VALUES (?, ?, ?, ?, ?)', ('target', sys.argv[2], 'Synthetic target', timestamp, timestamp)) +connection.execute('INSERT INTO workspaces (id, target_id, created_at, updated_at) VALUES (?, ?, ?, ?)', ('workspace', 'target', timestamp, timestamp)) +connection.executemany('''INSERT INTO scans ( + id, workspace_id, target_id, target_path, target_revision, scope, mode, scan_dir, + status, phase, started_at, created_at, updated_at +) VALUES (?, 'workspace', 'target', ?, 'unversioned', '.', 'standard', ?, 'complete', 'reporting', ?, ?, ?)''', ( + (f'scan-{index:03d}', sys.argv[2], str(Path(sys.argv[2]) / f'scan-{index:03d}'), timestamp, timestamp, timestamp) + for index in range(200) +)) +connection.executemany('INSERT INTO scan_comparisons VALUES (?, ?, ?, ?, ?)', ( + (f'scan-{before:03d}', f'scan-{after:03d}', json.dumps({'matches': [], 'uncertain': []}), timestamp, timestamp) + for after in range(200) for before in range(after) +)) +def rows(): + return [tuple(row) for row in connection.execute('SELECT * FROM scan_comparisons ORDER BY before_scan_id, after_scan_id')] +original = rows() +migrate(MIGRATIONS) +migrate(MIGRATIONS) +query = '''SELECT before_scan_id, after_scan_id, result_json FROM scan_comparisons + WHERE before_scan_id = ? OR after_scan_id = ? ORDER BY before_scan_id, after_scan_id''' +plan = [row['detail'] for row in connection.execute('EXPLAIN QUERY PLAN ' + query, ('scan-100', 'scan-100'))] +identity_plan = [row['detail'] for row in connection.execute( + 'EXPLAIN QUERY PLAN SELECT id FROM finding_occurrences WHERE finding_id = ?', ('synthetic-finding',))] +indexes = { + name: [row['name'] for row in connection.execute(f'PRAGMA index_info({name})')] + for name in ('finding_occurrences_by_finding', 'scan_comparisons_by_after_scan') +} +def identity(target, scan): + finding = {'ruleId': 'synthetic-control', 'identity': {'anchor': 'synthetic-control'}} + return _derived_finding_identity_rows( + {'scan': {'id': scan, 'target': {'targetId': target}}}, + {'scanId': scan, 'findings': [finding]})[0][2:4] +first_identity = identity('target', 'first') +recurring_identity = identity('target', 'second') +other_identity = identity('another-target', 'third') +print(json.dumps({'unchanged': rows() == original, 'comparisons': len(original), + 'plan': plan, 'identityPlan': identity_plan, 'indexes': indexes, + 'stableIdentity': first_identity[0] == recurring_identity[0], + 'distinctOccurrences': first_identity[1] != recurring_identity[1], + 'targetScopedIdentity': first_identity[0] != other_identity[0], + 'foreignKeyErrors': len(connection.execute('PRAGMA foreign_key_check').fetchall())})) +`; + const observed = (await runPythonProbe( + probe, + join(tmpdir(), "codex-security-index-fixture"), + )) as { + plan: string[]; + identityPlan: string[]; + }; + expect(observed).toMatchObject({ + unchanged: true, + comparisons: 19_900, + foreignKeyErrors: 0, + stableIdentity: true, + distinctOccurrences: true, + targetScopedIdentity: true, + indexes: { + finding_occurrences_by_finding: ["finding_id", "id"], + scan_comparisons_by_after_scan: ["after_scan_id", "before_scan_id"], + }, + }); + expect(observed.plan.some((step) => step.includes("before_scan_id=?"))).toBe( + true, + ); + expect(observed.plan.some((step) => step.includes("after_scan_id=?"))).toBe( + true, + ); + expect( + observed.plan.some((step) => step.startsWith("SCAN scan_comparisons")), + ).toBe(false); + expect( + observed.identityPlan.some((step) => + step.includes("finding_occurrences_by_finding"), + ), + ).toBe(true); +}); +test("loads each scan once and scopes saved links to uncached history", async () => { const probe = [ "import argparse, json, sqlite3, sys", "sys.path.insert(0, sys.argv[1])", @@ -18,6 +307,7 @@ test("loads each scan's matching findings once across historical batches", async "CREATE TABLE security_targets (id TEXT, current_path TEXT);", "CREATE TABLE scans (id TEXT, target_path TEXT, target_id TEXT, status TEXT, started_at TEXT);", "CREATE TABLE scan_comparisons (before_scan_id TEXT, after_scan_id TEXT);", + "CREATE TABLE scan_comparison_matches (before_scan_id TEXT, after_scan_id TEXT, before_occurrence_id TEXT, after_occurrence_id TEXT);", "CREATE TABLE finding_occurrences (id TEXT, finding_id TEXT, scan_id TEXT, details_json TEXT, remediation TEXT, severity TEXT, summary TEXT, title TEXT);", "CREATE TABLE finding_triage (occurrence_id TEXT, status TEXT, close_reason TEXT);", "CREATE TABLE finding_locations (occurrence_id TEXT, relative_path TEXT, role TEXT, sort_order INTEGER);", @@ -30,26 +320,81 @@ test("loads each scan's matching findings once across historical batches", async "connection.set_trace_callback(queries.append)", "backfilled = []", "result = history.list_unmatched_scan_pairs(connection, argparse.Namespace(repository=sys.argv[2], force=False), backfill_finding_details=lambda _connection, scan: backfilled.append(scan['id']), read_coverage=lambda _scan: {})", - "print(json.dumps({'result': result, 'backfilled': backfilled, 'findingQueries': sum('FROM finding_occurrences AS occurrences' in query for query in queries)}))", + "finding_queries = sum('FROM finding_occurrences AS occurrences' in query for query in queries)", + "connection.executemany('INSERT INTO scan_comparisons VALUES (?, ?)', [('scan-0', 'scan-1'), ('scan-0', 'scan-2'), ('scan-1', 'scan-2')])", + "queries.clear()", + "cached = history.list_unmatched_scan_pairs(connection, argparse.Namespace(repository=sys.argv[2], force=False), backfill_finding_details=lambda *_: None, read_coverage=lambda _scan: {})", + "cached_link_queries = sum('FROM scan_comparison_matches' in query for query in queries)", + "for name in ('foreign-a', 'foreign-b'):", + " connection.execute('INSERT INTO finding_occurrences VALUES (?, ?, ?, ?, ?, ?, ?, ?)', (name, name, name, '{}', 'fix', 'high', 'summary', 'title'))", + "connection.executemany('INSERT INTO scan_comparison_matches VALUES (?, ?, ?, ?)', [('scan-0', 'scan-1', 'scan-0', 'scan-1'), ('foreign-a', 'foreign-b', 'foreign-a', 'foreign-b')])", + "queries.clear()", + "scoped = history._saved_finding_links(connection, {'scan-0', 'scan-1'})", + "link_queries = [query for query in queries if 'FROM scan_comparison_matches' in query]", + "for index in (3, 4):", + " scan = f'scan-{index}'", + " connection.execute('INSERT INTO scans VALUES (?, ?, NULL, ?, ?)', (scan, sys.argv[2], 'complete', str(index)))", + " connection.execute('INSERT INTO finding_occurrences VALUES (?, ?, ?, ?, ?, ?, ?, ?)', (scan, f'scan-{index - 3}', scan, '{}', 'fix', 'high', 'summary', 'title'))", + "def coverage(scan):", + " if scan['id'] in {'scan-0', 'scan-1', 'scan-2'}:", + " raise SystemExit('Synthetic unavailable artifacts')", + " return {}", + "unavailable = history.list_unmatched_scan_pairs(connection, argparse.Namespace(repository=sys.argv[2], force=False), backfill_finding_details=lambda *_: None, read_coverage=coverage)", + "forced = history.list_unmatched_scan_pairs(connection, argparse.Namespace(repository=sys.argv[2], force=True), backfill_finding_details=lambda *_: None, read_coverage=coverage)", + "connection.executemany('INSERT INTO scan_comparison_matches VALUES (?, ?, ?, ?)', [('scan-1', 'scan-2', 'scan-1', 'scan-2'), ('scan-2', 'scan-0', 'scan-2', 'scan-0'), ('scan-0', 'foreign-a', 'scan-0', 'foreign-a'), ('foreign-a', 'scan-1', 'foreign-a', 'scan-1')])", + "limited = hasattr(connection, 'setlimit')", + "if limited:", + " old_limit = connection.setlimit(sqlite3.SQLITE_LIMIT_VARIABLE_NUMBER, 2)", + "queries.clear()", + "batched = history._saved_finding_links(connection, {'scan-2', 'scan-0', 'scan-1'})", + "batched_queries = len(queries)", + "if limited:", + " connection.setlimit(sqlite3.SQLITE_LIMIT_VARIABLE_NUMBER, old_limit)", + "queries.clear()", + "empty = history._saved_finding_links(connection, set())", + "print(json.dumps({", + " 'result': result, 'backfilled': backfilled, 'findingQueries': finding_queries,", + " 'cached': cached, 'cachedLinkQueries': cached_link_queries,", + " 'scopedLinks': [dict(row) for row in scoped], 'scopedQueryCount': len(link_queries),", + " 'unscopedQueries': sum('WHERE matches.before_scan_id' not in query for query in link_queries),", + " 'unavailable': unavailable, 'forcedKnownGroups': [batch.get('knownFindingGroups') for batch in forced['batches']],", + " 'batchedLinks': [[row['before_scan_id'], row['after_scan_id']] for row in batched],", + " 'batchedQueryCount': batched_queries, 'expectedBatchedQueryCount': 2 if limited else 1,", + " 'emptyLinks': empty, 'emptyQueryCount': len(queries),", + "}))", ].join("\n"); - const result = spawnSync( - python, - [ - "-I", - "-B", - "-", - join(PLUGIN_ROOT, "scripts"), - join(tmpdir(), "codex-security-matching-fixture"), - ], - { input: probe, encoding: "utf8", timeout: 10_000, windowsHide: true }, + const observed = await runPythonProbe( + probe, + join(tmpdir(), "codex-security-matching-fixture"), ); - - expect(result.status, result.stderr || result.error?.message).toBe(0); - expect(result.stderr).toBe(""); - expect(JSON.parse(result.stdout)).toMatchObject({ + expect(observed).toMatchObject({ backfilled: ["scan-0", "scan-1", "scan-2"], findingQueries: 3, + cached: { batches: [], skippedPairs: 3 }, + cachedLinkQueries: 0, + scopedLinks: [{ before_finding_id: "scan-0", after_finding_id: "scan-1" }], + scopedQueryCount: 1, + unscopedQueries: 0, + batchedLinks: [ + ["scan-0", "scan-1"], + ["scan-1", "scan-2"], + ["scan-2", "scan-0"], + ], + emptyLinks: [], + emptyQueryCount: 0, + unavailable: { + scanCount: 5, + unavailableScans: 3, + batches: [ + { + afterScanId: "scan-4", + beforeScans: [{ scanId: "scan-3" }], + knownFindingGroups: [["scan-0", "scan-1"]], + }, + ], + }, + forcedKnownGroups: [null], result: { scanCount: 3, batches: [ @@ -61,4 +406,233 @@ test("loads each scan's matching findings once across historical batches", async ], }, }); + expect(observed["batchedQueryCount"]).toBe( + observed["expectedBatchedQueryCount"], + ); +}); + +test("reconciles cached statuses without losing grouped coverage or uncertainty", async () => { + const probe = ` +import argparse, json, sqlite3, sys +sys.path.insert(0, sys.argv[1]) +import workbench_scan_history as history +connection = sqlite3.connect(':memory:') +connection.row_factory = sqlite3.Row +connection.executescript(''' +CREATE TABLE scans (id TEXT PRIMARY KEY, target_path TEXT, target_id TEXT, status TEXT); +CREATE TABLE finding_occurrences ( + id TEXT PRIMARY KEY, finding_id TEXT, scan_id TEXT, title TEXT, severity TEXT +); +CREATE TABLE finding_triage (occurrence_id TEXT, status TEXT, close_reason TEXT); +CREATE TABLE finding_locations (occurrence_id TEXT, relative_path TEXT, role TEXT, sort_order INTEGER); +CREATE TABLE scan_comparisons ( + before_scan_id TEXT, after_scan_id TEXT, result_json TEXT, + PRIMARY KEY(before_scan_id, after_scan_id) +); +CREATE TABLE scan_comparison_matches ( + before_scan_id TEXT, after_scan_id TEXT, before_occurrence_id TEXT, after_occurrence_id TEXT +); +CREATE INDEX matches_before ON scan_comparison_matches(before_occurrence_id); +CREATE INDEX matches_after ON scan_comparison_matches(after_occurrence_id); +''') +for scan in ('before', 'after', 'later', 'latest'): + connection.execute('INSERT INTO scans VALUES (?, ?, ?, ?)', (scan, sys.argv[2], 'target', 'complete')) +for scan, names in [('before', ('a1', 'a2')), ('after', ('b1', 'b2')), + ('later', ('c1', 'c2')), ('latest', ('d1',))]: + for name in names: + severity = 'low' if name.endswith('1') else 'high' + path = 'src/excluded.py' if name == 'a1' else 'src/covered.py' + connection.execute('INSERT INTO finding_occurrences VALUES (?, ?, ?, ?, ?)', (name, name, scan, name, severity)) + connection.execute('INSERT INTO finding_locations VALUES (?, ?, ?, ?)', (name, path, 'root_control', 0)) +def link(before, after): + connection.execute('''INSERT INTO scan_comparison_matches + SELECT previous.scan_id, current.scan_id, previous.id, current.id + FROM finding_occurrences AS previous, finding_occurrences AS current + WHERE previous.id = ? AND current.id = ?''', (before, after)) +for before, after in [('a1', 'c1'), ('a2', 'c1'), ('b1', 'c2'), ('b2', 'c2')]: + link(before, after) +payload = { + 'matches': [], + 'uncertain': [{'beforeOccurrenceId': 'a1', 'afterOccurrenceId': 'b1', 'reason': 'Synthetic uncertainty.'}], + 'related': [{'beforeOccurrenceId': 'a2', 'afterOccurrenceId': 'b2', 'reason': 'Separate synthetic controls.'}] +} +def cache(): + connection.execute('INSERT OR REPLACE INTO scan_comparisons VALUES (?, ?, ?)', ('before', 'after', json.dumps(payload))) +coverage = {'completeness': 'complete', 'includePaths': ['src'], + 'excludePaths': ['src/excluded.py'], 'explicitExclusions': []} +def compare(): + return history.compare_scans( + connection, argparse.Namespace(before_scan_id='before', after_scan_id='after'), + require_scan=lambda db, scan: db.execute('SELECT * FROM scans WHERE id = ?', (scan,)).fetchone(), + read_coverage=lambda _: coverage, require_matches=True) +cache() +uncertain = compare() +payload['uncertain'] = [] +cache() +excluded = compare() +coverage['excludePaths'] = [] +resolved = compare() +connection.execute('INSERT INTO finding_triage VALUES (?, ?, ?)', ('a1', 'closed', 'already_fixed')) +link('c1', 'd1') +link('c2', 'd1') +linked = compare() +unchanged = json.loads(connection.execute('SELECT result_json FROM scan_comparisons').fetchone()[0]) == payload +connection.execute("DELETE FROM scan_comparison_matches WHERE after_scan_id = 'latest'") +restored = compare() +print(json.dumps({'uncertain': uncertain, 'excluded': excluded, 'resolved': resolved, + 'linked': linked, 'unchanged': unchanged, 'restored': restored})) +`; + const observed = await runPythonProbe( + probe, + join(tmpdir(), "codex-security-comparison-fixture"), + ); + expect(observed).toMatchObject({ + uncertain: { + summary: { new: 0, resolved: 0, unknown: 2 }, + findings: [ + { + findingId: "a2", + beforeOccurrenceIds: ["a1", "a2"], + severity: "high", + status: "unknown", + reason: "Synthetic uncertainty.", + }, + { + findingId: "b2", + afterOccurrenceIds: ["b1", "b2"], + status: "unknown", + reason: "Synthetic uncertainty.", + }, + ], + }, + excluded: { summary: { new: 1, resolved: 0, unknown: 1 } }, + resolved: { summary: { new: 1, resolved: 1, unknown: 0 } }, + linked: { + summary: { new: 0, persisting: 0, reopened: 1, resolved: 0, unknown: 0 }, + findings: [ + { + beforeOccurrenceIds: ["a1", "a2"], + afterOccurrenceIds: ["b1", "b2"], + matchReason: expect.any(String), + status: "reopened", + }, + ], + }, + unchanged: true, + }); + expect(observed["linked"]).not.toHaveProperty("related"); + expect(observed["restored"]).toEqual(observed["resolved"]); +}); + +test("loads displayed relations in bulk and follows current confirmed identities", async () => { + const probe = ` +import json, sqlite3, sys +sys.path.insert(0, sys.argv[1]) +import workbench_scan_history as history +connection = sqlite3.connect(':memory:') +connection.row_factory = sqlite3.Row +connection.executescript(''' +CREATE TABLE scans (id TEXT PRIMARY KEY, target_id TEXT); +CREATE INDEX scans_by_target ON scans(target_id, id); +CREATE TABLE finding_occurrences ( + id TEXT PRIMARY KEY, finding_id TEXT, scan_id TEXT, title TEXT, + UNIQUE(scan_id, finding_id) +); +CREATE INDEX occurrences_by_finding ON finding_occurrences(finding_id, id); +CREATE TABLE scan_comparisons (before_scan_id TEXT, after_scan_id TEXT, result_json TEXT); +CREATE TABLE scan_comparison_matches ( + before_scan_id TEXT, after_scan_id TEXT, before_occurrence_id TEXT, after_occurrence_id TEXT +); +CREATE INDEX matches_before ON scan_comparison_matches(before_occurrence_id); +CREATE INDEX matches_after ON scan_comparison_matches(after_occurrence_id); +''') +connection.executemany('INSERT INTO scans VALUES (?, ?)', [ + ('one', 'target'), ('two', 'target'), ('three', 'clone'), + ('four', 'clone'), ('foreign-one', 'unrelated-target'), + ('foreign-two', 'unrelated-target') +]) +connection.executemany('INSERT INTO finding_occurrences VALUES (?, ?, ?, ?)', ( + (f'{side}-{index}', f'{side}-identity-{index}', scan, f'Synthetic {side} {index}') + for index in range(10_000) + for side, scan in [('left', 'one'), ('right', 'two')] +)) +payload = json.dumps({'matches': [], 'uncertain': [], 'related': [ + {'beforeOccurrenceId': f'left-{index}', 'afterOccurrenceId': f'right-{index}', + 'reason': 'Separate synthetic controls.'} + for index in range(10_000) +]}) +connection.execute('INSERT INTO scan_comparisons VALUES (?, ?, ?)', ('one', 'two', payload)) +queries = [] +connection.set_trace_callback(queries.append) +scoped = history.finding_relations(connection, 'one', ['left-0']) +scoped_queries = len(queries) +queries.clear() +empty = history.finding_relations(connection, 'one', []) +empty_queries = len(queries) +connection.executemany('INSERT INTO finding_occurrences VALUES (?, ?, ?, ?)', [ + ('recurring-left', 'left-identity-0', 'four', 'Recurring control'), + ('bridge', 'bridge-identity', 'three', 'Renamed control'), + ('foreign-a', 'foreign-identity-a', 'foreign-one', 'Unrelated A'), + ('foreign-b', 'foreign-identity-b', 'foreign-two', 'Unrelated B') +]) +connection.executemany('INSERT INTO scan_comparison_matches VALUES (?, ?, ?, ?)', [ + ('four', 'three', 'recurring-left', 'bridge'), + ('two', 'three', 'right-0', 'bridge'), + ('foreign-one', 'foreign-two', 'foreign-a', 'foreign-b') +]) +aliases = history._confirmed_finding_aliases(connection, ['left-0']) +forward = history.finding_relations(connection, 'one', ['left-0']) +reverse = history.finding_relations(connection, 'two', ['right-0']) +remaining = history.finding_relations(connection, 'one', ['left-1']) +unchanged = connection.execute('SELECT result_json FROM scan_comparisons').fetchone()[0] == payload +connection.execute('DELETE FROM scan_comparison_matches WHERE before_occurrence_id = ?', ('right-0',)) +restored = history.finding_relations(connection, 'one', ['left-0']) == scoped + +limited = hasattr(connection, 'setlimit') +if limited: + old_limit = connection.setlimit(sqlite3.SQLITE_LIMIT_VARIABLE_NUMBER, 8) +queries.clear() +batched = history.finding_relations(connection, 'one', [f'left-{index}' for index in range(1, 11)]) +batched_queries = len(queries) +if limited: + connection.setlimit(sqlite3.SQLITE_LIMIT_VARIABLE_NUMBER, old_limit) + +class LegacyConnection: + def execute(self, *args): + return connection.execute(*args) + +queries.clear() +legacy_rows = list(history._rows_for_ids( + LegacyConnection(), 'SELECT id FROM finding_occurrences WHERE id IN ({placeholders})', + (f'left-{index}' for index in range(1001)) +)) +print(json.dumps({ + 'scoped': scoped, 'scopedQueries': scoped_queries, 'empty': empty, + 'emptyQueries': empty_queries, 'aliases': sorted(aliases), 'forward': forward, + 'reverse': reverse, 'remaining': sorted(remaining), 'unchanged': unchanged, + 'restoredAfterUnlink': restored, + 'batchedCount': len(batched), 'batchedQueries': batched_queries, + 'expectedBatchedQueries': 6 if limited else 3, + 'legacyCount': len(legacy_rows), 'legacyQueries': len(queries) +})) +`; + const observed = await runPythonProbe(probe); + expect(observed).toMatchObject({ + scoped: { + "left-0": [{ occurrenceId: "right-0", scanId: "two" }], + }, + scopedQueries: 3, + empty: {}, + emptyQueries: 0, + aliases: ["bridge-identity", "left-identity-0", "right-identity-0"], + forward: {}, + reverse: {}, + remaining: ["left-1"], + unchanged: true, + restoredAfterUnlink: true, + batchedCount: 10, + legacyCount: 1001, + legacyQueries: 2, + }); + expect(observed["batchedQueries"]).toBe(observed["expectedBatchedQueries"]); });