verify_trace_attribution_dev.py

from __future__ import annotations

import json
import sys
from pathlib import Path
from typing import Any


ROOT = Path(__file__).resolve().parents[1]
sys.path.insert(0, str(ROOT))

from scripts.run_trace_attribution_dev import (
    EXPECTED_MODELS,
    EXPECTED_SEEDS,
    EXPECTED_STUDY_ID,
    _load_and_validate_config,
    _render_markdown,
    _summarize_reports,
)


def verify(root: Path = ROOT) -> dict[str, Any]:
    root = root.resolve()
    config_path = root / 'configs' / 'trace_attribution_dev_v1.json'
    config = _load_and_validate_config(config_path)
    summary_path = root / str(config['outputs']['report_json'])
    markdown_path = root / str(config['outputs']['report_markdown'])
    summary = json.loads(summary_path.read_text(encoding='utf-8'))

    if (
        summary.get('study_id') != EXPECTED_STUDY_ID
        or summary.get('status') != 'complete'
        or summary.get('evidence_class') != 'post_final_dev_diagnostic'
        or summary.get('split') != 'dev'
        or summary.get('test_data_accessed') is not False
        or summary.get('historical_results_read_or_rewritten') is not False
        or tuple(summary.get('seeds', ())) != EXPECTED_SEEDS
        or tuple(summary.get('models', {})) != EXPECTED_MODELS
    ):
        raise ValueError('committed summary violates the Dev-only evidence contract')

    created_at = summary.get('created_at')
    if not isinstance(created_at, str) or not created_at:
        raise ValueError('committed summary lacks created_at')
    recomputed = _summarize_reports(root, config, config_path)
    recomputed['created_at'] = created_at
    if recomputed != summary:
        raise ValueError('committed JSON differs from deterministic recomputation')

    expected_markdown = _render_markdown(recomputed)
    if markdown_path.read_text(encoding='utf-8') != expected_markdown:
        raise ValueError('committed Markdown differs from the verified JSON rendering')

    source_records = summary['integrity']['source_reports']
    return {
        'schema_version': 1,
        'status': 'pass',
        'study_id': EXPECTED_STUDY_ID,
        'evidence_class': 'post_final_dev_diagnostic',
        'split': 'dev',
        'test_data_accessed': False,
        'historical_results_read_or_rewritten': False,
        'report_json': str(config['outputs']['report_json']),
        'report_markdown': str(config['outputs']['report_markdown']),
        'checks': [
            'locked_config',
            'dev_only_source_reports',
            'source_report_sha256',
            'checkpoint_sha256',
            'per_user_artifact_sha256',
            'deterministic_statistics',
            'json_contract',
            'markdown_render',
        ],
        'source_reports': len(source_records),
        'artifacts': sum(
            len(source['artifacts']) for source in source_records
        ),
    }


def main() -> int:
    print(
        json.dumps(
            verify(ROOT),
            ensure_ascii=False,
            indent=2,
            allow_nan=False,
        )
    )
    return 0


if __name__ == '__main__':
    raise SystemExit(main())