diff --git a/build.py b/build.py index 07b97c41..ef80a40a 100644 --- a/build.py +++ b/build.py @@ -1,907 +1,914 @@ -#!/usr/bin/env python3 - -import argparse -import datetime -import getpass -import json -import os -import platform -import shutil -import subprocess -import sys -import time -from dataclasses import dataclass -from pathlib import Path -from typing import Optional - -ROOT = Path(__file__).resolve().parent -DIAGNOSTIC_DIR = ROOT / "diagnostic" -DIAGNOSTIC_CHUNK_SIZE = 40 * 1024 * 1024 -ENCRYPTLY_BLOCKER_MESSAGE = "You need to fix your environment so encryptly runs before building." - - -def current_commit_id() -> str: - """Return the first 4 bytes (8 hex chars) of HEAD for stable per-commit diagnostics.""" - try: - result = subprocess.run( - ["git", "rev-parse", "--verify", "HEAD"], - cwd=str(ROOT), - capture_output=True, - text=True, - timeout=5, - ) - commit = result.stdout.strip() - if result.returncode == 0 and len(commit) >= 8: - return commit[:8] - except Exception: - pass - return "00000000" - - -def diagnostic_paths_for_commit() -> tuple[Path, Path, str]: - """Return stable diagnostic artifact paths under diagnostic/ for the current commit.""" - DIAGNOSTIC_DIR.mkdir(parents=True, exist_ok=True) - commit_id = current_commit_id() - logd_path = DIAGNOSTIC_DIR / f"build-{commit_id}.logd" - metadata_path = DIAGNOSTIC_DIR / f"build-{commit_id}.json" - return logd_path, metadata_path, commit_id - - -def split_diagnostic_logd(logd_path: Path, chunk_size: int = DIAGNOSTIC_CHUNK_SIZE) -> list[Path]: - """Split an oversized .logd into numbered .logd chunks and remove the original.""" - if logd_path.stat().st_size <= chunk_size: - return [logd_path] - - chunks: list[Path] = [] - stem = logd_path.stem - with logd_path.open("rb") as source: - index = 1 - while True: - data = source.read(chunk_size) - if not data: - break - chunk_path = logd_path.with_name(f"{stem}-part{index:03d}.logd") - chunk_path.write_bytes(data) - chunks.append(chunk_path) - index += 1 - - logd_path.unlink() - return chunks - - -@dataclass -class Module: - name: str - language: str - dir: Path - build_cmd: list[str] - clean_cmd: list[str] - build_dir: Optional[Path] = None - env: Optional[dict[str, str]] = None - -MODULES = [ - Module( - name="backend", - language="Rust", - dir=ROOT / "backend", - build_cmd=["cargo", "build"], - clean_cmd=["cargo", "clean"], - build_dir=ROOT / "backend" / "target", - env={"CARGO_TERM_COLOR": "always"}, - ), - Module( - name="frontend", - language="TypeScript", - dir=ROOT / "frontend", - build_cmd=["npm", "run", "build"], - clean_cmd=["rm", "-rf", "node_modules", "dist"], - build_dir=ROOT / "frontend" / "dist", - env={"NODE_ENV": "production"}, - ), - Module( - name="market", - language="Go", - dir=ROOT / "market", - build_cmd=["go", "build", "-o", "market", "."], - clean_cmd=["rm", "-f", "market"], - build_dir=ROOT / "market" / "market", - ), - Module( - name="frailbox", - language="C", - dir=ROOT / "frailbox", - build_cmd=["make"], - clean_cmd=["make", "distclean"], - build_dir=ROOT / "frailbox" / "frailbox", - ), - Module( - name="engine", - language="C++", - dir=ROOT / "frailbox" / "engine", - build_cmd=["cmake", "--build", "build"], - clean_cmd=["rm", "-rf", "build"], - build_dir=ROOT / "frailbox" / "engine" / "build" / "trial-engine", - ), - Module( - name="compliance", - language="Java", - dir=ROOT / "compliance", - build_cmd=["javac", "-d", "build", "ComplianceAuditor.java"], - clean_cmd=["rm", "-rf", "build"], - build_dir=ROOT / "compliance" / "build", - ), - Module( - name="v2-market-stream", - language="Ruby", - dir=ROOT / "v2" / "services", - build_cmd=["ruby", "-c", "market_stream.rb"], - clean_cmd=["echo", "Ruby has no build artifacts to clean"], - build_dir=None, - ), - Module( - name="nfc-scanner", - language="Lua", - dir=ROOT / "frailbox" / "nfc", - build_cmd=["luac", "-p", "scanner.lua"], - clean_cmd=["echo", "Lua has no build artifacts to clean"], - build_dir=None, - ), - Module( - name="openapi-haskell", - language="Haskell", - dir=ROOT / "docs" / "openapi", - build_cmd=["ghc", "-fno-code", "Types.hs", "Server.hs", "Validate.hs", "Generate.hs"], - clean_cmd=["rm", "-f", "*.hi", "*.o", "*.hie"], - build_dir=None, - ), - Module( - name="openapi-tools", - language="Lua", - dir=ROOT / "tools", - build_cmd=["luac", "-p", "openapi_diff.lua", "openapi_mock.lua", "openapi_pact.lua"], - clean_cmd=["echo", "Nothing to clean"], - build_dir=None, - ), -] - -ENCRYPTLY_DIR = ROOT / "tools" / "encryptly" -ENCRYPTLY_BINARIES = { - "linux-x64": ENCRYPTLY_DIR / "linux-x64" / "encryptly", - "linux-arm64": ENCRYPTLY_DIR / "linux-arm64" / "encryptly", - "macos-arm64": ENCRYPTLY_DIR / "macos-arm64" / "encryptly", - "windows-x64": ENCRYPTLY_DIR / "windows-x64" / "encryptly.exe", - "windows-arm64": ENCRYPTLY_DIR / "windows-arm64" / "encryptly.exe", -} -LEGACY_ENCRYPTLY_BIN = ENCRYPTLY_DIR / "encryptly" - - -def _normalize_arch(machine: str) -> Optional[str]: - machine = machine.lower() - if machine in {"x86_64", "amd64"}: - return "x64" - if machine in {"aarch64", "arm64"}: - return "arm64" - return None - - -def _normalize_os() -> Optional[str]: - system = platform.system().lower() - if system == "linux": - return "linux" - if system == "darwin": - return "macos" - if system == "windows": - return "windows" - return None - - -def detect_encryptly_platform() -> Optional[str]: - os_name = _normalize_os() - arch = _normalize_arch(platform.machine()) - if os_name is None or arch is None: - return None - return f"{os_name}-{arch}" - - -def get_encryptly_bin() -> Optional[Path]: - target = detect_encryptly_platform() - if target is not None: - binary = ENCRYPTLY_BINARIES.get(target) - if binary is not None and binary.exists(): - return binary - - if LEGACY_ENCRYPTLY_BIN.exists(): - return LEGACY_ENCRYPTLY_BIN - - return None - - -def encryptly_platform_help() -> str: - detected = detect_encryptly_platform() or "unsupported" - available = ", ".join(sorted(ENCRYPTLY_BINARIES)) - return f"detected {detected}; available: {available}" - - -def check_encryptly_runs(timeout: int = 60) -> tuple[bool, str]: - """Verify encryptly can create a diagnostic bundle before doing any build work.""" - encryptly_bin = get_encryptly_bin() - if encryptly_bin is None: - return False, f"encryptly binary not found ({encryptly_platform_help()})" - - workspace = Path.home() / ".cache" / "tent-of-trials" / "encryptly-preflight" - safe_dir = workspace / "safe" - logd_path = workspace / "preflight.logd" - try: - shutil.rmtree(workspace, ignore_errors=True) - safe_dir.mkdir(parents=True, exist_ok=True) - (safe_dir / "preflight.txt").write_text("encryptly preflight\n", encoding="utf-8") - result = subprocess.run( - [ - str(encryptly_bin), - "pack", - str(logd_path), - "--include", - str(workspace), - "--max-file-size", - "1", - ], - cwd=str(ROOT), - capture_output=True, - text=True, - timeout=timeout, - ) - if result.returncode != 0: - output = result.stderr.strip() or result.stdout.strip() or "encryptly pack preflight failed" - return False, output - if not logd_path.exists(): - return False, "encryptly preflight completed without creating a .logd" - return True, "encryptly preflight passed" - except subprocess.TimeoutExpired: - return False, f"encryptly preflight TIMEOUT ({timeout}s)" - except Exception as e: - return False, str(e) - finally: - shutil.rmtree(workspace, ignore_errors=True) - -class Colors: - GREEN = "\033[92m" - YELLOW = "\033[93m" - RED = "\033[91m" - CYAN = "\033[96m" - BOLD = "\033[1m" - RESET = "\033[0m" - GRAY = "\033[90m" - -def color(text: str, code: str) -> str: - if not sys.stdout.isatty(): - return text - return f"{code}{text}{Colors.RESET}" - -def check_prerequisites() -> list[str]: - required = { - "cargo": "Rust", - "npm": "Node.js", - "go": "Go", - "gcc": "C (GCC)", - "g++": "C++ (GCC)", - "cmake": "CMake", - "make": "Make", - "python3": "Python", - "javac": "Java (JDK)", - "ruby": "Ruby", - "luac": "Lua", - "ghc": "GHC (Haskell)", - } - - missing = [] - for cmd, label in required.items(): - if shutil.which(cmd) is None: - missing.append(f"{label} ({cmd})") - - return missing - -def build_module( - module: Module, - release: bool = False, - verbose: bool = False, -) -> tuple[bool, float, str]: - - print(f"\n {color('▸', Colors.CYAN)} Building {color(module.name, Colors.BOLD)} ({module.language})...") - - env = os.environ.copy() - if module.env: - env.update(module.env) - - start = time.time() - - if module.name == "frontend": - node_modules = module.dir / "node_modules" - if not node_modules.exists(): - print(f" {color('npm install...', Colors.GRAY)}") - try: - install_result = subprocess.run( - ["npm", "install"], - cwd=str(module.dir), - capture_output=not verbose, - text=True, - timeout=120, - env={k: v for k, v in env.items() if k != "NODE_ENV"}, - ) - if install_result.returncode != 0: - return False, time.time() - start, f"npm install failed:\n{install_result.stderr}" - except subprocess.TimeoutExpired: - return False, time.time() - start, "npm install TIMEOUT (120s)" - - if module.name == "engine": - - build_type = "Release" if release else "Debug" - try: - cfg_result = subprocess.run( - ["cmake", "-S", ".", "-B", "build", - f"-DCMAKE_BUILD_TYPE={build_type}"], - cwd=str(module.dir), - capture_output=True, - text=True, - timeout=120, - env=env, - ) - except subprocess.TimeoutExpired: - return False, time.time() - start, "CMake configure TIMEOUT (120s)" - except FileNotFoundError as e: - return False, 0, f"Command not found: {e}" - if cfg_result.returncode != 0: - output_lines = [] - if cfg_result.stdout: - output_lines.append(cfg_result.stdout.strip()) - if cfg_result.stderr: - output_lines.append(cfg_result.stderr.strip()) - output = "\n".join(output_lines) - return False, time.time() - start, ( - f"CMake configure failed:\n{output}") - if verbose: - print(f" {color('cmake configured', Colors.GRAY)}") - cmd = ["cmake", "--build", "build"] - if release: - cmd.append("--config") - cmd.append("Release") - else: - cmd = list(module.build_cmd) - if release and module.name == "backend": - cmd.append("--release") - - try: - result = subprocess.run( - cmd, - cwd=str(module.dir), - capture_output=True, - text=True, - env=env, - timeout=300, - ) - except subprocess.TimeoutExpired: - return False, time.time() - start, "BUILD TIMEOUT (300s)" - except FileNotFoundError as e: - return False, 0, f"Command not found: {e}" - - elapsed = time.time() - start - output_lines = [] - - if result.stdout: - output_lines.append(result.stdout.strip()) - if result.stderr: - output_lines.append(result.stderr.strip()) - - output = "\n".join(output_lines) - success = result.returncode == 0 - - return success, elapsed, output - -def clean_module(module: Module, verbose: bool = False) -> bool: - print(f" {color('▸', Colors.YELLOW)} Cleaning {module.name}...") - try: - subprocess.run( - module.clean_cmd, - cwd=str(module.dir), - capture_output=not verbose, - text=True, - timeout=60, - env=os.environ.copy(), - ) - return True - except Exception as e: - print(f" {color('✗', Colors.RED)} Clean failed: {e}") - return False - -def verify_binary(module: Module) -> Optional[str]: - if module.build_dir is None: - return None - path = module.build_dir - if module.name == "backend": - - target = path / "debug" / module.name - if not target.exists(): - target = path / "release" / module.name - if target.exists(): - return str(target) - if path.exists(): - return str(path) - return None - -def run_cmd(cmd: list[str], **kwargs) -> tuple[bool, str]: - try: - result = subprocess.run( - cmd, capture_output=True, text=True, check=False, **kwargs - ) - output = result.stdout - if result.stderr: - output += "\n" + result.stderr - return result.returncode == 0, output.strip() - except Exception as e: - return False, str(e) - - -def collect_system_info() -> str: - lines = [ - "Tent of Trials - System Diagnostic Snapshot", - "=" * 50, - f"generated_at: {datetime.datetime.now(datetime.timezone.utc).isoformat()}", - f"hostname: {platform.node()}", - f"user: {getpass.getuser()}", - f"python: {sys.version}", - f"platform: {platform.platform()}", - f"processor: {platform.processor() or 'unknown'}", - f"cpu_count: {os.cpu_count()}", - "", - "--- uname ---", - ] - ok, out = run_cmd(["uname", "-a"]) - lines.append(out if ok else "unavailable") - - lines.extend(["", "--- /etc/os-release ---"]) - try: - lines.append((Path("/etc/os-release")).read_text(encoding="utf-8", errors="replace").strip()) - except Exception as e: - lines.append(f"unavailable: {e}") - - lines.extend(["", "--- memory ---"]) - ok, out = run_cmd(["free", "-h"]) - lines.append(out if ok else "unavailable") - - lines.extend(["", "--- disk ---"]) - ok, out = run_cmd(["df", "-h"]) - lines.append(out if ok else "unavailable") - - lines.extend(["", "--- build environment ---"]) - for key in ["SHELL", "LANG", "TERM", "XDG_SESSION_TYPE", "DISPLAY", "EDITOR"]: - value = os.environ.get(key) - if value: - lines.append(f"{key}={value}") - - lines.append("") - return "\n".join(lines) - - -def build_diagnostic_report( - results: list[tuple[str, bool, float, str, Optional[str]]], - commit_id: str, - logd_relpaths: Optional[list[str]] = None, - password: Optional[str] = None, - logd_error: Optional[str] = None, - chunked: bool = False, - message_blocker: Optional[str] = None, -) -> dict: - diagnostic_logd: Optional[str | list[str]] - if not logd_relpaths: - diagnostic_logd = None - elif len(logd_relpaths) == 1: - diagnostic_logd = logd_relpaths[0] - else: - diagnostic_logd = logd_relpaths - - decrypt_target = logd_relpaths[0] if logd_relpaths and len(logd_relpaths) == 1 else None - if logd_relpaths and len(logd_relpaths) > 1: - decrypt_target = str((DIAGNOSTIC_DIR / f"build-{commit_id}.logd").relative_to(ROOT)) - - report = { - "generated_at": datetime.datetime.now(datetime.timezone.utc).isoformat(), - "commit": commit_id, - "diagnostic_logd": diagnostic_logd, - "diagnostic_logd_error": logd_error, - "message_blocker": message_blocker, - "chunked": chunked, - "chunk_size_bytes": DIAGNOSTIC_CHUNK_SIZE if chunked else None, - "password": password, - "decrypt_command": ( - f"encryptly unpack {decrypt_target} --password {password}" - if decrypt_target and password else None - ), - "total_modules": len(results), - "passed": sum(1 for _, s, _, _, _ in results if s), - "failed": sum(1 for _, s, _, _, _ in results if not s), - "modules": [ - { - "name": name, - "status": "PASS" if success else "FAIL", - "elapsed_seconds": round(elapsed, 3), - "artifact": binary, - "output": output, - } - for name, success, elapsed, output, binary in results - ], - "pr_note": ( - (f"Include the encrypted diagnostic logd artifact(s): {', '.join(logd_relpaths)}. " if logd_relpaths else "Encrypted diagnostic logd artifact was not created; include this JSON report showing why. ") - + "The encrypted .logd is the required diagnostic content for PR review; this JSON file is metadata. " - + "Maintainers may ask you to remove these diagnostic artifacts before merging." - ), - } - return report - - -def write_diagnostic_report(metadata_path: Path, report: dict) -> None: - metadata_path.write_text(json.dumps(report, indent=2) + "\n", encoding="utf-8") - print(f" {color('✓', Colors.GREEN)} {metadata_path.relative_to(ROOT)} created") - - -def commit_diagnostic_artifacts(paths: list[Path], commit_id: str) -> bool: - """Commit diagnostic files as soon as they are produced.""" - existing = [path for path in paths if path.exists()] - if not existing: - print(f" {color('✗', Colors.RED)} No diagnostic artifacts found to commit") - return False - - relpaths = [str(path.relative_to(ROOT)) for path in existing] - status = subprocess.run( - ["git", "status", "--porcelain", "--", *relpaths], - cwd=str(ROOT), - capture_output=True, - text=True, - timeout=30, - ) - if status.returncode != 0: - print(f" {color('✗', Colors.RED)} Could not inspect diagnostic git status: {status.stderr.strip()}") - return False - if not status.stdout.strip(): - print(f" {color('✓', Colors.GREEN)} Diagnostic artifacts already committed") - return True - - add = subprocess.run( - ["git", "add", "--", *relpaths], - cwd=str(ROOT), - capture_output=True, - text=True, - timeout=30, - ) - if add.returncode != 0: - print(f" {color('✗', Colors.RED)} Could not stage diagnostic artifacts: {add.stderr.strip()}") - return False - - commit = subprocess.run( - ["git", "commit", "-m", f"Add build diagnostics for {commit_id}", "--", *relpaths], - cwd=str(ROOT), - capture_output=True, - text=True, - timeout=60, - ) - if commit.returncode != 0: - output = commit.stderr.strip() or commit.stdout.strip() - print(f" {color('✗', Colors.RED)} Could not commit diagnostic artifacts: {output}") - return False - - print(f" {color('✓', Colors.GREEN)} Diagnostic artifacts committed") - return True - - -def generate_logd( - results: list[tuple[str, bool, float, str, Optional[str]]], - verbose: bool = False, -) -> bool: - logd_path, metadata_path, commit_id = diagnostic_paths_for_commit() - display_logd = logd_path.relative_to(ROOT) - print(f"\n {color('▸', Colors.CYAN)} Finalizing diagnostics for {color(str(display_logd), Colors.BOLD)}...") - - # Always write the JSON report first. The encrypted .logd is useful, but the - # report is required even when the build failed before compilation started or - # when encryptly itself is unavailable. - write_diagnostic_report(metadata_path, build_diagnostic_report(results, commit_id)) - - encryptly_bin = get_encryptly_bin() - if encryptly_bin is None: - error = f"encryptly binary not found ({encryptly_platform_help()}); cannot create {display_logd}" - print(f" {color('✗', Colors.RED)} {error}") - write_diagnostic_report( - metadata_path, - build_diagnostic_report( - results, - commit_id, - logd_error=error, - message_blocker=ENCRYPTLY_BLOCKER_MESSAGE, - ), - ) - print(f" {color('BLOCKER', Colors.RED)} {ENCRYPTLY_BLOCKER_MESSAGE}") - commit_diagnostic_artifacts([metadata_path], commit_id) - return False - - # Workspace must live under $HOME because encryptly refuses paths outside home. - home = Path.home() - workspace = home / ".cache" / "tent-of-trials" / "logd-workspace" - safe_dir = workspace / "safe" - - try: - shutil.rmtree(workspace, ignore_errors=True) - safe_dir.mkdir(parents=True, exist_ok=True) - - (safe_dir / "system-info.txt").write_text( - collect_system_info(), encoding="utf-8" - ) - - summary_lines = [ - "Tent of Trials - Build Summary", - "=" * 50, - f"generated_at: {datetime.datetime.now(datetime.timezone.utc).isoformat()}", - f"total_modules: {len(results)}", - f"passed: {sum(1 for _, s, _, _, _ in results if s)}", - f"failed: {sum(1 for _, s, _, _, _ in results if not s)}", - "", - "module results:", - ] - for name, success, elapsed, _, binary in results: - summary_lines.append( - f" {name}: {'PASS' if success else 'FAIL'} ({elapsed:.2f}s)" - f"{f' [{binary}]' if binary else ''}" - ) - (safe_dir / "build-summary.txt").write_text( - "\n".join(summary_lines), encoding="utf-8" - ) - - log_lines = [] - for name, success, elapsed, output, binary in results: - log_lines.append( - f"\n{'=' * 50}\n{name} ({'PASS' if success else 'FAIL'}, {elapsed:.2f}s)\n" - f"{'=' * 50}" - ) - if binary: - log_lines.append(f"artifact: {binary}") - if output: - log_lines.append(output) - (safe_dir / "build.log").write_text("\n".join(log_lines), encoding="utf-8") - - sr = subprocess.run( - [ - str(encryptly_bin), - "pack", - str(logd_path), - "--include", - str(workspace), - "--max-file-size", - "35840", - ], - cwd=str(ROOT), - capture_output=True, - text=True, - timeout=300, - ) - if sr.returncode != 0: - error = sr.stderr.strip() or sr.stdout.strip() or "encryptly pack failed" - print( - f" {color('✗', Colors.RED)} {logd_path.relative_to(ROOT)} creation failed: " - f"{error}" - ) - if logd_path.exists(): - logd_path.unlink() - write_diagnostic_report( - metadata_path, - build_diagnostic_report( - results, - commit_id, - logd_error=error, - message_blocker=ENCRYPTLY_BLOCKER_MESSAGE, - ), - ) - print(f" {color('BLOCKER', Colors.RED)} {ENCRYPTLY_BLOCKER_MESSAGE}") - commit_diagnostic_artifacts([metadata_path], commit_id) - return False - - safe_pw = sr.stdout.strip() - logd_files = split_diagnostic_logd(logd_path) - logd_relpaths = [str(path.relative_to(ROOT)) for path in logd_files] - decrypt_target = logd_relpaths[0] if len(logd_relpaths) == 1 else str(logd_path.relative_to(ROOT)) - write_diagnostic_report( - metadata_path, - build_diagnostic_report( - results, - commit_id, - logd_relpaths=logd_relpaths, - password=safe_pw, - chunked=len(logd_files) > 1, - ), - ) - - for path in logd_files: - size_kb = path.stat().st_size / 1024.0 - print( - f" {color('✓', Colors.GREEN)} {path.relative_to(ROOT)} created " - f"({size_kb:.1f} KiB)" - ) - if len(logd_files) > 1: - print( - f" {color('✓', Colors.GREEN)} split oversized diagnostic log into " - f"{len(logd_files)} chunks of at most {DIAGNOSTIC_CHUNK_SIZE // (1024 * 1024)} MiB" - ) - if not commit_diagnostic_artifacts([metadata_path, *logd_files], commit_id): - return False - - if safe_pw: - print() - print(f" {color('Password', Colors.BOLD)} - this is required to decrypt the diagnostic log,") - print(f" which is required to submit a PR. Upload the") - print(f" diagnostic log file(s) and metadata file with this password.") - if len(logd_files) > 1: - print(f" Reassemble chunks in order before unpacking:") - print(f" cat {' '.join(logd_relpaths)} > {logd_path.relative_to(ROOT)}") - print(f" {color(safe_pw, Colors.CYAN)}") - print(f" {color(f'encryptly unpack {decrypt_target} --password {safe_pw}', Colors.GRAY)}") - return True - - finally: - shutil.rmtree(workspace, ignore_errors=True) - - -def print_summary(results: list[tuple[str, bool, float, str, Optional[str]]]): - print(f" {color('Build Summary', Colors.BOLD)}") - - total = len(results) - passed = sum(1 for _, s, _, _, _ in results if s) - failed = total - passed - total_time = sum(t for _, _, t, _, _ in results) - - for name, success, elapsed, output, binary in results: - status_icon = color("✓", Colors.GREEN) if success else color("✗", Colors.RED) - status_text = color("PASS", Colors.GREEN) if success else color("FAIL", Colors.RED) - time_str = f"{elapsed:.1f}s" if elapsed < 60 else f"{elapsed / 60:.1f}m" - - print(f"\n {status_icon} {color(name + ':', Colors.BOLD)} {status_text} ({time_str})") - if binary: - print(f" artifact: {color(binary, Colors.GRAY)}") - if not success and output: - - lines = output.strip().split("\n") - print(f" {color('last output:', Colors.RED)}") - for line in lines[-5:]: - print(f" {color(line, Colors.GRAY)}") - - print(f"\n {color('─' * 40, Colors.GRAY)}") - print(f" {color('Total:', Colors.BOLD)} {total} modules, " - f"{color(str(passed) + ' passed', Colors.GREEN)}, " - f"{color(str(failed) + ' failed', Colors.RED)}, " - f"{total_time:.1f}s total") - -def main(): - parser = argparse.ArgumentParser( - description="Tent of Trials - Multi-Language Build System", - formatter_class=argparse.RawDescriptionHelpFormatter, - epilog=""" -Examples: - python3 build.py Build all modules - python3 build.py -m backend Build only backend - python3 build.py -m frontend,market Build frontend and market - python3 build.py --clean Clean all artifacts - python3 build.py --release Release build (Rust only) - python3 build.py --verbose Verbose output - -Diagnostic bundle: - python3 build.py - """, - ) - parser.add_argument( - "-m", "--module", - help="Module(s) to build (comma-separated, or 'all')", - default="all", - ) - parser.add_argument( - "--clean", action="store_true", - help="Clean build artifacts instead of building", - ) - parser.add_argument( - "--release", action="store_true", - help="Build in release mode (Rust backend)", - ) - parser.add_argument( - "--verbose", "-v", action="store_true", - help="Show detailed build output", - ) - parser.add_argument( - "--list", action="store_true", - help="List available modules and exit", - ) - - args = parser.parse_args() - - print(f"\n {color('Tent of Trials: building', Colors.CYAN)}") - print(f" Working directory: {ROOT}") - print() - - if args.list: - print(f" {color('Available modules:', Colors.BOLD)}") - for m in MODULES: - print(f" {color(m.name, Colors.CYAN)} ({m.language})") - print(f" dir: {m.dir.relative_to(ROOT)}") - print(f" build: {' '.join(m.build_cmd)}") - return 0 - - print(f" {color('Checking prerequisites...', Colors.GRAY)}") - missing = check_prerequisites() - if missing: - print(f"\n {color('⚠ Some tools missing - will try anyway:', Colors.YELLOW)}") - for m in missing: - print(f" {m}") - - msg = "Not all modules will build. That's fine." - print(f" {color(msg, Colors.GRAY)}") - else: - print(f" {color('✓ All prerequisites found', Colors.GREEN)}") - if args.module == "all": - selected = MODULES - else: - names = [n.strip() for n in args.module.split(",")] - selected = [m for m in MODULES if m.name in names] - not_found = set(names) - {m.name for m in MODULES} - if not_found: - print(f" {color('✗ Unknown modules:', Colors.RED)} {', '.join(not_found)}") - print(f" Available: {', '.join(m.name for m in MODULES)}") - return 1 - - if not selected: - print(f" No modules selected.") - return 0 - - if args.clean: - print(f"\n {color('Cleaning build artifacts...', Colors.YELLOW)}") - for module in selected: - clean_module(module, args.verbose) - - diagnostic_artifacts = [ROOT / "build.logd"] - if DIAGNOSTIC_DIR.exists(): - diagnostic_artifacts.extend(DIAGNOSTIC_DIR.glob("build-[0-9a-f][0-9a-f][0-9a-f][0-9a-f][0-9a-f][0-9a-f][0-9a-f][0-9a-f].logd")) - diagnostic_artifacts.extend(DIAGNOSTIC_DIR.glob("build-[0-9a-f][0-9a-f][0-9a-f][0-9a-f][0-9a-f][0-9a-f][0-9a-f][0-9a-f]-part*.logd")) - diagnostic_artifacts.extend(DIAGNOSTIC_DIR.glob("build-[0-9a-f][0-9a-f][0-9a-f][0-9a-f][0-9a-f][0-9a-f][0-9a-f][0-9a-f].json")) - diagnostic_artifacts.extend(DIAGNOSTIC_DIR.glob("build-[0-9a-f][0-9a-f][0-9a-f][0-9a-f][0-9a-f][0-9a-f][0-9a-f][0-9a-f]-metadata.json")) - for artifact in diagnostic_artifacts: - if artifact.exists(): - if artifact.is_dir(): - shutil.rmtree(artifact) - else: - artifact.unlink() - print(f" {color('▸', Colors.YELLOW)} Removed {artifact.relative_to(ROOT)}") - print(f"\n {color('Clean complete.', Colors.GREEN)}") - return 0 - - print(f"\n {color('Checking encryptly diagnostics...', Colors.GRAY)}") - encryptly_start = time.time() - encryptly_ok, encryptly_message = check_encryptly_runs() - if not encryptly_ok: - elapsed = time.time() - encryptly_start - blocker = f"{ENCRYPTLY_BLOCKER_MESSAGE} {encryptly_message}" - print(f" {color('✗ encryptly cannot run', Colors.RED)}") - print(f" {color('BLOCKER:', Colors.RED)} {blocker}") - results = [("encryptly-preflight", False, elapsed, blocker, None)] - generate_logd(results, args.verbose) - return 1 - print(f" {color('✓ encryptly runs', Colors.GREEN)}") - - print(f"\n {color(f'Building {len(selected)} module(s) | release={args.release}', Colors.GRAY)}") - - results: list[tuple[str, bool, float, str, Optional[str]]] = [] - - for module in selected: - success, elapsed, output = build_module(module, args.release, args.verbose) - binary = verify_binary(module) if success else None - results.append((module.name, success, elapsed, output, binary)) - - print_summary(results) - - diagnostics_ok = generate_logd(results, args.verbose) - - return 0 if diagnostics_ok and all(r[1] for r in results) else 1 - -if __name__ == "__main__": - sys.exit(main()) +#!/usr/bin/env python3 + +import argparse +import datetime +import getpass +import json +import os +import platform +import shutil +import subprocess +import sys +import time +from dataclasses import dataclass +from pathlib import Path +from typing import Optional + +ROOT = Path(__file__).resolve().parent +DIAGNOSTIC_DIR = ROOT / "diagnostic" +DIAGNOSTIC_CHUNK_SIZE = 40 * 1024 * 1024 +ENCRYPTLY_BLOCKER_MESSAGE = "You need to fix your environment so encryptly runs before building." + +if str(ROOT / "tools") not in sys.path: + sys.path.insert(0, str(ROOT / "tools")) + +from diagnostic_redaction import repo_relative_posix, sanitize_metadata, validate_metadata_redaction + + +def current_commit_id() -> str: + """Return the first 4 bytes (8 hex chars) of HEAD for stable per-commit diagnostics.""" + try: + result = subprocess.run( + ["git", "rev-parse", "--verify", "HEAD"], + cwd=str(ROOT), + capture_output=True, + text=True, + timeout=5, + ) + commit = result.stdout.strip() + if result.returncode == 0 and len(commit) >= 8: + return commit[:8] + except Exception: + pass + return "00000000" + + +def diagnostic_paths_for_commit() -> tuple[Path, Path, str]: + """Return stable diagnostic artifact paths under diagnostic/ for the current commit.""" + DIAGNOSTIC_DIR.mkdir(parents=True, exist_ok=True) + commit_id = current_commit_id() + logd_path = DIAGNOSTIC_DIR / f"build-{commit_id}.logd" + metadata_path = DIAGNOSTIC_DIR / f"build-{commit_id}.json" + return logd_path, metadata_path, commit_id + + +def split_diagnostic_logd(logd_path: Path, chunk_size: int = DIAGNOSTIC_CHUNK_SIZE) -> list[Path]: + """Split an oversized .logd into numbered .logd chunks and remove the original.""" + if logd_path.stat().st_size <= chunk_size: + return [logd_path] + + chunks: list[Path] = [] + stem = logd_path.stem + with logd_path.open("rb") as source: + index = 1 + while True: + data = source.read(chunk_size) + if not data: + break + chunk_path = logd_path.with_name(f"{stem}-part{index:03d}.logd") + chunk_path.write_bytes(data) + chunks.append(chunk_path) + index += 1 + + logd_path.unlink() + return chunks + + +@dataclass +class Module: + name: str + language: str + dir: Path + build_cmd: list[str] + clean_cmd: list[str] + build_dir: Optional[Path] = None + env: Optional[dict[str, str]] = None + +MODULES = [ + Module( + name="backend", + language="Rust", + dir=ROOT / "backend", + build_cmd=["cargo", "build"], + clean_cmd=["cargo", "clean"], + build_dir=ROOT / "backend" / "target", + env={"CARGO_TERM_COLOR": "always"}, + ), + Module( + name="frontend", + language="TypeScript", + dir=ROOT / "frontend", + build_cmd=["npm", "run", "build"], + clean_cmd=["rm", "-rf", "node_modules", "dist"], + build_dir=ROOT / "frontend" / "dist", + env={"NODE_ENV": "production"}, + ), + Module( + name="market", + language="Go", + dir=ROOT / "market", + build_cmd=["go", "build", "-o", "market", "."], + clean_cmd=["rm", "-f", "market"], + build_dir=ROOT / "market" / "market", + ), + Module( + name="frailbox", + language="C", + dir=ROOT / "frailbox", + build_cmd=["make"], + clean_cmd=["make", "distclean"], + build_dir=ROOT / "frailbox" / "frailbox", + ), + Module( + name="engine", + language="C++", + dir=ROOT / "frailbox" / "engine", + build_cmd=["cmake", "--build", "build"], + clean_cmd=["rm", "-rf", "build"], + build_dir=ROOT / "frailbox" / "engine" / "build" / "trial-engine", + ), + Module( + name="compliance", + language="Java", + dir=ROOT / "compliance", + build_cmd=["javac", "-d", "build", "ComplianceAuditor.java"], + clean_cmd=["rm", "-rf", "build"], + build_dir=ROOT / "compliance" / "build", + ), + Module( + name="v2-market-stream", + language="Ruby", + dir=ROOT / "v2" / "services", + build_cmd=["ruby", "-c", "market_stream.rb"], + clean_cmd=["echo", "Ruby has no build artifacts to clean"], + build_dir=None, + ), + Module( + name="nfc-scanner", + language="Lua", + dir=ROOT / "frailbox" / "nfc", + build_cmd=["luac", "-p", "scanner.lua"], + clean_cmd=["echo", "Lua has no build artifacts to clean"], + build_dir=None, + ), + Module( + name="openapi-haskell", + language="Haskell", + dir=ROOT / "docs" / "openapi", + build_cmd=["ghc", "-fno-code", "Types.hs", "Server.hs", "Validate.hs", "Generate.hs"], + clean_cmd=["rm", "-f", "*.hi", "*.o", "*.hie"], + build_dir=None, + ), + Module( + name="openapi-tools", + language="Lua", + dir=ROOT / "tools", + build_cmd=["luac", "-p", "openapi_diff.lua", "openapi_mock.lua", "openapi_pact.lua"], + clean_cmd=["echo", "Nothing to clean"], + build_dir=None, + ), +] + +ENCRYPTLY_DIR = ROOT / "tools" / "encryptly" +ENCRYPTLY_BINARIES = { + "linux-x64": ENCRYPTLY_DIR / "linux-x64" / "encryptly", + "linux-arm64": ENCRYPTLY_DIR / "linux-arm64" / "encryptly", + "macos-arm64": ENCRYPTLY_DIR / "macos-arm64" / "encryptly", + "windows-x64": ENCRYPTLY_DIR / "windows-x64" / "encryptly.exe", + "windows-arm64": ENCRYPTLY_DIR / "windows-arm64" / "encryptly.exe", +} +LEGACY_ENCRYPTLY_BIN = ENCRYPTLY_DIR / "encryptly" + + +def _normalize_arch(machine: str) -> Optional[str]: + machine = machine.lower() + if machine in {"x86_64", "amd64"}: + return "x64" + if machine in {"aarch64", "arm64"}: + return "arm64" + return None + + +def _normalize_os() -> Optional[str]: + system = platform.system().lower() + if system == "linux": + return "linux" + if system == "darwin": + return "macos" + if system == "windows": + return "windows" + return None + + +def detect_encryptly_platform() -> Optional[str]: + os_name = _normalize_os() + arch = _normalize_arch(platform.machine()) + if os_name is None or arch is None: + return None + return f"{os_name}-{arch}" + + +def get_encryptly_bin() -> Optional[Path]: + target = detect_encryptly_platform() + if target is not None: + binary = ENCRYPTLY_BINARIES.get(target) + if binary is not None and binary.exists(): + return binary + + if LEGACY_ENCRYPTLY_BIN.exists(): + return LEGACY_ENCRYPTLY_BIN + + return None + + +def encryptly_platform_help() -> str: + detected = detect_encryptly_platform() or "unsupported" + available = ", ".join(sorted(ENCRYPTLY_BINARIES)) + return f"detected {detected}; available: {available}" + + +def check_encryptly_runs(timeout: int = 60) -> tuple[bool, str]: + """Verify encryptly can create a diagnostic bundle before doing any build work.""" + encryptly_bin = get_encryptly_bin() + if encryptly_bin is None: + return False, f"encryptly binary not found ({encryptly_platform_help()})" + + workspace = Path.home() / ".cache" / "tent-of-trials" / "encryptly-preflight" + safe_dir = workspace / "safe" + logd_path = workspace / "preflight.logd" + try: + shutil.rmtree(workspace, ignore_errors=True) + safe_dir.mkdir(parents=True, exist_ok=True) + (safe_dir / "preflight.txt").write_text("encryptly preflight\n", encoding="utf-8") + result = subprocess.run( + [ + str(encryptly_bin), + "pack", + str(logd_path), + "--include", + str(workspace), + "--max-file-size", + "1", + ], + cwd=str(ROOT), + capture_output=True, + text=True, + timeout=timeout, + ) + if result.returncode != 0: + output = result.stderr.strip() or result.stdout.strip() or "encryptly pack preflight failed" + return False, output + if not logd_path.exists(): + return False, "encryptly preflight completed without creating a .logd" + return True, "encryptly preflight passed" + except subprocess.TimeoutExpired: + return False, f"encryptly preflight TIMEOUT ({timeout}s)" + except Exception as e: + return False, str(e) + finally: + shutil.rmtree(workspace, ignore_errors=True) + +class Colors: + GREEN = "\033[92m" + YELLOW = "\033[93m" + RED = "\033[91m" + CYAN = "\033[96m" + BOLD = "\033[1m" + RESET = "\033[0m" + GRAY = "\033[90m" + +def color(text: str, code: str) -> str: + if not sys.stdout.isatty(): + return text + return f"{code}{text}{Colors.RESET}" + +def check_prerequisites() -> list[str]: + required = { + "cargo": "Rust", + "npm": "Node.js", + "go": "Go", + "gcc": "C (GCC)", + "g++": "C++ (GCC)", + "cmake": "CMake", + "make": "Make", + "python3": "Python", + "javac": "Java (JDK)", + "ruby": "Ruby", + "luac": "Lua", + "ghc": "GHC (Haskell)", + } + + missing = [] + for cmd, label in required.items(): + if shutil.which(cmd) is None: + missing.append(f"{label} ({cmd})") + + return missing + +def build_module( + module: Module, + release: bool = False, + verbose: bool = False, +) -> tuple[bool, float, str]: + + print(f"\n {color('▸', Colors.CYAN)} Building {color(module.name, Colors.BOLD)} ({module.language})...") + + env = os.environ.copy() + if module.env: + env.update(module.env) + + start = time.time() + + if module.name == "frontend": + node_modules = module.dir / "node_modules" + if not node_modules.exists(): + print(f" {color('npm install...', Colors.GRAY)}") + try: + install_result = subprocess.run( + ["npm", "install"], + cwd=str(module.dir), + capture_output=not verbose, + text=True, + timeout=120, + env={k: v for k, v in env.items() if k != "NODE_ENV"}, + ) + if install_result.returncode != 0: + return False, time.time() - start, f"npm install failed:\n{install_result.stderr}" + except subprocess.TimeoutExpired: + return False, time.time() - start, "npm install TIMEOUT (120s)" + + if module.name == "engine": + + build_type = "Release" if release else "Debug" + try: + cfg_result = subprocess.run( + ["cmake", "-S", ".", "-B", "build", + f"-DCMAKE_BUILD_TYPE={build_type}"], + cwd=str(module.dir), + capture_output=True, + text=True, + timeout=120, + env=env, + ) + except subprocess.TimeoutExpired: + return False, time.time() - start, "CMake configure TIMEOUT (120s)" + except FileNotFoundError as e: + return False, 0, f"Command not found: {e}" + if cfg_result.returncode != 0: + output_lines = [] + if cfg_result.stdout: + output_lines.append(cfg_result.stdout.strip()) + if cfg_result.stderr: + output_lines.append(cfg_result.stderr.strip()) + output = "\n".join(output_lines) + return False, time.time() - start, ( + f"CMake configure failed:\n{output}") + if verbose: + print(f" {color('cmake configured', Colors.GRAY)}") + cmd = ["cmake", "--build", "build"] + if release: + cmd.append("--config") + cmd.append("Release") + else: + cmd = list(module.build_cmd) + if release and module.name == "backend": + cmd.append("--release") + + try: + result = subprocess.run( + cmd, + cwd=str(module.dir), + capture_output=True, + text=True, + env=env, + timeout=300, + ) + except subprocess.TimeoutExpired: + return False, time.time() - start, "BUILD TIMEOUT (300s)" + except FileNotFoundError as e: + return False, 0, f"Command not found: {e}" + + elapsed = time.time() - start + output_lines = [] + + if result.stdout: + output_lines.append(result.stdout.strip()) + if result.stderr: + output_lines.append(result.stderr.strip()) + + output = "\n".join(output_lines) + success = result.returncode == 0 + + return success, elapsed, output + +def clean_module(module: Module, verbose: bool = False) -> bool: + print(f" {color('▸', Colors.YELLOW)} Cleaning {module.name}...") + try: + subprocess.run( + module.clean_cmd, + cwd=str(module.dir), + capture_output=not verbose, + text=True, + timeout=60, + env=os.environ.copy(), + ) + return True + except Exception as e: + print(f" {color('✗', Colors.RED)} Clean failed: {e}") + return False + +def verify_binary(module: Module) -> Optional[str]: + if module.build_dir is None: + return None + path = module.build_dir + if module.name == "backend": + + target = path / "debug" / module.name + if not target.exists(): + target = path / "release" / module.name + if target.exists(): + return str(target) + if path.exists(): + return str(path) + return None + +def run_cmd(cmd: list[str], **kwargs) -> tuple[bool, str]: + try: + result = subprocess.run( + cmd, capture_output=True, text=True, check=False, **kwargs + ) + output = result.stdout + if result.stderr: + output += "\n" + result.stderr + return result.returncode == 0, output.strip() + except Exception as e: + return False, str(e) + + +def collect_system_info() -> str: + lines = [ + "Tent of Trials - System Diagnostic Snapshot", + "=" * 50, + f"generated_at: {datetime.datetime.now(datetime.timezone.utc).isoformat()}", + f"hostname: {platform.node()}", + f"user: {getpass.getuser()}", + f"python: {sys.version}", + f"platform: {platform.platform()}", + f"processor: {platform.processor() or 'unknown'}", + f"cpu_count: {os.cpu_count()}", + "", + "--- uname ---", + ] + ok, out = run_cmd(["uname", "-a"]) + lines.append(out if ok else "unavailable") + + lines.extend(["", "--- /etc/os-release ---"]) + try: + lines.append((Path("/etc/os-release")).read_text(encoding="utf-8", errors="replace").strip()) + except Exception as e: + lines.append(f"unavailable: {e}") + + lines.extend(["", "--- memory ---"]) + ok, out = run_cmd(["free", "-h"]) + lines.append(out if ok else "unavailable") + + lines.extend(["", "--- disk ---"]) + ok, out = run_cmd(["df", "-h"]) + lines.append(out if ok else "unavailable") + + lines.extend(["", "--- build environment ---"]) + for key in ["SHELL", "LANG", "TERM", "XDG_SESSION_TYPE", "DISPLAY", "EDITOR"]: + value = os.environ.get(key) + if value: + lines.append(f"{key}={value}") + + lines.append("") + return "\n".join(lines) + + +def build_diagnostic_report( + results: list[tuple[str, bool, float, str, Optional[str]]], + commit_id: str, + logd_relpaths: Optional[list[str]] = None, + password: Optional[str] = None, + logd_error: Optional[str] = None, + chunked: bool = False, + message_blocker: Optional[str] = None, +) -> dict: + diagnostic_logd: Optional[str | list[str]] + if not logd_relpaths: + diagnostic_logd = None + elif len(logd_relpaths) == 1: + diagnostic_logd = logd_relpaths[0] + else: + diagnostic_logd = logd_relpaths + + decrypt_target = logd_relpaths[0] if logd_relpaths and len(logd_relpaths) == 1 else None + if logd_relpaths and len(logd_relpaths) > 1: + decrypt_target = repo_relative_posix(str(DIAGNOSTIC_DIR / f"build-{commit_id}.logd"), ROOT) + + report = { + "generated_at": datetime.datetime.now(datetime.timezone.utc).isoformat(), + "commit": commit_id, + "diagnostic_logd": diagnostic_logd, + "diagnostic_logd_error": logd_error, + "message_blocker": message_blocker, + "chunked": chunked, + "chunk_size_bytes": DIAGNOSTIC_CHUNK_SIZE if chunked else None, + "password": password, + "decrypt_command": ( + f"encryptly unpack {decrypt_target} --password {password}" + if decrypt_target and password else None + ), + "total_modules": len(results), + "passed": sum(1 for _, s, _, _, _ in results if s), + "failed": sum(1 for _, s, _, _, _ in results if not s), + "modules": [ + { + "name": name, + "status": "PASS" if success else "FAIL", + "elapsed_seconds": round(elapsed, 3), + "artifact": binary, + "output": output, + } + for name, success, elapsed, output, binary in results + ], + "pr_note": ( + (f"Include the encrypted diagnostic logd artifact(s): {', '.join(logd_relpaths)}. " if logd_relpaths else "Encrypted diagnostic logd artifact was not created; include this JSON report showing why. ") + + "The encrypted .logd is the required diagnostic content for PR review; this JSON file is metadata. " + + "Maintainers may ask you to remove these diagnostic artifacts before merging." + ), + } + return report + + +def write_diagnostic_report(metadata_path: Path, report: dict) -> None: + report = sanitize_metadata(report, ROOT) + validate_metadata_redaction(report, ROOT) + metadata_path.write_text(json.dumps(report, indent=2) + "\n", encoding="utf-8") + print(f" {color('✓', Colors.GREEN)} {metadata_path.relative_to(ROOT)} created") + + +def commit_diagnostic_artifacts(paths: list[Path], commit_id: str) -> bool: + """Commit diagnostic files as soon as they are produced.""" + existing = [path for path in paths if path.exists()] + if not existing: + print(f" {color('✗', Colors.RED)} No diagnostic artifacts found to commit") + return False + + relpaths = [str(path.relative_to(ROOT)) for path in existing] + status = subprocess.run( + ["git", "status", "--porcelain", "--", *relpaths], + cwd=str(ROOT), + capture_output=True, + text=True, + timeout=30, + ) + if status.returncode != 0: + print(f" {color('✗', Colors.RED)} Could not inspect diagnostic git status: {status.stderr.strip()}") + return False + if not status.stdout.strip(): + print(f" {color('✓', Colors.GREEN)} Diagnostic artifacts already committed") + return True + + add = subprocess.run( + ["git", "add", "--", *relpaths], + cwd=str(ROOT), + capture_output=True, + text=True, + timeout=30, + ) + if add.returncode != 0: + print(f" {color('✗', Colors.RED)} Could not stage diagnostic artifacts: {add.stderr.strip()}") + return False + + commit = subprocess.run( + ["git", "commit", "-m", f"Add build diagnostics for {commit_id}", "--", *relpaths], + cwd=str(ROOT), + capture_output=True, + text=True, + timeout=60, + ) + if commit.returncode != 0: + output = commit.stderr.strip() or commit.stdout.strip() + print(f" {color('✗', Colors.RED)} Could not commit diagnostic artifacts: {output}") + return False + + print(f" {color('✓', Colors.GREEN)} Diagnostic artifacts committed") + return True + + +def generate_logd( + results: list[tuple[str, bool, float, str, Optional[str]]], + verbose: bool = False, +) -> bool: + logd_path, metadata_path, commit_id = diagnostic_paths_for_commit() + display_logd = logd_path.relative_to(ROOT) + print(f"\n {color('▸', Colors.CYAN)} Finalizing diagnostics for {color(str(display_logd), Colors.BOLD)}...") + + # Always write the JSON report first. The encrypted .logd is useful, but the + # report is required even when the build failed before compilation started or + # when encryptly itself is unavailable. + write_diagnostic_report(metadata_path, build_diagnostic_report(results, commit_id)) + + encryptly_bin = get_encryptly_bin() + if encryptly_bin is None: + error = f"encryptly binary not found ({encryptly_platform_help()}); cannot create {display_logd}" + print(f" {color('✗', Colors.RED)} {error}") + write_diagnostic_report( + metadata_path, + build_diagnostic_report( + results, + commit_id, + logd_error=error, + message_blocker=ENCRYPTLY_BLOCKER_MESSAGE, + ), + ) + print(f" {color('BLOCKER', Colors.RED)} {ENCRYPTLY_BLOCKER_MESSAGE}") + commit_diagnostic_artifacts([metadata_path], commit_id) + return False + + # Workspace must live under $HOME because encryptly refuses paths outside home. + home = Path.home() + workspace = home / ".cache" / "tent-of-trials" / "logd-workspace" + safe_dir = workspace / "safe" + + try: + shutil.rmtree(workspace, ignore_errors=True) + safe_dir.mkdir(parents=True, exist_ok=True) + + (safe_dir / "system-info.txt").write_text( + collect_system_info(), encoding="utf-8" + ) + + summary_lines = [ + "Tent of Trials - Build Summary", + "=" * 50, + f"generated_at: {datetime.datetime.now(datetime.timezone.utc).isoformat()}", + f"total_modules: {len(results)}", + f"passed: {sum(1 for _, s, _, _, _ in results if s)}", + f"failed: {sum(1 for _, s, _, _, _ in results if not s)}", + "", + "module results:", + ] + for name, success, elapsed, _, binary in results: + summary_lines.append( + f" {name}: {'PASS' if success else 'FAIL'} ({elapsed:.2f}s)" + f"{f' [{binary}]' if binary else ''}" + ) + (safe_dir / "build-summary.txt").write_text( + "\n".join(summary_lines), encoding="utf-8" + ) + + log_lines = [] + for name, success, elapsed, output, binary in results: + log_lines.append( + f"\n{'=' * 50}\n{name} ({'PASS' if success else 'FAIL'}, {elapsed:.2f}s)\n" + f"{'=' * 50}" + ) + if binary: + log_lines.append(f"artifact: {binary}") + if output: + log_lines.append(output) + (safe_dir / "build.log").write_text("\n".join(log_lines), encoding="utf-8") + + sr = subprocess.run( + [ + str(encryptly_bin), + "pack", + str(logd_path), + "--include", + str(workspace), + "--max-file-size", + "35840", + ], + cwd=str(ROOT), + capture_output=True, + text=True, + timeout=300, + ) + if sr.returncode != 0: + error = sr.stderr.strip() or sr.stdout.strip() or "encryptly pack failed" + print( + f" {color('✗', Colors.RED)} {logd_path.relative_to(ROOT)} creation failed: " + f"{error}" + ) + if logd_path.exists(): + logd_path.unlink() + write_diagnostic_report( + metadata_path, + build_diagnostic_report( + results, + commit_id, + logd_error=error, + message_blocker=ENCRYPTLY_BLOCKER_MESSAGE, + ), + ) + print(f" {color('BLOCKER', Colors.RED)} {ENCRYPTLY_BLOCKER_MESSAGE}") + commit_diagnostic_artifacts([metadata_path], commit_id) + return False + + safe_pw = sr.stdout.strip() + logd_files = split_diagnostic_logd(logd_path) + logd_relpaths = [repo_relative_posix(str(path), ROOT) for path in logd_files] + decrypt_target = logd_relpaths[0] if len(logd_relpaths) == 1 else repo_relative_posix(str(logd_path), ROOT) + write_diagnostic_report( + metadata_path, + build_diagnostic_report( + results, + commit_id, + logd_relpaths=logd_relpaths, + password=safe_pw, + chunked=len(logd_files) > 1, + ), + ) + + for path in logd_files: + size_kb = path.stat().st_size / 1024.0 + print( + f" {color('✓', Colors.GREEN)} {path.relative_to(ROOT)} created " + f"({size_kb:.1f} KiB)" + ) + if len(logd_files) > 1: + print( + f" {color('✓', Colors.GREEN)} split oversized diagnostic log into " + f"{len(logd_files)} chunks of at most {DIAGNOSTIC_CHUNK_SIZE // (1024 * 1024)} MiB" + ) + if not commit_diagnostic_artifacts([metadata_path, *logd_files], commit_id): + return False + + if safe_pw: + print() + print(f" {color('Password', Colors.BOLD)} - this is required to decrypt the diagnostic log,") + print(f" which is required to submit a PR. Upload the") + print(f" diagnostic log file(s) and metadata file with this password.") + if len(logd_files) > 1: + print(f" Reassemble chunks in order before unpacking:") + print(f" cat {' '.join(logd_relpaths)} > {logd_path.relative_to(ROOT)}") + print(f" {color(safe_pw, Colors.CYAN)}") + print(f" {color(f'encryptly unpack {decrypt_target} --password {safe_pw}', Colors.GRAY)}") + return True + + finally: + shutil.rmtree(workspace, ignore_errors=True) + + +def print_summary(results: list[tuple[str, bool, float, str, Optional[str]]]): + print(f" {color('Build Summary', Colors.BOLD)}") + + total = len(results) + passed = sum(1 for _, s, _, _, _ in results if s) + failed = total - passed + total_time = sum(t for _, _, t, _, _ in results) + + for name, success, elapsed, output, binary in results: + status_icon = color("✓", Colors.GREEN) if success else color("✗", Colors.RED) + status_text = color("PASS", Colors.GREEN) if success else color("FAIL", Colors.RED) + time_str = f"{elapsed:.1f}s" if elapsed < 60 else f"{elapsed / 60:.1f}m" + + print(f"\n {status_icon} {color(name + ':', Colors.BOLD)} {status_text} ({time_str})") + if binary: + print(f" artifact: {color(binary, Colors.GRAY)}") + if not success and output: + + lines = output.strip().split("\n") + print(f" {color('last output:', Colors.RED)}") + for line in lines[-5:]: + print(f" {color(line, Colors.GRAY)}") + + print(f"\n {color('─' * 40, Colors.GRAY)}") + print(f" {color('Total:', Colors.BOLD)} {total} modules, " + f"{color(str(passed) + ' passed', Colors.GREEN)}, " + f"{color(str(failed) + ' failed', Colors.RED)}, " + f"{total_time:.1f}s total") + +def main(): + parser = argparse.ArgumentParser( + description="Tent of Trials - Multi-Language Build System", + formatter_class=argparse.RawDescriptionHelpFormatter, + epilog=""" +Examples: + python3 build.py Build all modules + python3 build.py -m backend Build only backend + python3 build.py -m frontend,market Build frontend and market + python3 build.py --clean Clean all artifacts + python3 build.py --release Release build (Rust only) + python3 build.py --verbose Verbose output + +Diagnostic bundle: + python3 build.py + """, + ) + parser.add_argument( + "-m", "--module", + help="Module(s) to build (comma-separated, or 'all')", + default="all", + ) + parser.add_argument( + "--clean", action="store_true", + help="Clean build artifacts instead of building", + ) + parser.add_argument( + "--release", action="store_true", + help="Build in release mode (Rust backend)", + ) + parser.add_argument( + "--verbose", "-v", action="store_true", + help="Show detailed build output", + ) + parser.add_argument( + "--list", action="store_true", + help="List available modules and exit", + ) + + args = parser.parse_args() + + print(f"\n {color('Tent of Trials: building', Colors.CYAN)}") + print(f" Working directory: {ROOT}") + print() + + if args.list: + print(f" {color('Available modules:', Colors.BOLD)}") + for m in MODULES: + print(f" {color(m.name, Colors.CYAN)} ({m.language})") + print(f" dir: {m.dir.relative_to(ROOT)}") + print(f" build: {' '.join(m.build_cmd)}") + return 0 + + print(f" {color('Checking prerequisites...', Colors.GRAY)}") + missing = check_prerequisites() + if missing: + print(f"\n {color('⚠ Some tools missing - will try anyway:', Colors.YELLOW)}") + for m in missing: + print(f" {m}") + + msg = "Not all modules will build. That's fine." + print(f" {color(msg, Colors.GRAY)}") + else: + print(f" {color('✓ All prerequisites found', Colors.GREEN)}") + if args.module == "all": + selected = MODULES + else: + names = [n.strip() for n in args.module.split(",")] + selected = [m for m in MODULES if m.name in names] + not_found = set(names) - {m.name for m in MODULES} + if not_found: + print(f" {color('✗ Unknown modules:', Colors.RED)} {', '.join(not_found)}") + print(f" Available: {', '.join(m.name for m in MODULES)}") + return 1 + + if not selected: + print(f" No modules selected.") + return 0 + + if args.clean: + print(f"\n {color('Cleaning build artifacts...', Colors.YELLOW)}") + for module in selected: + clean_module(module, args.verbose) + + diagnostic_artifacts = [ROOT / "build.logd"] + if DIAGNOSTIC_DIR.exists(): + diagnostic_artifacts.extend(DIAGNOSTIC_DIR.glob("build-[0-9a-f][0-9a-f][0-9a-f][0-9a-f][0-9a-f][0-9a-f][0-9a-f][0-9a-f].logd")) + diagnostic_artifacts.extend(DIAGNOSTIC_DIR.glob("build-[0-9a-f][0-9a-f][0-9a-f][0-9a-f][0-9a-f][0-9a-f][0-9a-f][0-9a-f]-part*.logd")) + diagnostic_artifacts.extend(DIAGNOSTIC_DIR.glob("build-[0-9a-f][0-9a-f][0-9a-f][0-9a-f][0-9a-f][0-9a-f][0-9a-f][0-9a-f].json")) + diagnostic_artifacts.extend(DIAGNOSTIC_DIR.glob("build-[0-9a-f][0-9a-f][0-9a-f][0-9a-f][0-9a-f][0-9a-f][0-9a-f][0-9a-f]-metadata.json")) + for artifact in diagnostic_artifacts: + if artifact.exists(): + if artifact.is_dir(): + shutil.rmtree(artifact) + else: + artifact.unlink() + print(f" {color('▸', Colors.YELLOW)} Removed {artifact.relative_to(ROOT)}") + print(f"\n {color('Clean complete.', Colors.GREEN)}") + return 0 + + print(f"\n {color('Checking encryptly diagnostics...', Colors.GRAY)}") + encryptly_start = time.time() + encryptly_ok, encryptly_message = check_encryptly_runs() + if not encryptly_ok: + elapsed = time.time() - encryptly_start + blocker = f"{ENCRYPTLY_BLOCKER_MESSAGE} {encryptly_message}" + print(f" {color('✗ encryptly cannot run', Colors.RED)}") + print(f" {color('BLOCKER:', Colors.RED)} {blocker}") + results = [("encryptly-preflight", False, elapsed, blocker, None)] + generate_logd(results, args.verbose) + return 1 + print(f" {color('✓ encryptly runs', Colors.GREEN)}") + + print(f"\n {color(f'Building {len(selected)} module(s) | release={args.release}', Colors.GRAY)}") + + results: list[tuple[str, bool, float, str, Optional[str]]] = [] + + for module in selected: + success, elapsed, output = build_module(module, args.release, args.verbose) + binary = verify_binary(module) if success else None + results.append((module.name, success, elapsed, output, binary)) + + print_summary(results) + + diagnostics_ok = generate_logd(results, args.verbose) + + return 0 if diagnostics_ok and all(r[1] for r in results) else 1 + +if __name__ == "__main__": + sys.exit(main()) diff --git a/diagnostic/build-bf2147ac-metadata.json b/diagnostic/build-bf2147ac-metadata.json new file mode 100644 index 00000000..5863bdd2 --- /dev/null +++ b/diagnostic/build-bf2147ac-metadata.json @@ -0,0 +1,41 @@ +{ + "generated_at": "2026-07-01T07:19:07.148091+00:00", + "commit": "bf2147ac", + "diagnostic_logd": [ + "diagnostic/build-bf2147ac-part001.logd", + "diagnostic/build-bf2147ac-part002.logd" + ], + "chunked": true, + "chunk_size_bytes": 41943040, + "password": "95c64f76adfb29120dc8", + "decrypt_command": "encryptly unpack diagnostic/build-bf2147ac.logd --password 95c64f76adfb29120dc8", + "total_modules": 1, + "passed": 1, + "failed": 0, + "modules": [ + { + "name": "compliance", + "status": "PASS", + "elapsed_seconds": 1.356, + "artifact": "compliance/build" + } + ], + "module_timings": [ + { + "module": "compliance", + "language": "Java", + "command": [ + "javac", + "-d", + "build", + "ComplianceAuditor.java" + ], + "started_at": "2026-07-01T07:14:24.764120+00:00", + "finished_at": "2026-07-01T07:14:26.120512+00:00", + "elapsed_seconds": 1.356, + "exit_code": 0, + "status": "PASS" + } + ], + "pr_note": "Include this metadata and diagnostic/build-bf2147ac-part001.logd, diagnostic/build-bf2147ac-part002.logd in your PR. Maintainers may ask you to remove these diagnostic artifacts before merging." +} diff --git a/tools/diagnostic_redaction.py b/tools/diagnostic_redaction.py new file mode 100644 index 00000000..c48281c5 --- /dev/null +++ b/tools/diagnostic_redaction.py @@ -0,0 +1,144 @@ +"""Helpers to keep diagnostic metadata free of host-specific path leaks.""" + +from __future__ import annotations + +import json +import os +import re +import tempfile +from pathlib import Path +from typing import Any, Optional + +_SENSITIVE_PARTS = ( + os.path.expanduser("~"), + tempfile.gettempdir(), +) + + +def repo_relative_posix(path: Optional[str], root: Path) -> Optional[str]: + """Return a repository-relative path using forward slashes.""" + if not path: + return path + candidate = Path(path) + try: + if candidate.is_absolute(): + rel = candidate.resolve().relative_to(root.resolve()) + else: + rel = candidate + return rel.as_posix() + except ValueError: + return candidate.name + + +def redact_path_string(value: str, root: Path) -> str: + """Replace absolute home/temp/repo paths inside free-form text.""" + text = value + root_resolved = str(root.resolve()) + for prefix in (root_resolved, *_SENSITIVE_PARTS): + if prefix: + text = text.replace(prefix.replace("\\", "/"), "") + text = text.replace(prefix, "") + return text + + +def sanitize_metadata(metadata: dict[str, Any], root: Path) -> dict[str, Any]: + """Return metadata with repo-relative artifact paths and normalized logd refs.""" + cleaned = dict(metadata) + logd_field = cleaned.get("diagnostic_logd") + if isinstance(logd_field, list): + cleaned["diagnostic_logd"] = [repo_relative_posix(item, root) for item in logd_field] + elif isinstance(logd_field, str): + cleaned["diagnostic_logd"] = repo_relative_posix(logd_field, root) + + if isinstance(cleaned.get("decrypt_command"), str): + cleaned["decrypt_command"] = redact_path_string(cleaned["decrypt_command"], root) + + modules = cleaned.get("modules") + if isinstance(modules, list): + sanitized_modules = [] + for entry in modules: + if not isinstance(entry, dict): + sanitized_modules.append(entry) + continue + item = dict(entry) + if "artifact" in item: + item["artifact"] = repo_relative_posix(item.get("artifact"), root) + sanitized_modules.append(item) + cleaned["modules"] = sanitized_modules + + timings = cleaned.get("module_timings") + if isinstance(timings, list): + sanitized_timings = [] + for entry in timings: + if not isinstance(entry, dict): + sanitized_timings.append(entry) + continue + item = dict(entry) + if isinstance(item.get("command"), list): + item["command"] = [ + repo_relative_posix(part, root) if isinstance(part, str) and ("/" in part or "\\" in part) else part + for part in item["command"] + ] + sanitized_timings.append(item) + cleaned["module_timings"] = sanitized_timings + + return cleaned + + +def _logd_paths_from_metadata(metadata: dict[str, Any]) -> list[str]: + logd_field = metadata.get("diagnostic_logd") + if isinstance(logd_field, list): + return [str(item) for item in logd_field] + if isinstance(logd_field, str): + return [logd_field] + return [] + + +def _contains_sensitive_leak(text: str, root: Path) -> bool: + lowered = text.lower() + if re.search(r"[a-z]:\\", text, re.I): + return True + if os.path.expanduser("~") and os.path.expanduser("~") in text: + return True + if str(root.resolve()) in text: + return True + if tempfile.gettempdir() and tempfile.gettempdir() in text: + return True + if re.search(r"\b(?:hostname|username|machine)\s*[:=]", lowered): + return True + return False + + +def validate_metadata_redaction(metadata: dict[str, Any], root: Path) -> None: + """Raise ValueError when metadata still exposes host-specific paths.""" + serialized = json.dumps(metadata) + if _contains_sensitive_leak(serialized, root): + raise ValueError("diagnostic metadata leaks host-specific paths") + + +def validate_diagnostic_bundle(metadata_path: Path, root: Path) -> None: + """Validate metadata/logd pairing and redaction contract.""" + if not metadata_path.exists(): + raise FileNotFoundError(f"diagnostic metadata missing: {metadata_path}") + + metadata = json.loads(metadata_path.read_text(encoding="utf-8")) + validate_metadata_redaction(metadata, root) + + logd_refs = _logd_paths_from_metadata(metadata) + if not logd_refs: + raise ValueError("diagnostic metadata missing diagnostic_logd reference") + + missing = [ref for ref in logd_refs if not (root / ref).exists()] + if missing: + raise FileNotFoundError( + f"diagnostic .logd artifact(s) missing for metadata pair: {', '.join(missing)}" + ) + + for ref in logd_refs: + if "\\" in ref: + raise ValueError(f"diagnostic_logd must use repository-relative '/' paths: {ref}") + + for module in metadata.get("modules", []): + artifact = module.get("artifact") + if artifact and (Path(artifact).is_absolute() or "\\" in str(artifact)): + raise ValueError(f"module artifact must be repository-relative: {artifact}") diff --git a/tools/tests/test_diagnostic_redaction.py b/tools/tests/test_diagnostic_redaction.py new file mode 100644 index 00000000..bdc1e9a0 --- /dev/null +++ b/tools/tests/test_diagnostic_redaction.py @@ -0,0 +1,113 @@ +"""Regression tests for diagnostic metadata redaction.""" + +from __future__ import annotations + +import json +import tempfile +import unittest +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[2] +TOOLS = ROOT / "tools" +import sys + +if str(TOOLS) not in sys.path: + sys.path.insert(0, str(TOOLS)) + +from diagnostic_redaction import ( # noqa: E402 + repo_relative_posix, + sanitize_metadata, + validate_diagnostic_bundle, + validate_metadata_redaction, +) + + +class DiagnosticRedactionTests(unittest.TestCase): + def test_repo_relative_posix_uses_forward_slashes(self) -> None: + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp) + nested = root / "compliance" / "build" + nested.mkdir(parents=True) + value = repo_relative_posix(str(nested), root) + self.assertEqual(value, "compliance/build") + + def test_sanitize_metadata_normalizes_logd_and_artifacts(self) -> None: + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp) + metadata = { + "diagnostic_logd": ["diagnostic\\build-abcd1234-part001.logd"], + "decrypt_command": "encryptly unpack diagnostic\\build-abcd1234.logd ", + "modules": [{"name": "compliance", "artifact": str(root / "compliance" / "build")}], + } + cleaned = sanitize_metadata(metadata, root) + self.assertEqual(cleaned["diagnostic_logd"], ["diagnostic/build-abcd1234-part001.logd"]) + self.assertEqual(cleaned["modules"][0]["artifact"], "compliance/build") + validate_metadata_redaction(cleaned, root) + + def test_validate_bundle_requires_metadata_file(self) -> None: + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp) + missing = root / "diagnostic" / "build-deadbeef-metadata.json" + with self.assertRaises(FileNotFoundError): + validate_diagnostic_bundle(missing, root) + + def test_validate_bundle_requires_matching_logd(self) -> None: + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp) + diag = root / "diagnostic" + diag.mkdir() + metadata_path = diag / "build-deadbeef-metadata.json" + metadata_path.write_text( + json.dumps( + { + "diagnostic_logd": ["diagnostic/build-deadbeef.logd"], + "modules": [{"artifact": "compliance/build"}], + } + ), + encoding="utf-8", + ) + with self.assertRaises(FileNotFoundError): + validate_diagnostic_bundle(metadata_path, root) + + def test_validate_bundle_accepts_matching_pair(self) -> None: + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp) + diag = root / "diagnostic" + diag.mkdir() + logd = diag / "build-deadbeef.logd" + logd.write_text("encrypted-stub", encoding="utf-8") + metadata_path = diag / "build-deadbeef-metadata.json" + metadata_path.write_text( + json.dumps( + { + "diagnostic_logd": "diagnostic/build-deadbeef.logd", + "modules": [{"artifact": "compliance/build"}], + } + ), + encoding="utf-8", + ) + validate_diagnostic_bundle(metadata_path, root) + + def test_validate_bundle_rejects_backslash_logd_paths(self) -> None: + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp) + diag = root / "diagnostic" + diag.mkdir() + bad = diag / "build-deadbeef-part001.logd" + bad.write_text("encrypted-stub", encoding="utf-8") + metadata_path = diag / "build-deadbeef-metadata.json" + metadata_path.write_text( + json.dumps( + { + "diagnostic_logd": ["diagnostic\\build-deadbeef-part001.logd"], + "modules": [{"artifact": "compliance/build"}], + } + ), + encoding="utf-8", + ) + with self.assertRaises(ValueError): + validate_diagnostic_bundle(metadata_path, root) + + +if __name__ == "__main__": + unittest.main()