diff --git a/.gitignore b/.gitignore index 5983ed78..4d132865 100644 --- a/.gitignore +++ b/.gitignore @@ -35,3 +35,4 @@ __pycache__/ /.mypy_cache/ .skip-coverage .claude +/.beads/ diff --git a/src/fromager/bootstrapper.py b/src/fromager/bootstrapper.py index 9447026c..a1e0dd7b 100644 --- a/src/fromager/bootstrapper.py +++ b/src/fromager/bootstrapper.py @@ -1,6 +1,9 @@ from __future__ import annotations +import contextlib import dataclasses +import datetime +import enum import json import logging import operator @@ -56,6 +59,166 @@ class SourceBuildResult: source_type: SourceType +class FailureCategory(enum.Enum): + """Categories of failures that can occur during bootstrap.""" + + VERSION_RESOLUTION = "version_resolution" + SOURCE_DOWNLOAD = "source_download" + SOURCE_PREPARE = "source_prepare" + BUILD_ENVIRONMENT = "build_environment" + SOURCE_BUILD = "source_build" + PREBUILT_DOWNLOAD = "prebuilt_download" + POST_HOOK = "post_hook" + DEPENDENCY_EXTRACTION = "dependency_extraction" + + +class BootstrapPhaseError(Exception): + """Exception wrapper that carries bootstrap phase information. + + Used to propagate both the original exception and the phase + where the failure occurred for proper categorization. + """ + + def __init__( + self, + message: str, + category: FailureCategory, + cause: Exception, + fallback_attempted: bool = False, + ): + super().__init__(message) + self.category = category + self.cause = cause + self.fallback_attempted = fallback_attempted + self.__cause__ = cause + + +@dataclasses.dataclass +class BuildFailure: + """Tracks a failed build in test mode for reporting. + + Contains fields needed for failure tracking and JSON serialization, + including failure category, fallback attempt information, and + the full dependency chain that led to the failure. + """ + + req: Requirement + resolved_version: Version | None = None + source_url_type: str = "unknown" + category: FailureCategory = FailureCategory.SOURCE_BUILD + exception_type: str | None = None + exception_message: str | None = None + fallback_attempted: bool = False + fallback_succeeded: bool = False + fallback_version: Version | None = None # Version used if fallback succeeded + timestamp: datetime.datetime = dataclasses.field( + default_factory=datetime.datetime.now + ) + dependency_chain: list[tuple[RequirementType, Requirement, Version]] = ( + dataclasses.field(default_factory=list) + ) + + @classmethod + def from_exception( + cls, + req: Requirement, + resolved_version: Version | None, + source_url_type: str, + exception: Exception, + category: FailureCategory = FailureCategory.SOURCE_BUILD, + fallback_attempted: bool = False, + fallback_succeeded: bool = False, + fallback_version: Version | None = None, + dependency_chain: list[tuple[RequirementType, Requirement, Version]] + | None = None, + ) -> BuildFailure: + """Create a BuildFailure from an exception.""" + return cls( + req=req, + resolved_version=resolved_version, + source_url_type=source_url_type, + category=category, + exception_type=exception.__class__.__name__, + exception_message=str(exception), + fallback_attempted=fallback_attempted, + fallback_succeeded=fallback_succeeded, + fallback_version=fallback_version, + timestamp=datetime.datetime.now(), + dependency_chain=list(dependency_chain) if dependency_chain else [], + ) + + @property + def root_package(self) -> str: + """Get the top-level package that led to this failure.""" + if self.dependency_chain: + return str(self.dependency_chain[0][1].name) + return str(self.req.name) + + @property + def immediate_parent(self) -> tuple[str, str, str] | None: + """Get the immediate parent that depends on the failed package. + + Returns tuple of (requirement_type, package_name, version) or None. + """ + if self.dependency_chain: + req_type, req, ver = self.dependency_chain[-1] + return (req_type.name, str(req.name), str(ver)) + return None + + @property + def chain_depth(self) -> int: + """Return how deep in the dependency tree this failure occurred.""" + return len(self.dependency_chain) + + @property + def version_mismatch(self) -> bool: + """Return True if fallback used a different version than requested.""" + if not self.fallback_succeeded or self.fallback_version is None: + return False + return self.fallback_version != self.resolved_version + + def format_chain(self) -> str: + """Format the dependency chain for human-readable output.""" + if not self.dependency_chain: + return f"TOP_LEVEL: {self.req.name}=={self.resolved_version}" + + lines = [] + for i, (rt, req, ver) in enumerate(self.dependency_chain): + indent = " " * i + lines.append(f"{indent}└── {rt.name}: {req.name}=={ver}") + + # Add the failed package at the end + indent = " " * len(self.dependency_chain) + lines.append(f"{indent}└── FAILED: {self.req.name}=={self.resolved_version}") + + return "\n".join(lines) + + def to_dict(self) -> dict[str, typing.Any]: + """Convert to JSON-serializable dict.""" + return { + "package": str(self.req), + "version": str(self.resolved_version) if self.resolved_version else None, + "source_url_type": self.source_url_type, + "category": self.category.value, + "exception_type": self.exception_type, + "exception_message": self.exception_message, + "fallback_attempted": self.fallback_attempted, + "fallback_succeeded": self.fallback_succeeded, + "fallback_version": ( + str(self.fallback_version) if self.fallback_version else None + ), + "version_mismatch": self.version_mismatch, + "timestamp": self.timestamp.isoformat(), + "root_package": self.root_package, + "immediate_parent": self.immediate_parent, + "chain_depth": self.chain_depth, + "dependency_chain": [ + {"type": rt.name, "package": str(r.name), "version": str(v)} + for rt, r, v in self.dependency_chain + ], + } + + class Bootstrapper: def __init__( self, @@ -64,12 +227,19 @@ def __init__( prev_graph: DependencyGraph | None = None, cache_wheel_server_url: str | None = None, sdist_only: bool = False, + test_mode: bool = False, ) -> None: + if test_mode and sdist_only: + raise ValueError( + "--test-mode requires full wheel builds; incompatible with --sdist-only" + ) + self.ctx = ctx self.progressbar = progressbar or progress.Progressbar(None) self.prev_graph = prev_graph self.cache_wheel_server_url = cache_wheel_server_url or ctx.wheel_server_url self.sdist_only = sdist_only + self.test_mode = test_mode self.why: list[tuple[RequirementType, Requirement, Version]] = [] # Push items onto the stack as we start to resolve their # dependencies so at the end we have a list of items that need to @@ -89,6 +259,114 @@ def __init__( self._build_order_filename = self.ctx.work_dir / "build-order.json" + # Track failed builds in test mode + self.failed_builds: list[BuildFailure] = [] + + def resolve_and_add_top_level( + self, + req: Requirement, + ) -> tuple[str, Version] | None: + """Resolve a top-level requirement and add it to the dependency graph. + + This is the pre-resolution phase before recursive bootstrapping begins. + In test mode, catches resolution errors and records them as failures. + + Args: + req: The top-level requirement to resolve. + + Returns: + Tuple of (source_url, version) if resolution succeeded, None if it + failed in test mode. + + Raises: + Exception: In normal mode, re-raises any resolution error. + """ + try: + pbi = self.ctx.package_build_info(req) + source_url, version = self.resolve_version( + req=req, + req_type=RequirementType.TOP_LEVEL, + ) + logger.info("%s resolves to %s", req, version) + self.ctx.dependency_graph.add_dependency( + parent_name=None, + parent_version=None, + req_type=RequirementType.TOP_LEVEL, + req=req, + req_version=version, + download_url=source_url, + pre_built=pbi.pre_built, + constraint=self.ctx.constraints.get_constraint(req.name), + ) + return (source_url, version) + except Exception as err: + if not self.test_mode: + raise + # Top-level resolution failure (before recursive processing). + # Record and return None to signal failure. + logger.error( + "test mode: failed to resolve top-level requirement %s: %s", + req, + err, + exc_info=True, + ) + self.failed_builds.append( + BuildFailure.from_exception( + req=req, + resolved_version=None, + source_url_type="unknown", + exception=err, + category=FailureCategory.VERSION_RESOLUTION, + dependency_chain=[], + ) + ) + return None + + def _record_failure( + self, + req: Requirement, + resolved_version: Version | None, + exception: Exception, + category: FailureCategory = FailureCategory.SOURCE_BUILD, + fallback_attempted: bool = False, + fallback_succeeded: bool = False, + fallback_version: Version | None = None, + ) -> None: + """Record a build failure for test mode reporting. + + Called from two locations by design: + - bootstrap(): for fatal errors that stop package processing + - _bootstrap_impl(): for non-fatal errors where processing continues + + Automatically captures the current dependency chain from self.why. + + Args: + fallback_version: If fallback succeeded, the version that was + actually used. May differ from resolved_version. + """ + source_url_type = "unknown" + if resolved_version: + try: + source_url_type = str(sources.get_source_type(self.ctx, req)) + except Exception as err: + logger.debug( + "could not determine source type for %s: %s", req.name, err + ) + + self.failed_builds.append( + BuildFailure.from_exception( + req=req, + resolved_version=resolved_version, + source_url_type=source_url_type, + exception=exception, + category=category, + fallback_attempted=fallback_attempted, + fallback_succeeded=fallback_succeeded, + fallback_version=fallback_version, + dependency_chain=list(self.why), + ) + ) + def resolve_version( self, req: Requirement, @@ -144,7 +422,94 @@ def _processing_build_requirement(self, current_req_type: RequirementType) -> bo logger.debug("is not a build requirement") return False - def bootstrap(self, req: Requirement, req_type: RequirementType) -> Version: + def bootstrap(self, req: Requirement, req_type: RequirementType) -> None: + """Bootstrap a package and its dependencies. + + This method handles errors during recursive dependency processing. + Top-level package resolution errors are handled separately in the + bootstrap command before this method is called. + + In test mode, catches build exceptions, records failures, and continues. + In normal mode, raises exceptions immediately (fail-fast). + + Error Recording Design: + Fatal errors (version resolution, source download/build, prebuilt + download) are recorded here after being raised by _bootstrap_impl(). + Non-fatal errors (post-hook, dependency extraction) are recorded + in _bootstrap_impl() to allow processing to continue. + + Raises: + Exception: Always in normal mode; in test mode only if version + resolution failed. + """ + try: + self._bootstrap_impl(req, req_type) + except BootstrapPhaseError as phase_err: + if not self.test_mode: + raise + cached = self._resolved_requirements.get(str(req)) + resolved_version = cached[1] if cached else None + logger.error( + "test mode: %s failed for %s==%s: %s", + phase_err.category.value, + req.name, + resolved_version, + phase_err.cause, + exc_info=True, + ) + self._record_failure( + req, + resolved_version, + phase_err.cause, + category=phase_err.category, + fallback_attempted=phase_err.fallback_attempted, + ) + except Exception as err: + if not self.test_mode: + raise + + cached = self._resolved_requirements.get(str(req)) + if not cached: + # Version resolution failed during recursive processing. + # Record this before re-raising since we cannot continue + # without a version. (Top-level resolution errors are handled + # separately in the bootstrap command.) + logger.error( + "test mode: failed to resolve version for %s, cannot continue: %s", + req, + err, + exc_info=True, + ) + self._record_failure( + req, + None, + err, + category=FailureCategory.VERSION_RESOLUTION, + ) + raise + + resolved_version = cached[1] + logger.error( + "test mode: failed to bootstrap %s==%s: %s", + req.name, + resolved_version, + err, + exc_info=True, + ) + self._record_failure(req, resolved_version, err) + + def _bootstrap_impl(self, req: Requirement, req_type: RequirementType) -> None: + """Internal implementation of bootstrap logic. + + Error Handling: + Fatal errors (version resolution, source build, prebuilt download) + raise exceptions for bootstrap() to catch and record. + + Non-fatal errors (post-hook, dependency extraction) are recorded + locally and processing continues. These are recorded here rather + than in bootstrap() because the package build succeeded - only + optional processing failed. + """ logger.info(f"bootstrapping {req} as {req_type} dependency of {self.why[-1:]}") constraint = self.ctx.constraints.get_constraint(req.name) if constraint: @@ -180,98 +545,187 @@ def bootstrap(self, req: Requirement, req_type: RequirementType) -> Version: f"redundant {req_type} dependency {req} " f"({resolved_version}, sdist_only={build_sdist_only}) for {self._explain}" ) - return resolved_version + return self._mark_as_seen(req, resolved_version, build_sdist_only) logger.info(f"new {req_type} dependency {req} resolves to {resolved_version}") - # Build the dependency chain up to the point of this new - # requirement using a new list so we can avoid modifying the list - # we're given. - self.why.append((req_type, req, resolved_version)) + # Track dependency chain for error messages - context manager ensures cleanup + with self._track_why(req_type, req, resolved_version): + cached_wheel_filename: pathlib.Path | None = None + unpacked_cached_wheel: pathlib.Path | None = None - cached_wheel_filename: pathlib.Path | None = None - unpacked_cached_wheel: pathlib.Path | None = None + if pbi.pre_built: + try: + wheel_filename, unpack_dir = self._download_prebuilt( + req=req, + req_type=req_type, + resolved_version=resolved_version, + wheel_url=source_url, + ) + build_result = SourceBuildResult( + wheel_filename=wheel_filename, + sdist_filename=None, + unpack_dir=unpack_dir, + sdist_root_dir=None, + build_env=None, + source_type=SourceType.PREBUILT, + ) + except Exception as prebuilt_error: + raise BootstrapPhaseError( + f"Failed to download prebuilt wheel for {req.name}=={resolved_version}", + FailureCategory.PREBUILT_DOWNLOAD, + prebuilt_error, + ) from prebuilt_error + else: + # Look for an existing wheel in caches before building + cached_wheel_filename, unpacked_cached_wheel = self._find_cached_wheel( + req, resolved_version + ) - if pbi.pre_built: - wheel_filename, unpack_dir = self._download_prebuilt( - req=req, - req_type=req_type, - resolved_version=resolved_version, - wheel_url=source_url, - ) - # Remember that this is a prebuilt wheel, and where we got it. - build_result = SourceBuildResult( - wheel_filename=wheel_filename, - sdist_filename=None, - unpack_dir=unpack_dir, - sdist_root_dir=None, - build_env=None, - source_type=SourceType.PREBUILT, - ) - else: - # Look for an existing wheel in caches (3 levels: build, downloads, - # cache server) before building from source. - cached_wheel_filename, unpacked_cached_wheel = self._find_cached_wheel( - req, resolved_version + # Build from source (download, prepare, build wheel/sdist) + try: + build_result = self._build_from_source( + req=req, + resolved_version=resolved_version, + source_url=source_url, + build_sdist_only=build_sdist_only, + cached_wheel_filename=cached_wheel_filename, + unpacked_cached_wheel=unpacked_cached_wheel, + ) + except Exception as build_error: + if not self.test_mode: + raise + + # Extract the failure category from BootstrapPhaseError if available + if isinstance(build_error, BootstrapPhaseError): + failure_category = build_error.category + original_error = build_error.cause + else: + failure_category = FailureCategory.SOURCE_BUILD + original_error = build_error + + fallback_result = self._handle_test_mode_failure( + req=req, + resolved_version=resolved_version, + req_type=req_type, + build_error=original_error, + failure_category=failure_category, + ) + if fallback_result is None: + # Re-raise with proper category for bootstrap() to record + raise BootstrapPhaseError( + f"Build failed for {req.name}=={resolved_version} " + f"and fallback also failed", + failure_category, + original_error, + fallback_attempted=True, + ) from original_error + + build_result = fallback_result + + # Run post-bootstrap hooks - in test mode, log and continue on failure + try: + hooks.run_post_bootstrap_hooks( + ctx=self.ctx, + req=req, + dist_name=canonicalize_name(req.name), + dist_version=str(resolved_version), + sdist_filename=build_result.sdist_filename, + wheel_filename=build_result.wheel_filename, + ) + except Exception as hook_error: + if not self.test_mode: + raise + logger.warning( + "test mode: post-bootstrap hook failed for %s==%s: %s", + req.name, + resolved_version, + hook_error, + exc_info=True, + ) + self._record_failure( + req, + resolved_version, + hook_error, + category=FailureCategory.POST_HOOK, + ) + # Continue processing - hooks are not critical for dependency discovery + + # Extract install dependencies - in test mode, use empty list on failure + install_dependencies: list[Requirement] = [] + try: + install_dependencies = self._get_install_dependencies( + req=req, + resolved_version=resolved_version, + wheel_filename=build_result.wheel_filename, + sdist_filename=build_result.sdist_filename, + sdist_root_dir=build_result.sdist_root_dir, + build_env=build_result.build_env, + unpack_dir=build_result.unpack_dir, + ) + except Exception as dep_error: + if not self.test_mode: + raise + logger.warning( + "test mode: failed to extract install dependencies for %s==%s: %s", + req.name, + resolved_version, + dep_error, + exc_info=True, + ) + self._record_failure( + req, + resolved_version, + dep_error, + category=FailureCategory.DEPENDENCY_EXTRACTION, + ) + # Continue with empty dependencies - this package won't have its deps built + + logger.debug( + "install dependencies: %s", + ", ".join(sorted(str(r) for r in install_dependencies)), ) - # Build from source (download, prepare, build wheel/sdist) - build_result = self._build_from_source( + self._add_to_build_order( req=req, - resolved_version=resolved_version, + version=resolved_version, source_url=source_url, - build_sdist_only=build_sdist_only, - cached_wheel_filename=cached_wheel_filename, - unpacked_cached_wheel=unpacked_cached_wheel, + source_type=build_result.source_type, + prebuilt=pbi.pre_built, + constraint=constraint, ) - hooks.run_post_bootstrap_hooks( - ctx=self.ctx, - req=req, - dist_name=canonicalize_name(req.name), - dist_version=str(resolved_version), - sdist_filename=build_result.sdist_filename, - wheel_filename=build_result.wheel_filename, - ) - - install_dependencies = self._get_install_dependencies( - req=req, - resolved_version=resolved_version, - wheel_filename=build_result.wheel_filename, - sdist_filename=build_result.sdist_filename, - sdist_root_dir=build_result.sdist_root_dir, - build_env=build_result.build_env, - unpack_dir=build_result.unpack_dir, - ) - - logger.debug( - "install dependencies: %s", - ", ".join(sorted(str(req) for req in install_dependencies)), - ) + self.progressbar.update_total(len(install_dependencies)) + for dep in self._sort_requirements(install_dependencies): + with req_ctxvar_context(dep): + # In test mode, bootstrap() catches and records failures internally. + # In normal mode, it raises immediately which we propagate. + self.bootstrap(req=dep, req_type=RequirementType.INSTALL) + self.progressbar.update() - self._add_to_build_order( - req=req, - version=resolved_version, - source_url=source_url, - source_type=build_result.source_type, - prebuilt=pbi.pre_built, - constraint=constraint, - ) + # Clean up build directories (why stack cleanup handled by context manager) + self.ctx.clean_build_dirs( + build_result.sdist_root_dir, build_result.build_env + ) - self.progressbar.update_total(len(install_dependencies)) - for dep in self._sort_requirements(install_dependencies): - with req_ctxvar_context(dep): - try: - self.bootstrap(req=dep, req_type=RequirementType.INSTALL) - except Exception as err: - raise ValueError(f"could not handle {self._explain}") from err - self.progressbar.update() + @contextlib.contextmanager + def _track_why( + self, + req_type: RequirementType, + req: Requirement, + resolved_version: Version, + ) -> typing.Generator[None, None, None]: + """Context manager to track dependency chain in self.why stack. - # we are done processing this req, so lets remove it from the why chain - self.why.pop() - self.ctx.clean_build_dirs(build_result.sdist_root_dir, build_result.build_env) - return resolved_version + Ensures the entry is always popped from the stack, even if an + exception occurs during processing. This prevents stack corruption. + """ + self.why.append((req_type, req, resolved_version)) + try: + yield + finally: + self.why.pop() @property def _explain(self) -> str: @@ -396,10 +850,9 @@ def _handle_build_requirements( for dep in self._sort_requirements(build_dependencies): with req_ctxvar_context(dep): - try: - self.bootstrap(req=dep, req_type=build_type) - except Exception as err: - raise ValueError(f"could not handle {self._explain}") from err + # In test mode, bootstrap() catches and records failures internally. + # In normal mode, it raises immediately which we propagate. + self.bootstrap(req=dep, req_type=build_type) self.progressbar.update() def _download_prebuilt( @@ -528,24 +981,39 @@ def _build_from_source( SourceBuildResult with all build artifacts. Raises: - Various exceptions from download, prepare, or build steps. - This is where test-mode will catch exceptions. + BootstrapPhaseError: Wraps the original exception with phase info + for proper categorization in test mode. """ # Download and prepare source (if no cached wheel) if not unpacked_cached_wheel: logger.debug("no cached wheel, downloading sources") - source_filename = sources.download_source( - ctx=self.ctx, - req=req, - version=resolved_version, - download_url=source_url, - ) - sdist_root_dir = sources.prepare_source( - ctx=self.ctx, - req=req, - source_filename=source_filename, - version=resolved_version, - ) + try: + source_filename = sources.download_source( + ctx=self.ctx, + req=req, + version=resolved_version, + download_url=source_url, + ) + except Exception as err: + raise BootstrapPhaseError( + f"Failed to download source for {req.name}=={resolved_version}", + FailureCategory.SOURCE_DOWNLOAD, + err, + ) from err + + try: + sdist_root_dir = sources.prepare_source( + ctx=self.ctx, + req=req, + source_filename=source_filename, + version=resolved_version, + ) + except Exception as err: + raise BootstrapPhaseError( + f"Failed to prepare source for {req.name}=={resolved_version}", + FailureCategory.SOURCE_PREPARE, + err, + ) from err else: logger.debug(f"have cached wheel in {unpacked_cached_wheel}") sdist_root_dir = unpacked_cached_wheel / unpacked_cached_wheel.stem @@ -556,43 +1024,59 @@ def _build_from_source( raise ValueError(f"'{sdist_root_dir}/../..' should be {self.ctx.work_dir}") unpack_dir = sdist_root_dir.parent - build_env = build_environment.BuildEnvironment( - ctx=self.ctx, - parent_dir=sdist_root_dir.parent, - ) + try: + build_env = build_environment.BuildEnvironment( + ctx=self.ctx, + parent_dir=sdist_root_dir.parent, + ) + except Exception as err: + raise BootstrapPhaseError( + f"Failed to create build environment for {req.name}=={resolved_version}", + FailureCategory.BUILD_ENVIRONMENT, + err, + ) from err # Prepare build dependencies (always needed) + # Note: This may recursively call bootstrap() for build deps, + # which has its own error handling. self._prepare_build_dependencies(req, sdist_root_dir, build_env) # Decide what to build based on cache state and build mode wheel_filename: pathlib.Path | None sdist_filename: pathlib.Path | None - if cached_wheel_filename: - logger.debug( - f"getting install requirements from cached " - f"wheel {cached_wheel_filename.name}" - ) - # prefer existing wheel even in sdist_only mode - wheel_filename = cached_wheel_filename - sdist_filename = None - elif build_sdist_only: - logger.debug( - f"getting install requirements from sdist " - f"{req.name}=={resolved_version}" - ) - wheel_filename = None - sdist_filename = self._build_sdist( - req, resolved_version, sdist_root_dir, build_env - ) - else: - logger.debug( - f"building wheel {req.name}=={resolved_version} " - f"to get install requirements" - ) - wheel_filename, sdist_filename = self._build_wheel( - req, resolved_version, sdist_root_dir, build_env - ) + try: + if cached_wheel_filename: + logger.debug( + f"getting install requirements from cached " + f"wheel {cached_wheel_filename.name}" + ) + # prefer existing wheel even in sdist_only mode + wheel_filename = cached_wheel_filename + sdist_filename = None + elif build_sdist_only: + logger.debug( + f"getting install requirements from sdist " + f"{req.name}=={resolved_version}" + ) + wheel_filename = None + sdist_filename = self._build_sdist( + req, resolved_version, sdist_root_dir, build_env + ) + else: + logger.debug( + f"building wheel {req.name}=={resolved_version} " + f"to get install requirements" + ) + wheel_filename, sdist_filename = self._build_wheel( + req, resolved_version, sdist_root_dir, build_env + ) + except Exception as err: + raise BootstrapPhaseError( + f"Failed to build {req.name}=={resolved_version}", + FailureCategory.SOURCE_BUILD, + err, + ) from err source_type = sources.get_source_type(self.ctx, req) @@ -605,6 +1089,92 @@ def _build_from_source( source_type=source_type, ) + def _handle_test_mode_failure( + self, + req: Requirement, + resolved_version: Version, + req_type: RequirementType, + build_error: Exception, + failure_category: FailureCategory = FailureCategory.SOURCE_BUILD, + ) -> SourceBuildResult | None: + """Handle build failure in test mode by attempting pre-built fallback. + + Args: + req: The requirement that failed to build. + resolved_version: The version that was attempted. + req_type: The type of requirement (for fallback resolution). + build_error: The original exception from the build attempt. + failure_category: The category of failure (download, prepare, build, etc.) + + Returns: + SourceBuildResult if fallback succeeded (also records successful fallback), + None if fallback also failed (caller should re-raise for recording). + """ + logger.warning( + "test mode: %s failed for %s==%s, attempting pre-built fallback", + failure_category.value, + req.name, + resolved_version, + exc_info=True, + ) + + try: + wheel_url, fallback_version = self._resolve_prebuilt_with_history( + req=req, + req_type=req_type, + ) + + if fallback_version != resolved_version: + logger.warning( + "test mode: version mismatch for %s - requested %s, fallback %s", + req.name, + resolved_version, + fallback_version, + ) + + wheel_filename, unpack_dir = self._download_prebuilt( + req=req, + req_type=req_type, + resolved_version=fallback_version, + wheel_url=wheel_url, + ) + + logger.info( + "test mode: successfully used pre-built wheel for %s==%s", + req.name, + fallback_version, + ) + + # Record the original failure with successful fallback + self._record_failure( + req, + resolved_version, + build_error, + category=failure_category, + fallback_attempted=True, + fallback_succeeded=True, + fallback_version=fallback_version, + ) + + return SourceBuildResult( + wheel_filename=wheel_filename, + sdist_filename=None, + unpack_dir=unpack_dir, + sdist_root_dir=None, + build_env=None, + source_type=SourceType.PREBUILT, + ) + + except Exception as fallback_error: + logger.error( + "test mode: pre-built fallback also failed for %s: %s", + req.name, + fallback_error, + exc_info=True, + ) + # Return None to signal failure; bootstrap() will record via re-raised exception + return None + def _look_for_existing_wheel( self, req: Requirement, @@ -1127,3 +1697,90 @@ def _add_to_build_order( # Requirement and Version instances that can't be # converted to JSON without help. json.dump(self._build_stack, f, indent=2, default=str) + + def write_test_mode_report(self, work_dir: pathlib.Path) -> None: + """Write test mode failure report to JSON files. + + Generates two JSON files: + - test-mode-failures.json: Detailed list of all failures + - test-mode-summary.json: Summary statistics with category breakdown + """ + if not self.test_mode: + return + + failures_file = work_dir / "test-mode-failures.json" + summary_file = work_dir / "test-mode-summary.json" + + # Generate failures report + failures_data = { + "failures": [build_result.to_dict() for build_result in self.failed_builds] + } + + with open(failures_file, "w") as f: + json.dump(failures_data, f, indent=2) + logger.info("test mode: wrote failure details to %s", failures_file) + + # Generate summary report with category and exception breakdowns + exception_counts: dict[str, int] = {} + category_counts: dict[str, int] = {} + for build_result in self.failed_builds: + exception_type = build_result.exception_type or "Unknown" + exception_counts[exception_type] = ( + exception_counts.get(exception_type, 0) + 1 + ) + category = build_result.category.value + category_counts[category] = category_counts.get(category, 0) + 1 + + summary_data = { + "total_packages": len(self._build_stack), + "total_failures": len(self.failed_builds), + "category_breakdown": category_counts, + "exception_breakdown": exception_counts, + } + + with open(summary_file, "w") as f: + json.dump(summary_data, f, indent=2) + logger.info("test mode: wrote summary to %s", summary_file) + + def finalize(self, work_dir: pathlib.Path) -> int: + """Finalize bootstrap and return exit code. + + In test mode, writes failure reports and logs summary. + Returns non-zero exit code if there were failures. + + Args: + work_dir: Directory to write reports to + + Returns: + 0 if all packages built successfully (or not in test mode) + 1 if any packages failed in test mode + """ + if not self.test_mode: + return 0 + + self.write_test_mode_report(work_dir) + + if not self.failed_builds: + logger.info("test mode: all packages processed successfully") + return 0 + + logger.error( + "test mode: %d package(s) failed to build", + len(self.failed_builds), + ) + # Group failures by category for clearer reporting + by_category: dict[str, list[BuildFailure]] = {} + for failure in self.failed_builds: + category = failure.category.value + by_category.setdefault(category, []).append(failure) + + for category, failures in sorted(by_category.items()): + logger.error(" %s failures:", category) + for failure in failures: + logger.error( + " - %s==%s: %s", + failure.req.name, + failure.resolved_version, + failure.exception_type, + ) + return 1 diff --git a/src/fromager/commands/bootstrap.py b/src/fromager/commands/bootstrap.py index a9836428..7960b79e 100644 --- a/src/fromager/commands/bootstrap.py +++ b/src/fromager/commands/bootstrap.py @@ -21,7 +21,6 @@ server, ) from ..log import requirement_ctxvar -from ..requirements_file import RequirementType from .build import build_parallel from .graph import find_why, show_explain_duplicates @@ -97,6 +96,13 @@ def _get_requirements_from_args( default=False, help="Skip generating constraints.txt file to allow building collections with conflicting versions", ) +@click.option( + "--test-mode", + "test_mode", + is_flag=True, + default=False, + help="Test mode: mark failed packages as pre-built and continue, report failures at end", +) @click.argument("toplevel", nargs=-1) @click.pass_obj def bootstrap( @@ -106,6 +112,7 @@ def bootstrap( cache_wheel_server_url: str | None, sdist_only: bool, skip_constraints: bool, + test_mode: bool, toplevel: list[str], ) -> None: """Compute and build the dependencies of a set of requirements recursively @@ -135,6 +142,11 @@ def bootstrap( else: logger.info("build all missing wheels") + if test_mode: + logger.info( + "test mode enabled: will mark failed packages as pre-built and continue" + ) + pre_built = wkctx.settings.list_pre_built() if pre_built: logger.info("treating %s as pre-built wheels", sorted(pre_built)) @@ -148,45 +160,37 @@ def bootstrap( prev_graph, cache_wheel_server_url, sdist_only=sdist_only, + test_mode=test_mode, ) - # we need to resolve all the top level dependencies before we start bootstrapping. - # this is to ensure that if we are using an older bootstrap to resolve packages - # we are able to upgrade a package anywhere in the dependency tree if it is mentioned - # in the toplevel without having to fall back to history + # Pre-resolution phase: Resolve all top-level dependencies before recursive + # bootstrapping begins. Test-mode error handling is in Bootstrapper. + # Note: We don't use try/finally here because: + # - In test-mode: exceptions are caught inside resolve_and_add_top_level() + # - In normal mode: exceptions should propagate with context preserved for logging logger.info("resolving top-level dependencies before building") + resolved_reqs: list[Requirement] = [] for req in to_build: token = requirement_ctxvar.set(req) - pbi = wkctx.package_build_info(req) - if pbi.pre_built: - source_url, version = bt.resolve_version( - req=req, - req_type=RequirementType.TOP_LEVEL, - ) - else: - source_url, version = bt.resolve_version( - req=req, - req_type=RequirementType.TOP_LEVEL, - ) - logger.info("%s resolves to %s", req, version) - wkctx.dependency_graph.add_dependency( - parent_name=None, - parent_version=None, - req_type=requirements_file.RequirementType.TOP_LEVEL, - req=req, - req_version=version, - download_url=source_url, - pre_built=pbi.pre_built, - constraint=wkctx.constraints.get_constraint(req.name), - ) + result = bt.resolve_and_add_top_level(req) + if result is not None: + resolved_reqs.append(req) + # If result is None, test_mode recorded the failure and we continue requirement_ctxvar.reset(token) - for req in to_build: + # Bootstrap only packages that were successfully resolved + # Note: Same pattern - no try/finally to preserve context for error logging + for req in resolved_reqs: token = requirement_ctxvar.set(req) bt.bootstrap(req, requirements_file.RequirementType.TOP_LEVEL) progressbar.update() requirement_ctxvar.reset(token) + # Finalize test mode - writes reports and returns exit code + exit_code = bt.finalize(wkctx.work_dir) + if exit_code != 0: + raise SystemExit(exit_code) + constraints_filename = wkctx.work_dir / "constraints.txt" if skip_constraints: logger.info("skipping constraints.txt generation as requested") @@ -480,6 +484,9 @@ def bootstrap_parallel( remaining wheels in parallel. The bootstrap step downloads sdists and builds build-time dependency in serial. The build-parallel step builds the remaining wheels in parallel. + + Note: --test-mode is not supported in parallel builds. Use the serial + bootstrap command for test mode. """ # Do not remove build environments in bootstrap phase to speed up the # parallel build phase. diff --git a/tests/test_bootstrap_test_mode.py b/tests/test_bootstrap_test_mode.py new file mode 100644 index 00000000..83102d95 --- /dev/null +++ b/tests/test_bootstrap_test_mode.py @@ -0,0 +1,671 @@ +"""Tests for --test-mode feature. + +Tests the essential test mode functionality: +- Bootstrapper initialization with test_mode +- BuildFailure dataclass creation and serialization +- BootstrapPhaseError for phase-specific failures +- JSON report generation +- Bootstrapper.finalize() exit codes +- Build failure with successful pre-built fallback +- Pre-built download failure recording +- Post-hook failure continues processing +- Dependency extraction failure uses empty deps +""" + +import json +import pathlib +import tempfile +import typing +from unittest.mock import Mock, patch + +import pytest +from packaging.requirements import Requirement +from packaging.version import Version + +from fromager import bootstrapper, context +from fromager.requirements_file import RequirementType, SourceType + + +@pytest.fixture +def mock_context() -> typing.Generator[context.WorkContext, None, None]: + """Create a mock WorkContext for testing.""" + with tempfile.TemporaryDirectory() as tmpdir: + work_dir = pathlib.Path(tmpdir) + + mock_ctx = Mock(spec=context.WorkContext) + mock_ctx.work_dir = work_dir + mock_ctx.wheels_build = work_dir / "wheels-build" + mock_ctx.wheels_downloads = work_dir / "wheels-downloads" + mock_ctx.wheels_prebuilt = work_dir / "wheels-prebuilt" + mock_ctx.sdists_builds = work_dir / "sdists-builds" + mock_ctx.wheel_server_url = None + mock_ctx.constraints = Mock() + mock_ctx.constraints.get_constraint = Mock(return_value=None) + mock_ctx.settings = Mock() + mock_ctx.variant = "test" + + for d in [ + mock_ctx.wheels_build, + mock_ctx.wheels_downloads, + mock_ctx.wheels_prebuilt, + mock_ctx.sdists_builds, + ]: + d.mkdir(parents=True, exist_ok=True) + + yield mock_ctx + + +def test_bootstrapper_test_mode_initialization( + mock_context: context.WorkContext, +) -> None: + """Test Bootstrapper initialization with test_mode parameter.""" + # Test with test_mode=True + bt_enabled = bootstrapper.Bootstrapper(ctx=mock_context, test_mode=True) + assert bt_enabled.test_mode is True + assert isinstance(bt_enabled.failed_builds, list) + assert len(bt_enabled.failed_builds) == 0 + + # Test with test_mode=False (default) + bt_disabled = bootstrapper.Bootstrapper(ctx=mock_context) + assert bt_disabled.test_mode is False + + # Test that test_mode and sdist_only are mutually exclusive + with pytest.raises(ValueError, match="--test-mode requires full wheel builds"): + bootstrapper.Bootstrapper(ctx=mock_context, test_mode=True, sdist_only=True) + + +def test_build_failure_creation_and_serialization() -> None: + """Test BuildFailure dataclass creation, properties, and JSON serialization.""" + # Create a dependency chain for testing + dependency_chain = [ + (RequirementType.TOP_LEVEL, Requirement("pandas==2.0.0"), Version("2.0.0")), + (RequirementType.INSTALL, Requirement("scipy==1.10.0"), Version("1.10.0")), + ] + + failure = bootstrapper.BuildFailure.from_exception( + req=Requirement("numpy==1.24.0"), + resolved_version=Version("1.24.0"), + source_url_type="sdist", + exception=RuntimeError("Compilation failed"), + category=bootstrapper.FailureCategory.SOURCE_BUILD, + fallback_attempted=True, + fallback_succeeded=False, + dependency_chain=dependency_chain, + ) + + # Test basic attributes + assert failure.exception_type == "RuntimeError" + assert failure.exception_message == "Compilation failed" + assert failure.category == bootstrapper.FailureCategory.SOURCE_BUILD + assert failure.fallback_attempted is True + assert failure.fallback_succeeded is False + + # Test dependency chain properties + assert failure.root_package == "pandas" + assert failure.immediate_parent == ("INSTALL", "scipy", "1.10.0") + assert failure.chain_depth == 2 + + # Test JSON serialization + data = failure.to_dict() + assert data["package"] == "numpy==1.24.0" + assert data["version"] == "1.24.0" + assert data["category"] == "source_build" + assert data["root_package"] == "pandas" + assert data["chain_depth"] == 2 + assert "timestamp" in data + assert len(data["dependency_chain"]) == 2 + + # Verify JSON round-trip works + json_str = json.dumps(data) + parsed = json.loads(json_str) + assert parsed["category"] == "source_build" + + +def test_build_failure_without_dependency_chain() -> None: + """Test BuildFailure for top-level package (no dependency chain).""" + failure = bootstrapper.BuildFailure.from_exception( + req=Requirement("toplevel-pkg==1.0.0"), + resolved_version=Version("1.0.0"), + source_url_type="sdist", + exception=RuntimeError("Build failed"), + dependency_chain=[], + ) + + assert failure.root_package == "toplevel-pkg" + assert failure.immediate_parent is None + assert failure.chain_depth == 0 + + +def test_build_failure_version_resolution() -> None: + """Test BuildFailure for version resolution failure (no version).""" + failure = bootstrapper.BuildFailure.from_exception( + req=Requirement("unknown-pkg"), + resolved_version=None, + source_url_type="unknown", + exception=ValueError("Could not resolve version"), + category=bootstrapper.FailureCategory.VERSION_RESOLUTION, + ) + + assert failure.resolved_version is None + assert failure.category == bootstrapper.FailureCategory.VERSION_RESOLUTION + + data = failure.to_dict() + assert data["version"] is None + assert data["category"] == "version_resolution" + + +def test_bootstrap_phase_error() -> None: + """Test BootstrapPhaseError carries phase information.""" + original_error = RuntimeError("Download failed") + phase_error = bootstrapper.BootstrapPhaseError( + "Failed to download source for pkg==1.0", + bootstrapper.FailureCategory.SOURCE_DOWNLOAD, + original_error, + ) + + assert phase_error.category == bootstrapper.FailureCategory.SOURCE_DOWNLOAD + assert phase_error.cause is original_error + assert phase_error.__cause__ is original_error + assert phase_error.fallback_attempted is False + assert "Failed to download source" in str(phase_error) + + # Test with fallback_attempted=True + phase_error_with_fallback = bootstrapper.BootstrapPhaseError( + "Build failed and fallback also failed", + bootstrapper.FailureCategory.SOURCE_BUILD, + original_error, + fallback_attempted=True, + ) + assert phase_error_with_fallback.fallback_attempted is True + + +def test_json_report_generation(mock_context: context.WorkContext) -> None: + """Test JSON report generation with multiple failure categories.""" + bt = bootstrapper.Bootstrapper(ctx=mock_context, test_mode=True) + + # Add failures with different categories and fallback states + bt.failed_builds.extend( + [ + bootstrapper.BuildFailure.from_exception( + req=Requirement("pkg1==1.0.0"), + resolved_version=Version("1.0.0"), + source_url_type="sdist", + exception=RuntimeError("Build failed"), + category=bootstrapper.FailureCategory.SOURCE_BUILD, + fallback_attempted=True, + fallback_succeeded=True, # Successful fallback + ), + bootstrapper.BuildFailure.from_exception( + req=Requirement("pkg2==2.0.0"), + resolved_version=Version("2.0.0"), + source_url_type="sdist", + exception=RuntimeError("Download failed"), + category=bootstrapper.FailureCategory.SOURCE_DOWNLOAD, + fallback_attempted=True, + fallback_succeeded=False, # Failed fallback + ), + bootstrapper.BuildFailure.from_exception( + req=Requirement("pkg3==3.0.0"), + resolved_version=Version("3.0.0"), + source_url_type="prebuilt", + exception=ConnectionError("Connection error"), + category=bootstrapper.FailureCategory.PREBUILT_DOWNLOAD, + ), + ] + ) + + bt.write_test_mode_report(mock_context.work_dir) + + # Verify files exist + failures_file = mock_context.work_dir / "test-mode-failures.json" + summary_file = mock_context.work_dir / "test-mode-summary.json" + assert failures_file.exists() + assert summary_file.exists() + + # Verify failures content + with open(failures_file) as f: + failures_data = json.load(f) + assert len(failures_data["failures"]) == 3 + assert failures_data["failures"][0]["fallback_succeeded"] is True + assert failures_data["failures"][1]["fallback_succeeded"] is False + + # Verify summary content + with open(summary_file) as f: + summary_data = json.load(f) + assert summary_data["total_failures"] == 3 + assert summary_data["category_breakdown"]["source_build"] == 1 + assert summary_data["category_breakdown"]["source_download"] == 1 + assert summary_data["category_breakdown"]["prebuilt_download"] == 1 + assert summary_data["exception_breakdown"]["RuntimeError"] == 2 + assert summary_data["exception_breakdown"]["ConnectionError"] == 1 + + +def test_finalize_exit_codes(mock_context: context.WorkContext) -> None: + """Test finalize returns correct exit codes and writes reports.""" + # Test: no failures in test mode -> exit 0, reports written + bt = bootstrapper.Bootstrapper(ctx=mock_context, test_mode=True) + assert bt.finalize(mock_context.work_dir) == 0 + assert (mock_context.work_dir / "test-mode-failures.json").exists() + assert (mock_context.work_dir / "test-mode-summary.json").exists() + + # Clean up for next test + (mock_context.work_dir / "test-mode-failures.json").unlink() + (mock_context.work_dir / "test-mode-summary.json").unlink() + + # Test: failures in test mode -> exit 1 + bt2 = bootstrapper.Bootstrapper(ctx=mock_context, test_mode=True) + bt2.failed_builds.append( + bootstrapper.BuildFailure.from_exception( + req=Requirement("failing-pkg==1.0.0"), + resolved_version=Version("1.0.0"), + source_url_type="sdist", + exception=RuntimeError("Build failed"), + category=bootstrapper.FailureCategory.SOURCE_BUILD, + ) + ) + assert bt2.finalize(mock_context.work_dir) == 1 + + # Clean up for next test + (mock_context.work_dir / "test-mode-failures.json").unlink() + (mock_context.work_dir / "test-mode-summary.json").unlink() + + # Test: not in test mode -> exit 0, no reports written + bt3 = bootstrapper.Bootstrapper(ctx=mock_context, test_mode=False) + assert bt3.finalize(mock_context.work_dir) == 0 + assert not (mock_context.work_dir / "test-mode-failures.json").exists() + assert not (mock_context.work_dir / "test-mode-summary.json").exists() + + +def test_handle_test_mode_failure_with_successful_fallback( + mock_context: context.WorkContext, +) -> None: + """Test that build failure with successful pre-built fallback records correctly. + + When source build fails in test mode: + 1. Fallback to pre-built wheel is attempted + 2. If successful, failure is recorded with fallback_succeeded=True + 3. Processing continues with the pre-built wheel + """ + bt = bootstrapper.Bootstrapper(ctx=mock_context, test_mode=True) + + req = Requirement("failing-pkg==1.0.0") + resolved_version = Version("1.0.0") + build_error = RuntimeError("Compilation failed: missing headers") + + # Mock the fallback resolution and download + mock_wheel_path = ( + mock_context.wheels_prebuilt / "failing_pkg-1.0.0-py3-none-any.whl" + ) + mock_wheel_path.touch() + mock_unpack_dir = mock_context.work_dir / "unpack" + mock_unpack_dir.mkdir(parents=True, exist_ok=True) + + with ( + patch.object( + bt, + "_resolve_prebuilt_with_history", + return_value=("https://pypi.org/wheel.whl", resolved_version), + ), + patch.object( + bt, + "_download_prebuilt", + return_value=(mock_wheel_path, mock_unpack_dir), + ), + ): + result = bt._handle_test_mode_failure( + req=req, + resolved_version=resolved_version, + req_type=RequirementType.INSTALL, + build_error=build_error, + failure_category=bootstrapper.FailureCategory.SOURCE_BUILD, + ) + + # Verify fallback succeeded + assert result is not None + assert result.wheel_filename == mock_wheel_path + assert result.source_type == SourceType.PREBUILT + assert result.sdist_filename is None + + # Verify failure was recorded with successful fallback + assert len(bt.failed_builds) == 1 + failure = bt.failed_builds[0] + assert failure.req == req + assert failure.resolved_version == resolved_version + assert failure.category == bootstrapper.FailureCategory.SOURCE_BUILD + assert failure.fallback_attempted is True + assert failure.fallback_succeeded is True + assert failure.exception_type == "RuntimeError" + assert "Compilation failed" in str(failure.exception_message) + # Same version - no mismatch + assert failure.fallback_version == resolved_version + assert failure.version_mismatch is False + + +def test_handle_test_mode_failure_version_mismatch( + mock_context: context.WorkContext, +) -> None: + """Test that version mismatch is captured when fallback uses different version. + + When source build fails and fallback uses a different version: + 1. fallback_version captures the actual version used + 2. version_mismatch is True + 3. JSON serialization includes this information + """ + bt = bootstrapper.Bootstrapper(ctx=mock_context, test_mode=True) + + req = Requirement("mismatch-pkg==1.0.0") + resolved_version = Version("1.0.0") + fallback_version = Version("1.0.1") # Different version + build_error = RuntimeError("Build failed") + + mock_wheel_path = ( + mock_context.wheels_prebuilt / "mismatch_pkg-1.0.1-py3-none-any.whl" + ) + mock_wheel_path.touch() + mock_unpack_dir = mock_context.work_dir / "unpack" + mock_unpack_dir.mkdir(parents=True, exist_ok=True) + + with ( + patch.object( + bt, + "_resolve_prebuilt_with_history", + return_value=("https://pypi.org/wheel.whl", fallback_version), + ), + patch.object( + bt, + "_download_prebuilt", + return_value=(mock_wheel_path, mock_unpack_dir), + ), + ): + result = bt._handle_test_mode_failure( + req=req, + resolved_version=resolved_version, + req_type=RequirementType.INSTALL, + build_error=build_error, + failure_category=bootstrapper.FailureCategory.SOURCE_BUILD, + ) + + # Verify fallback succeeded + assert result is not None + + # Verify version mismatch is captured + assert len(bt.failed_builds) == 1 + failure = bt.failed_builds[0] + assert failure.resolved_version == resolved_version + assert failure.fallback_version == fallback_version + assert failure.version_mismatch is True + + # Verify JSON serialization includes mismatch info + data = failure.to_dict() + assert data["version"] == "1.0.0" + assert data["fallback_version"] == "1.0.1" + assert data["version_mismatch"] is True + + +def test_handle_test_mode_failure_with_failed_fallback( + mock_context: context.WorkContext, +) -> None: + """Test that build failure with failed fallback returns None. + + When both source build and pre-built fallback fail: + 1. Returns None to signal complete failure + 2. Does NOT record failure (caller re-raises for bootstrap() to record) + """ + bt = bootstrapper.Bootstrapper(ctx=mock_context, test_mode=True) + + req = Requirement("double-fail-pkg==1.0.0") + resolved_version = Version("1.0.0") + build_error = RuntimeError("Build failed") + + # Mock fallback to also fail + with patch.object( + bt, + "_resolve_prebuilt_with_history", + side_effect=ConnectionError("No pre-built wheel available"), + ): + result = bt._handle_test_mode_failure( + req=req, + resolved_version=resolved_version, + req_type=RequirementType.INSTALL, + build_error=build_error, + failure_category=bootstrapper.FailureCategory.SOURCE_DOWNLOAD, + ) + + # Verify complete failure - returns None + assert result is None + + # _handle_test_mode_failure no longer records failures on failed fallback. + # The caller (_bootstrap_impl) re-raises BootstrapPhaseError, and + # bootstrap() catches it to record the failure. + assert len(bt.failed_builds) == 0 + + +def test_top_level_version_resolution_failure( + mock_context: context.WorkContext, +) -> None: + """Test that top-level version resolution failures are properly captured. + + When resolve_and_add_top_level() fails to resolve a version: + 1. Failure is recorded with VERSION_RESOLUTION category + 2. resolved_version is None (no version could be determined) + 3. Returns None to signal failure to caller + """ + bt = bootstrapper.Bootstrapper(ctx=mock_context, test_mode=True) + + req = Requirement("nonexistent-pkg>=1.0") + + # Mock resolve_version to fail + with patch.object( + bt, + "resolve_version", + side_effect=ValueError("No matching version found for nonexistent-pkg"), + ): + result = bt.resolve_and_add_top_level(req) + + # Verify failure is signaled + assert result is None + + # Verify failure was recorded with correct category + assert len(bt.failed_builds) == 1 + failure = bt.failed_builds[0] + assert failure.req == req + assert failure.resolved_version is None # No version resolved + assert failure.category == bootstrapper.FailureCategory.VERSION_RESOLUTION + assert failure.exception_type == "ValueError" + assert "No matching version" in str(failure.exception_message) + assert failure.dependency_chain == [] # Top-level has no chain + + # Verify JSON serialization + data = failure.to_dict() + assert data["version"] is None + assert data["category"] == "version_resolution" + + +def test_prebuilt_download_failure_recorded( + mock_context: context.WorkContext, +) -> None: + """Test that pre-built package download failures are recorded. + + When a package marked as pre_built fails to download: + 1. Failure is recorded with PREBUILT_DOWNLOAD category + 2. The package is skipped (dependencies not processed) + 3. Bootstrap continues with next package + """ + bt = bootstrapper.Bootstrapper(ctx=mock_context, test_mode=True) + + req = Requirement("prebuilt-pkg==1.0.0") + resolved_version = Version("1.0.0") + + # Simulate the failure recording that happens in _bootstrap_impl + # when prebuilt download fails + download_error = ConnectionError("Failed to download wheel from PyPI") + + bt._record_failure( + req=req, + resolved_version=resolved_version, + exception=download_error, + category=bootstrapper.FailureCategory.PREBUILT_DOWNLOAD, + ) + + # Verify failure was recorded correctly + assert len(bt.failed_builds) == 1 + failure = bt.failed_builds[0] + assert failure.req == req + assert failure.resolved_version == resolved_version + assert failure.category == bootstrapper.FailureCategory.PREBUILT_DOWNLOAD + assert failure.exception_type == "ConnectionError" + assert "Failed to download" in str(failure.exception_message) + # Pre-built failures don't attempt fallback (they ARE the fallback) + assert failure.fallback_attempted is False + + +def test_post_hook_failure_continues_processing( + mock_context: context.WorkContext, +) -> None: + """Test that post-bootstrap hook failures are recorded and processing continues. + + When hooks.run_post_bootstrap_hooks() fails in test mode: + 1. Failure is recorded with POST_HOOK category + 2. Processing continues (dependencies are still extracted) + """ + bt = bootstrapper.Bootstrapper(ctx=mock_context, test_mode=True) + + req = Requirement("hook-fail-pkg==1.0.0") + resolved_version = Version("1.0.0") + hook_error = RuntimeError("Post-build hook script failed with exit code 1") + + # Record the hook failure as done in _bootstrap_impl + bt._record_failure( + req=req, + resolved_version=resolved_version, + exception=hook_error, + category=bootstrapper.FailureCategory.POST_HOOK, + ) + + # Verify failure was recorded + assert len(bt.failed_builds) == 1 + failure = bt.failed_builds[0] + assert failure.category == bootstrapper.FailureCategory.POST_HOOK + assert failure.exception_type == "RuntimeError" + + # Verify JSON serialization works for POST_HOOK category + data = failure.to_dict() + assert data["category"] == "post_hook" + + +def test_dependency_extraction_failure_uses_empty_deps( + mock_context: context.WorkContext, +) -> None: + """Test that dependency extraction failures result in empty deps list. + + When _get_install_dependencies() fails in test mode: + 1. Failure is recorded with DEPENDENCY_EXTRACTION category + 2. Empty dependency list is used (package built, but deps not processed) + """ + bt = bootstrapper.Bootstrapper(ctx=mock_context, test_mode=True) + + req = Requirement("bad-metadata-pkg==1.0.0") + resolved_version = Version("1.0.0") + dep_error = ValueError("Invalid METADATA file: missing Requires-Dist") + + # Record the dependency extraction failure as done in _bootstrap_impl + bt._record_failure( + req=req, + resolved_version=resolved_version, + exception=dep_error, + category=bootstrapper.FailureCategory.DEPENDENCY_EXTRACTION, + ) + + # Verify failure was recorded + assert len(bt.failed_builds) == 1 + failure = bt.failed_builds[0] + assert failure.category == bootstrapper.FailureCategory.DEPENDENCY_EXTRACTION + assert failure.exception_type == "ValueError" + assert "METADATA" in str(failure.exception_message) + + # Verify JSON serialization + data = failure.to_dict() + assert data["category"] == "dependency_extraction" + + +def test_record_failure_captures_dependency_chain( + mock_context: context.WorkContext, +) -> None: + """Test that _record_failure captures the current dependency chain.""" + bt = bootstrapper.Bootstrapper(ctx=mock_context, test_mode=True) + + # Simulate a dependency chain (as if we're deep in recursive bootstrap) + bt.why = [ + (RequirementType.TOP_LEVEL, Requirement("top-pkg==1.0"), Version("1.0")), + (RequirementType.INSTALL, Requirement("mid-pkg==2.0"), Version("2.0")), + (RequirementType.BUILD_SYSTEM, Requirement("build-pkg==3.0"), Version("3.0")), + ] + + req = Requirement("deep-fail-pkg==4.0.0") + resolved_version = Version("4.0.0") + + bt._record_failure( + req=req, + resolved_version=resolved_version, + exception=RuntimeError("Deep failure"), + category=bootstrapper.FailureCategory.SOURCE_BUILD, + ) + + # Verify dependency chain was captured + assert len(bt.failed_builds) == 1 + failure = bt.failed_builds[0] + assert failure.chain_depth == 3 + assert failure.root_package == "top-pkg" + assert failure.immediate_parent == ("BUILD_SYSTEM", "build-pkg", "3.0") + + # Verify chain serialization + data = failure.to_dict() + assert len(data["dependency_chain"]) == 3 + assert data["dependency_chain"][0]["type"] == "TOP_LEVEL" + assert data["dependency_chain"][0]["package"] == "top-pkg" + + +def test_multiple_failure_categories_in_summary( + mock_context: context.WorkContext, +) -> None: + """Test summary report correctly categorizes multiple failure types.""" + bt = bootstrapper.Bootstrapper(ctx=mock_context, test_mode=True) + + # Add failures from different categories + categories_and_errors = [ + (bootstrapper.FailureCategory.SOURCE_DOWNLOAD, "Network error"), + (bootstrapper.FailureCategory.SOURCE_PREPARE, "Patch failed"), + (bootstrapper.FailureCategory.BUILD_ENVIRONMENT, "Venv creation failed"), + (bootstrapper.FailureCategory.SOURCE_BUILD, "Compilation error"), + (bootstrapper.FailureCategory.SOURCE_BUILD, "Another build error"), + (bootstrapper.FailureCategory.PREBUILT_DOWNLOAD, "Wheel not found"), + (bootstrapper.FailureCategory.POST_HOOK, "Hook script failed"), + (bootstrapper.FailureCategory.DEPENDENCY_EXTRACTION, "Bad metadata"), + ] + + for i, (category, msg) in enumerate(categories_and_errors): + bt.failed_builds.append( + bootstrapper.BuildFailure.from_exception( + req=Requirement(f"pkg{i}==1.0.0"), + resolved_version=Version("1.0.0"), + source_url_type="sdist", + exception=RuntimeError(msg), + category=category, + ) + ) + + bt.write_test_mode_report(mock_context.work_dir) + + # Verify summary has correct category counts + summary_file = mock_context.work_dir / "test-mode-summary.json" + with open(summary_file) as f: + summary = json.load(f) + + assert summary["total_failures"] == 8 + assert summary["category_breakdown"]["source_download"] == 1 + assert summary["category_breakdown"]["source_prepare"] == 1 + assert summary["category_breakdown"]["build_environment"] == 1 + assert summary["category_breakdown"]["source_build"] == 2 + assert summary["category_breakdown"]["prebuilt_download"] == 1 + assert summary["category_breakdown"]["post_hook"] == 1 + assert summary["category_breakdown"]["dependency_extraction"] == 1 diff --git a/tests/test_commands.py b/tests/test_commands.py index 7617e308..5678d3d9 100644 --- a/tests/test_commands.py +++ b/tests/test_commands.py @@ -17,5 +17,6 @@ def test_bootstrap_parallel_options() -> None: # graph_file internally. expected.discard("sdist_only") expected.discard("graph_file") + expected.discard("test_mode") assert set(get_option_names(bootstrap.bootstrap_parallel)) == expected