diff --git a/.gitignore b/.gitignore index 91922c6..0961dfa 100644 --- a/.gitignore +++ b/.gitignore @@ -36,3 +36,5 @@ dmypy.json # Temporary files .tmp/ +docs/design/fallback-support.md +docs/release-posts.md diff --git a/src/catsu/__init__.py b/src/catsu/__init__.py index 2220f5b..d7afae0 100644 --- a/src/catsu/__init__.py +++ b/src/catsu/__init__.py @@ -8,17 +8,44 @@ from .catalog import ModelCatalog from .client import Client from .models import EmbedResponse, ModelInfo, TokenizeResponse, Usage +from .utils.errors import ( + AuthenticationError, + CatsuError, + ConfigurationError, + FallbackExhaustedError, + InvalidInputError, + ModelNotFoundError, + NetworkError, + ProviderError, + RateLimitError, + TimeoutError, + UnsupportedFeatureError, +) __version__ = "0.0.2" __author__ = "Chonkie, Inc." __all__ = [ + # Client and models "Client", "EmbedResponse", "Usage", "TokenizeResponse", "ModelInfo", "ModelCatalog", + # Exceptions + "CatsuError", + "ProviderError", + "AuthenticationError", + "RateLimitError", + "TimeoutError", + "NetworkError", + "ModelNotFoundError", + "InvalidInputError", + "UnsupportedFeatureError", + "ConfigurationError", + "FallbackExhaustedError", + # Package metadata "__version__", "__author__", ] diff --git a/src/catsu/client.py b/src/catsu/client.py index 836bff0..2a56bfa 100644 --- a/src/catsu/client.py +++ b/src/catsu/client.py @@ -4,7 +4,9 @@ providers through a single API. """ +import asyncio import os +import time from typing import Any, Dict, List, Literal, Optional, Tuple, Union import httpx @@ -12,7 +14,22 @@ from .catalog import ModelCatalog from .models import EmbedResponse, TokenizeResponse from .providers import BaseProvider, registry -from .utils.errors import InvalidInputError, ModelNotFoundError, UnsupportedFeatureError +from .utils.errors import ( + AuthenticationError, + ConfigurationError, + FallbackExhaustedError, + InvalidInputError, + ModelNotFoundError, + NetworkError, + ProviderError, + RateLimitError, + TimeoutError, + UnsupportedFeatureError, +) + +# Type aliases for fallback configuration +FallbackConfig = Dict[str, Any] # {"model": "...", **provider_overrides} +Fallbacks = Union[str, FallbackConfig, List[Union[str, FallbackConfig]], None] class Client: @@ -57,13 +74,36 @@ def __init__( max_retries: int = 3, timeout: int = 30, api_keys: Optional[Dict[str, str]] = None, + fallbacks: Fallbacks = None, + allow_unsafe_fallback: bool = False, + base_delay: float = 1.0, + max_delay: float = 10.0, ) -> None: - """Initialize the Catsu client.""" + """Initialize the Catsu client. + + Args: + verbose: Enable verbose logging (default: False) + max_retries: Maximum number of retry attempts (default: 3) + timeout: Request timeout in seconds (default: 30) + api_keys: Optional dict of API keys by provider name + fallbacks: Default fallback models for all requests + allow_unsafe_fallback: If True, allow fallback to models with + different dimensions (default: False, only dimension-compatible) + base_delay: Base delay between retry cycles in seconds + max_delay: Maximum delay between retry cycles in seconds + + """ self.verbose = verbose self.max_retries = max_retries self.timeout = timeout self._api_keys = api_keys or {} + # Fallback configuration + self.fallbacks = fallbacks + self.allow_unsafe_fallback = allow_unsafe_fallback + self.base_delay = base_delay + self.max_delay = max_delay + # Initialize HTTP clients for sync and async self._http_client = httpx.Client(timeout=timeout) self._async_http_client = httpx.AsyncClient(timeout=timeout) @@ -199,6 +239,457 @@ def _get_provider(self, provider_name: str) -> BaseProvider: return self._providers[provider_name] + # ------------------------------------------------------------------------- + # Fallback helper methods + # ------------------------------------------------------------------------- + + def _normalize_fallbacks(self, fallbacks: Fallbacks) -> List[FallbackConfig]: + """Convert any fallback input format to List[FallbackConfig]. + + Args: + fallbacks: Fallback specification (str, dict, list, or None) + + Returns: + Normalized list of fallback configurations + + """ + if fallbacks is None: + return [] + if isinstance(fallbacks, str): + return [{"model": fallbacks}] + if isinstance(fallbacks, dict): + return [fallbacks] + # List - normalize each item + return [{"model": f} if isinstance(f, str) else f for f in fallbacks] + + def _get_env_var_for_provider(self, provider: str) -> str: + """Get the environment variable name for a provider's API key.""" + env_var_map = { + "voyageai": "VOYAGE_API_KEY", + "openai": "OPENAI_API_KEY", + "cohere": "COHERE_API_KEY", + "gemini": "GEMINI_API_KEY", + "jinaai": "JINA_API_KEY", + "mistral": "MISTRAL_API_KEY", + "nomic": "NOMIC_API_KEY", + "cloudflare": "CLOUDFLARE_API_KEY", + "deepinfra": "DEEPINFRA_API_KEY", + "mixedbread": "MIXEDBREAD_API_KEY", + "togetherai": "TOGETHERAI_API_KEY", + } + return env_var_map.get(provider, f"{provider.upper()}_API_KEY") + + def _validate_fallback_credentials( + self, + primary_model: str, + fallback_configs: List[FallbackConfig], + ) -> None: + """Validate API keys exist for primary and all fallback models. + + Args: + primary_model: The primary model string + fallback_configs: Normalized fallback configurations + + Raises: + ConfigurationError: If any required API keys are missing + + """ + all_models = [primary_model] + [f["model"] for f in fallback_configs] + missing = [] + + for model in all_models: + provider_name, _ = self._parse_model_string(model) + api_key = self._get_api_key(provider_name) + if not api_key: + env_var = self._get_env_var_for_provider(provider_name) + missing.append({"model": model, "provider": provider_name, "env_var": env_var}) + + if missing: + details = "\n".join( + f" - {m['model']} ({m['provider']}): set {m['env_var']}" + for m in missing + ) + raise ConfigurationError( + f"Missing API keys for models:\n{details}", + details={"missing_credentials": missing}, + ) + + def _get_target_dimensions( + self, + provider_name: str, + model_name: str, + kwargs: Dict[str, Any], + ) -> int: + """Get the target dimensions (from kwargs or model default). + + Args: + provider_name: Provider name + model_name: Model name + kwargs: Request kwargs that may contain dimensions + + Returns: + Target dimensions value + + """ + if "dimensions" in kwargs and kwargs["dimensions"] is not None: + return kwargs["dimensions"] + model_info = self._catalog.get_model_info(provider_name, model_name) + return model_info.dimensions + + def _can_produce_dimensions( + self, + provider_name: str, + model_name: str, + target_dims: int, + ) -> bool: + """Check if a model can produce the target dimensions. + + Args: + provider_name: Provider name + model_name: Model name + target_dims: Required dimensions + + Returns: + True if model can produce target dimensions + + """ + model_info = self._catalog.get_model_info(provider_name, model_name) + + # If model supports custom dimensions, it can produce any size + # (we assume the user knows valid ranges) + if model_info.supports_dimensions: + return True + + # Otherwise, model must have matching default dimensions + return model_info.dimensions == target_dims + + def _filter_compatible_fallbacks( + self, + target_dims: int, + fallback_configs: List[FallbackConfig], + ) -> List[FallbackConfig]: + """Filter fallbacks to only those that can produce target dimensions. + + Args: + target_dims: Required dimensions + fallback_configs: Normalized fallback configurations + + Returns: + Filtered list of compatible fallback configurations + + """ + compatible = [] + + for config in fallback_configs: + model = config["model"] + provider_name, model_name = self._parse_model_string(model) + + if self._can_produce_dimensions(provider_name, model_name, target_dims): + # If model supports custom dims and none specified, add target + model_info = self._catalog.get_model_info(provider_name, model_name) + if model_info.supports_dimensions and "dimensions" not in config: + config = {**config, "dimensions": target_dims} + compatible.append(config) + elif self.verbose: + print(f"[catsu] Skipping {model}: incompatible dimensions") + + return compatible + + def _is_fallback_error(self, error: Exception) -> bool: + """Check if an error should trigger fallback. + + Fallback is triggered for transient errors (rate limits, timeouts, + network issues, server errors). Client errors (auth, invalid input) + should not trigger fallback. + + Args: + error: The exception to check + + Returns: + True if fallback should be attempted + + """ + # These errors should trigger fallback + if isinstance(error, (RateLimitError, TimeoutError, NetworkError)): + return True + + # Provider errors with 5xx status codes should trigger fallback + if isinstance(error, ProviderError): + if error.status_code and error.status_code >= 500: + return True + + # Auth errors and client errors should NOT trigger fallback + if isinstance(error, (AuthenticationError, InvalidInputError)): + return False + + # Provider errors with 4xx should NOT trigger fallback + if isinstance(error, ProviderError): + if error.status_code and 400 <= error.status_code < 500: + return False + + # Default: don't fallback for unknown errors + return False + + # ------------------------------------------------------------------------- + # Fallback orchestration + # ------------------------------------------------------------------------- + + def _embed_with_fallbacks( + self, + primary_model: str, + inputs: List[str], + fallback_configs: List[FallbackConfig], + allow_unsafe: bool, + input_type: Optional[Literal["query", "document"]] = None, + dimensions: Optional[int] = None, + api_key: Optional[str] = None, + **kwargs: Any, + ) -> EmbedResponse: + """Execute embed with fallback chain using round-robin retries. + + Args: + primary_model: Primary model string + inputs: List of input texts + fallback_configs: Normalized fallback configurations + allow_unsafe: Whether to allow dimension-incompatible fallbacks + input_type: Optional input type hint + dimensions: Optional output dimensions + api_key: Optional API key override + **kwargs: Additional provider-specific parameters + + Returns: + EmbedResponse from successful model + + Raises: + FallbackExhaustedError: If all models fail after all cycles + + """ + # 1. Validate all credentials upfront (fail fast) + self._validate_fallback_credentials(primary_model, fallback_configs) + + # 2. Get target dimensions from primary model + primary_provider, primary_model_name = self._parse_model_string(primary_model) + target_dims = self._get_target_dimensions( + primary_provider, primary_model_name, {"dimensions": dimensions} + ) + + # 3. Filter by dimension compatibility if safe mode + if not allow_unsafe: + fallback_configs = self._filter_compatible_fallbacks(target_dims, fallback_configs) + + # 4. Build execution chain: [primary, ...fallbacks] + chain: List[FallbackConfig] = [ + {"model": primary_model, "dimensions": dimensions, "api_key": api_key, **kwargs} + ] + for config in fallback_configs: + chain.append({**config}) + + errors: Dict[str, Exception] = {} + cycle = 1 + + # 5. Round-robin with global retries + while cycle <= self.max_retries: + if self.verbose: + print(f"[catsu] Starting cycle {cycle}/{self.max_retries}") + + # Try each model in the chain (1 attempt each) + for config in chain: + model = config["model"] + model_kwargs = {k: v for k, v in config.items() if k != "model"} + + try: + # Parse model and get provider + provider_name, model_name = self._parse_model_string(model) + model_info = self._catalog.get_model_info(provider_name, model_name) + + # Validate features for this model + model_dims = model_kwargs.get("dimensions") + if model_dims is not None and not model_info.supports_dimensions: + # Skip this model if it doesn't support requested dimensions + if self.verbose: + print(f"[catsu] Skipping {model}: doesn't support custom dimensions") + continue + + model_input_type = input_type + if model_input_type is not None and not model_info.supports_input_type: + # Use model without input_type + model_input_type = None + + # Get provider and make request + provider_instance = self._get_provider(provider_name) + response = provider_instance.embed( + model=model_name, + inputs=inputs, + input_type=model_input_type, + dimensions=model_dims, + api_key=model_kwargs.get("api_key"), + **{k: v for k, v in model_kwargs.items() if k not in ("dimensions", "api_key")}, + ) + + # Add fallback metadata if not primary + if model != primary_model: + response.requested_model = primary_model + response.fallback_used = True + primary_error = errors.get(primary_model) + response.fallback_reason = ( + type(primary_error).__name__ if primary_error else "Unknown" + ) + + if self.verbose and model != primary_model: + print(f"[catsu] Fallback succeeded with {model}") + + return response + + except Exception as e: + errors[model] = e + if self.verbose: + print(f"[catsu] Cycle {cycle}: {model} failed: {e}") + + # Check if this error should trigger fallback + if not self._is_fallback_error(e): + # Re-raise non-fallback errors immediately + raise + + # Continue to next model in chain + + # All models failed this cycle - backoff before next cycle + if cycle < self.max_retries: + backoff = min( + self.base_delay * (2 ** (cycle - 1)), + self.max_delay, + ) + if self.verbose: + print(f"[catsu] All models failed cycle {cycle}, backing off {backoff}s...") + time.sleep(backoff) + + cycle += 1 + + # 6. All cycles exhausted + raise FallbackExhaustedError( + primary_model=primary_model, + fallbacks=[f["model"] for f in fallback_configs], + errors=errors, + cycles_attempted=self.max_retries, + ) + + async def _aembed_with_fallbacks( + self, + primary_model: str, + inputs: List[str], + fallback_configs: List[FallbackConfig], + allow_unsafe: bool, + input_type: Optional[Literal["query", "document"]] = None, + dimensions: Optional[int] = None, + api_key: Optional[str] = None, + **kwargs: Any, + ) -> EmbedResponse: + """Async version of _embed_with_fallbacks.""" + # 1. Validate all credentials upfront (fail fast) + self._validate_fallback_credentials(primary_model, fallback_configs) + + # 2. Get target dimensions from primary model + primary_provider, primary_model_name = self._parse_model_string(primary_model) + target_dims = self._get_target_dimensions( + primary_provider, primary_model_name, {"dimensions": dimensions} + ) + + # 3. Filter by dimension compatibility if safe mode + if not allow_unsafe: + fallback_configs = self._filter_compatible_fallbacks(target_dims, fallback_configs) + + # 4. Build execution chain: [primary, ...fallbacks] + chain: List[FallbackConfig] = [ + {"model": primary_model, "dimensions": dimensions, "api_key": api_key, **kwargs} + ] + for config in fallback_configs: + chain.append({**config}) + + errors: Dict[str, Exception] = {} + cycle = 1 + + # 5. Round-robin with global retries + while cycle <= self.max_retries: + if self.verbose: + print(f"[catsu] Starting cycle {cycle}/{self.max_retries}") + + # Try each model in the chain (1 attempt each) + for config in chain: + model = config["model"] + model_kwargs = {k: v for k, v in config.items() if k != "model"} + + try: + # Parse model and get provider + provider_name, model_name = self._parse_model_string(model) + model_info = self._catalog.get_model_info(provider_name, model_name) + + # Validate features for this model + model_dims = model_kwargs.get("dimensions") + if model_dims is not None and not model_info.supports_dimensions: + if self.verbose: + print(f"[catsu] Skipping {model}: doesn't support custom dimensions") + continue + + model_input_type = input_type + if model_input_type is not None and not model_info.supports_input_type: + model_input_type = None + + # Get provider and make async request + provider_instance = self._get_provider(provider_name) + response = await provider_instance.aembed( + model=model_name, + inputs=inputs, + input_type=model_input_type, + dimensions=model_dims, + api_key=model_kwargs.get("api_key"), + **{k: v for k, v in model_kwargs.items() if k not in ("dimensions", "api_key")}, + ) + + # Add fallback metadata if not primary + if model != primary_model: + response.requested_model = primary_model + response.fallback_used = True + primary_error = errors.get(primary_model) + response.fallback_reason = ( + type(primary_error).__name__ if primary_error else "Unknown" + ) + + if self.verbose and model != primary_model: + print(f"[catsu] Fallback succeeded with {model}") + + return response + + except Exception as e: + errors[model] = e + if self.verbose: + print(f"[catsu] Cycle {cycle}: {model} failed: {e}") + + if not self._is_fallback_error(e): + raise + + # All models failed this cycle - backoff before next cycle + if cycle < self.max_retries: + backoff = min( + self.base_delay * (2 ** (cycle - 1)), + self.max_delay, + ) + if self.verbose: + print(f"[catsu] All models failed cycle {cycle}, backing off {backoff}s...") + await asyncio.sleep(backoff) + + cycle += 1 + + # 6. All cycles exhausted + raise FallbackExhaustedError( + primary_model=primary_model, + fallbacks=[f["model"] for f in fallback_configs], + errors=errors, + cycles_attempted=self.max_retries, + ) + + # ------------------------------------------------------------------------- + # Public API + # ------------------------------------------------------------------------- + def embed( self, model: str, @@ -207,6 +698,8 @@ def embed( input_type: Optional[Literal["query", "document"]] = None, dimensions: Optional[int] = None, api_key: Optional[str] = None, + fallbacks: Fallbacks = None, + allow_unsafe_fallback: Optional[bool] = None, **kwargs: Any, ) -> EmbedResponse: """Generate embeddings for input text(s). @@ -219,6 +712,12 @@ def embed( Used by some providers like VoyageAI dimensions: Optional output dimensions (model must support this feature) api_key: Optional API key override for this specific request + fallbacks: Fallback model(s) to try if primary fails. Can be: + - str: Single model name (e.g., "text-embedding-3-small") + - dict: Model with config (e.g., {"model": "...", "api_key": "..."}) + - list: Multiple fallbacks in order of preference + allow_unsafe_fallback: If True, allow fallback to models with different + dimensions (overrides client-level setting) **kwargs: Additional provider-specific parameters Returns: @@ -230,20 +729,49 @@ def embed( ModelNotFoundError: Model not found AmbiguousModelError: Model name is ambiguous UnsupportedFeatureError: Model doesn't support requested feature + ConfigurationError: Missing API keys for fallback models + FallbackExhaustedError: All fallback models failed Example: >>> client = Client() >>> result = client.embed( ... model="voyage-3", ... input="hello world", - ... provider="voyageai", - ... dimensions=512 + ... fallbacks=["text-embedding-3-small", "embed-english-v3.0"] ... ) >>> print(result.embeddings) # [[0.1, 0.2, ...]] - >>> print(result.usage.total_tokens) # 2 - >>> print(result.usage.total_cost) # 0.0000002 + >>> print(result.fallback_used) # True if fallback was used """ + # Normalize input to list + inputs = [input] if isinstance(input, str) else input + + # Determine effective fallbacks (per-request overrides client-level) + effective_fallbacks = fallbacks if fallbacks is not None else self.fallbacks + effective_allow_unsafe = ( + allow_unsafe_fallback + if allow_unsafe_fallback is not None + else self.allow_unsafe_fallback + ) + + # If fallbacks are configured, use fallback logic + if effective_fallbacks is not None: + # Construct full model string with provider if specified + full_model = f"{provider}:{model}" if provider and ":" not in model else model + + fallback_configs = self._normalize_fallbacks(effective_fallbacks) + return self._embed_with_fallbacks( + primary_model=full_model, + inputs=inputs, + fallback_configs=fallback_configs, + allow_unsafe=effective_allow_unsafe, + input_type=input_type, + dimensions=dimensions, + api_key=api_key, + **kwargs, + ) + + # No fallbacks - use original logic # Parse provider and model provider_name, model_name = self._parse_model_string(model, provider) @@ -276,9 +804,6 @@ def embed( # Get provider instance provider_instance = self._get_provider(provider_name) - # Normalize input to list - inputs = [input] if isinstance(input, str) else input - # Call provider's embed method return provider_instance.embed( model=model_name, @@ -297,6 +822,8 @@ async def aembed( input_type: Optional[Literal["query", "document"]] = None, dimensions: Optional[int] = None, api_key: Optional[str] = None, + fallbacks: Fallbacks = None, + allow_unsafe_fallback: Optional[bool] = None, **kwargs: Any, ) -> EmbedResponse: """Async version of embed(). @@ -310,6 +837,9 @@ async def aembed( input_type: Optional input type hint ("query" or "document") dimensions: Optional output dimensions (model must support this feature) api_key: Optional API key override for this specific request + fallbacks: Fallback model(s) to try if primary fails + allow_unsafe_fallback: If True, allow fallback to models with different + dimensions (overrides client-level setting) **kwargs: Additional provider-specific parameters Returns: @@ -317,6 +847,8 @@ async def aembed( Raises: UnsupportedFeatureError: Model doesn't support requested feature + ConfigurationError: Missing API keys for fallback models + FallbackExhaustedError: All fallback models failed Example: >>> import asyncio @@ -324,11 +856,39 @@ async def aembed( >>> result = await client.aembed( ... model="voyage-3", ... input="hello world", - ... provider="voyageai", - ... dimensions=512 + ... fallbacks=["text-embedding-3-small"] ... ) """ + # Normalize input to list + inputs = [input] if isinstance(input, str) else input + + # Determine effective fallbacks (per-request overrides client-level) + effective_fallbacks = fallbacks if fallbacks is not None else self.fallbacks + effective_allow_unsafe = ( + allow_unsafe_fallback + if allow_unsafe_fallback is not None + else self.allow_unsafe_fallback + ) + + # If fallbacks are configured, use fallback logic + if effective_fallbacks is not None: + # Construct full model string with provider if specified + full_model = f"{provider}:{model}" if provider and ":" not in model else model + + fallback_configs = self._normalize_fallbacks(effective_fallbacks) + return await self._aembed_with_fallbacks( + primary_model=full_model, + inputs=inputs, + fallback_configs=fallback_configs, + allow_unsafe=effective_allow_unsafe, + input_type=input_type, + dimensions=dimensions, + api_key=api_key, + **kwargs, + ) + + # No fallbacks - use original logic # Parse provider and model provider_name, model_name = self._parse_model_string(model, provider) @@ -361,9 +921,6 @@ async def aembed( # Get provider instance provider_instance = self._get_provider(provider_name) - # Normalize input to list - inputs = [input] if isinstance(input, str) else input - # Call provider's aembed method return await provider_instance.aembed( model=model_name, @@ -443,6 +1000,44 @@ def tokenize( **kwargs, ) + def get_compatible_fallbacks(self, model: str) -> List[str]: + """Get models with dimensions compatible with the given model. + + Returns models that can produce embeddings with the same dimensions + as the specified model, making them suitable fallback candidates. + + Args: + model: Model name or "provider:model" string + + Returns: + List of compatible model names (in "provider:model" format) + + Example: + >>> client = Client() + >>> fallbacks = client.get_compatible_fallbacks("voyage-3") + >>> print(fallbacks) + ['voyageai:voyage-3-lite', 'cohere:embed-english-v3.0', ...] + + """ + # Get target model's dimensions + provider_name, model_name = self._parse_model_string(model) + model_info = self._catalog.get_model_info(provider_name, model_name) + target_dims = model_info.dimensions + + compatible = [] + all_models = self._catalog.list_models() + + for m in all_models: + # Skip the same model + if m.provider == provider_name and m.name == model_name: + continue + + # Check if model can produce target dimensions + if m.supports_dimensions or m.dimensions == target_dims: + compatible.append(f"{m.provider}:{m.name}") + + return compatible + def close(self) -> None: """Close sync HTTP client.""" self._http_client.close() diff --git a/src/catsu/models.py b/src/catsu/models.py index 6660bae..5acadd0 100644 --- a/src/catsu/models.py +++ b/src/catsu/models.py @@ -95,6 +95,9 @@ class EmbedResponse(BaseModel): latency_ms: Request latency in milliseconds input_count: Number of input texts processed input_type: Type of input ("query" or "document") + requested_model: Original model requested (if fallback was used) + fallback_used: Whether a fallback model was used + fallback_reason: Reason why fallback was triggered Example: >>> response = EmbedResponse( @@ -123,6 +126,19 @@ class EmbedResponse(BaseModel): default="document", description='Input type: "query" or "document"', ) + # Fallback metadata + requested_model: Optional[str] = Field( + default=None, + description="Original model requested (set when fallback was used)", + ) + fallback_used: bool = Field( + default=False, + description="Whether a fallback model was used", + ) + fallback_reason: Optional[str] = Field( + default=None, + description="Reason why fallback was triggered (e.g., 'RateLimitError')", + ) @field_validator("dimensions") @classmethod @@ -202,12 +218,15 @@ def to_numpy(self) -> "np.ndarray": def __repr__(self) -> str: """Return string representation of EmbedResponse.""" - return ( + base = ( f"EmbedResponse(model='{self.model}', provider='{self.provider}', " f"input_count={self.input_count}, dimensions={self.dimensions}, " f"tokens={self.usage.tokens}, cost=${self.usage.cost:.6f}, " - f"latency={self.latency_ms:.2f}ms)" + f"latency={self.latency_ms:.2f}ms" ) + if self.fallback_used: + base += f", fallback_used=True, requested_model='{self.requested_model}'" + return base + ")" class ModelInfo(BaseModel): diff --git a/src/catsu/utils/errors.py b/src/catsu/utils/errors.py index 049cf46..b18e5e4 100644 --- a/src/catsu/utils/errors.py +++ b/src/catsu/utils/errors.py @@ -4,7 +4,7 @@ easier to handle and debug issues when working with embedding providers. """ -from typing import Any, Dict, Optional +from typing import Any, Dict, List, Optional class CatsuError(Exception): @@ -378,3 +378,85 @@ def __init__( details["feature"] = feature super().__init__(message, details) + + +class ConfigurationError(CatsuError): + """Exception raised for configuration issues detected before making requests. + + Raised when there's a configuration problem that would prevent requests from + succeeding, such as missing API keys for fallback models. + + Example: + >>> client.embed(model="voyage-3", input="test", fallbacks=["text-embedding-3-small"]) + ConfigurationError: Missing API keys for fallback models: text-embedding-3-small (openai: set OPENAI_API_KEY) + + """ + + def __init__( + self, + message: str, + details: Optional[Dict[str, Any]] = None, + ) -> None: + """Initialize ConfigurationError. + + Args: + message: Error message + details: Optional additional error details + + """ + super().__init__(message, details) + + +class FallbackExhaustedError(CatsuError): + """Exception raised when all fallback models have failed. + + Raised when the primary model and all configured fallback models fail + after exhausting all retry cycles. + + Attributes: + primary_model: The originally requested model + fallbacks: List of fallback models that were tried + errors: Dict mapping model names to their exceptions + cycles_attempted: Number of retry cycles attempted + + Example: + >>> client.embed(model="voyage-3", input="test", fallbacks=["text-embedding-3-small"]) + FallbackExhaustedError: All models failed: voyage-3 and 1 fallbacks + + """ + + def __init__( + self, + primary_model: str, + fallbacks: List[str], + errors: Dict[str, Exception], + cycles_attempted: int = 1, + details: Optional[Dict[str, Any]] = None, + ) -> None: + """Initialize FallbackExhaustedError. + + Args: + primary_model: The originally requested model + fallbacks: List of fallback model names that were tried + errors: Dict mapping model names to their exceptions + cycles_attempted: Number of retry cycles attempted + details: Optional additional error details + + """ + self.primary_model = primary_model + self.fallbacks = fallbacks + self.errors = errors + self.cycles_attempted = cycles_attempted + + fallback_count = len(fallbacks) + message = f"All models failed: {primary_model} and {fallback_count} fallback(s) after {cycles_attempted} cycle(s)" + + details = details or {} + details["primary_model"] = primary_model + details["fallbacks"] = fallbacks + details["cycles_attempted"] = cycles_attempted + details["error_summary"] = { + model: str(error) for model, error in errors.items() + } + + super().__init__(message, details) diff --git a/tests/test_client.py b/tests/test_client.py index f04b17d..21b5c95 100644 --- a/tests/test_client.py +++ b/tests/test_client.py @@ -1,14 +1,19 @@ """Tests for Client class.""" import os +from unittest.mock import MagicMock, patch import pytest from catsu import Client -from catsu.models import EmbedResponse +from catsu.models import EmbedResponse, Usage from catsu.utils.errors import ( + ConfigurationError, + FallbackExhaustedError, InvalidInputError, ModelNotFoundError, + NetworkError, + RateLimitError, UnsupportedFeatureError, ) @@ -239,3 +244,266 @@ async def test_context_manager_async(self): """Test async context manager.""" async with Client() as client: assert client is not None + + +class TestFallbackNormalization: + """Tests for fallback normalization.""" + + def test_normalize_fallbacks_none(self): + """Test normalizing None returns empty list.""" + client = Client() + result = client._normalize_fallbacks(None) + assert result == [] + + def test_normalize_fallbacks_string(self): + """Test normalizing string returns single-item list.""" + client = Client() + result = client._normalize_fallbacks("text-embedding-3-small") + assert result == [{"model": "text-embedding-3-small"}] + + def test_normalize_fallbacks_dict(self): + """Test normalizing dict returns single-item list.""" + client = Client() + config = {"model": "text-embedding-3-small", "api_key": "test-key"} + result = client._normalize_fallbacks(config) + assert result == [config] + + def test_normalize_fallbacks_list_of_strings(self): + """Test normalizing list of strings.""" + client = Client() + result = client._normalize_fallbacks(["model-a", "model-b"]) + assert result == [{"model": "model-a"}, {"model": "model-b"}] + + def test_normalize_fallbacks_mixed_list(self): + """Test normalizing mixed list of strings and dicts.""" + client = Client() + fallbacks = [ + "model-a", + {"model": "model-b", "api_key": "key-b"}, + ] + result = client._normalize_fallbacks(fallbacks) + assert result == [ + {"model": "model-a"}, + {"model": "model-b", "api_key": "key-b"}, + ] + + +class TestFallbackCredentialValidation: + """Tests for fallback credential validation.""" + + def test_validate_credentials_missing_primary(self, monkeypatch): + """Test ConfigurationError raised when primary model API key missing.""" + # Clear all API keys + for key in ["VOYAGE_API_KEY", "OPENAI_API_KEY", "COHERE_API_KEY"]: + monkeypatch.delenv(key, raising=False) + + client = Client() + with pytest.raises(ConfigurationError) as exc_info: + client._validate_fallback_credentials( + "voyageai:voyage-3", + [{"model": "openai:text-embedding-3-small"}], + ) + assert "VOYAGE_API_KEY" in str(exc_info.value) + + def test_validate_credentials_missing_fallback(self, monkeypatch): + """Test ConfigurationError raised when fallback API key missing.""" + monkeypatch.setenv("VOYAGE_API_KEY", "test-key") + monkeypatch.delenv("OPENAI_API_KEY", raising=False) + + client = Client() + with pytest.raises(ConfigurationError) as exc_info: + client._validate_fallback_credentials( + "voyageai:voyage-3", + [{"model": "openai:text-embedding-3-small"}], + ) + assert "OPENAI_API_KEY" in str(exc_info.value) + + def test_validate_credentials_all_present(self, monkeypatch): + """Test no error when all API keys are present.""" + monkeypatch.setenv("VOYAGE_API_KEY", "key-1") + monkeypatch.setenv("OPENAI_API_KEY", "key-2") + + client = Client() + # Should not raise + client._validate_fallback_credentials( + "voyageai:voyage-3", + [{"model": "openai:text-embedding-3-small"}], + ) + + +class TestFallbackDimensionFiltering: + """Tests for fallback dimension filtering.""" + + def test_can_produce_dimensions_exact_match(self): + """Test model with exact matching dimensions is compatible.""" + client = Client() + # embed-english-v3.0 has 1024 dims and doesn't support custom + assert client._can_produce_dimensions("cohere", "embed-english-v3.0", 1024) + + def test_can_produce_dimensions_supports_custom(self): + """Test model with custom dimension support is compatible.""" + client = Client() + # voyage-3 supports custom dimensions + assert client._can_produce_dimensions("voyageai", "voyage-3", 512) + + def test_can_produce_dimensions_mismatch(self): + """Test model with mismatched dimensions is not compatible.""" + client = Client() + # text-embedding-ada-002 has 1536 dims and doesn't support custom + assert not client._can_produce_dimensions("openai", "text-embedding-ada-002", 1024) + + def test_filter_compatible_fallbacks(self): + """Test filtering fallbacks by dimension compatibility.""" + client = Client() + fallbacks = [ + {"model": "voyageai:voyage-3-lite"}, # Supports custom dims + {"model": "openai:text-embedding-ada-002"}, # 1536 fixed + ] + compatible = client._filter_compatible_fallbacks(1024, fallbacks) + # Only voyage-3-lite should be compatible (supports custom dims) + assert len(compatible) == 1 + assert compatible[0]["model"] == "voyageai:voyage-3-lite" + + +class TestFallbackErrorClassification: + """Tests for fallback error classification.""" + + def test_rate_limit_error_triggers_fallback(self): + """Test RateLimitError triggers fallback.""" + client = Client() + error = RateLimitError("Rate limited", provider="test") + assert client._is_fallback_error(error) + + def test_network_error_triggers_fallback(self): + """Test NetworkError triggers fallback.""" + client = Client() + error = NetworkError("Connection failed", provider="test") + assert client._is_fallback_error(error) + + def test_invalid_input_error_does_not_trigger_fallback(self): + """Test InvalidInputError does not trigger fallback.""" + client = Client() + error = InvalidInputError("Bad input") + assert not client._is_fallback_error(error) + + +class TestFallbackClientConfiguration: + """Tests for client-level fallback configuration.""" + + def test_client_fallback_defaults(self): + """Test client initializes with fallback defaults.""" + client = Client() + assert client.fallbacks is None + assert client.allow_unsafe_fallback is False + assert client.base_delay == 1.0 + assert client.max_delay == 10.0 + + def test_client_fallback_custom_config(self): + """Test client with custom fallback config.""" + client = Client( + fallbacks=["text-embedding-3-small"], + allow_unsafe_fallback=True, + base_delay=2.0, + max_delay=20.0, + ) + assert client.fallbacks == ["text-embedding-3-small"] + assert client.allow_unsafe_fallback is True + assert client.base_delay == 2.0 + assert client.max_delay == 20.0 + + +class TestGetCompatibleFallbacks: + """Tests for get_compatible_fallbacks helper.""" + + def test_get_compatible_fallbacks_returns_list(self): + """Test get_compatible_fallbacks returns a list.""" + client = Client() + fallbacks = client.get_compatible_fallbacks("voyage-3") + assert isinstance(fallbacks, list) + + def test_get_compatible_fallbacks_excludes_same_model(self): + """Test get_compatible_fallbacks excludes the input model.""" + client = Client() + fallbacks = client.get_compatible_fallbacks("voyage-3") + assert "voyageai:voyage-3" not in fallbacks + + +class TestFallbackExhaustedError: + """Tests for FallbackExhaustedError.""" + + def test_fallback_exhausted_error_attributes(self): + """Test FallbackExhaustedError has correct attributes.""" + error = FallbackExhaustedError( + primary_model="voyage-3", + fallbacks=["model-a", "model-b"], + errors={"voyage-3": RateLimitError("Rate limited")}, + cycles_attempted=3, + ) + assert error.primary_model == "voyage-3" + assert error.fallbacks == ["model-a", "model-b"] + assert "voyage-3" in error.errors + assert error.cycles_attempted == 3 + assert "3 cycle(s)" in str(error) + + +class TestConfigurationError: + """Tests for ConfigurationError.""" + + def test_configuration_error_message(self): + """Test ConfigurationError has correct message.""" + error = ConfigurationError("Missing API key") + assert "Missing API key" in str(error) + + +class TestEmbedResponseFallbackMetadata: + """Tests for EmbedResponse fallback metadata.""" + + def test_embed_response_default_fallback_fields(self): + """Test EmbedResponse has default fallback field values.""" + response = EmbedResponse( + embeddings=[[0.1, 0.2, 0.3]], + model="voyage-3", + provider="voyageai", + dimensions=3, + usage=Usage(tokens=10, cost=0.000001), + latency_ms=100.0, + input_count=1, + ) + assert response.fallback_used is False + assert response.requested_model is None + assert response.fallback_reason is None + + def test_embed_response_with_fallback_metadata(self): + """Test EmbedResponse with fallback metadata.""" + response = EmbedResponse( + embeddings=[[0.1, 0.2, 0.3]], + model="text-embedding-3-small", + provider="openai", + dimensions=3, + usage=Usage(tokens=10, cost=0.000001), + latency_ms=100.0, + input_count=1, + fallback_used=True, + requested_model="voyage-3", + fallback_reason="RateLimitError", + ) + assert response.fallback_used is True + assert response.requested_model == "voyage-3" + assert response.fallback_reason == "RateLimitError" + + def test_embed_response_repr_with_fallback(self): + """Test EmbedResponse repr includes fallback info when used.""" + response = EmbedResponse( + embeddings=[[0.1, 0.2, 0.3]], + model="text-embedding-3-small", + provider="openai", + dimensions=3, + usage=Usage(tokens=10, cost=0.000001), + latency_ms=100.0, + input_count=1, + fallback_used=True, + requested_model="voyage-3", + ) + repr_str = repr(response) + assert "fallback_used=True" in repr_str + assert "requested_model='voyage-3'" in repr_str