config.py

"""
Configuration management for Arabic text normalizer.
"""
from dataclasses import dataclass
from typing import Optional
import json


@dataclass
class Config:
    """Simple configuration for Arabic text normalization."""
    
    # LLM settings  
    llm_model: str = "gemini-2.0-flash"
    max_tokens: int = 8000
    temperature: float = 0.05
    chunk_size: int = 4000
    
    # Processing options
    preserve_sudanese_expressions: bool = True
    
    # Text filtering
    min_paragraph_length: int = 20
    max_paragraph_length: Optional[int] = 1000
    min_arabic_ratio: float = 0.7
    
    # Rate limiting for massive processing
    api_delay: float = 3.0  # seconds between API calls
    requests_per_minute: int = 15  # production rate
    max_retries: int = 5  # retry failed requests
    exponential_backoff_base: float = 2.0  # backoff multiplier
    
    # Checkpointing for huge datasets
    checkpoint_every: int = 100  # save progress every N chunks
    enable_resume: bool = True  # resume from checkpoints
    
    @classmethod
    def load_from_file(cls, config_path: str) -> 'Config':
        """Load configuration from JSON file."""
        try:
            with open(config_path, 'r', encoding='utf-8') as f:
                config_dict = json.load(f)
            
            # Filter out comment fields
            filtered_config = {k: v for k, v in config_dict.items() 
                             if not k.startswith('_')}
            
            # Handle nested config structure
            if 'llm_settings' in filtered_config:
                flat_config = {}
                for section in filtered_config.values():
                    if isinstance(section, dict):
                        flat_config.update(section)
                return cls(**flat_config)
            
            return cls(**filtered_config)
        except Exception as e:
            print(f"Warning: Could not load config from {config_path}: {e}")
            return cls()
    
    def save_to_file(self, config_path: str):
        """Save configuration to JSON file."""
        config_dict = {
            'llm_model': self.llm_model,
            'max_tokens': self.max_tokens,
            'temperature': self.temperature,
            'chunk_size': self.chunk_size,
            'preserve_sudanese_expressions': self.preserve_sudanese_expressions,
            'min_paragraph_length': self.min_paragraph_length,
            'max_paragraph_length': self.max_paragraph_length,
            'min_arabic_ratio': self.min_arabic_ratio,
            'api_delay': self.api_delay,
            'requests_per_minute': self.requests_per_minute,
            'max_retries': self.max_retries,
            'exponential_backoff_base': self.exponential_backoff_base,
            'checkpoint_every': self.checkpoint_every,
            'enable_resume': self.enable_resume
        }
        
        with open(config_path, 'w', encoding='utf-8') as f:
            json.dump(config_dict, f, ensure_ascii=False, indent=2)