operations.py

"""
Core operations for the synthesis pool: feed, observe, harvest, query.
"""
import os
import pandas as pd
from datetime import date, datetime
from typing import Optional, List
from pathlib import Path
from collections import Counter

from models import (
    MemoryMatter, SynthesisOutput, PoolState,
    TimestampConfidence, DecayState, SynthesisType, PoolHealth
)


# Use absolute path relative to this file's location
DATA_DIR = Path(__file__).parent / "data"
MEMORY_MATTER_CSV = DATA_DIR / "memory_matter.csv"
SYNTHESIS_OUTPUTS_CSV = DATA_DIR / "synthesis_outputs.csv"
POOL_STATES_CSV = DATA_DIR / "pool_states.csv"


def ensure_data_files():
    """Initialize CSV files if they don't exist."""
    DATA_DIR.mkdir(exist_ok=True)
    
    if not MEMORY_MATTER_CSV.exists():
        df = pd.DataFrame(columns=[
            'id', 'timestamp_actual', 'timestamp_confidence', 'source_type',
            'source_date', 'raw', 'layers', 'tags', 'toxicity', 'nutrient_density',
            'decay_state', 'name', 'pool_reaction', 'linked_matter'
        ])
        df.to_csv(MEMORY_MATTER_CSV, index=False)
    
    if not SYNTHESIS_OUTPUTS_CSV.exists():
        df = pd.DataFrame(columns=[
            'id', 'type', 'derived_from', 'concentration', 'label',
            'description', 'application', 'harvest_date'
        ])
        df.to_csv(SYNTHESIS_OUTPUTS_CSV, index=False)
    
    if not POOL_STATES_CSV.exists():
        df = pd.DataFrame(columns=[
            'snapshot_date', 'health', 'matter_count', 'active_synthesis',
            'distress_signals', 'contamination'
        ])
        df.to_csv(POOL_STATES_CSV, index=False)


def get_next_id(prefix: str, csv_path: Path) -> str:
    """Generate next sequential ID."""
    if not csv_path.exists() or os.path.getsize(csv_path) == 0:
        return f"{prefix}_001"
    
    df = pd.read_csv(csv_path)
    if len(df) == 0:
        return f"{prefix}_001"
    
    # Extract numeric part from existing IDs
    ids = df['id'].tolist()
    numbers = [int(id.split('_')[1]) for id in ids if '_' in id]
    
    if not numbers:
        return f"{prefix}_001"
    
    next_num = max(numbers) + 1
    return f"{prefix}_{next_num:03d}"


def feed_memory(
    timestamp_actual: date,
    timestamp_confidence: TimestampConfidence,
    source_type: str,
    raw: str,
    layers: str,
    tags: str,
    toxicity: int,
    nutrient_density: int,
    decay_state: DecayState = DecayState.FRESH,
    name: Optional[str] = None,
    pool_reaction: Optional[str] = None,
    linked_matter: Optional[str] = None,
    auto_name: bool = True
) -> MemoryMatter:
    """
    FEED — Add new memory_matter entry.

    Args:
        auto_name: If True and name is None, generate name via LLM
    """
    ensure_data_files()

    # Auto-generate ID
    new_id = get_next_id("mm", MEMORY_MATTER_CSV)

    # Auto-generate name if not provided
    if name is None and auto_name:
        try:
            from llm import generate_matter_name
            name = generate_matter_name(raw, tags, layers)
        except Exception as e:
            print(f"[feed] Name generation failed: {e}")
            name = None

    # Create memory matter object
    memory = MemoryMatter(
        id=new_id,
        timestamp_actual=timestamp_actual,
        timestamp_confidence=timestamp_confidence,
        source_type=source_type,
        source_date=date.today(),
        raw=raw,
        layers=layers,
        tags=tags,
        toxicity=toxicity,
        nutrient_density=nutrient_density,
        decay_state=decay_state,
        name=name,
        pool_reaction=pool_reaction,
        linked_matter=linked_matter
    )

    # Append to CSV
    df = pd.read_csv(MEMORY_MATTER_CSV)
    new_row = pd.DataFrame([memory.to_dict()])
    df = pd.concat([df, new_row], ignore_index=True)
    df.to_csv(MEMORY_MATTER_CSV, index=False)

    # Add to vector index (non-blocking)
    try:
        from embeddings import embeddings
        if embeddings.is_available():
            metadata = {
                'timestamp_actual': timestamp_actual.isoformat(),
                'layers': layers,
                'tags': tags,
                'toxicity': toxicity,
                'nutrient_density': nutrient_density,
                'source_type': source_type
            }
            embeddings.add_memory(memory.id, raw, metadata)
    except Exception as e:
        print(f"[feed] Embedding failed (non-critical): {e}")

    return memory


def observe_pool(as_of_date: Optional[date] = None) -> PoolState:
    """
    OBSERVE — Generate pool_state snapshot.
    Analyzes current state and detects distress signals.

    Args:
        as_of_date: Optional date to analyze pool state as it was at that time.
                   If None, uses today and analyzes current state.
    """
    ensure_data_files()

    # Use today if no date specified
    snapshot_date = as_of_date if as_of_date else date.today()

    df = pd.read_csv(MEMORY_MATTER_CSV)

    # Convert dates first
    if len(df) > 0:
        df['timestamp_actual'] = pd.to_datetime(df['timestamp_actual'])
        df['source_date'] = pd.to_datetime(df['source_date'])

        # Filter to memories that existed at the snapshot date
        if as_of_date:
            df = df[df['source_date'] <= pd.Timestamp(as_of_date)]

    if len(df) == 0:
        state = PoolState(
            snapshot_date=snapshot_date,
            health=PoolHealth.DISSOCIATED,
            matter_count=0,
            active_synthesis="None - pool is empty" if not as_of_date else "None - pool was empty",
            distress_signals="No data in pool" if not as_of_date else "No data in pool at this date",
            contamination="None"
        )
    else:
        matter_count = len(df)
        
        # Analyze toxicity/nutrient distributions
        avg_toxicity = df['toxicity'].mean()
        avg_nutrient = df['nutrient_density'].mean()
        
        # Detect health state (thresholds calibrated to actual data distribution)
        # Data ranges: toxicity 0-5 (median=2.0), nutrient 0-5 (median=4.0)
        if avg_toxicity > 2.5:
            health = PoolHealth.TRAUMATIZED  # High stress/pain
        elif avg_toxicity >= 1.8 or (avg_nutrient <= 4.0 and avg_toxicity >= 1.5):
            health = PoolHealth.STRESSED  # Moderate difficulty or mixed state
        elif avg_nutrient > 4.0 and avg_toxicity < 1.8:
            health = PoolHealth.THRIVING  # High insight, low pain
        else:
            health = PoolHealth.STRESSED  # Default cautious state
        
        # Detect distress signals
        distress_signals = []
        
        # Check for timeline gaps
        df_sorted = df.sort_values('timestamp_actual')
        dates = df_sorted['timestamp_actual'].dt.date.tolist()
        if len(dates) > 1:
            gaps = []
            for i in range(1, len(dates)):
                delta = (dates[i] - dates[i-1]).days
                if delta > 30:  # More than a month gap
                    gaps.append(f"{dates[i-1]} to {dates[i]} ({delta} days)")
            if gaps:
                distress_signals.append(f"Timeline gaps: {'; '.join(gaps[:3])}")
        
        # Check for layer imbalances
        all_layers = []
        for layers_str in df['layers'].dropna():
            all_layers.extend(layers_str.split('|'))
        layer_counts = Counter(all_layers)
        if layer_counts:
            max_layer = max(layer_counts.values())
            min_layer = min(layer_counts.values())
            if max_layer > min_layer * 3:
                distress_signals.append(f"Layer imbalance: {dict(layer_counts)}")
        
        # Check for high toxicity concentration
        high_tox_count = len(df[df['toxicity'] >= 4])
        if high_tox_count > matter_count * 0.3:
            distress_signals.append(f"High toxicity: {high_tox_count}/{matter_count} entries >= 4")
        
        # Active synthesis (recent high-nutrient entries relative to snapshot_date)
        week_before = snapshot_date - pd.Timedelta(days=7)
        recent = df[df['source_date'] >= pd.Timestamp(week_before)]
        if len(recent) > 0:
            high_nutrient = recent[recent['nutrient_density'] >= 4]
            if len(high_nutrient) > 0:
                active_synthesis = f"Processing {len(high_nutrient)} high-nutrient entries from past week"
            else:
                active_synthesis = f"Integrating {len(recent)} recent entries"
        else:
            active_synthesis = "No recent activity"
        
        # Contamination detection
        contamination = []
        if avg_toxicity > 3.5:
            contamination.append(f"High average toxicity: {avg_toxicity:.2f}")
        if avg_nutrient < 2:
            contamination.append(f"Low nutrient density: {avg_nutrient:.2f}")
        
        state = PoolState(
            snapshot_date=snapshot_date,
            health=health,
            matter_count=matter_count,
            active_synthesis=active_synthesis,
            distress_signals='; '.join(distress_signals) if distress_signals else "None detected",
            contamination='; '.join(contamination) if contamination else "None detected"
        )
    
    # Append to CSV
    df_states = pd.read_csv(POOL_STATES_CSV)
    new_row = pd.DataFrame([state.to_dict()])
    df_states = pd.concat([df_states, new_row], ignore_index=True)
    df_states.to_csv(POOL_STATES_CSV, index=False)
    
    return state


def harvest_synthesis(
    synthesis_type: SynthesisType,
    derived_from: str,  # comma-separated ids
    concentration: int,
    label: str,
    description: str,
    application: str
) -> SynthesisOutput:
    """
    HARVEST — Create synthesis_output from memory matter.
    """
    ensure_data_files()
    
    # Auto-generate ID
    new_id = get_next_id("so", SYNTHESIS_OUTPUTS_CSV)
    
    # Create synthesis output
    synthesis = SynthesisOutput(
        id=new_id,
        type=synthesis_type,
        derived_from=derived_from,
        concentration=concentration,
        label=label,
        description=description,
        application=application,
        harvest_date=date.today()
    )
    
    # Append to CSV
    df = pd.read_csv(SYNTHESIS_OUTPUTS_CSV)
    new_row = pd.DataFrame([synthesis.to_dict()])
    df = pd.concat([df, new_row], ignore_index=True)
    df.to_csv(SYNTHESIS_OUTPUTS_CSV, index=False)
    
    return synthesis


def query_memory(
    date_after: Optional[date] = None,
    date_before: Optional[date] = None,
    layers: Optional[List[str]] = None,
    tags: Optional[List[str]] = None,
    min_toxicity: Optional[int] = None,
    max_toxicity: Optional[int] = None,
    min_nutrient: Optional[int] = None,
    max_nutrient: Optional[int] = None,
    source_type: Optional[str] = None
) -> pd.DataFrame:
    """
    QUERY — Filter/search memory_matter.
    Returns DataFrame of matching entries.
    """
    ensure_data_files()
    
    df = pd.read_csv(MEMORY_MATTER_CSV)
    
    if len(df) == 0:
        return df
    
    # Convert dates
    df['timestamp_actual'] = pd.to_datetime(df['timestamp_actual'])
    
    # Apply filters
    if date_after:
        df = df[df['timestamp_actual'] >= pd.Timestamp(date_after)]
    
    if date_before:
        df = df[df['timestamp_actual'] <= pd.Timestamp(date_before)]
    
    if layers:
        # Filter by any matching layer
        mask = df['layers'].apply(
            lambda x: any(layer in str(x).split('|') for layer in layers)
        )
        df = df[mask]
    
    if tags:
        # Filter by any matching tag
        mask = df['tags'].apply(
            lambda x: any(tag in str(x).split(',') for tag in tags)
        )
        df = df[mask]
    
    if min_toxicity is not None:
        df = df[df['toxicity'] >= min_toxicity]
    
    if max_toxicity is not None:
        df = df[df['toxicity'] <= max_toxicity]
    
    if min_nutrient is not None:
        df = df[df['nutrient_density'] >= min_nutrient]
    
    if max_nutrient is not None:
        df = df[df['nutrient_density'] <= max_nutrient]
    
    if source_type:
        df = df[df['source_type'] == source_type.value]
    
    return df