diff --git a/.gitignore b/.gitignore index 5ffa009..e6096e3 100644 --- a/.gitignore +++ b/.gitignore @@ -79,3 +79,6 @@ npm-debug.log* yarn-debug.log* yarn-error.log*web/.next/ CLAUDE.md +web/.next/ +web/tsconfig.tsbuildinfo +scenarios-market-analysis.md diff --git a/AGENTS.md b/AGENTS.md new file mode 100644 index 0000000..f6e5397 --- /dev/null +++ b/AGENTS.md @@ -0,0 +1,40 @@ +# Repository Guidelines + +## Project Structure & Module Organization +- `api/`: FastAPI backend (Alembic migrations, Celery, tests under `api/tests`). +- `web/`: Next.js 15 + TypeScript frontend (TailwindCSS, ESLint). +- `scripts/`: Helper scripts for seeding and demos. +- `sample_dataset_templates/`: CSV templates for datasets. +- `docker-compose*.yml`: Local dev and service orchestration. +- `Makefile`: Common developer commands. + +## Build, Test, and Development Commands +- Backend (Dockerized): `make dev` — start API, DB, Redis for local dev. +- Full demo: `make demo` — migrate, start, seed, quick API checks. +- DB migrations: `make migrate` — run Alembic upgrades. +- Seed data: `make seed` — insert demo evaluators/scenarios/datasets. +- Inspect: `make logs` (API logs), `make status` (container status). +- Stop/clean: `make clean` — down + prune volumes. +- Frontend: `cd web && npm run dev | build | start | lint`. +- Backend tests: `cd api && pytest -v` (use `-m unit|integration|auth|api`). + +## Coding Style & Naming Conventions +- Python (api): 4-space indentation; PEP 8; modules `snake_case.py`; classes `PascalCase`; functions/vars `snake_case`. +- TypeScript/React (web): components export `PascalCase`; files in `web/src/components` use kebab-case (e.g., `scenario-preview-modal.tsx`). +- Imports: prefer relative within package boundaries; avoid circular deps. +- Formatting: use project defaults (ESLint for `web/` via `npm run lint`). + +## Testing Guidelines +- Framework: Pytest (`api/pytest.ini` configured). Test files: `test_*.py`, classes `Test*`, functions `test_*`. +- Run: `cd api && pytest -v` or targeted (e.g., `pytest tests/test_auth.py -m auth`). +- Optional coverage: `pytest --cov=.` if `pytest-cov` is available. +- Add minimal, focused tests near the code under test; prefer unit over integration unless necessary. + +## Commit & Pull Request Guidelines +- Commits: concise imperative subject (≤72 chars). Example: `feat(api): add PII regex evaluator`. +- Include rationale and scope in body; reference issues (`Closes #123`). +- PRs: clear description, steps to test, screenshots for UI changes, and linked issue(s). Keep changes scoped. + +## Security & Configuration Tips +- Never commit secrets. Use `.env.example` to document new variables. Local envs: root `.env`, `api/.env`, `web/.env.local`. +- Default local URLs: API `http://localhost:8000`, Web `http://localhost:3000`. diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md new file mode 100644 index 0000000..30bc0ec --- /dev/null +++ b/CONTRIBUTING.md @@ -0,0 +1,37 @@ +# Contributing + +Thanks for your interest in improving EvalWise! This guide summarizes how to get set up, develop, and submit changes. For detailed project conventions, see `AGENTS.md`. + +## Getting Started +- Prereqs: Docker + Docker Compose, Make, Node.js (for `web/`). +- Environment: copy `.env.example` to `.env` (and `api/.env`, `web/.env.local` as needed). Never commit secrets. + +## Local Development +- Start services: `make dev` (API, Postgres, Redis) or `make demo` (migrate + seed + smoke checks). +- Useful targets: `make migrate`, `make seed`, `make logs`, `make status`, `make clean`. +- Frontend: `cd web && npm run dev` (and `npm run lint` before PRs). + +## Testing +- Backend tests: `cd api && pytest -v`. +- Markers: `-m unit|integration|auth|api` (e.g., `pytest -m auth`). +- Optional coverage: `pytest --cov=.` if `pytest-cov` is available. + +## Coding Standards +- Python: PEP 8, 4-space indentation, `snake_case` for files/functions/variables, `PascalCase` for classes. +- TypeScript/React: `PascalCase` components, kebab-case component filenames in `web/src/components`. +- See `AGENTS.md` for structure, naming, and additional tips. + +## Commit & PR Process +- Branch: `feature/`, `fix/`, or `chore/`. +- Commits: imperative and scoped. Example: `feat(api): add PII regex evaluator`. +- PRs should include: + - Purpose and summary of changes + - Testing notes (commands, expected outputs) + - Screenshots for UI changes + - Linked issue references (e.g., `Closes #123`) + +## Resources +- Project guidelines: `AGENTS.md` +- Makefile commands: `make help` +- API docs (local): `http://localhost:8000/docs` +- Web app (local): `http://localhost:3000` diff --git a/api/adapters/factory.py b/api/adapters/factory.py index 6b44926..35c8daf 100644 --- a/api/adapters/factory.py +++ b/api/adapters/factory.py @@ -3,6 +3,9 @@ from .openai_adapter import OpenAIAdapter from .ollama_adapter import OllamaAdapter import os +from sqlalchemy.orm import Session +from sqlalchemy import create_engine +from database import get_db class ModelAdapterFactory: """Factory for creating model adapters""" @@ -14,19 +17,52 @@ class ModelAdapterFactory: "ollama": OllamaAdapter, } + @classmethod + def get_organization_api_key(cls, provider: str, organization_id: str, db: Session) -> Optional[str]: + """Retrieve and decrypt API key for provider from organization's stored keys""" + try: + from auth.models import EncryptedApiKey + from utils.encryption import encryption + + # Find the API key for this provider and organization + encrypted_key_record = db.query(EncryptedApiKey).filter( + EncryptedApiKey.provider == provider, + EncryptedApiKey.organization_id == organization_id, + EncryptedApiKey.is_active == True + ).first() + + if not encrypted_key_record: + return None + + # Decrypt the API key + return encryption.decrypt_api_key(encrypted_key_record.encrypted_key) + + except Exception as e: + print(f"Failed to retrieve API key for {provider}: {str(e)}") + return None + @classmethod def create_adapter( - self, + cls, provider: str, api_key: Optional[str] = None, - base_url: Optional[str] = None + base_url: Optional[str] = None, + organization_id: Optional[str] = None, + db: Optional[Session] = None ) -> BaseModelAdapter: """Create adapter for the specified provider""" - if provider not in self._adapters: + if provider not in cls._adapters: raise ValueError(f"Unsupported provider: {provider}") - adapter_class = self._adapters[provider] + adapter_class = cls._adapters[provider] + + # Try to get API key from organization's stored keys first + if not api_key and organization_id and db: + stored_api_key = cls.get_organization_api_key(provider, organization_id, db) + if stored_api_key: + api_key = stored_api_key + print(f"Using stored API key for {provider} from organization {organization_id}") # Handle provider-specific configurations if provider == "openai": diff --git a/api/alembic/versions/b8f5c9a3d2e1_remove_usage_tracking_from_api_keys.py b/api/alembic/versions/b8f5c9a3d2e1_remove_usage_tracking_from_api_keys.py new file mode 100644 index 0000000..de77dc4 --- /dev/null +++ b/api/alembic/versions/b8f5c9a3d2e1_remove_usage_tracking_from_api_keys.py @@ -0,0 +1,28 @@ +"""remove_usage_tracking_from_api_keys + +Revision ID: b8f5c9a3d2e1 +Revises: 9b8d8990d1c0 +Create Date: 2025-01-05 10:00:00.000000 + +""" +from alembic import op +import sqlalchemy as sa + + +# revision identifiers, used by Alembic. +revision = 'b8f5c9a3d2e1' +down_revision = '9b8d8990d1c0' +branch_labels = None +depends_on = None + + +def upgrade() -> None: + # Remove usage tracking columns from encrypted_api_keys table + op.drop_column('encrypted_api_keys', 'last_used') + op.drop_column('encrypted_api_keys', 'usage_count') + + +def downgrade() -> None: + # Add the columns back if we need to rollback + op.add_column('encrypted_api_keys', sa.Column('usage_count', sa.Integer(), nullable=True, default=0)) + op.add_column('encrypted_api_keys', sa.Column('last_used', sa.DateTime(), nullable=True)) \ No newline at end of file diff --git a/api/alembic/versions/c9d4e8f7a6b5_add_llm_providers_table.py b/api/alembic/versions/c9d4e8f7a6b5_add_llm_providers_table.py new file mode 100644 index 0000000..a915867 --- /dev/null +++ b/api/alembic/versions/c9d4e8f7a6b5_add_llm_providers_table.py @@ -0,0 +1,44 @@ +"""add_llm_providers_table + +Revision ID: c9d4e8f7a6b5 +Revises: b8f5c9a3d2e1 +Create Date: 2025-01-05 10:30:00.000000 + +""" +from alembic import op +import sqlalchemy as sa + + +# revision identifiers, used by Alembic. +revision = 'c9d4e8f7a6b5' +down_revision = 'b8f5c9a3d2e1' +branch_labels = None +depends_on = None + + +def upgrade() -> None: + # Create llm_providers table + op.create_table('llm_providers', + sa.Column('id', sa.UUID(), nullable=False), + sa.Column('organization_id', sa.UUID(), nullable=False), + sa.Column('name', sa.String(), nullable=False), + sa.Column('provider_type', sa.String(), nullable=False), + sa.Column('encrypted_api_key', sa.Text(), nullable=True), + sa.Column('base_url', sa.String(), nullable=True), + sa.Column('default_model_name', sa.String(), nullable=False), + sa.Column('default_temperature', sa.Float(), nullable=False), + sa.Column('default_max_tokens', sa.Integer(), nullable=False), + sa.Column('is_default', sa.Boolean(), nullable=True), + sa.Column('is_active', sa.Boolean(), nullable=True), + sa.Column('created_at', sa.DateTime(), nullable=True), + sa.Column('updated_at', sa.DateTime(), nullable=True), + sa.Column('created_by', sa.UUID(), nullable=False), + sa.ForeignKeyConstraint(['created_by'], ['users.id'], ), + sa.ForeignKeyConstraint(['organization_id'], ['organizations.id'], ), + sa.PrimaryKeyConstraint('id') + ) + + +def downgrade() -> None: + # Drop the table if we need to rollback + op.drop_table('llm_providers') \ No newline at end of file diff --git a/api/auth/models.py b/api/auth/models.py index 0dc03bf..e327597 100644 --- a/api/auth/models.py +++ b/api/auth/models.py @@ -1,4 +1,4 @@ -from sqlalchemy import Column, String, DateTime, Boolean, Text, ForeignKey, Integer +from sqlalchemy import Column, String, DateTime, Boolean, Text, ForeignKey, Integer, Float from sqlalchemy.dialects.postgresql import UUID from sqlalchemy.orm import relationship from datetime import datetime @@ -24,6 +24,7 @@ class Organization(Base): # Relationships user_organizations = relationship("UserOrganization", back_populates="organization") encrypted_api_keys = relationship("EncryptedApiKey", back_populates="organization") + llm_providers = relationship("LLMProvider", back_populates="organization") def __repr__(self): return f"" @@ -97,10 +98,6 @@ class EncryptedApiKey(Base): updated_at = Column(DateTime, default=datetime.utcnow, onupdate=datetime.utcnow) created_by = Column(UUID(as_uuid=True), ForeignKey("users.id"), nullable=False) - # Usage tracking - last_used = Column(DateTime, nullable=True) - usage_count = Column(Integer, default=0) - # Relationships organization = relationship("Organization", back_populates="encrypted_api_keys") @@ -108,6 +105,36 @@ def __repr__(self): return f"" +class LLMProvider(Base): + """LLM Provider configurations for organizations""" + __tablename__ = "llm_providers" + + id = Column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4) + organization_id = Column(UUID(as_uuid=True), ForeignKey("organizations.id"), nullable=False) + name = Column(String, nullable=False) # User-friendly name like "My OpenAI Account" + provider_type = Column(String, nullable=False) # openai, ollama, azure_openai, etc. + encrypted_api_key = Column(Text, nullable=True) # AES encrypted API key (nullable for local providers) + base_url = Column(String, nullable=True) # Custom base URL + + # Model defaults + default_model_name = Column(String, nullable=False) + default_temperature = Column(Float, nullable=False, default=0.7) + default_max_tokens = Column(Integer, nullable=False, default=1000) + + # Metadata + is_default = Column(Boolean, default=False) + is_active = Column(Boolean, default=True) + created_at = Column(DateTime, default=datetime.utcnow) + updated_at = Column(DateTime, default=datetime.utcnow, onupdate=datetime.utcnow) + created_by = Column(UUID(as_uuid=True), ForeignKey("users.id"), nullable=False) + + # Relationships + organization = relationship("Organization", back_populates="llm_providers") + + def __repr__(self): + return f"" + + class LoginAttempt(Base): """Track failed login attempts for rate limiting""" __tablename__ = "login_attempts" diff --git a/api/main_v2.py b/api/main_v2.py index 3cd9f78..0119e96 100644 --- a/api/main_v2.py +++ b/api/main_v2.py @@ -13,17 +13,18 @@ # Import database and models from database import get_db, engine from models import Base, Dataset, Item, Scenario, Evaluator, Run, Result, Evaluation -from auth.models import User, Organization, UserOrganization +from auth.models import User, Organization, UserOrganization, LLMProvider from schemas_simple import DatasetCreate, ScenarioCreate, EvaluatorCreate, RunCreate, PlaygroundRequest # Import authentication from auth.routes import router as auth_router from auth.admin_routes import router as admin_router -from auth.security import get_current_user_flexible +from auth.security import get_current_user_flexible, get_current_user # Import logging and error handling from utils.logging import get_logger, RequestContext from utils.errors import NotFoundError, ValidationError, ErrorResponse, ErrorDetail, InternalServerError +# Import middleware from middleware import RequestTrackingMiddleware, ErrorHandlingMiddleware from middleware.security import SecurityHeadersMiddleware from middleware.rate_limiting import RateLimitingMiddleware @@ -45,7 +46,7 @@ # Add middleware (order matters - last added runs first) app.add_middleware(SecurityHeadersMiddleware) -app.add_middleware(APIValidationMiddleware) +# app.add_middleware(APIValidationMiddleware) # Temporarily disabled for debugging app.add_middleware(RateLimitingMiddleware) app.add_middleware(ErrorHandlingMiddleware) app.add_middleware(RequestTrackingMiddleware) @@ -55,7 +56,7 @@ CORSMiddleware, allow_origins=settings.cors_origins, allow_credentials=True, - allow_methods=["GET", "POST", "PUT", "DELETE", "OPTIONS"], + allow_methods=["GET", "POST", "PUT", "PATCH", "DELETE", "OPTIONS"], allow_headers=["Content-Type", "Authorization", "X-Requested-With", "Accept"], ) @@ -858,29 +859,138 @@ def test_evaluators( "message": "Evaluator test started" } -# Playground endpoint (simplified for now) +# Playground endpoint @app.post("/playground/test") -def playground_test( +async def playground_test( request: PlaygroundRequest, + db: Session = Depends(get_db), current_user = Depends(get_current_user_flexible) ): - """Test a single prompt (placeholder implementation)""" - return { - "output": f"Mock response to: {request.prompt[:50]}...", - "latency_ms": 1000, - "token_input": 10, - "token_output": 20, - "cost_usd": 0.001, - "evaluations": [ - { - "evaluator_id": eid, - "score_float": 0.8, - "pass_bool": True, - "notes_text": "Mock evaluation result" - } - for eid in request.evaluator_ids[:2] # Limit to first 2 - ] - } + """Test a single prompt with real LLM and evaluators""" + request_id = str(uuid.uuid4()) + + try: + # Import adapters and evaluators + from adapters.factory import ModelAdapterFactory + from evaluators.factory import EvaluatorFactory + + # Get model configuration from request + model_provider = getattr(request, 'model_provider', 'openai') + model_name = getattr(request, 'model_name', 'gpt-3.5-turbo') + + # Get user's organization for API key lookup + user_org = db.query(UserOrganization).filter( + UserOrganization.user_id == current_user.id, + UserOrganization.is_active == True + ).first() + + organization_id = str(user_org.organization_id) if user_org else None + + # Create model adapter + try: + adapter = ModelAdapterFactory.create_adapter( + provider=model_provider, + api_key=None, # Will try stored keys first, then environment variables + organization_id=organization_id, + db=db + ) + except Exception as e: + logger.error(f"Failed to create adapter for {model_provider}: {str(e)}") + raise ValidationError( + message=f"Failed to configure model provider: {model_provider}", + details=[ErrorDetail( + code="MODEL_CONFIGURATION_ERROR", + message=str(e), + field="model_provider" + )], + request_id=request_id + ) + + # Generate response from LLM + try: + response = await adapter.generate( + prompt=request.prompt, + model_name=model_name, + temperature=0.7, + max_tokens=1000 + ) + except Exception as e: + logger.error(f"Failed to generate response: {str(e)}") + raise ValidationError( + message="Failed to generate response from model", + details=[ErrorDetail( + code="MODEL_GENERATION_ERROR", + message=str(e), + field="prompt" + )], + request_id=request_id + ) + + # Run evaluators if any are specified + evaluations = [] + if request.evaluator_ids: + for evaluator_id in request.evaluator_ids: + try: + # Get evaluator from database + evaluator = db.query(Evaluator).filter(Evaluator.id == evaluator_id).first() + if not evaluator: + logger.warning(f"Evaluator {evaluator_id} not found") + continue + + # Create evaluator instance + evaluator_instance = EvaluatorFactory.create_evaluator( + evaluator.type, + evaluator.config or {} + ) + + # Run evaluation + eval_result = await evaluator_instance.evaluate( + input_text=request.prompt, + output_text=response.content, + expected_output=None, + metadata=None + ) + + evaluations.append({ + "evaluator_id": str(evaluator_id), + "evaluator_name": evaluator.name, + "score_float": eval_result.score, + "pass_bool": eval_result.pass_fail, + "notes_text": eval_result.notes or "" + }) + + except Exception as e: + logger.error(f"Failed to run evaluator {evaluator_id}: {str(e)}") + evaluations.append({ + "evaluator_id": str(evaluator_id), + "evaluator_name": f"Evaluator {evaluator_id}", + "score_float": None, + "pass_bool": None, + "notes_text": f"Evaluation failed: {str(e)}" + }) + + return { + "output": response.content, + "latency_ms": response.latency_ms, + "token_input": response.token_input, + "token_output": response.token_output, + "cost_usd": response.cost_usd, + "evaluations": evaluations + } + + except ValidationError: + raise # Re-raise validation errors + except Exception as e: + logger.error(f"Unexpected error in playground test: {str(e)}") + raise ValidationError( + message="Internal server error during playground test", + details=[ErrorDetail( + code="INTERNAL_ERROR", + message="An unexpected error occurred", + field=None + )], + request_id=request_id + ) # Organization endpoints @app.post("/organizations") @@ -948,6 +1058,1277 @@ async def list_organizations( return organizations +# User profile endpoints +@app.patch("/users/me/password") +async def change_password( + request: Request, + current_password: str = Body(...), + new_password: str = Body(...), + db: Session = Depends(get_db), + current_user = Depends(get_current_user_flexible) +): + """Change user password""" + request_id = getattr(request.state, "request_id", str(uuid.uuid4())) if request else str(uuid.uuid4()) + + # Validate current password + from auth.security import verify_password, get_password_hash + + if not verify_password(current_password, current_user.hashed_password): + raise ValidationError( + message="Current password is incorrect", + details=[ErrorDetail( + code="INVALID_PASSWORD", + message="The current password you entered is incorrect", + field="current_password" + )], + request_id=request_id + ) + + # Validate new password strength + if len(new_password) < 8: + raise ValidationError( + message="New password must be at least 8 characters long", + details=[ErrorDetail( + code="PASSWORD_TOO_SHORT", + message="Password must be at least 8 characters long", + field="new_password" + )], + request_id=request_id + ) + + # Update password + current_user.hashed_password = get_password_hash(new_password) + current_user.updated_at = datetime.utcnow() + db.commit() + + logger.info(f"Password changed successfully for user {current_user.id}") + + return { + "message": "Password changed successfully", + "updated_at": current_user.updated_at.isoformat() + } + +# API Key management endpoints +@app.post("/users/me/api-keys") +async def generate_api_key( + request: Request, + name: str = Body(...), + description: Optional[str] = Body(None), + db: Session = Depends(get_db), + current_user = Depends(get_current_user_flexible) +): + """Generate a new API key for the user""" + import secrets + import string + + request_id = getattr(request.state, "request_id", str(uuid.uuid4())) if request else str(uuid.uuid4()) + + # Generate secure API key + alphabet = string.ascii_letters + string.digits + api_key = 'ew_' + ''.join(secrets.choice(alphabet) for _ in range(32)) + + # Hash the API key for storage (you'll need to create APIKey model) + from auth.security import get_password_hash + api_key_hash = get_password_hash(api_key) + + # For now, we'll store in a simple format until APIKey model is created + # This is a placeholder - you should create a proper APIKey model + api_key_data = { + "id": str(uuid.uuid4()), + "name": name, + "description": description, + "key_preview": f"{api_key[:8]}...{api_key[-4:]}", + "hash": api_key_hash, + "user_id": current_user.id, + "created_at": datetime.utcnow().isoformat(), + "last_used": None, + "is_active": True + } + + logger.info(f"API key generated for user {current_user.id}: {name}") + + return { + "message": "API key generated successfully", + "api_key": api_key, # Only returned once + "key_info": { + "id": api_key_data["id"], + "name": api_key_data["name"], + "description": api_key_data["description"], + "preview": api_key_data["key_preview"], + "created_at": api_key_data["created_at"] + } + } + +@app.get("/users/me/api-keys") +async def list_api_keys( + request: Request, + db: Session = Depends(get_db), + current_user = Depends(get_current_user_flexible) +): + """List user's API keys (without the actual keys)""" + # Placeholder implementation - return empty list until APIKey model exists + return [] + +# LLM Provider API Keys management endpoints +@app.get("/organizations/{org_id}/llm-keys") +async def list_llm_provider_keys( + org_id: str, + request: Request, + db: Session = Depends(get_db), + current_user = Depends(get_current_user_flexible) +): + """List LLM provider API keys for an organization""" + from auth.models import EncryptedApiKey + request_id = getattr(request.state, 'request_id', str(uuid.uuid4())) + # Use module logger with context + + try: + org_uuid = uuid.UUID(org_id) + except ValueError: + raise ValidationError( + message="Invalid organization ID format", + details=[ErrorDetail( + code="INVALID_UUID", + message="Organization ID must be a valid UUID", + field="org_id" + )], + request_id=request_id + ) + + # Check organization access + user_org = db.query(UserOrganization).filter( + UserOrganization.user_id == current_user.id, + UserOrganization.organization_id == org_uuid, + UserOrganization.is_active == True + ).first() + + if not user_org: + raise NotFoundError( + message="Organization not found or access denied", + resource_type="organization", + resource_id=org_id, + request_id=request_id + ) + + # Get LLM provider keys for the organization + keys = db.query(EncryptedApiKey).filter( + EncryptedApiKey.organization_id == org_uuid, + EncryptedApiKey.is_active == True + ).all() + + logger.info(f"Listed {len(keys)} LLM provider keys for organization {org_id}") + + return [ + { + "id": str(key.id), + "provider": key.provider, + "key_name": key.key_name, + "created_at": key.created_at.isoformat() + } + for key in keys + ] + +@app.post("/organizations/{org_id}/llm-keys") +async def create_llm_provider_key( + org_id: str, + request: Request, + provider: str = Body(...), + key_name: str = Body(...), + api_key: Optional[str] = Body(None), + endpoint_url: Optional[str] = Body(None), + model_deployment_name: Optional[str] = Body(None), + api_version: Optional[str] = Body(None), + db: Session = Depends(get_db), + current_user = Depends(get_current_user_flexible) +): + """Add a new LLM provider API key""" + from auth.models import EncryptedApiKey + from utils.encryption import encryption + + request_id = getattr(request.state, 'request_id', str(uuid.uuid4())) + # Use module logger with context + + print(f"DEBUG: Starting create_llm_key for org_id: {org_id}") + + try: + org_uuid = uuid.UUID(org_id) + print(f"DEBUG: Parsed org_uuid: {org_uuid}") + except ValueError: + raise ValidationError( + message="Invalid organization ID format", + details=[ErrorDetail( + code="INVALID_UUID", + message="Organization ID must be a valid UUID", + field="org_id" + )], + request_id=request_id + ) + + # Validate provider + print(f"DEBUG: Validating provider: {provider}") + valid_providers = ["openai", "azure_openai", "local_openai", "ollama"] + if provider not in valid_providers: + raise ValidationError( + message="Invalid provider", + details=[ErrorDetail( + code="INVALID_PROVIDER", + message=f"Provider must be one of: {', '.join(valid_providers)}", + field="provider" + )], + request_id=request_id + ) + + # Check admin access to organization (allow both admin and member for now) + print(f"DEBUG: Checking organization membership for user {current_user.id}") + admin_membership = db.query(UserOrganization).filter( + UserOrganization.user_id == current_user.id, + UserOrganization.organization_id == org_uuid, + UserOrganization.role.in_(["admin", "member"]), + UserOrganization.is_active == True + ).first() + print(f"DEBUG: Found membership: {admin_membership}") + + if not admin_membership: + raise NotFoundError( + message="Organization not found or insufficient permissions", + resource_type="organization", + resource_id=org_id, + request_id=request_id + ) + + # Validate API key requirements based on provider + if provider in ["openai", "azure_openai"] and not api_key: + raise ValidationError( + message=f"API key is required for {provider}", + details=[ErrorDetail( + code="MISSING_API_KEY", + message=f"{provider} requires an API key", + field="api_key" + )], + request_id=request_id + ) + + # Validate API key format for OpenAI + if provider == "openai" and api_key and not api_key.startswith("sk-"): + raise ValidationError( + message="Invalid OpenAI API key format", + details=[ErrorDetail( + code="INVALID_API_KEY_FORMAT", + message="OpenAI API keys must start with 'sk-'", + field="api_key" + )], + request_id=request_id + ) + + # For local providers, use a default dummy key if none provided + if provider in ["local_openai", "ollama"] and not api_key: + api_key = "dummy-api-key" # Default for local servers + + if provider == "azure_openai" and not endpoint_url: + raise ValidationError( + message="Azure OpenAI requires endpoint URL", + details=[ErrorDetail( + code="MISSING_ENDPOINT", + message="Azure OpenAI provider requires endpoint_url", + field="endpoint_url" + )], + request_id=request_id + ) + + # Check for duplicate provider keys + existing_key = db.query(EncryptedApiKey).filter( + EncryptedApiKey.organization_id == org_uuid, + EncryptedApiKey.provider == provider, + EncryptedApiKey.key_name == key_name, + EncryptedApiKey.is_active == True + ).first() + + if existing_key: + raise ValidationError( + message="API key with this provider and name already exists", + details=[ErrorDetail( + code="DUPLICATE_KEY", + message="A key with this provider and name already exists", + field="key_name" + )], + request_id=request_id + ) + + # Encrypt the API key + try: + encrypted_key = encryption.encrypt_api_key(api_key) + except Exception as e: + logger.error(f"Failed to encrypt API key: {str(e)}") + raise InternalServerError( + message="Failed to encrypt API key", + request_id=request_id + ) + + # Create metadata for the key + metadata = {} + if endpoint_url: + metadata["endpoint_url"] = endpoint_url + if model_deployment_name: + metadata["model_deployment_name"] = model_deployment_name + if api_version: + metadata["api_version"] = api_version + + # Create the encrypted API key record + new_key = EncryptedApiKey( + organization_id=org_uuid, + provider=provider, + encrypted_key=encrypted_key, + key_name=key_name, + created_by=current_user.id, + is_active=True + ) + + db.add(new_key) + db.commit() + db.refresh(new_key) + + logger.info(f"Created LLM provider key for {provider} in organization {org_id}") + + return { + "message": "API key added successfully", + "key_id": str(new_key.id), + "provider": provider, + "key_name": key_name, + "created_at": new_key.created_at.isoformat() + } + +@app.delete("/organizations/{org_id}/llm-keys/{key_id}") +async def delete_llm_provider_key( + org_id: str, + key_id: str, + request: Request, + db: Session = Depends(get_db), + current_user = Depends(get_current_user_flexible) +): + """Delete a LLM provider API key""" + from auth.models import EncryptedApiKey + + request_id = getattr(request.state, 'request_id', str(uuid.uuid4())) + # Use module logger with context + + try: + org_uuid = uuid.UUID(org_id) + key_uuid = uuid.UUID(key_id) + except ValueError: + raise ValidationError( + message="Invalid UUID format", + details=[ErrorDetail( + code="INVALID_UUID", + message="Organization ID and Key ID must be valid UUIDs", + field="ids" + )], + request_id=request_id + ) + + # Check admin access + admin_membership = db.query(UserOrganization).filter( + UserOrganization.user_id == current_user.id, + UserOrganization.organization_id == org_uuid, + UserOrganization.role.in_(["admin"]), + UserOrganization.is_active == True + ).first() + + if not admin_membership: + raise NotFoundError( + message="Organization not found or insufficient permissions", + resource_type="organization", + resource_id=org_id, + request_id=request_id + ) + + # Find the key + key = db.query(EncryptedApiKey).filter( + EncryptedApiKey.id == key_uuid, + EncryptedApiKey.organization_id == org_uuid, + EncryptedApiKey.is_active == True + ).first() + + if not key: + raise NotFoundError( + message="API key not found", + resource_type="encrypted_api_key", + resource_id=key_id, + request_id=request_id + ) + + # Soft delete + key.is_active = False + key.updated_at = datetime.utcnow() + db.commit() + + logger.info(f"Deleted LLM provider key {key_id} from organization {org_id}") + + return { + "message": "API key deleted successfully", + "deleted_at": key.updated_at.isoformat() + } + +# Team Members management endpoints +@app.get("/organizations/{org_id}/members") +async def list_organization_members( + org_id: str, + request: Request, + db: Session = Depends(get_db), + current_user = Depends(get_current_user_flexible) +): + """List all members of an organization""" + request_id = getattr(request.state, 'request_id', str(uuid.uuid4())) + # Use module logger with context + + try: + org_uuid = uuid.UUID(org_id) + except ValueError: + raise ValidationError( + message="Invalid organization ID format", + details=[ErrorDetail( + code="INVALID_UUID", + message="Organization ID must be a valid UUID", + field="org_id" + )], + request_id=request_id + ) + + # Check if user has access to this organization + user_org = db.query(UserOrganization).filter( + UserOrganization.user_id == current_user.id, + UserOrganization.organization_id == org_uuid, + UserOrganization.is_active == True + ).first() + + if not user_org: + raise NotFoundError( + message="Organization not found or access denied", + resource_type="organization", + resource_id=org_id, + request_id=request_id + ) + + # Get all active members of the organization + members = db.query(UserOrganization).join(User).filter( + UserOrganization.organization_id == org_uuid, + UserOrganization.is_active == True, + User.is_active == True + ).all() + + logger.info(f"Listed {len(members)} members for organization {org_id}") + + return [ + { + "id": str(member.user.id), + "email": member.user.email, + "username": member.user.username, + "full_name": member.user.full_name, + "role": member.role, + "joined_at": member.created_at.isoformat(), + "last_login": member.user.last_login.isoformat() if member.user.last_login else None, + "is_current_user": member.user_id == current_user.id + } + for member in members + ] + +@app.post("/organizations/{org_id}/members/invite") +async def invite_organization_member( + org_id: str, + request: Request, + email: str = Body(...), + role: str = Body("member"), + db: Session = Depends(get_db), + current_user = Depends(get_current_user_flexible) +): + """Invite a new member to an organization""" + request_id = getattr(request.state, 'request_id', str(uuid.uuid4())) + # Use module logger with context + + try: + org_uuid = uuid.UUID(org_id) + except ValueError: + raise ValidationError( + message="Invalid organization ID format", + details=[ErrorDetail( + code="INVALID_UUID", + message="Organization ID must be a valid UUID", + field="org_id" + )], + request_id=request_id + ) + + # Validate role + valid_roles = ["admin", "member", "viewer"] + if role not in valid_roles: + raise ValidationError( + message="Invalid role specified", + details=[ErrorDetail( + code="INVALID_ROLE", + message=f"Role must be one of: {', '.join(valid_roles)}", + field="role" + )], + request_id=request_id + ) + + # Check if current user has admin access to this organization + user_org = db.query(UserOrganization).filter( + UserOrganization.user_id == current_user.id, + UserOrganization.organization_id == org_uuid, + UserOrganization.role == "admin", + UserOrganization.is_active == True + ).first() + + if not user_org: + raise NotFoundError( + message="Organization not found or insufficient permissions", + resource_type="organization", + resource_id=org_id, + request_id=request_id + ) + + # Check if user exists + target_user = db.query(User).filter(User.email == email, User.is_active == True).first() + if not target_user: + raise NotFoundError( + message="User not found or inactive", + resource_type="user", + resource_id=email, + request_id=request_id + ) + + # Check if user is already a member + existing_membership = db.query(UserOrganization).filter( + UserOrganization.user_id == target_user.id, + UserOrganization.organization_id == org_uuid + ).first() + + if existing_membership: + if existing_membership.is_active: + raise ValidationError( + message="User is already a member of this organization", + details=[ErrorDetail( + code="ALREADY_MEMBER", + message="User is already an active member", + field="email" + )], + request_id=request_id + ) + else: + # Reactivate existing membership + existing_membership.is_active = True + existing_membership.role = role + existing_membership.updated_at = datetime.utcnow() + db.commit() + + logger.info(f"Reactivated membership for user {target_user.id} in organization {org_id}") + + return { + "message": "User successfully added to organization", + "user_id": str(target_user.id), + "email": target_user.email, + "role": role, + "status": "reactivated" + } + + # Create new membership + new_membership = UserOrganization( + user_id=target_user.id, + organization_id=org_uuid, + role=role, + is_active=True + ) + + db.add(new_membership) + db.commit() + + logger.info(f"Added user {target_user.id} to organization {org_id} with role {role}") + + return { + "message": "User successfully added to organization", + "user_id": str(target_user.id), + "email": target_user.email, + "role": role, + "status": "new" + } + +@app.post("/organizations/{org_id}/members/create") +async def create_organization_member( + org_id: str, + request: Request, + first_name: str = Body(...), + last_name: str = Body(...), + email: str = Body(...), + password: str = Body(...), + role: str = Body("member"), + db: Session = Depends(get_db), + current_user = Depends(get_current_user_flexible) +): + """Create a new user and add them to the organization""" + from auth.security import get_password_hash + + request_id = getattr(request.state, 'request_id', str(uuid.uuid4())) + # Use module logger with context + + try: + org_uuid = uuid.UUID(org_id) + except ValueError: + raise ValidationError( + message="Invalid organization ID format", + details=[ErrorDetail( + code="INVALID_UUID", + message="Organization ID must be a valid UUID", + field="org_id" + )], + request_id=request_id + ) + + # Check if current user has admin permissions + user_org = db.query(UserOrganization).filter( + UserOrganization.user_id == current_user.id, + UserOrganization.organization_id == org_uuid, + UserOrganization.role.in_(["admin"]), # Only admins can create users + UserOrganization.is_active == True + ).first() + + if not user_org: + raise NotFoundError( + message="Organization not found or insufficient permissions", + resource_type="organization", + resource_id=org_id, + request_id=request_id + ) + + # Validate role + valid_roles = ["admin", "member", "viewer"] + if role not in valid_roles: + raise ValidationError( + message="Invalid role", + details=[ErrorDetail( + code="INVALID_ROLE", + message=f"Role must be one of: {', '.join(valid_roles)}", + field="role" + )], + request_id=request_id + ) + + # Validate email format + import re + email_pattern = r'^[a-zA-Z0-9._%+-]+@[a-zA-Z0-9.-]+\.[a-zA-Z]{2,}$' + if not re.match(email_pattern, email): + raise ValidationError( + message="Invalid email format", + details=[ErrorDetail( + code="INVALID_EMAIL", + message="Please provide a valid email address", + field="email" + )], + request_id=request_id + ) + + # Check if user already exists + existing_user = db.query(User).filter(User.email == email).first() + if existing_user: + raise ValidationError( + message="User with this email already exists", + details=[ErrorDetail( + code="EMAIL_EXISTS", + message="A user with this email address already exists", + field="email" + )], + request_id=request_id + ) + + # Validate password strength + if len(password) < 8: + raise ValidationError( + message="Password too weak", + details=[ErrorDetail( + code="PASSWORD_TOO_SHORT", + message="Password must be at least 8 characters long", + field="password" + )], + request_id=request_id + ) + + # Create new user + password_hash = get_password_hash(password) + # Generate username from email (everything before @) + username = email.split('@')[0] + # Ensure username is unique by appending numbers if needed + base_username = username + counter = 1 + while db.query(User).filter(User.username == username).first(): + username = f"{base_username}{counter}" + counter += 1 + + new_user = User( + email=email, + username=username, + hashed_password=password_hash, + full_name=f"{first_name.strip()} {last_name.strip()}", + is_active=True + ) + + db.add(new_user) + db.flush() # Get the ID without committing + + # Add user to organization + new_membership = UserOrganization( + user_id=new_user.id, + organization_id=org_uuid, + role=role, + is_active=True + ) + + db.add(new_membership) + db.commit() + + logger.info(f"Created new user {new_user.id} ({email}) and added to organization {org_id} with role {role}") + + return { + "message": "User created and added to organization successfully", + "user_id": str(new_user.id), + "email": new_user.email, + "username": new_user.username, + "full_name": new_user.full_name, + "role": role + } + +@app.patch("/organizations/{org_id}/members/{user_id}") +async def update_organization_member( + org_id: str, + user_id: str, + request: Request, + role: Optional[str] = Body(None), + is_active: Optional[bool] = Body(None), + db: Session = Depends(get_db), + current_user = Depends(get_current_user_flexible) +): + """Update a member's role or status in an organization""" + request_id = getattr(request.state, 'request_id', str(uuid.uuid4())) + # Use module logger with context + + try: + org_uuid = uuid.UUID(org_id) + target_user_uuid = uuid.UUID(user_id) + except ValueError: + raise ValidationError( + message="Invalid UUID format", + details=[ErrorDetail( + code="INVALID_UUID", + message="Organization ID and User ID must be valid UUIDs", + field="ids" + )], + request_id=request_id + ) + + # Validate role if provided + if role and role not in ["admin", "member", "viewer"]: + raise ValidationError( + message="Invalid role specified", + details=[ErrorDetail( + code="INVALID_ROLE", + message="Role must be one of: admin, member, viewer", + field="role" + )], + request_id=request_id + ) + + # Check if current user has admin access + admin_membership = db.query(UserOrganization).filter( + UserOrganization.user_id == current_user.id, + UserOrganization.organization_id == org_uuid, + UserOrganization.role == "admin", + UserOrganization.is_active == True + ).first() + + if not admin_membership: + raise NotFoundError( + message="Organization not found or insufficient permissions", + resource_type="organization", + resource_id=org_id, + request_id=request_id + ) + + # Find the target membership + target_membership = db.query(UserOrganization).filter( + UserOrganization.user_id == target_user_uuid, + UserOrganization.organization_id == org_uuid + ).first() + + if not target_membership: + raise NotFoundError( + message="Member not found in organization", + resource_type="user_organization", + resource_id=f"{user_id}_{org_id}", + request_id=request_id + ) + + # Prevent admins from deactivating themselves + if target_user_uuid == current_user.id and is_active == False: + raise ValidationError( + message="Cannot deactivate your own membership", + details=[ErrorDetail( + code="SELF_DEACTIVATION", + message="You cannot deactivate your own organization membership", + field="user_id" + )], + request_id=request_id + ) + + # Update the membership + if role: + target_membership.role = role + if is_active is not None: + target_membership.is_active = is_active + target_membership.updated_at = datetime.utcnow() + + db.commit() + + action = "updated" if role else ("activated" if is_active else "deactivated") + logger.info(f"Member {user_id} {action} in organization {org_id}") + + return { + "message": f"Member successfully {action}", + "user_id": user_id, + "role": target_membership.role, + "is_active": target_membership.is_active, + "updated_at": target_membership.updated_at.isoformat() + } + +@app.delete("/organizations/{org_id}/members/{user_id}") +async def remove_organization_member( + org_id: str, + user_id: str, + request: Request, + db: Session = Depends(get_db), + current_user = Depends(get_current_user_flexible) +): + """Remove a member from an organization""" + request_id = getattr(request.state, 'request_id', str(uuid.uuid4())) + # Use module logger with context + + try: + org_uuid = uuid.UUID(org_id) + target_user_uuid = uuid.UUID(user_id) + except ValueError: + raise ValidationError( + message="Invalid UUID format", + details=[ErrorDetail( + code="INVALID_UUID", + message="Organization ID and User ID must be valid UUIDs", + field="ids" + )], + request_id=request_id + ) + + # Check admin access + admin_membership = db.query(UserOrganization).filter( + UserOrganization.user_id == current_user.id, + UserOrganization.organization_id == org_uuid, + UserOrganization.role == "admin", + UserOrganization.is_active == True + ).first() + + if not admin_membership: + raise NotFoundError( + message="Organization not found or insufficient permissions", + resource_type="organization", + resource_id=org_id, + request_id=request_id + ) + + # Prevent self-removal + if target_user_uuid == current_user.id: + raise ValidationError( + message="Cannot remove yourself from organization", + details=[ErrorDetail( + code="SELF_REMOVAL", + message="You cannot remove your own organization membership", + field="user_id" + )], + request_id=request_id + ) + + # Find and deactivate membership + target_membership = db.query(UserOrganization).filter( + UserOrganization.user_id == target_user_uuid, + UserOrganization.organization_id == org_uuid + ).first() + + if not target_membership or not target_membership.is_active: + raise NotFoundError( + message="Active member not found in organization", + resource_type="user_organization", + resource_id=f"{user_id}_{org_id}", + request_id=request_id + ) + + # Soft delete by deactivating + target_membership.is_active = False + target_membership.updated_at = datetime.utcnow() + db.commit() + + logger.info(f"Removed member {user_id} from organization {org_id}") + + return { + "message": "Member successfully removed from organization", + "user_id": user_id, + "removed_at": target_membership.updated_at.isoformat() + } + +# =============================== +# LLM Provider Management Endpoints +# =============================== + +@app.get("/organizations/{org_id}/llm-providers") +async def list_llm_providers( + org_id: str, + current_user: User = Depends(get_current_user), + db: Session = Depends(get_db) +): + """List all LLM providers for an organization""" + request_id = str(uuid.uuid4()) + + # Verify user belongs to organization + try: + org_uuid = uuid.UUID(org_id) + except ValueError: + raise ValidationError( + message="Invalid organization ID format", + details=[ErrorDetail(code="INVALID_UUID", message=f"'{org_id}' is not a valid UUID")], + request_id=request_id + ) + + membership = db.query(UserOrganization).filter( + UserOrganization.user_id == current_user.id, + UserOrganization.organization_id == org_uuid, + UserOrganization.is_active == True + ).first() + if not membership: + raise PermissionError("User does not belong to this organization") + + try: + org_uuid = uuid.UUID(org_id) + except ValueError: + raise ValidationError( + message="Invalid organization ID format", + details=[ErrorDetail( + code="INVALID_UUID", + message="Organization ID must be a valid UUID", + field="org_id" + )], + request_id=request_id + ) + + # Get all active providers for the organization + providers = db.query(LLMProvider).filter( + LLMProvider.organization_id == org_uuid, + LLMProvider.is_active == True + ).order_by(LLMProvider.is_default.desc(), LLMProvider.created_at.desc()).all() + + logger.info(f"Listed {len(providers)} LLM providers for organization {org_id}") + + return [ + { + "id": str(provider.id), + "name": provider.name, + "provider_type": provider.provider_type, + "base_url": provider.base_url, + "model_defaults": { + "model_name": provider.default_model_name, + "temperature": provider.default_temperature, + "max_tokens": provider.default_max_tokens + }, + "is_default": provider.is_default, + "created_at": provider.created_at.isoformat() + } + for provider in providers + ] + +@app.post("/organizations/{org_id}/llm-providers") +async def create_llm_provider( + org_id: str, + name: str = Body(...), + provider_type: str = Body(...), + api_key: Optional[str] = Body(None), + base_url: Optional[str] = Body(None), + model_defaults: dict = Body(...), + is_default: bool = Body(False), + current_user: User = Depends(get_current_user), + db: Session = Depends(get_db) +): + """Create a new LLM provider for an organization""" + request_id = str(uuid.uuid4()) + + # Verify user belongs to organization with admin rights + try: + org_uuid = uuid.UUID(org_id) + except ValueError: + raise ValidationError( + message="Invalid organization ID format", + details=[ErrorDetail(code="INVALID_UUID", message=f"'{org_id}' is not a valid UUID")], + request_id=request_id + ) + + membership = db.query(UserOrganization).filter( + UserOrganization.user_id == current_user.id, + UserOrganization.organization_id == org_uuid, + UserOrganization.is_active == True + ).first() + if not membership or membership.role not in ["owner", "admin"]: + raise PermissionError("User does not have permission to create providers") + + try: + org_uuid = uuid.UUID(org_id) + except ValueError: + raise ValidationError( + message="Invalid organization ID format", + details=[ErrorDetail( + code="INVALID_UUID", + message="Organization ID must be a valid UUID", + field="org_id" + )], + request_id=request_id + ) + + # Validate required fields + if not name.strip(): + raise ValidationError( + message="Provider name is required", + details=[ErrorDetail( + code="MISSING_NAME", + message="Provider name cannot be empty", + field="name" + )], + request_id=request_id + ) + + if provider_type not in ["openai", "azure_openai", "local_openai", "ollama", "anthropic", "cohere", "google", "custom"]: + raise ValidationError( + message="Invalid provider type", + details=[ErrorDetail( + code="INVALID_PROVIDER_TYPE", + message="Provider type must be one of: openai, azure_openai, local_openai, ollama, anthropic, cohere, google, custom", + field="provider_type" + )], + request_id=request_id + ) + + # Validate API key for cloud providers + if provider_type in ["openai", "azure_openai", "anthropic", "cohere", "google"] and not api_key: + raise ValidationError( + message=f"API key is required for {provider_type}", + details=[ErrorDetail( + code="MISSING_API_KEY", + message=f"{provider_type} requires an API key", + field="api_key" + )], + request_id=request_id + ) + + # Validate model defaults + if "model_name" not in model_defaults or not model_defaults["model_name"]: + raise ValidationError( + message="Default model name is required", + details=[ErrorDetail( + code="MISSING_MODEL_NAME", + message="model_defaults.model_name is required", + field="model_defaults" + )], + request_id=request_id + ) + + # Encrypt API key if provided + encrypted_api_key = None + if api_key: + from utils.encryption import encryption + encrypted_api_key = encryption.encrypt_api_key(api_key) + + # If this is set as default, unset other defaults + if is_default: + db.query(LLMProvider).filter( + LLMProvider.organization_id == org_uuid + ).update({LLMProvider.is_default: False}) + + # Create new provider + new_provider = LLMProvider( + organization_id=org_uuid, + name=name.strip(), + provider_type=provider_type, + encrypted_api_key=encrypted_api_key, + base_url=base_url, + default_model_name=model_defaults["model_name"], + default_temperature=model_defaults.get("temperature", 0.7), + default_max_tokens=model_defaults.get("max_tokens", 1000), + is_default=is_default, + created_by=current_user.id + ) + + db.add(new_provider) + db.commit() + db.refresh(new_provider) + + logger.info(f"Created LLM provider {new_provider.id} for organization {org_id}") + + return { + "id": str(new_provider.id), + "name": new_provider.name, + "provider_type": new_provider.provider_type, + "created_at": new_provider.created_at.isoformat() + } + +@app.put("/organizations/{org_id}/llm-providers/{provider_id}") +async def update_llm_provider( + org_id: str, + provider_id: str, + name: Optional[str] = Body(None), + api_key: Optional[str] = Body(None), + base_url: Optional[str] = Body(None), + model_defaults: Optional[dict] = Body(None), + is_default: Optional[bool] = Body(None), + current_user: User = Depends(get_current_user), + db: Session = Depends(get_db) +): + """Update an LLM provider""" + request_id = str(uuid.uuid4()) + + # Verify user belongs to organization with admin rights + membership = get_organization_membership(current_user.id, org_id, db) + if not membership or membership.role not in ["owner", "admin"]: + raise PermissionError("User does not have permission to update providers") + + try: + org_uuid = uuid.UUID(org_id) + provider_uuid = uuid.UUID(provider_id) + except ValueError: + raise ValidationError( + message="Invalid UUID format", + details=[ErrorDetail( + code="INVALID_UUID", + message="IDs must be valid UUIDs", + field="id" + )], + request_id=request_id + ) + + # Get the provider + provider = db.query(LLMProvider).filter( + LLMProvider.id == provider_uuid, + LLMProvider.organization_id == org_uuid, + LLMProvider.is_active == True + ).first() + + if not provider: + raise ValidationError( + message="LLM provider not found", + details=[ErrorDetail( + code="PROVIDER_NOT_FOUND", + message="Provider not found or access denied", + field="provider_id" + )], + request_id=request_id + ) + + # Update fields if provided + if name is not None: + provider.name = name.strip() + + if api_key is not None: + from utils.encryption import encryption + provider.encrypted_api_key = encryption.encrypt_api_key(api_key) if api_key else None + + if base_url is not None: + provider.base_url = base_url + + if model_defaults is not None: + if "model_name" in model_defaults: + provider.default_model_name = model_defaults["model_name"] + if "temperature" in model_defaults: + provider.default_temperature = model_defaults["temperature"] + if "max_tokens" in model_defaults: + provider.default_max_tokens = model_defaults["max_tokens"] + + if is_default is not None: + if is_default: + # Unset other defaults + db.query(LLMProvider).filter( + LLMProvider.organization_id == org_uuid, + LLMProvider.id != provider_uuid + ).update({LLMProvider.is_default: False}) + provider.is_default = is_default + + provider.updated_at = datetime.utcnow() + db.commit() + + logger.info(f"Updated LLM provider {provider_id} for organization {org_id}") + + return { + "id": str(provider.id), + "name": provider.name, + "updated_at": provider.updated_at.isoformat() + } + +@app.delete("/organizations/{org_id}/llm-providers/{provider_id}") +async def delete_llm_provider( + org_id: str, + provider_id: str, + current_user: User = Depends(get_current_user), + db: Session = Depends(get_db) +): + """Delete an LLM provider (soft delete)""" + request_id = str(uuid.uuid4()) + + # Verify user belongs to organization with admin rights + membership = get_organization_membership(current_user.id, org_id, db) + if not membership or membership.role not in ["owner", "admin"]: + raise PermissionError("User does not have permission to delete providers") + + try: + org_uuid = uuid.UUID(org_id) + provider_uuid = uuid.UUID(provider_id) + except ValueError: + raise ValidationError( + message="Invalid UUID format", + details=[ErrorDetail( + code="INVALID_UUID", + message="IDs must be valid UUIDs", + field="id" + )], + request_id=request_id + ) + + # Get the provider + provider = db.query(LLMProvider).filter( + LLMProvider.id == provider_uuid, + LLMProvider.organization_id == org_uuid, + LLMProvider.is_active == True + ).first() + + if not provider: + raise ValidationError( + message="LLM provider not found", + details=[ErrorDetail( + code="PROVIDER_NOT_FOUND", + message="Provider not found or access denied", + field="provider_id" + )], + request_id=request_id + ) + + # Soft delete + provider.is_active = False + provider.updated_at = datetime.utcnow() + db.commit() + + logger.info(f"Deleted LLM provider {provider_id} for organization {org_id}") + + return { + "message": "Provider successfully deleted", + "provider_id": provider_id, + "deleted_at": provider.updated_at.isoformat() + } + if __name__ == "__main__": import uvicorn uvicorn.run(app, host="0.0.0.0", port=8000) diff --git a/api/middleware/rate_limiting.py b/api/middleware/rate_limiting.py index 575bcf7..4d544b9 100644 --- a/api/middleware/rate_limiting.py +++ b/api/middleware/rate_limiting.py @@ -22,10 +22,10 @@ class RateLimitingMiddleware(BaseHTTPMiddleware): def __init__(self, app): super().__init__(app) - # Rate limits (requests per minute) - self.unauthenticated_limit = 20 # Very restrictive for unauthenticated - self.authenticated_limit = 100 # More generous for authenticated users - self.admin_limit = 500 # High limit for admin users + # Rate limits (requests per minute) - Increased 10x for testing + self.unauthenticated_limit = 200 # 20 * 10 for easier testing + self.authenticated_limit = 1000 # 100 * 10 for authenticated users + self.admin_limit = 5000 # 500 * 10 for admin users # Window size in seconds self.window_size = 60 diff --git a/api/tasks/simple_evaluation.py b/api/tasks/simple_evaluation.py index 442ac3a..7c2e89e 100644 --- a/api/tasks/simple_evaluation.py +++ b/api/tasks/simple_evaluation.py @@ -11,7 +11,7 @@ from models import Run, Result, Evaluation, Item, Scenario, Evaluator from evaluators.factory import EvaluatorFactory -from scenarios.factory import ScenarioFactory +from scenarios.factory import ScenarioGeneratorFactory from adapters.factory import ModelAdapterFactory # Database setup for Celery tasks @@ -144,8 +144,8 @@ def process_simple_item(db, run: Run, item: Item, scenario: Scenario, evaluators final_prompt = input_text if scenario: try: - scenario_generator = ScenarioFactory.create_scenario( - scenario.type, + scenario_generator = ScenarioGeneratorFactory.create_generator( + scenario.type, scenario.params_json or {} ) final_prompt = scenario_generator.generate_prompt(input_text, item.metadata_json) diff --git a/api/utils/encryption.py b/api/utils/encryption.py index 6437de3..fb6275a 100644 --- a/api/utils/encryption.py +++ b/api/utils/encryption.py @@ -3,7 +3,7 @@ from cryptography.fernet import Fernet from cryptography.hazmat.primitives import hashes from cryptography.hazmat.primitives.kdf.pbkdf2 import PBKDF2HMAC -from typing import str +# str is a built-in type, no import needed class APIKeyEncryption: """Utility for encrypting/decrypting API keys""" diff --git a/cookies.txt b/cookies.txt new file mode 100644 index 0000000..9961c86 --- /dev/null +++ b/cookies.txt @@ -0,0 +1,5 @@ +# Netscape HTTP Cookie File +# https://curl.se/docs/http-cookies.html +# This file was generated by libcurl! Edit at your own risk. + +#HttpOnly_localhost FALSE / TRUE 1757715649 refresh_token eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJzdWIiOiJiOThjNWZhMi03ODhkLTQ0Y2YtYjQzNS1iNDMyMWQxYmFiNmUiLCJleHAiOjE3NTc3MTU2NDksImlhdCI6MTc1NzExMDg0OSwiaXNzIjoiZXZhbHdpc2UtYXBpIiwiYXVkIjoiZXZhbHdpc2UtY2xpZW50IiwidHlwZSI6InJlZnJlc2giLCJqdGkiOiJOQ2RNTDJSUTVZVWlXNkpZSld6T0JnIn0.DhxJ-dojDe0pXYupCoUR7_izymwdUHKrnoJub484Jv0 diff --git a/create_admin.py b/create_admin.py new file mode 100644 index 0000000..3ec2e87 --- /dev/null +++ b/create_admin.py @@ -0,0 +1,71 @@ +#!/usr/bin/env python3 +""" +Create a fresh admin user for testing +""" +import sys +import os +sys.path.append(os.path.join(os.path.dirname(__file__), 'api')) + +from database import SessionLocal, engine +from auth.models import User, Organization, UserOrganization +from auth.security import get_password_hash + +def create_admin(): + db = SessionLocal() + try: + # Delete existing admin user if exists + existing_admin = db.query(User).filter(User.username == "admin").first() + if existing_admin: + print("Deleting existing admin user...") + db.delete(existing_admin) + db.commit() + + # Get default organization + default_org = db.query(Organization).filter(Organization.name == "Default Organization").first() + if not default_org: + default_org = Organization( + name="Default Organization", + description="Default organization", + max_users=100, + max_datasets=1000, + max_runs_per_month=10000 + ) + db.add(default_org) + db.flush() + + # Create new admin user with known password + admin_user = User( + username="admin", + email="admin@evalwise.local", + hashed_password=get_password_hash("admin123"), + full_name="System Administrator", + is_active=True, + is_superuser=True, + rate_limit_tier="enterprise" + ) + db.add(admin_user) + db.flush() + + # Add to organization + user_org = UserOrganization( + user_id=admin_user.id, + organization_id=default_org.id, + role="admin" + ) + db.add(user_org) + + db.commit() + print(f"✅ Created admin user: {admin_user.username}") + print(f" Email: {admin_user.email}") + print(f" Password: admin123") + print(f" Superuser: {admin_user.is_superuser}") + + except Exception as e: + db.rollback() + print(f"❌ Error creating admin user: {e}") + raise + finally: + db.close() + +if __name__ == "__main__": + create_admin() \ No newline at end of file diff --git a/docker-compose.yml b/docker-compose.yml index 7e9439a..80ec1a7 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -4,7 +4,7 @@ services: environment: POSTGRES_DB: ${POSTGRES_DB:-evalwise} POSTGRES_USER: ${POSTGRES_USER:-evalwise} - POSTGRES_PASSWORD: ${POSTGRES_PASSWORD} # Required - no default + POSTGRES_PASSWORD: ${POSTGRES_PASSWORD:-evalwise123} ports: - "5432:5432" volumes: @@ -32,12 +32,16 @@ services: ports: - "8003:8000" env_file: - - .env + - ./api/.env environment: - DATABASE_URL: postgresql://${POSTGRES_USER:-evalwise}:${POSTGRES_PASSWORD}@postgres:5432/${POSTGRES_DB:-evalwise} + DATABASE_URL: postgresql://${POSTGRES_USER:-evalwise}:${POSTGRES_PASSWORD:-evalwise123}@postgres:5432/${POSTGRES_DB:-evalwise} REDIS_URL: redis://redis:6379/0 CELERY_BROKER_URL: redis://redis:6379/0 CELERY_RESULT_BACKEND: redis://redis:6379/0 + API_ENCRYPTION_KEY: bab7ada6bbadbd20e52c01f8a891caf151f3e3c9f3b8a0e34b486ca6bda0b866 + POSTGRES_PASSWORD: evalwise123 + SECRET_KEY: your-super-secret-key-for-sessions-change-in-production + JWT_SECRET_KEY: your-super-secret-jwt-key-change-in-production depends_on: postgres: condition: service_healthy @@ -52,7 +56,7 @@ services: context: ./api dockerfile: Dockerfile env_file: - - .env + - ./api/.env environment: DATABASE_URL: postgresql://${POSTGRES_USER:-evalwise}:${POSTGRES_PASSWORD}@postgres:5432/${POSTGRES_DB:-evalwise} REDIS_URL: redis://redis:6379/0 diff --git a/web/src/app/admin/page.tsx b/web/src/app/admin/page.tsx index 0680119..84dda99 100644 --- a/web/src/app/admin/page.tsx +++ b/web/src/app/admin/page.tsx @@ -6,6 +6,7 @@ import { Button } from "@/components/ui/button" import { Input } from "@/components/ui/input" import { Label } from "@/components/ui/label" import { Badge } from "@/components/ui/badge" +import ProtectedRoute from '@/components/protected-route' import { Users, Building2, @@ -21,6 +22,7 @@ import { RefreshCw } from "lucide-react" import { useAuth } from '@/contexts/auth-context' +import CreateUserModal from '@/components/create-user-modal' const API_BASE_URL = process.env.NEXT_PUBLIC_API_URL || 'http://localhost:8000' @@ -56,23 +58,15 @@ interface NewUser { rate_limit_tier: string } -export default function AdminPage() { +function AdminPageContent() { const { user, token } = useAuth() const [stats, setStats] = useState(null) const [users, setUsers] = useState([]) const [loading, setLoading] = useState(true) const [searchTerm, setSearchTerm] = useState('') - const [showCreateUser, setShowCreateUser] = useState(false) + const [showCreateUserModal, setShowCreateUserModal] = useState(false) const [showPasswords, setShowPasswords] = useState(false) - const [newUser, setNewUser] = useState({ - email: '', - username: '', - password: '', - full_name: '', - is_active: true, - is_superuser: false, - rate_limit_tier: 'basic' - }) + const [createUserLoading, setCreateUserLoading] = useState(false) const [error, setError] = useState('') const fetchStats = async () => { @@ -114,8 +108,9 @@ export default function AdminPage() { } } - const createUser = async () => { + const createUser = async (userData: NewUser) => { setError('') + setCreateUserLoading(true) try { const response = await fetch(`${API_BASE_URL}/admin/users`, { method: 'POST', @@ -123,28 +118,24 @@ export default function AdminPage() { 'Authorization': `Bearer ${token}`, 'Content-Type': 'application/json' }, - body: JSON.stringify(newUser) + body: JSON.stringify({...userData, rate_limit_tier: 'basic'}) }) if (response.ok) { - setNewUser({ - email: '', - username: '', - password: '', - full_name: '', - is_active: true, - is_superuser: false, - rate_limit_tier: 'basic' - }) - setShowCreateUser(false) await fetchUsers() await fetchStats() + setShowCreateUserModal(false) } else { const errorData = await response.json() setError(errorData.detail?.message || 'Failed to create user') + throw new Error(errorData.detail?.message || 'Failed to create user') } } catch (err) { - setError('Network error occurred') + const errorMessage = 'Network error occurred' + setError(errorMessage) + throw new Error(errorMessage) + } finally { + setCreateUserLoading(false) } } @@ -211,7 +202,7 @@ export default function AdminPage() { } return ( -
+
{/* Header */}

Admin Dashboard

@@ -287,7 +278,7 @@ export default function AdminPage() { Manage system users and their permissions
- @@ -312,86 +303,6 @@ export default function AdminPage() {
- {/* Create User Form */} - {showCreateUser && ( - - - Create New User - - - {error && ( -
- {error} -
- )} - -
-
- - setNewUser({...newUser, email: e.target.value})} - /> -
-
- - setNewUser({...newUser, username: e.target.value})} - /> -
-
- - setNewUser({...newUser, password: e.target.value})} - /> -
-
- - setNewUser({...newUser, full_name: e.target.value})} - /> -
-
- -
- - -
- -
- - -
- - - )} {/* Users Table */}
@@ -442,6 +353,26 @@ export default function AdminPage() {
+ + {/* Create User Modal */} + { + setShowCreateUserModal(false) + setError('') + }} + onSubmit={createUser} + isLoading={createUserLoading} + error={error} + />
) +} + +export default function AdminPage() { + return ( + + + + ) } \ No newline at end of file diff --git a/web/src/app/datasets/page.tsx b/web/src/app/datasets/page.tsx index 8dbd212..9041c1c 100644 --- a/web/src/app/datasets/page.tsx +++ b/web/src/app/datasets/page.tsx @@ -4,6 +4,7 @@ import { useState, useEffect } from 'react' import { Upload, Plus, Database, FileText, Table, ChevronLeft } from 'lucide-react' import { datasetApi } from '@/lib/api' import DatasetTableEditor from '@/components/dataset-table-editor' +import ProtectedRoute from '@/components/protected-route' interface Dataset { id: string @@ -15,7 +16,7 @@ interface Dataset { item_count?: number } -export default function DatasetsPage() { +function DatasetsPageContent() { const [datasets, setDatasets] = useState([]) const [loading, setLoading] = useState(true) const [showCreateForm, setShowCreateForm] = useState(false) @@ -64,12 +65,17 @@ export default function DatasetsPage() { const handleCreateWithTable = async () => { try { - // Create a basic dataset first - const dataset = await createDataset(new Event('submit') as any) + // Avoid relying on pending state updates; create directly + const payload = { name: 'New Dataset', tags: [], is_synthetic: false } + const response = await datasetApi.create(payload) + const dataset = response.data + // Refresh list and open table editor for the new dataset + await loadDatasets() setEditingDataset(dataset) setShowTableEditor(true) } catch (error) { console.error('Failed to create dataset for table editing:', error) + alert('Failed to create dataset') } } @@ -156,7 +162,7 @@ export default function DatasetsPage() { } return ( -
+

Datasets

@@ -171,10 +177,7 @@ export default function DatasetsPage() { Upload CSV