From 2887c408b38d75b284923c6f9a08632044d06ddf Mon Sep 17 00:00:00 2001 From: hackertron Date: Tue, 7 Apr 2026 17:29:30 +0530 Subject: [PATCH 1/6] feat: worker lane split, dirty flag optimization, and lease-based sync Port sync infrastructure overhaul from prod: Architecture: - Replace QStash queue with lease-based connection_sync_state control plane - Add worker lane split (realtime vs reconcile claim modes) - Add dedicated webhook ingress service (webhooks_index.py) - Extract app_factory.py from monolithic index.py - Add Railway sync worker entrypoint (sync_worker.py) New modules: - sync_dispatcher.py: shared dispatch layer for marking streams dirty - sync_state_store.py: lease claim/heartbeat/complete/fail helpers - stream_worker.py: Railway worker loop for lease-based sync - failure_policy.py: auto-quarantine for permanently broken connections - google_webhook_security.py: HMAC token signing for Calendar webhooks - google_retry.py: Google API retry helpers Database migrations: - connection_sync_state table with RPCs (mark dirty, claim, heartbeat, complete, fail) - Skip redundant dirty-mark writes (dedup Pub/Sub bursts) - Per-stream next_reconcile_at replacing broad sweep - Claim mode split (dirty_only/reconcile_only/any) with jitter - Autovacuum tuning for high-churn sync state table Router changes: - Webhooks: ingress dedup cache, Pub/Sub JWT verification, dirty marking - Workers: lease-based execution wrapper, mode dispatch - Cron: watch-health detection, stream-aware setup-missing-watches - Calendar/Email: manual sync via mark_stream_dirty (priority 200) - Auth service: initial sync uses dirty marking instead of QStash --- core-api/api/app_factory.py | 259 +++++++ core-api/api/config.py | 26 + core-api/api/routers/calendar.py | 105 +-- core-api/api/routers/cron.py | 707 ++++++++++-------- core-api/api/routers/email.py | 199 ++--- core-api/api/routers/webhooks.py | 454 +++++++---- core-api/api/routers/workers.py | 201 ++++- core-api/api/services/auth.py | 106 ++- core-api/api/services/syncs/failure_policy.py | 106 +++ .../api/services/syncs/google_error_utils.py | 3 + .../api/services/syncs/google_services.py | 29 +- core-api/api/services/syncs/stream_worker.py | 344 +++++++++ .../api/services/syncs/sync_dispatcher.py | 93 +++ core-api/api/services/syncs/sync_gmail.py | 7 +- .../api/services/syncs/sync_gmail_cron.py | 5 +- .../services/syncs/sync_google_calendar.py | 3 +- .../syncs/sync_google_calendar_cron.py | 20 +- .../api/services/syncs/sync_state_store.py | 313 ++++++++ core-api/api/services/syncs/watch_manager.py | 243 +++--- core-api/api/services/webhooks/__init__.py | 4 +- .../api/services/webhooks/calendar_webhook.py | 7 +- .../api/services/webhooks/gmail_webhook.py | 157 +++- core-api/index.py | 203 +---- core-api/lib/google_retry.py | 78 ++ core-api/lib/google_webhook_security.py | 41 + core-api/setup_pubsub_subscription.py | 308 +++++--- ...nforce_single_active_push_subscription.sql | 28 + .../20260401000002_connection_sync_state.sql | 538 +++++++++++++ ...00001_skip_redundant_dirty_mark_writes.sql | 121 +++ .../20260402000002_next_reconcile_at.sql | 397 ++++++++++ ...000001_reconcile_claim_mode_and_jitter.sql | 257 +++++++ ...00002_connection_sync_state_autovacuum.sql | 13 + core-api/sync_worker.py | 17 + .../unit/test_auth_initial_sync_queue.py | 81 +- .../unit/test_calendar_sync_dirty_only.py | 99 +++ .../tests/unit/test_cron_batch_mode_toggle.py | 138 +--- .../unit/test_cron_google_oauth_failures.py | 78 ++ core-api/tests/unit/test_cron_watch_health.py | 147 ++++ .../tests/unit/test_email_sync_queue_only.py | 106 +++ core-api/tests/unit/test_failure_policy.py | 25 + .../unit/test_google_webhook_security.py | 43 ++ core-api/tests/unit/test_image_proxy.py | 2 +- .../unit/test_notify_attendees_google.py | 4 +- core-api/tests/unit/test_share_link_access.py | 67 +- core-api/tests/unit/test_stream_worker.py | 407 ++++++++++ core-api/tests/unit/test_sync_dispatcher.py | 76 ++ .../test_sync_google_calendar_cron_tokens.py | 99 +++ core-api/tests/unit/test_sync_state_store.py | 313 ++++++++ core-api/tests/unit/test_token_encryption.py | 387 ++++++++++ .../unit/test_token_encryption_regressions.py | 244 ++++++ .../tests/unit/test_user_profile_helpers.py | 4 +- .../test_watch_manager_permanent_failures.py | 127 ++++ .../tests/unit/test_webhook_dirty_marking.py | 288 +++++++ .../tests/unit/test_workers_mode_dispatch.py | 169 ++++- .../tests/unit/test_workspace_default_apps.py | 6 +- .../test_workspace_invitations_service.py | 4 +- core-api/webhooks_index.py | 11 + 57 files changed, 6879 insertions(+), 1438 deletions(-) create mode 100644 core-api/api/app_factory.py create mode 100644 core-api/api/services/syncs/failure_policy.py create mode 100644 core-api/api/services/syncs/stream_worker.py create mode 100644 core-api/api/services/syncs/sync_dispatcher.py create mode 100644 core-api/api/services/syncs/sync_state_store.py create mode 100644 core-api/lib/google_retry.py create mode 100644 core-api/lib/google_webhook_security.py create mode 100644 core-api/supabase/migrations/20260401000001_enforce_single_active_push_subscription.sql create mode 100644 core-api/supabase/migrations/20260401000002_connection_sync_state.sql create mode 100644 core-api/supabase/migrations/20260402000001_skip_redundant_dirty_mark_writes.sql create mode 100644 core-api/supabase/migrations/20260402000002_next_reconcile_at.sql create mode 100644 core-api/supabase/migrations/20260403000001_reconcile_claim_mode_and_jitter.sql create mode 100644 core-api/supabase/migrations/20260403000002_connection_sync_state_autovacuum.sql create mode 100644 core-api/sync_worker.py create mode 100644 core-api/tests/unit/test_calendar_sync_dirty_only.py create mode 100644 core-api/tests/unit/test_cron_watch_health.py create mode 100644 core-api/tests/unit/test_email_sync_queue_only.py create mode 100644 core-api/tests/unit/test_failure_policy.py create mode 100644 core-api/tests/unit/test_google_webhook_security.py create mode 100644 core-api/tests/unit/test_stream_worker.py create mode 100644 core-api/tests/unit/test_sync_dispatcher.py create mode 100644 core-api/tests/unit/test_sync_google_calendar_cron_tokens.py create mode 100644 core-api/tests/unit/test_sync_state_store.py create mode 100644 core-api/tests/unit/test_token_encryption.py create mode 100644 core-api/tests/unit/test_token_encryption_regressions.py create mode 100644 core-api/tests/unit/test_webhook_dirty_marking.py create mode 100644 core-api/webhooks_index.py diff --git a/core-api/api/app_factory.py b/core-api/api/app_factory.py new file mode 100644 index 00000000..cc31f7d7 --- /dev/null +++ b/core-api/api/app_factory.py @@ -0,0 +1,259 @@ +"""Shared FastAPI app factories for the main API and webhook ingress.""" + +from __future__ import annotations + +import logging +import time +import traceback +from datetime import datetime + +import sentry_sdk +from fastapi import FastAPI, HTTPException, Request +from fastapi.middleware.cors import CORSMiddleware +from fastapi.responses import JSONResponse +from slowapi import _rate_limit_exceeded_handler +from slowapi.errors import RateLimitExceeded +from slowapi.middleware import SlowAPIMiddleware + +from api.config import settings +from api.rate_limit import limiter +from api.schemas import HealthResponse +from lib.supabase_client import start_supabase_request_scope, reset_supabase_request_scope + +logger = logging.getLogger(__name__) +_SENTRY_INITIALIZED = False + + +def _sentry_filter_noise(event, hint): + """Drop expected HTTP errors (4xx) from Sentry to reduce noise.""" + exc = hint.get("exc_info", (None, None, None))[1] + if isinstance(exc, HTTPException) and exc.status_code < 500: + return None + return event + + +def _ensure_sentry_initialized() -> None: + global _SENTRY_INITIALIZED + if _SENTRY_INITIALIZED: + return + + sentry_sdk.init( + dsn=settings.sentry_dsn, + environment=settings.api_env, + traces_sample_rate=0.05, + send_default_pii=True, + before_send=_sentry_filter_noise, + ) + _SENTRY_INITIALIZED = True + + +def _install_exception_handler(app: FastAPI) -> None: + async def global_exception_handler(request: Request, exc: Exception): + logger.error( + f"Unhandled exception on {request.method} {request.url.path}: {exc}\n" + f"{''.join(traceback.format_exception(type(exc), exc, exc.__traceback__))}" + ) + sentry_sdk.capture_exception(exc) + return JSONResponse( + status_code=500, + content={ + "detail": "Internal server error", + "error_type": type(exc).__name__, + }, + ) + + app.add_exception_handler(Exception, global_exception_handler) + + +def _install_middlewares( + app: FastAPI, + *, + include_cors: bool, + include_rate_limit: bool, +) -> None: + if include_rate_limit: + app.state.limiter = limiter + app.add_middleware(SlowAPIMiddleware) + app.add_exception_handler(RateLimitExceeded, _rate_limit_exceeded_handler) + + if include_cors: + app.add_middleware( + CORSMiddleware, + allow_origins=settings.get_allowed_origins, + allow_credentials=True, + allow_methods=["GET", "POST", "PUT", "PATCH", "DELETE", "OPTIONS"], + allow_headers=["*"], + ) + + @app.middleware("http") + async def supabase_request_scope_middleware(request: Request, call_next): + scope_token = start_supabase_request_scope() + try: + return await call_next(request) + finally: + reset_supabase_request_scope(scope_token) + + @app.middleware("http") + async def security_headers_middleware(request: Request, call_next): + response = await call_next(request) + response.headers["X-Content-Type-Options"] = "nosniff" + response.headers["X-Frame-Options"] = "DENY" + response.headers["Strict-Transport-Security"] = "max-age=31536000; includeSubDomains" + response.headers["Referrer-Policy"] = "strict-origin-when-cross-origin" + response.headers["Permissions-Policy"] = "camera=(), microphone=(), geolocation=()" + return response + + @app.middleware("http") + async def timing_middleware(request: Request, call_next): + start_time = time.perf_counter() + + try: + response = await call_next(request) + except Exception as exc: + process_time_ms = (time.perf_counter() - start_time) * 1000 + logger.error( + f"[PERF] {request.method} {request.url.path} - {process_time_ms:.2f}ms - " + f"EXCEPTION: {type(exc).__name__}" + ) + raise + + process_time_ms = (time.perf_counter() - start_time) * 1000 + response.headers["X-Process-Time-Ms"] = f"{process_time_ms:.2f}" + logger.info( + f"[PERF] {request.method} {request.url.path} - {process_time_ms:.2f}ms - " + f"Status: {response.status_code}" + ) + return response + + +def _install_health_routes( + app: FastAPI, + *, + service_name: str, + root_message: str, +) -> None: + @app.get("/", response_model=HealthResponse) + async def root(): + return { + "status": "healthy", + "service": service_name, + "message": root_message, + "version": settings.app_version, + } + + @app.get("/api/health", response_model=HealthResponse) + async def health_check(): + return { + "status": "healthy", + "service": service_name, + "timestamp": datetime.utcnow().isoformat() + "Z", + } + + +def _create_base_app( + *, + service_name: str, + description: str, + root_message: str, + include_cors: bool, + include_rate_limit: bool, +) -> FastAPI: + _ensure_sentry_initialized() + + app = FastAPI( + title=settings.app_name, + description=description, + version=settings.app_version, + debug=settings.debug, + ) + + _install_exception_handler(app) + _install_middlewares( + app, + include_cors=include_cors, + include_rate_limit=include_rate_limit, + ) + _install_health_routes( + app, + service_name=service_name, + root_message=root_message, + ) + return app + + +def create_full_app() -> FastAPI: + """Build the full application used by the main API deployment.""" + from api.routers import ( + app_drawer, + auth, + builder, + calendar, + chat, + chat_attachments, + cron, + documents, + email, + files, + init, + invitations, + messages, + notifications, + permissions, + preferences, + projects, + public, + sync, + users, + webhooks, + workers, + workspaces, + ) + + app = _create_base_app( + service_name="core-api", + description="FastAPI backend for the all-in-one productivity app", + root_message="Core Productivity API is running", + include_cors=True, + include_rate_limit=True, + ) + + app.include_router(auth.router) + app.include_router(workspaces.router) + app.include_router(invitations.router) + app.include_router(calendar.router) + app.include_router(email.router) + app.include_router(documents.router) + app.include_router(files.router) + if settings.enable_webhook_routes: + app.include_router(webhooks.router) + app.include_router(cron.router) + app.include_router(sync.router) + app.include_router(chat.router) + app.include_router(chat_attachments.router) + app.include_router(app_drawer.router) + app.include_router(preferences.router) + app.include_router(messages.router) + app.include_router(users.router) + app.include_router(projects.router) + app.include_router(notifications.router) + app.include_router(permissions.router) + app.include_router(init.router) + app.include_router(public.router) + app.include_router(workers.router) + app.include_router(builder.router) + return app + + +def create_webhooks_app() -> FastAPI: + """Build the minimal public webhook ingress service.""" + from api.routers import webhooks + + app = _create_base_app( + service_name="core-webhooks", + description="Dedicated public webhook ingress for provider notifications", + root_message="Core webhook ingress is running", + include_cors=False, + include_rate_limit=False, + ) + app.include_router(webhooks.router) + return app diff --git a/core-api/api/config.py b/core-api/api/config.py index d4046208..10a3fbd5 100644 --- a/core-api/api/config.py +++ b/core-api/api/config.py @@ -117,6 +117,32 @@ def get_allowed_origins(self) -> List[str]: # Webhook URLs (set in production) webhook_base_url: str = "" # Set to your deployed API URL (e.g., https://your-api.vercel.app) + enable_webhook_routes: bool = True # Disable on Vercel after ingress is cut over to Railway + google_pubsub_push_service_account_email: str = "" # Expected Pub/Sub push OIDC service account email + google_pubsub_push_audience: str = "" # Optional explicit Pub/Sub push audience override + google_calendar_webhook_secret: str = "" # Shared secret used to sign Calendar channel tokens + + @property + def normalized_webhook_base_url(self) -> str: + """Return WEBHOOK_BASE_URL without a trailing slash.""" + return self.webhook_base_url.rstrip("/") + + @property + def gmail_webhook_url(self) -> str: + """Return the public Gmail webhook URL.""" + return f"{self.normalized_webhook_base_url}/api/webhooks/gmail" + + @property + def calendar_webhook_url(self) -> str: + """Return the public Calendar webhook URL.""" + return f"{self.normalized_webhook_base_url}/api/webhooks/calendar" + + @property + def resolved_google_pubsub_push_audience(self) -> str: + """Return the expected audience for authenticated Pub/Sub pushes.""" + if self.google_pubsub_push_audience: + return self.google_pubsub_push_audience + return self.gmail_webhook_url # Cron job authentication cron_secret: str = "" # Secret for authenticating cron job requests diff --git a/core-api/api/routers/calendar.py b/core-api/api/routers/calendar.py index 88d66bc4..37f492a2 100644 --- a/core-api/api/routers/calendar.py +++ b/core-api/api/routers/calendar.py @@ -22,7 +22,6 @@ delete_event, respond_to_event, ) -from api.services.syncs.sync_google_calendar import sync_google_calendar from api.dependencies import get_current_user_jwt, get_current_user_id from api.exceptions import handle_api_exception import logging @@ -168,6 +167,7 @@ class CalendarSyncResponse(BaseModel): total_events: Optional[int] = None total_fetched: Optional[int] = None jobs_enqueued: Optional[int] = None + streams_marked: Optional[int] = None class Config: extra = "allow" @@ -413,57 +413,70 @@ async def sync_google_calendar_endpoint( """ Sync calendar events from connected providers. - When QStash is configured, enqueues per-connection sync jobs and returns - 202 Accepted immediately. Falls back to inline processing otherwise. + Marks per-connection calendar streams dirty and returns 202 immediately. + + Sync execution happens on the background worker plane. Requires: Authorization header with user's Supabase JWT """ try: - from lib.queue import queue_client from lib.supabase_client import get_authenticated_supabase_client + from lib.supabase_client import get_service_role_client from fastapi.responses import JSONResponse + from api.services.syncs.sync_dispatcher import MANUAL_SYNC_PRIORITY, mark_stream_dirty + from api.services.syncs.sync_state_store import SYNC_KIND_CALENDAR + + auth_supabase = get_authenticated_supabase_client(user_jwt) + service_supabase = get_service_role_client() + connections_result = auth_supabase.table('ext_connections')\ + .select('id, provider')\ + .eq('user_id', user_id)\ + .eq('is_active', True)\ + .in_('provider', ['google', 'microsoft'])\ + .execute() + + connections = connections_result.data or [] + streams_marked = 0 + + for conn in connections: + cid = conn.get('id') + if not cid: + continue + if conn.get('provider') == 'google': + if mark_stream_dirty( + service_supabase, + cid, + "google", + SYNC_KIND_CALENDAR, + priority=MANUAL_SYNC_PRIORITY, + metadata={"source": "manual-calendar-sync"}, + ): + streams_marked += 1 + elif conn.get('provider') == 'microsoft': + if mark_stream_dirty( + service_supabase, + cid, + "microsoft", + SYNC_KIND_CALENDAR, + priority=MANUAL_SYNC_PRIORITY, + metadata={"source": "manual-calendar-sync"}, + ): + streams_marked += 1 + + if streams_marked <= 0: + raise HTTPException( + status_code=status.HTTP_503_SERVICE_UNAVAILABLE, + detail="Calendar sync is temporarily unavailable. Please try again shortly.", + ) - # --- Queue path --- - if queue_client.available: - auth_supabase = get_authenticated_supabase_client(user_jwt) - connections_result = auth_supabase.table('ext_connections')\ - .select('id, provider')\ - .eq('user_id', user_id)\ - .eq('is_active', True)\ - .in_('provider', ['google', 'microsoft'])\ - .execute() - - connections = connections_result.data or [] - jobs_enqueued = 0 - - for conn in connections: - cid = conn.get('id') - if not cid: - continue - if conn.get('provider') == 'google': - if queue_client.enqueue_sync_for_connection(cid, "sync-calendar"): - jobs_enqueued += 1 - elif conn.get('provider') == 'microsoft': - if queue_client.enqueue_sync_for_connection(cid, "sync-outlook-calendar"): - jobs_enqueued += 1 - - if jobs_enqueued > 0: - logger.info(f"βœ… Enqueued {jobs_enqueued} calendar sync jobs for user {user_id[:8]}...") - return JSONResponse( - status_code=202, - content={ - "status": "queued", - "jobs_enqueued": jobs_enqueued, - } - ) - - # All enqueues failed β€” fall through to inline processing - logger.warning(f"⚠️ QStash available but all publishes failed for user {user_id[:8]}..., falling back to inline") - - # --- Fallback: inline processing --- - logger.info(f"πŸ”„ Syncing Google Calendar for user {user_id}") - result = sync_google_calendar(user_id, user_jwt) - logger.info(f"βœ… Sync completed for user {user_id}") - return result + logger.info(f"βœ… Marked {streams_marked} calendar streams dirty for user {user_id[:8]}...") + return JSONResponse( + status_code=202, + content={ + "status": "accepted", + "jobs_enqueued": streams_marked, + "streams_marked": streams_marked, + } + ) except Exception as e: handle_api_exception(e, "Failed to sync calendar", logger) diff --git a/core-api/api/routers/cron.py b/core-api/api/routers/cron.py index 3531b939..47c10aa6 100644 --- a/core-api/api/routers/cron.py +++ b/core-api/api/routers/cron.py @@ -5,29 +5,35 @@ CRON JOB SCHEDULE: ================== -1. /api/cron/incremental-sync (Every 15 minutes) - - Safety net for missed webhook notifications - - Runs incremental sync for all active users - - Catches any emails/events that webhooks missed +1. /api/cron/incremental-sync (Disabled rollback stub) + - Broad sweep reconciliation has been replaced by per-stream scheduling + - Route remains available as a fast rollback hook during migration + - Should not be scheduled in Vercel cron -2. /api/cron/renew-watches (Every 6 hours) +2. /api/cron/renew-watches (Every hour) - CRITICAL: Prevents watch subscriptions from expiring - Gmail watches expire after 7 days - Calendar watches expire after configured time - Automatically renews watches before they expire + - Batch size configurable via RENEWAL_BATCH_SIZE (default 50) -3. /api/cron/setup-missing-watches (Every hour) - - Ensures all users have active watches - - Sets up watches for new users - - Recovers from watch setup failures +3. /api/cron/setup-missing-watches (Every 6 hours) + - Recreates watches for active connections that have no active subscription + - Catches cases where watches expired and renewal missed them + - Batch size limited to prevent thundering herd -4. /api/cron/daily-verification (Daily at 2am) +4. /api/cron/watch-health (Disabled / manual recovery) + - Queues targeted recovery work for suspiciously silent watches + - Avoids a broad fleet-wide sweep + - Capped oldest-first batch to limit blast radius + +5. /api/cron/daily-verification (Daily at 2am) - Full sync for data integrity verification - Catches any edge cases or long-term drift - Runs full sync for a subset of users each day """ from fastapi import APIRouter, HTTPException, status, Header -from typing import Optional +from typing import Any, Dict, List, Optional import hashlib import hmac import logging @@ -39,13 +45,11 @@ from lib.supabase_client import get_service_role_client from lib.token_encryption import decrypt_ext_connection_tokens from api.services.syncs import ( - sync_gmail_cron, - sync_google_calendar_cron, renew_watch_service_role, start_gmail_watch_service_role, start_calendar_watch_service_role ) -from api.services.syncs.google_error_utils import is_permanent_google_api_error +from api.services.syncs.sync_dispatcher import mark_stream_dirty from api.services.syncs.google_services import get_google_services_for_connection from api.services.microsoft.microsoft_oauth_provider import ( MicrosoftReauthRequiredError, @@ -54,10 +58,13 @@ from api.services.microsoft.microsoft_webhook_provider import renew_microsoft_subscription from pydantic import BaseModel -from typing import List logger = logging.getLogger(__name__) router = APIRouter(prefix="/api/cron", tags=["cron"]) +WATCH_PROVIDER_TO_SYNC_KIND = { + "gmail": "email", + "calendar": "calendar", +} # ============================================================================ @@ -78,6 +85,7 @@ class IncrementalSyncResponse(BaseModel): jobs_failed: Optional[int] = None batch_mode: Optional[bool] = None connections_considered: Optional[int] = None + streams_marked: Optional[int] = None class RenewWatchesResponse(BaseModel): @@ -100,6 +108,16 @@ class SetupWatchesResponse(BaseModel): errors: Optional[int] = None +class WatchHealthResponse(BaseModel): + """Response for targeted watch-health recovery cron job.""" + status: str + message: Optional[str] = None + duration_seconds: Optional[float] = None + checked: int = 0 + queued: int = 0 + errors: int = 0 + + class DailyVerificationResponse(BaseModel): """Response for daily verification cron job.""" status: str @@ -194,6 +212,156 @@ def get_cron_batch_size() -> int: return batch_size if 1 <= batch_size <= 250 else 100 +def get_reconciliation_stale_minutes() -> int: + """Return stale threshold for reconciliation in minutes. + + Streams synced more recently than this are skipped. + Default 45 min. Tunable via RECONCILIATION_STALE_MINUTES. + """ + value = os.getenv("RECONCILIATION_STALE_MINUTES", "45").strip() + try: + minutes = int(value) + except (TypeError, ValueError): + return 45 + return max(5, min(minutes, 1440)) + + +def get_renewal_batch_size() -> int: + """Return watch-renewal batch size per cron run. + + Default 50. Tunable via RENEWAL_BATCH_SIZE. + """ + value = os.getenv("RENEWAL_BATCH_SIZE", "50").strip() + try: + size = int(value) + except (TypeError, ValueError): + return 50 + return max(1, min(size, 200)) + + +def get_watch_health_batch_size() -> int: + """Return the max number of watch-health recovery marks per run.""" + value = os.getenv("WATCH_HEALTH_BATCH_SIZE", "25").strip() + try: + size = int(value) + except (TypeError, ValueError): + return 25 + return max(1, min(size, 100)) + + +def get_watch_health_stale_hours() -> int: + """Return the stale-notification threshold for active watches.""" + value = os.getenv("WATCH_HEALTH_STALE_HOURS", "24").strip() + try: + hours = int(value) + except (TypeError, ValueError): + return 24 + return max(1, min(hours, 7 * 24)) + + +def get_watch_health_initial_grace_hours() -> int: + """Return how long a never-notified watch gets before recovery.""" + value = os.getenv("WATCH_HEALTH_INITIAL_GRACE_HOURS", "12").strip() + try: + hours = int(value) + except (TypeError, ValueError): + return 12 + return max(1, min(hours, 7 * 24)) + + +def get_watch_health_priority() -> int: + """Return the priority used for watch-health dirty marks.""" + value = os.getenv("WATCH_HEALTH_PRIORITY", "25").strip() + try: + priority = int(value) + except (TypeError, ValueError): + return 25 + return max(0, min(priority, 99)) + + +def _parse_optional_datetime(value: Optional[Any]) -> Optional[datetime]: + if value is None or value == "": + return None + if isinstance(value, datetime): + return value if value.tzinfo is not None else value.replace(tzinfo=timezone.utc) + try: + parsed = datetime.fromisoformat(str(value).replace("Z", "+00:00")) + except ValueError: + return None + return parsed if parsed.tzinfo is not None else parsed.replace(tzinfo=timezone.utc) + + +def _has_pending_sync_state(sync_state: Optional[Dict[str, Any]], *, now: datetime) -> bool: + if not sync_state: + return False + if sync_state.get("dirty"): + return True + if int(sync_state.get("retry_count") or 0) > 0: + return True + lease_expires_at = _parse_optional_datetime(sync_state.get("lease_expires_at")) + return bool(lease_expires_at and lease_expires_at > now) + + +def _classify_watch_health_candidate( + watch: Dict[str, Any], + sync_state: Optional[Dict[str, Any]], + *, + now: datetime, + stale_after: timedelta, + initial_grace_after: timedelta, +) -> Optional[str]: + """ + Return a recovery reason when a watch looks silently unhealthy. + + This intentionally avoids a blanket "no notification for N minutes" rule. + Quiet inboxes/calendars are normal. We only target watches that either used + to notify and then went silent, or have never notified after a long grace + window, and only when the corresponding sync stream is not already active. + """ + if watch.get("provider") not in WATCH_PROVIDER_TO_SYNC_KIND: + return None + + expiration = _parse_optional_datetime(watch.get("expiration")) + if expiration and expiration <= now + timedelta(hours=24): + return None + + if _has_pending_sync_state(sync_state, now=now): + return None + + last_sync_finished_at = _parse_optional_datetime((sync_state or {}).get("last_sync_finished_at")) + notification_count = int(watch.get("notification_count") or 0) + last_notification_or_creation = ( + _parse_optional_datetime(watch.get("last_notification_at")) + or _parse_optional_datetime(watch.get("created_at")) + ) + + if notification_count > 0: + if ( + last_notification_or_creation + and last_notification_or_creation <= now - stale_after + and ( + last_sync_finished_at is None + or last_sync_finished_at <= now - stale_after + ) + ): + return "stale_notifications" + return None + + created_at = _parse_optional_datetime(watch.get("created_at")) + if ( + notification_count == 0 + and created_at + and created_at <= now - initial_grace_after + and ( + last_sync_finished_at is None + or last_sync_finished_at <= now - initial_grace_after + ) + ): + return "never_notified" + + return None + + def _get_batch_bucket(now: Optional[datetime] = None) -> str: timestamp = now or datetime.now(timezone.utc) return timestamp.strftime("%Y%m%d%H%M") @@ -218,299 +386,34 @@ def _chunk_connection_ids(connection_ids: List[str], batch_size: int) -> List[Li @router.get("/incremental-sync", response_model=IncrementalSyncResponse) async def cron_incremental_sync(authorization: str = Header(None)): """ - CRON JOB: Incremental sync for all active users - - RUNS: Every 15 minutes - - PURPOSE: Safety net to catch any missed webhook notifications - - Runs incremental sync for all users with active connections - - Only syncs emails/events since last sync (efficient) - - Ensures no data is lost if webhooks fail - - This job processes users in batches to handle rate limits gracefully. - - NOTE: Uses GET because Vercel cron jobs send GET requests by default + Disabled rollback stub for the old broad-sweep reconciliation cron. + + Per-stream safety-net reconciliation is now scheduled via + connection_sync_state.next_reconcile_at and claimed directly by workers. """ logger.info("=" * 80) - logger.info("πŸ• CRON: Starting incremental sync for all users") + logger.info("πŸ• CRON: Incremental sweep endpoint invoked") logger.info(f"⏰ Timestamp: {datetime.now(timezone.utc).isoformat()}") logger.info(f"πŸ”‘ Authorization header present: {bool(authorization)}") logger.info(f"🌍 Environment: {settings.api_env}") - + # Verify authorization if not verify_cron_auth(authorization): logger.warning("⚠️ Unauthorized cron attempt - authorization failed") raise HTTPException(status_code=status.HTTP_401_UNAUTHORIZED, detail="Unauthorized") - - logger.info("βœ… Authorization verified") - start_time = datetime.now(timezone.utc) - - check_in_id = capture_checkin(monitor_slug="incremental-sync", status=MonitorStatus.IN_PROGRESS) - - try: - from lib.queue import queue_client - - # Use service role client to access all user connections - service_supabase = get_service_role_client() - - # Get all active connections (Google + Microsoft) - connections = service_supabase.table('ext_connections')\ - .select('user_id, id, last_synced, provider')\ - .in_('provider', ['google', 'microsoft'])\ - .eq('is_active', True)\ - .execute() - - if not connections.data: - logger.info("ℹ️ No active connections to sync") - capture_checkin(monitor_slug="incremental-sync", check_in_id=check_in_id, status=MonitorStatus.OK) - return { - "status": "completed", - "message": "No active connections", - "users_processed": 0 - } - - total_connections = len(connections.data) - logger.info(f"πŸ‘₯ Found {total_connections} active connections to sync") - - jobs_enqueued = 0 - jobs_failed = 0 - skipped_count = 0 - error_count = 0 - success_count = 0 - - # Pre-filter stale connections once so all modes share skip behavior. - stale_connections = [] - for conn in connections.data: - last_synced = conn.get('last_synced') - if last_synced: - last_sync_dt = datetime.fromisoformat(last_synced.replace('Z', '+00:00')) - if datetime.now(timezone.utc) - last_sync_dt < timedelta(minutes=10): - skipped_count += 1 - continue - stale_connections.append(conn) - - if not stale_connections: - duration = (datetime.now(timezone.utc) - start_time).total_seconds() - capture_checkin(monitor_slug="incremental-sync", check_in_id=check_in_id, status=MonitorStatus.OK) - return { - "status": "completed", - "message": "No stale connections to sync", - "duration_seconds": duration, - "total_users": total_connections, - "success": 0, - "skipped": skipped_count, - "errors": 0, - "jobs_enqueued": 0, - "jobs_failed": 0, - } - - use_queue = queue_client.available - batch_mode = is_cron_batch_mode_enabled() - - # --- Queue path: batched fanout (default) --- - if use_queue and batch_mode: - google_ids = sorted(str(c['id']) for c in stale_connections if c.get('provider') == 'google') - microsoft_ids = sorted(str(c['id']) for c in stale_connections if c.get('provider') == 'microsoft') - scheduled_connection_ids = set() - dedup_bucket = _get_batch_bucket() - batch_size = get_cron_batch_size() - - batch_jobs = [ - ("sync-gmail", google_ids), - ("sync-calendar", google_ids), - ("sync-outlook", microsoft_ids), - ("sync-outlook-calendar", microsoft_ids), - ] - - for job_type, ids in batch_jobs: - if not ids: - continue - for chunk_ids in _chunk_connection_ids(ids, batch_size): - batch_token = _build_batch_token(chunk_ids, dedup_bucket) - dedup_id = f"batch-{job_type}-{batch_token}" - if queue_client.enqueue_batch( - job_type, - chunk_ids, - extra={"batch_token": batch_token}, - dedup_id=dedup_id, - ): - jobs_enqueued += 1 - scheduled_connection_ids.update(chunk_ids) - else: - jobs_failed += 1 - error_count += 1 - logger.warning( - f"⚠️ Queue publish failed for batch job {job_type} " - f"({len(chunk_ids)} IDs, token={batch_token})" - ) - - success_count = len(scheduled_connection_ids) - - else: - # --- Legacy path: per-connection queue OR inline fallback --- - for conn in stale_connections: - user_id = conn['user_id'] - provider = conn.get('provider') - connection_id = conn['id'] - - try: - if use_queue: - conn_enqueued = 0 - conn_failed = 0 - if provider == 'google': - if queue_client.enqueue_sync_for_connection(connection_id, "sync-gmail"): - conn_enqueued += 1 - else: - conn_failed += 1 - if queue_client.enqueue_sync_for_connection(connection_id, "sync-calendar"): - conn_enqueued += 1 - else: - conn_failed += 1 - elif provider == 'microsoft': - if queue_client.enqueue_sync_for_connection(connection_id, "sync-outlook"): - conn_enqueued += 1 - else: - conn_failed += 1 - if queue_client.enqueue_sync_for_connection(connection_id, "sync-outlook-calendar"): - conn_enqueued += 1 - else: - conn_failed += 1 - - jobs_enqueued += conn_enqueued - jobs_failed += conn_failed - - if conn_enqueued > 0: - success_count += 1 - - if conn_failed > 0: - logger.warning( - f"⚠️ Queue publish failures for connection {connection_id[:8]}...: " - f"{conn_failed} failed, {conn_enqueued} enqueued" - ) - error_count += conn_failed - continue - - # Inline fallback path (legacy behavior) - if provider == 'microsoft': - skipped_count += 1 - continue - - logger.info(f"πŸ”„ Syncing connection {connection_id[:8]}... (user {user_id[:8]}...)") - - gmail_service, calendar_service, _ = get_google_services_for_connection( - connection_id, - service_supabase - ) - - if not gmail_service and not calendar_service: - logger.warning(f"⚠️ Could not get Google services for connection {connection_id[:8]}...") - skipped_count += 1 - continue - - synced_gmail = False - synced_calendar = False - - if gmail_service: - try: - result = sync_gmail_cron( - gmail_service=gmail_service, - connection_id=connection_id, - user_id=user_id, - service_supabase=service_supabase, - days_back=7 - ) - if result.get('status') == 'success': - synced_gmail = True - else: - if is_permanent_google_api_error(result.get('error')): - logger.warning(f"⚠️ Gmail sync permanently unavailable for user {user_id[:8]}...: {result.get('error')}") - else: - logger.error(f"❌ Gmail sync returned error: {result.get('error')}") - except Exception as e: - if is_permanent_google_api_error(e): - logger.warning(f"⚠️ Gmail sync permanently unavailable for user {user_id[:8]}...: {str(e)}") - else: - logger.error(f"❌ Gmail sync failed for user {user_id[:8]}...: {str(e)}") - logger.exception("Full traceback:") - - if calendar_service: - try: - result = sync_google_calendar_cron( - calendar_service=calendar_service, - connection_id=connection_id, - user_id=user_id, - service_supabase=service_supabase, - days_past=30, - days_future=90 - ) - if result.get('status') == 'success': - synced_calendar = True - else: - if is_permanent_google_api_error(result.get('error')): - logger.warning(f"⚠️ Calendar sync permanently unavailable for user {user_id[:8]}...: {result.get('error')}") - else: - logger.error(f"❌ Calendar sync returned error: {result.get('error')}") - except Exception as e: - if is_permanent_google_api_error(e): - logger.warning(f"⚠️ Calendar sync permanently unavailable for user {user_id[:8]}...: {str(e)}") - else: - logger.error(f"❌ Calendar sync failed for user {user_id[:8]}...: {str(e)}") - logger.exception("Full traceback:") - - if synced_gmail or synced_calendar: - service_supabase.table('ext_connections')\ - .update({'last_synced': datetime.now(timezone.utc).isoformat()})\ - .eq('id', connection_id)\ - .execute() - success_count += 1 - else: - skipped_count += 1 - - except Exception as e: - logger.error(f"❌ Error syncing user {user_id[:8]}...: {str(e)}") - error_count += 1 - continue - - duration = (datetime.now(timezone.utc) - start_time).total_seconds() - - if use_queue: - logger.info(f"βœ… CRON: Enqueued {jobs_enqueued} sync jobs in {duration:.2f}s") - else: - logger.info(f"βœ… CRON: Incremental sync completed in {duration:.2f}s") - logger.info( - f"πŸ“Š Results: {success_count} success, {skipped_count} skipped, {error_count} errors, " - f"{jobs_enqueued} enqueued, {jobs_failed} failed enqueues" - ) - logger.info("=" * 80) - - # Mark check-in as degraded whenever queue publishing has failures. - checkin_status = MonitorStatus.OK - if use_queue and jobs_failed > 0: - checkin_status = MonitorStatus.ERROR - capture_checkin(monitor_slug="incremental-sync", check_in_id=check_in_id, status=checkin_status) - return { - "status": "completed", - "duration_seconds": duration, - "total_users": total_connections, - "success": success_count, - "skipped": skipped_count, - "errors": error_count, - "jobs_enqueued": jobs_enqueued, - "jobs_failed": jobs_failed, - "batch_mode": bool(use_queue and batch_mode), - "connections_considered": len(stale_connections), - } - - except Exception as e: - capture_checkin(monitor_slug="incremental-sync", check_in_id=check_in_id, status=MonitorStatus.ERROR) - logger.error(f"❌ CRON: Incremental sync failed: {str(e)}") - logger.exception("Full traceback:") - logger.info("=" * 80) - raise HTTPException( - status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, - detail=f"Sync failed: {str(e)}" - ) + logger.info("βœ… Authorization verified") + logger.info("ℹ️ Incremental sweep remains disabled; workers reconcile via next_reconcile_at") + logger.info("=" * 80) + return { + "status": "disabled", + "message": "Broad sweep reconciliation has been replaced by worker-driven next_reconcile_at scheduling", + "users_processed": 0, + "jobs_enqueued": 0, + "jobs_failed": 0, + "streams_marked": 0, + "batch_mode": False, + } @router.get("/renew-watches", response_model=RenewWatchesResponse) @@ -518,14 +421,16 @@ async def cron_renew_watches(authorization: str = Header(None)): """ CRON JOB: Renew expiring watch subscriptions - RUNS: Every 6 hours - + RUNS: Every hour (configurable via vercel.json) + + PURPOSE: CRITICAL - Prevents watch subscriptions from expiring - Gmail watches expire after 7 days - Calendar watches expire after configured time - This job renews any watches expiring within 24 hours + - Batch size is configurable via RENEWAL_BATCH_SIZE (default 50) - Ensures continuous real-time notifications - + Without this job, push notifications will stop working after 7 days! NOTE: Uses GET because Vercel cron jobs send GET requests by default @@ -548,13 +453,20 @@ async def cron_renew_watches(authorization: str = Header(None)): # Use service role to access all subscriptions service_supabase = get_service_role_client() - # Get subscriptions expiring within 24 hours + # Get subscriptions expiring within 24 hours. + # Limit batch size and renew soonest-first to smooth provider load + # across hourly runs instead of renewing large batches all at once. + # At 1.4k users with ~2 watches each, 50/hour provides 1200 renewals + # per day, which comfortably covers the steady-state requirement. + RENEWAL_BATCH_SIZE = get_renewal_batch_size() threshold_time = datetime.now(timezone.utc) + timedelta(hours=24) result = service_supabase.table('push_subscriptions')\ .select('*, ext_connections!push_subscriptions_ext_connection_id_fkey!inner(id, user_id, is_active, access_token, refresh_token, token_expires_at, metadata)')\ .eq('is_active', True)\ .lt('expiration', threshold_time.isoformat())\ + .order('expiration', desc=False)\ + .limit(RENEWAL_BATCH_SIZE)\ .execute() expiring_subs = result.data @@ -722,13 +634,13 @@ async def cron_setup_missing_watches(authorization: str = Header(None)): """ CRON JOB: Set up watches for users who don't have them - RUNS: Every hour + RUNS: Disabled / manual recovery - PURPOSE: Ensures all users have active watch subscriptions + PURPOSE: Controlled recovery path for missing watch subscriptions - Sets up watches for new users who just connected Google - Recovers from watch setup failures - - Ensures no users are left without push notifications - + - Intentionally left unscheduled during backlog stabilization + NOTE: Uses GET because Vercel cron jobs send GET requests by default """ logger.info("=" * 80) @@ -768,6 +680,12 @@ async def cron_setup_missing_watches(authorization: str = Header(None)): setup_count = 0 error_count = 0 + # Limit how many watches we set up per cron run to avoid a burst + # of Google notifications when all watches fire their initial sync. + # At 20/hour, full recovery from a mass deactivation takes a few + # hours instead of creating a thundering herd. + SETUP_BATCH_SIZE = 50 + for conn in connections.data: user_id = conn['user_id'] connection_id = conn['id'] @@ -792,6 +710,10 @@ async def cron_setup_missing_watches(authorization: str = Header(None)): needs_setup = not gmail_watch.data or not calendar_watch.data + if needs_setup and setup_count + error_count >= SETUP_BATCH_SIZE: + logger.info(f"⏸️ Batch limit reached ({SETUP_BATCH_SIZE}), deferring remaining setups to next run") + break + if needs_setup: logger.info(f"πŸ”§ Setting up watches for connection {connection_id[:8]}... (user {user_id[:8]}...)") @@ -873,6 +795,164 @@ async def cron_setup_missing_watches(authorization: str = Header(None)): ) +@router.get("/watch-health", response_model=WatchHealthResponse) +async def cron_watch_health(authorization: str = Header(None)): + """ + CRON JOB: Queue targeted recovery work for suspiciously silent watches. + + This is intentionally not a blanket "no notification in N minutes" sweep. + It only marks streams dirty when the watch looks stale and the stream is + otherwise idle, so quiet inboxes/calendars are not spam-synced. + """ + logger.info("=" * 80) + logger.info("🩺 CRON: Starting watch-health recovery scan") + logger.info(f"⏰ Timestamp: {datetime.now(timezone.utc).isoformat()}") + + if not verify_cron_auth(authorization): + logger.warning("⚠️ Unauthorized cron attempt") + raise HTTPException(status_code=status.HTTP_401_UNAUTHORIZED, detail="Unauthorized") + + start_time = datetime.now(timezone.utc) + check_in_id = capture_checkin(monitor_slug="watch-health", status=MonitorStatus.IN_PROGRESS) + + try: + service_supabase = get_service_role_client() + now = datetime.now(timezone.utc) + stale_after = timedelta(hours=get_watch_health_stale_hours()) + initial_grace_after = timedelta(hours=get_watch_health_initial_grace_hours()) + batch_size = get_watch_health_batch_size() + priority = get_watch_health_priority() + + watch_result = service_supabase.table("push_subscriptions")\ + .select( + "id, ext_connection_id, provider, created_at, expiration, " + "notification_count, last_notification_at" + )\ + .in_("provider", list(WATCH_PROVIDER_TO_SYNC_KIND.keys()))\ + .eq("is_active", True)\ + .execute() + + active_watches = watch_result.data or [] + if not active_watches: + duration = (datetime.now(timezone.utc) - start_time).total_seconds() + capture_checkin(monitor_slug="watch-health", check_in_id=check_in_id, status=MonitorStatus.OK) + return { + "status": "completed", + "message": "No active Google watches", + "duration_seconds": duration, + "checked": 0, + "queued": 0, + "errors": 0, + } + + connection_ids = sorted( + { + str(watch["ext_connection_id"]) + for watch in active_watches + if watch.get("ext_connection_id") + } + ) + + state_by_key: Dict[tuple[str, str], Dict[str, Any]] = {} + if connection_ids: + state_result = service_supabase.table("connection_sync_state")\ + .select( + "connection_id, provider, sync_kind, dirty, retry_count, " + "next_retry_at, lease_expires_at, last_sync_finished_at" + )\ + .in_("connection_id", connection_ids)\ + .in_("sync_kind", list(WATCH_PROVIDER_TO_SYNC_KIND.values()))\ + .execute() + + state_by_key = { + (row["connection_id"], row["sync_kind"]): row + for row in (state_result.data or []) + } + + queued = 0 + errors = 0 + ordered_watches = sorted( + active_watches, + key=lambda watch: ( + _parse_optional_datetime(watch.get("last_notification_at")) + or _parse_optional_datetime(watch.get("created_at")) + or now + ), + ) + + for watch in ordered_watches: + if queued + errors >= batch_size: + break + + connection_id = watch.get("ext_connection_id") + watch_provider = watch.get("provider") + sync_kind = WATCH_PROVIDER_TO_SYNC_KIND.get(watch_provider) + if not connection_id or not sync_kind: + continue + + sync_state = state_by_key.get((connection_id, sync_kind)) + reason = _classify_watch_health_candidate( + watch, + sync_state, + now=now, + stale_after=stale_after, + initial_grace_after=initial_grace_after, + ) + if not reason: + continue + + stream_provider = (sync_state or {}).get("provider") or "google" + try: + marked = mark_stream_dirty( + service_supabase, + connection_id, + stream_provider, + sync_kind, + priority=priority, + metadata={ + "source": "watch-health-recovery", + "reason": reason, + "watch_id": watch.get("id"), + "watch_provider": watch_provider, + }, + ) + if marked: + queued += 1 + logger.info( + "🩺 queued watch-health recovery for %s/%s (%s)", + connection_id[:8], + sync_kind, + reason, + ) + except Exception: + errors += 1 + logger.exception( + "❌ Failed watch-health recovery mark for %s/%s", + connection_id[:8], + sync_kind, + ) + + duration = (datetime.now(timezone.utc) - start_time).total_seconds() + capture_checkin(monitor_slug="watch-health", check_in_id=check_in_id, status=MonitorStatus.OK) + return { + "status": "completed", + "duration_seconds": duration, + "checked": len(active_watches), + "queued": queued, + "errors": errors, + } + + except Exception as e: + capture_checkin(monitor_slug="watch-health", check_in_id=check_in_id, status=MonitorStatus.ERROR) + logger.error(f"❌ CRON: Watch health scan failed: {str(e)}") + logger.exception("Full traceback:") + logger.info("=" * 80) + raise HTTPException( + status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, + detail=f"Watch health scan failed: {str(e)}" + ) + + @router.get("/daily-verification", response_model=DailyVerificationResponse) async def cron_daily_verification(authorization: str = Header(None)): """ @@ -1229,18 +1309,23 @@ async def cron_health(): "jobs": [ { "name": "incremental-sync", - "schedule": "Every 15 minutes", - "description": "Safety net for missed webhooks" + "schedule": "Disabled", + "description": "Broad sweep replaced by worker-driven next_reconcile_at reconciliation" }, { "name": "renew-watches", - "schedule": "Every 6 hours", + "schedule": "Every hour", "description": "CRITICAL: Renews expiring watch subscriptions" }, { "name": "setup-missing-watches", - "schedule": "Every hour", - "description": "Ensures all users have active watches" + "schedule": "Disabled / manual recovery", + "description": "Controlled recovery path for restoring missing watches" + }, + { + "name": "watch-health", + "schedule": "Disabled / manual recovery", + "description": "Queues targeted recovery work for suspiciously silent watches" }, { "name": "daily-verification", diff --git a/core-api/api/routers/email.py b/core-api/api/routers/email.py index cf7b86c4..e1cb9d32 100644 --- a/core-api/api/routers/email.py +++ b/core-api/api/routers/email.py @@ -28,7 +28,6 @@ search_emails_with_providers, fetch_remote_email, ) -from api.services.syncs import sync_gmail, sync_outlook from api.dependencies import get_current_user_jwt, get_current_user_id from api.exceptions import handle_api_exception from lib.supabase_client import get_authenticated_supabase_client, get_authenticated_async_client @@ -156,6 +155,7 @@ class EmailSyncResponse(BaseModel): updated_emails: Optional[int] = None ai_analyzed_count: Optional[int] = None jobs_enqueued: Optional[int] = None + streams_marked: Optional[int] = None class Config: extra = "allow" @@ -964,21 +964,24 @@ async def sync_emails_endpoint( user_id: str = Depends(get_current_user_id) ): """ - Sync emails from connected providers (Google and Microsoft) and run AI analysis. + Sync emails from connected providers (Google and Microsoft). - When QStash is configured, enqueues per-connection sync jobs and returns - 202 Accepted immediately. Falls back to inline processing otherwise. + Marks per-connection email streams dirty and returns 202 immediately. + + Sync execution happens on the background worker plane. Requires: Authorization header with user's Supabase JWT """ logger.info(f"πŸ“§ Email sync requested for user {user_id[:8]}...") - from api.services.email.analyze_email_ai import analyze_unanalyzed_emails - from lib.queue import queue_client + from lib.supabase_client import get_service_role_client from fastapi.responses import JSONResponse + from api.services.syncs.sync_dispatcher import MANUAL_SYNC_PRIORITY, mark_stream_dirty + from api.services.syncs.sync_state_store import SYNC_KIND_EMAIL try: auth_supabase = get_authenticated_supabase_client(user_jwt) + service_supabase = get_service_role_client() connections_result = auth_supabase.table('ext_connections')\ .select('id, provider, provider_email, access_token, refresh_token, token_expires_at, metadata, is_primary')\ .eq('user_id', user_id)\ @@ -993,153 +996,51 @@ async def sync_emails_endpoint( if not google_connections and not microsoft_connections: raise ValueError("No active email connection found. Please sign in with Google or Microsoft first.") - # --- Queue path: enqueue per-connection jobs and return 202 --- - if queue_client.available: - jobs_enqueued = 0 - - for conn in google_connections: - if queue_client.enqueue_sync_for_connection(conn['id'], "sync-gmail"): - jobs_enqueued += 1 - - for conn in microsoft_connections: - if conn.get('id'): - if queue_client.enqueue_sync_for_connection(conn['id'], "sync-outlook"): - jobs_enqueued += 1 - - if jobs_enqueued > 0: - # Enqueue email analysis only when sync jobs were accepted - queue_client.enqueue( - "analyze-emails", - {"user_id": user_id, "limit": 100}, - dedup_id=f"analyze-emails-{user_id}", - ) - - logger.info(f"βœ… Enqueued {jobs_enqueued} sync jobs for user {user_id[:8]}...") - return JSONResponse( - status_code=202, - content={ - "status": "queued", - "jobs_enqueued": jobs_enqueued, - "new_emails": 0, - "updated_emails": 0, - } - ) - - # All enqueues failed β€” fall through to inline processing - logger.warning(f"⚠️ QStash available but all publishes failed for user {user_id[:8]}..., falling back to inline") - - # --- Fallback path: inline processing (existing behavior) --- - logger.info( - f"πŸ”„ Starting provider sync for user {user_id[:8]}... " - f"({len(google_connections)} Google, {len(microsoft_connections)} Microsoft)" - ) - - result: Dict[str, Any] = { - "new_emails": 0, - "updated_emails": 0, - "provider_results": {}, - "providers_synced": [] - } - sync_errors: List[Dict[str, str]] = [] - - # Google sync (existing behavior: sync primary/selected Google connection) - if google_connections: - try: - google_result = await asyncio.to_thread(sync_gmail, user_id, user_jwt) - result["provider_results"]["google"] = google_result - result["new_emails"] += int(google_result.get('new_emails') or 0) - result["updated_emails"] += int(google_result.get('updated_emails') or 0) - result["providers_synced"].append("google") - except Exception as e: - error_msg = str(e) - logger.error(f"❌ Google sync failed for user {user_id[:8]}...: {error_msg}") - sync_errors.append({"provider": "google", "error": error_msg}) - result["provider_results"]["google"] = {"success": False, "error": error_msg} - - # Microsoft sync (all active Microsoft connections) - if microsoft_connections: - microsoft_result: Dict[str, Any] = { + streams_marked = 0 + + for conn in google_connections: + if mark_stream_dirty( + service_supabase, + conn['id'], + "google", + SYNC_KIND_EMAIL, + priority=MANUAL_SYNC_PRIORITY, + metadata={"source": "manual-email-sync"}, + ): + streams_marked += 1 + + for conn in microsoft_connections: + if conn.get('id'): + if mark_stream_dirty( + service_supabase, + conn['id'], + "microsoft", + SYNC_KIND_EMAIL, + priority=MANUAL_SYNC_PRIORITY, + metadata={"source": "manual-email-sync"}, + ): + streams_marked += 1 + + if streams_marked <= 0: + logger.error( + f"❌ Failed to mark any email streams dirty for user {user_id[:8]}..." + ) + raise HTTPException( + status_code=status.HTTP_503_SERVICE_UNAVAILABLE, + detail="Email sync is temporarily unavailable. Please try again shortly.", + ) + + logger.info(f"βœ… Marked {streams_marked} email streams dirty for user {user_id[:8]}...") + return JSONResponse( + status_code=202, + content={ + "status": "accepted", + "jobs_enqueued": streams_marked, + "streams_marked": streams_marked, "new_emails": 0, "updated_emails": 0, - "accounts": [] } - microsoft_had_success = False - - for conn in microsoft_connections: - connection_id = conn.get('id') - connection_email = conn.get('provider_email') or 'unknown' - - if not connection_id: - error_msg = "Missing connection id" - logger.error(f"❌ Microsoft sync skipped: {error_msg}") - microsoft_result["accounts"].append({ - "connection_id": None, - "email": connection_email, - "success": False, - "error": error_msg - }) - sync_errors.append({"provider": "microsoft", "error": error_msg}) - continue - - try: - conn_result = await asyncio.to_thread( - sync_outlook, - user_id=user_id, - connection_id=connection_id, - connection_data=conn - ) - new_count = int(conn_result.get('new_emails') or 0) - updated_count = int(conn_result.get('updated_emails') or 0) - - microsoft_result["new_emails"] += new_count - microsoft_result["updated_emails"] += updated_count - microsoft_result["accounts"].append({ - "connection_id": connection_id, - "email": connection_email, - "success": True, - "new_emails": new_count, - "updated_emails": updated_count - }) - microsoft_had_success = True - except Exception as e: - error_msg = str(e) - logger.error(f"❌ Microsoft sync failed for {connection_email}: {error_msg}") - microsoft_result["accounts"].append({ - "connection_id": connection_id, - "email": connection_email, - "success": False, - "error": error_msg - }) - sync_errors.append({"provider": "microsoft", "error": error_msg}) - - result["provider_results"]["microsoft"] = microsoft_result - result["new_emails"] += microsoft_result["new_emails"] - result["updated_emails"] += microsoft_result["updated_emails"] - if microsoft_had_success: - result["providers_synced"].append("microsoft") - - if not result["providers_synced"]: - raise ValueError("Failed to sync any connected provider") - - logger.info( - f"βœ… Sync completed for user {user_id[:8]}...: " - f"{result.get('new_emails', 0)} new, {result.get('updated_emails', 0)} updated" ) - - # After sync, analyze any unanalyzed emails for this user - logger.info("πŸ€– Analyzing unanalyzed emails...") - analyzed_count = await asyncio.to_thread(analyze_unanalyzed_emails, user_id=user_id, limit=100) - - if analyzed_count > 0: - logger.info(f"βœ… Analyzed {analyzed_count} previously unanalyzed emails") - else: - logger.debug("All emails already analyzed") - result['ai_analyzed_count'] = analyzed_count - - if sync_errors: - result["errors"] = sync_errors - - return result except Exception as e: handle_api_exception(e, "Failed to sync emails", logger) diff --git a/core-api/api/routers/webhooks.py b/core-api/api/routers/webhooks.py index 3f8f49a3..0490c8dd 100644 --- a/core-api/api/routers/webhooks.py +++ b/core-api/api/routers/webhooks.py @@ -2,26 +2,36 @@ Webhooks router - Receives push notifications from external services Thin layer that handles HTTP concerns and delegates to services. """ -from fastapi import APIRouter, Request, Header, Query, Response +from fastapi import APIRouter, Request, Header, HTTPException, Query, Response, status from fastapi.responses import PlainTextResponse -from typing import Optional import asyncio -import logging -import json import base64 -from datetime import datetime, timezone +from collections import OrderedDict +import json +import logging +import threading +import time +from typing import Dict, Optional +from starlette.requests import ClientDisconnect -from api.services.webhooks import ( - process_gmail_notification, - process_calendar_notification -) +from api.config import settings +from api.schemas import HealthResponse +from api.services.syncs.sync_dispatcher import mark_stream_dirty +from api.services.syncs.sync_state_store import SYNC_KIND_CALENDAR, SYNC_KIND_EMAIL +from lib.google_webhook_security import verify_google_calendar_channel_token +from lib.supabase_client import get_service_role_client from lib.token_encryption import decrypt_ext_connection_tokens from pydantic import BaseModel -from api.schemas import HealthResponse logger = logging.getLogger(__name__) router = APIRouter(prefix="/api/webhooks", tags=["webhooks"]) +_PUBSUB_JWT_CACHE: Dict[str, tuple[dict, float]] = {} +_PUBSUB_JWT_CACHE_LOCK = threading.Lock() +_GMAIL_INGRESS_DEDUP_TTL_SECONDS = 120.0 +_GMAIL_INGRESS_DEDUP_MAX_ENTRIES = 20_000 +_GMAIL_INGRESS_DEDUP_CACHE: "OrderedDict[str, float]" = OrderedDict() +_GMAIL_INGRESS_DEDUP_LOCK = threading.Lock() # ============================================================================ @@ -41,8 +51,120 @@ class Config: # Endpoints # ============================================================================ + +def _extract_bearer_token(authorization: Optional[str]) -> Optional[str]: + if not authorization: + return None + scheme, _, token = authorization.partition(" ") + if scheme.lower() != "bearer" or not token: + return None + return token.strip() + + +def _gmail_ingress_dedup_key(email_address: str, history_id: str) -> str: + return f"{email_address}\n{history_id}" + + +def _prune_gmail_ingress_dedup_cache(now: float) -> None: + while _GMAIL_INGRESS_DEDUP_CACHE: + first_key = next(iter(_GMAIL_INGRESS_DEDUP_CACHE)) + expires_at = _GMAIL_INGRESS_DEDUP_CACHE[first_key] + if expires_at > now and len(_GMAIL_INGRESS_DEDUP_CACHE) <= _GMAIL_INGRESS_DEDUP_MAX_ENTRIES: + return + _GMAIL_INGRESS_DEDUP_CACHE.popitem(last=False) + + +def _was_recent_successful_gmail_notification(email_address: str, history_id: str) -> bool: + now = time.monotonic() + key = _gmail_ingress_dedup_key(email_address, history_id) + with _GMAIL_INGRESS_DEDUP_LOCK: + _prune_gmail_ingress_dedup_cache(now) + expires_at = _GMAIL_INGRESS_DEDUP_CACHE.get(key) + if expires_at is None: + return False + if expires_at <= now: + _GMAIL_INGRESS_DEDUP_CACHE.pop(key, None) + return False + _GMAIL_INGRESS_DEDUP_CACHE.move_to_end(key) + return True + + +def _remember_successful_gmail_notification(email_address: str, history_id: str) -> None: + now = time.monotonic() + key = _gmail_ingress_dedup_key(email_address, history_id) + with _GMAIL_INGRESS_DEDUP_LOCK: + _GMAIL_INGRESS_DEDUP_CACHE[key] = now + _GMAIL_INGRESS_DEDUP_TTL_SECONDS + _GMAIL_INGRESS_DEDUP_CACHE.move_to_end(key) + _prune_gmail_ingress_dedup_cache(now) + + +def _verify_google_pubsub_auth(authorization: Optional[str]) -> None: + """Verify Pub/Sub authenticated push JWT when the deployment requires it.""" + expected_email = settings.google_pubsub_push_service_account_email.strip() + configured_audience = settings.google_pubsub_push_audience.strip() + + if not expected_email and not configured_audience: + return + + expected_audience = settings.resolved_google_pubsub_push_audience.strip() + encoded_token = _extract_bearer_token(authorization) + if not encoded_token: + raise HTTPException( + status_code=status.HTTP_401_UNAUTHORIZED, + detail="Missing Pub/Sub authorization token", + ) + + now = time.time() + with _PUBSUB_JWT_CACHE_LOCK: + cached_claims = _PUBSUB_JWT_CACHE.get(encoded_token) + if cached_claims and cached_claims[1] > now + 30: + claims = cached_claims[0] + else: + from google.auth import exceptions as google_auth_exceptions + from google.auth.transport.requests import Request as GoogleAuthRequest + from google.oauth2 import id_token as google_id_token + + try: + claims = google_id_token.verify_oauth2_token( + encoded_token, + GoogleAuthRequest(), + audience=expected_audience, + clock_skew_in_seconds=3600, + ) + except (ValueError, google_auth_exceptions.GoogleAuthError) as exc: + logger.warning("Pub/Sub JWT verification failed: %s", exc) + raise HTTPException( + status_code=status.HTTP_401_UNAUTHORIZED, + detail="Invalid Pub/Sub authorization token", + ) from exc + expiry = float(claims.get("exp") or 0) + with _PUBSUB_JWT_CACHE_LOCK: + _PUBSUB_JWT_CACHE[encoded_token] = (claims, expiry) + expired_tokens = [ + token + for token, (_, token_expiry) in _PUBSUB_JWT_CACHE.items() + if token_expiry <= now + ] + for token in expired_tokens: + _PUBSUB_JWT_CACHE.pop(token, None) + + if claims.get("email_verified") is False: + raise HTTPException( + status_code=status.HTTP_401_UNAUTHORIZED, + detail="Pub/Sub token email is not verified", + ) + + if expected_email and claims.get("email") != expected_email: + raise HTTPException( + status_code=status.HTTP_401_UNAUTHORIZED, + detail="Unexpected Pub/Sub service account", + ) + @router.post("/gmail", response_model=WebhookProcessResponse) -async def gmail_webhook(request: Request): +async def gmail_webhook( + request: Request, + authorization: Optional[str] = Header(None), +): """ Receive Gmail push notifications from Google Cloud Pub/Sub. @@ -62,30 +184,51 @@ async def gmail_webhook(request: Request): "historyId": "12345" } - Returns 200 immediately to acknowledge receipt (required by Pub/Sub). + Returns 200 on success or non-actionable input. Returns 503 on transient + infrastructure failures (e.g. Supabase down) so Pub/Sub retries delivery. """ + try: + # Buffer the small Pub/Sub payload before JWT verification so a slow + # cert fetch or claim check does not turn a later body read into + # ClientDisconnect noise. + raw_body = await request.body() + except ClientDisconnect: + logger.warning("Gmail webhook client disconnected before request body was read") + return {"status": "error", "message": "Client disconnected before body read"} + + # Pub/Sub envelopes are ~200 bytes. Reject anything unreasonably large + # before spending time on auth verification. + if len(raw_body) > 65_536: + return {"status": "error", "message": "Payload too large"} + + # Run JWT verification in a thread pool so the blocking Google cert + # fetch does not freeze the event loop and starve concurrent requests. + await asyncio.to_thread(_verify_google_pubsub_auth, authorization) + try: # Parse Pub/Sub message format - body = await request.json() - - logger.info("πŸ“¬ Gmail webhook received") + body = json.loads(raw_body) logger.debug(f"Body keys: {body.keys() if body else 'empty'}") - + # Validate Pub/Sub message format if not body.get('message') or not body['message'].get('data'): logger.error(f"❌ Invalid Pub/Sub message format: {body}") return {"status": "error", "message": "Invalid Pub/Sub format"} - + # Decode base64 data message_data = body['message']['data'] try: decoded_data = base64.b64decode(message_data).decode('utf-8') payload = json.loads(decoded_data) - logger.info(f"πŸ“© Decoded payload: {payload}") + logger.debug( + "πŸ“© Decoded Gmail payload for %s historyId=%s", + payload.get("emailAddress"), + payload.get("historyId"), + ) except Exception as e: logger.error(f"❌ Failed to decode message data: {str(e)}") return {"status": "error", "message": "Failed to decode message"} - + # Extract notification data email_address = payload.get('emailAddress') raw_history_id = payload.get('historyId') @@ -95,58 +238,66 @@ async def gmail_webhook(request: Request): logger.error(f"❌ Missing required fields in payload: {payload}") return {"status": "error", "message": "Missing required fields"} - # Try to enqueue via QStash for async processing - try: - from lib.queue import queue_client - if queue_client.available: - from lib.supabase_client import get_service_role_client - service_supabase = get_service_role_client() - # Look up connection_id from provider_email - conn_result = service_supabase.table('ext_connections')\ - .select('id')\ - .eq('provider_email', email_address)\ - .eq('provider', 'google')\ - .eq('is_active', True)\ - .limit(1)\ - .execute() - if conn_result.data: - connection_id = conn_result.data[0]['id'] - dedup_id = f"wh-sync-gmail-{connection_id}-{history_id}" - if queue_client.enqueue_sync_for_connection( - connection_id, "sync-gmail", - extra={ - "history_id": history_id, - "email_address": email_address, - }, - dedup_id=dedup_id, - ): - return {"status": "ok", "message": "Enqueued to worker"} - # enqueue returned False β€” fall through to inline - except Exception as eq_err: - logger.warning(f"⚠️ Queue enqueue failed, falling back to inline: {eq_err}") - - # Fallback: process inline (existing behavior) - try: - result = await asyncio.to_thread(process_gmail_notification, email_address, history_id) - return {"status": "ok", **result} - except Exception as e: - logger.error(f"❌ Error processing Gmail notification: {str(e)}") - logger.exception("Full traceback:") - # Still return 200 to Pub/Sub - cron job will catch any missed emails - return {"status": "ok", "message": "Notification received with errors"} - + if _was_recent_successful_gmail_notification(email_address, history_id): + logger.debug( + "Skipping duplicate Gmail notification for %s historyId=%s before DB write", + email_address, + history_id, + ) + return {"status": "ok", "message": "Accepted duplicate notification"} + + service_supabase = get_service_role_client() + conn_result = service_supabase.table('ext_connections')\ + .select('id')\ + .eq('provider_email', email_address)\ + .eq('provider', 'google')\ + .eq('is_active', True)\ + .limit(1)\ + .execute() + + if not conn_result.data: + logger.debug(f"No active Gmail connection found for {email_address}; acknowledging webhook") + return {"status": "ok", "message": "Accepted without matching connection"} + + connection_id = conn_result.data[0]['id'] + marked = mark_stream_dirty( + service_supabase, + connection_id, + "google", + SYNC_KIND_EMAIL, + latest_seen_cursor=history_id, + priority=100, + metadata={ + "source": "gmail-webhook", + "email_address": email_address, + "history_id": history_id, + }, + ) + if marked: + _remember_successful_gmail_notification(email_address, history_id) + return {"status": "ok", "message": "Accepted for async sync"} + except Exception as e: - logger.error(f"❌ Error handling Gmail webhook: {str(e)}") - logger.exception("Full traceback:") - # Always return 200 to Pub/Sub, even on error - # We don't want Pub/Sub to think our endpoint is down - return {"status": "error", "message": str(e)} + logger.exception( + "❌ Error handling Gmail webhook after auth; acknowledging to avoid " + "global Pub/Sub push backoff: %s", + str(e), + ) + # Gmail Pub/Sub push backoff is global to the subscription. Once the + # request is authenticated and the payload parsed, prefer an ACK and + # rely on reconciliation as the safety net instead of slowing delivery + # for every mailbox on transient dirty-mark failures. + return { + "status": "ok", + "message": "Accepted without dirty mark; reconciliation will recover", + } @router.post("/calendar", response_model=WebhookProcessResponse) async def calendar_webhook( request: Request, x_goog_channel_id: Optional[str] = Header(None), + x_goog_channel_token: Optional[str] = Header(None), x_goog_resource_id: Optional[str] = Header(None), x_goog_resource_state: Optional[str] = Header(None), x_goog_message_number: Optional[str] = Header(None) @@ -163,61 +314,69 @@ async def calendar_webhook( - X-Goog-Resource-State: "sync" (initial) or "exists" (change notification) - X-Goog-Message-Number: Sequential message number - Returns 200 immediately to acknowledge receipt (required by Google). + Returns 200 on success. Returns 503 on transient infrastructure failures + so Google retries delivery instead of silently losing the notification. """ try: - logger.info(f"πŸ“… Calendar webhook received: channel={x_goog_channel_id}, state={x_goog_resource_state}") + logger.debug(f"Calendar webhook received: channel={x_goog_channel_id}, state={x_goog_resource_state}") - # Try to enqueue via QStash for async processing - try: - from lib.queue import queue_client - if queue_client.available and x_goog_channel_id: - from lib.supabase_client import get_service_role_client - service_supabase = get_service_role_client() - # Look up connection_id from push_subscriptions by channel_id - sub_result = service_supabase.table('push_subscriptions')\ - .select('ext_connection_id')\ - .eq('channel_id', x_goog_channel_id)\ - .eq('is_active', True)\ - .limit(1)\ - .execute() - if sub_result.data: - connection_id = sub_result.data[0]['ext_connection_id'] - dedup_suffix = x_goog_message_number or datetime.now(timezone.utc).strftime("%Y%m%d%H%M") - dedup_id = f"wh-sync-calendar-{connection_id}-{dedup_suffix}" - if queue_client.enqueue_sync_for_connection( - connection_id, - "sync-calendar", - extra={ - "channel_id": x_goog_channel_id, - "resource_state": x_goog_resource_state, - "message_number": x_goog_message_number, - }, - dedup_id=dedup_id, - ): - return {"status": "ok", "message": "Enqueued to worker"} - # enqueue returned False β€” fall through to inline - except Exception as eq_err: - logger.warning(f"⚠️ Queue enqueue failed, falling back to inline: {eq_err}") - - # Fallback: process inline (existing behavior) - try: - result = await asyncio.to_thread( - process_calendar_notification, - channel_id=x_goog_channel_id, - resource_state=x_goog_resource_state + if not x_goog_channel_id: + return {"status": "ok", "message": "Accepted without channel id"} + + service_supabase = get_service_role_client() + sub_result = service_supabase.table('push_subscriptions')\ + .select('ext_connection_id, resource_id')\ + .eq('channel_id', x_goog_channel_id)\ + .eq('is_active', True)\ + .limit(1)\ + .execute() + + if not sub_result.data: + logger.debug(f"No active Calendar subscription found for channel {x_goog_channel_id}") + return {"status": "ok", "message": "Accepted without matching subscription"} + + subscription = sub_result.data[0] + connection_id = subscription['ext_connection_id'] + stored_resource_id = subscription.get('resource_id') + if stored_resource_id and x_goog_resource_id and stored_resource_id != x_goog_resource_id: + raise HTTPException( + status_code=status.HTTP_401_UNAUTHORIZED, + detail="Unexpected Google Calendar resource id", ) - return {"status": "ok", **result} - except Exception as e: - logger.error(f"❌ Error processing Calendar notification: {str(e)}") - logger.exception("Full traceback:") - # Don't fail the webhook - cron will catch missed events - return {"status": "ok", "message": "Notification received with errors"} + if not verify_google_calendar_channel_token( + connection_id, + x_goog_channel_id, + x_goog_channel_token, + ): + raise HTTPException( + status_code=status.HTTP_401_UNAUTHORIZED, + detail="Invalid Google Calendar channel token", + ) + + mark_stream_dirty( + service_supabase, + connection_id, + "google", + SYNC_KIND_CALENDAR, + priority=100, + metadata={ + "source": "google-calendar-webhook", + "channel_id": x_goog_channel_id, + "resource_id": x_goog_resource_id, + "resource_state": x_goog_resource_state, + "message_number": x_goog_message_number, + }, + ) + return {"status": "ok", "message": "Accepted for async sync"} + except HTTPException: + raise except Exception as e: - logger.error(f"❌ Error handling Calendar webhook: {str(e)}") - # Always return 200 to Google - return {"status": "error", "message": str(e)} + logger.exception(f"❌ Error handling Calendar webhook: {str(e)}") + raise HTTPException( + status_code=status.HTTP_503_SERVICE_UNAVAILABLE, + detail="Transient error processing webhook", + ) @router.get("/gmail/verify", response_model=HealthResponse) @@ -261,7 +420,7 @@ async def microsoft_webhook_validation( validationToken query parameter. We MUST return the token as plain text within 10 seconds or subscription creation fails. """ - logger.info("πŸ“‘ [Microsoft] Subscription validation request received") + logger.debug("[Microsoft] Subscription validation request received") logger.debug(f"πŸ“‘ [Microsoft] Validation token: {validationToken[:20]}...") return PlainTextResponse( content=validationToken, @@ -284,13 +443,14 @@ async def microsoft_webhook_notification( subscriptionId, changeType, resource, clientState. We verify clientState matches what we stored and return 202 Accepted - to acknowledge receipt (async processing). + to acknowledge receipt. Returns 503 on transient infrastructure failures + so Microsoft retries delivery. """ try: # Microsoft may send validation as POST with ?validationToken= query param validation_token = request.query_params.get("validationToken") if validation_token: - logger.info("πŸ“‘ [Microsoft] Subscription validation via POST request") + logger.debug("[Microsoft] Subscription validation via POST request") return PlainTextResponse( content=validation_token, status_code=200, @@ -299,26 +459,18 @@ async def microsoft_webhook_notification( body = await request.json() - logger.info("πŸ“¬ [Microsoft] Webhook notification received") + logger.debug("[Microsoft] Webhook notification received") notifications = body.get("value", []) if not notifications: logger.warning("⚠️ [Microsoft] Empty notification payload") return Response(status_code=202) - logger.info(f"πŸ“¬ [Microsoft] Processing {len(notifications)} notification(s)") - - # Try queue-based processing first - from lib.queue import queue_client - use_queue = queue_client.available + logger.debug(f"[Microsoft] Processing {len(notifications)} notification(s)") results = [] for notification in notifications: - if use_queue: - result = await _enqueue_microsoft_notification(notification) - else: - result = await process_microsoft_notification(notification) - results.append(result) + results.append(await _mark_microsoft_notification_dirty(notification)) # Return 202 Accepted - we've acknowledged the notifications return Response( @@ -334,22 +486,15 @@ async def microsoft_webhook_notification( logger.error("❌ [Microsoft] Invalid JSON in webhook body") return Response(status_code=400) except Exception as e: - logger.error(f"❌ [Microsoft] Webhook error: {e}") - import traceback - logger.error(f"❌ [Microsoft] Traceback: {traceback.format_exc()}") - # Still return 202 to acknowledge - cron will catch any missed changes - return Response(status_code=202) + logger.exception(f"❌ [Microsoft] Webhook error: {e}") + return Response(status_code=503) -async def _enqueue_microsoft_notification(notification: dict) -> dict: +async def _mark_microsoft_notification_dirty(notification: dict) -> dict: """ - Enqueue a Microsoft notification as a worker job via QStash. - - Validates clientState and resolves connection_id before enqueuing. - Falls back to inline processing on any error. + Validate a Microsoft notification and mark the corresponding stream dirty. """ from lib.supabase_client import get_service_role_client - from lib.queue import queue_client from api.services.microsoft.microsoft_webhook_provider import MicrosoftWebhookProvider subscription_id = notification.get('subscriptionId') @@ -393,20 +538,32 @@ async def _enqueue_microsoft_notification(notification: dict) -> dict: resource_lower = resource.lower() if '/events' in resource_lower or '/calendar' in resource_lower: job_type = "sync-outlook-calendar" + sync_kind = SYNC_KIND_CALENDAR else: # Default to mail sync (covers /messages, /mailFolders, etc.) job_type = "sync-outlook" - - if queue_client.enqueue_sync_for_connection(connection_id, job_type): - return {"success": True, "enqueued": job_type} - - # enqueue returned False β€” fall through to inline - logger.warning(f"⚠️ [Microsoft] Queue publish failed for {connection_id[:8]}..., falling back to inline") - return await process_microsoft_notification(notification) + sync_kind = SYNC_KIND_EMAIL + + if mark_stream_dirty( + service_supabase, + connection_id, + "microsoft", + sync_kind, + priority=100, + metadata={ + "source": "microsoft-webhook", + "resource": resource, + "subscription_id": subscription_id, + }, + ): + return {"success": True, "marked": job_type} + + logger.warning(f"⚠️ [Microsoft] Failed to mark dirty for {connection_id[:8]}...") + return {"success": False, "error": "mark dirty failed"} except Exception as e: - logger.warning(f"⚠️ [Microsoft] Queue enqueue failed, falling back: {e}") - return await process_microsoft_notification(notification) + logger.warning(f"⚠️ [Microsoft] Failed to mark notification dirty: {e}") + return {"success": False, "error": str(e)} async def process_microsoft_notification(notification: dict) -> dict: @@ -419,11 +576,10 @@ async def process_microsoft_notification(notification: dict) -> dict: from api.services.microsoft.microsoft_webhook_provider import MicrosoftWebhookProvider subscription_id = notification.get('subscriptionId') - client_state = notification.get('clientState') change_type = notification.get('changeType') resource = notification.get('resource', '') - logger.info(f"πŸ“¬ [Microsoft] Notification: {change_type} on {resource[:50]}... (clientState: {'βœ“' if client_state else 'βœ—'})") + logger.debug(f"[Microsoft] Notification: {change_type} on {resource[:50]}...") if not subscription_id: logger.warning("⚠️ [Microsoft] No subscriptionId in notification") diff --git a/core-api/api/routers/workers.py b/core-api/api/routers/workers.py index f79f2acc..0d4789b9 100644 --- a/core-api/api/routers/workers.py +++ b/core-api/api/routers/workers.py @@ -9,6 +9,7 @@ import time from datetime import datetime, timezone from typing import Any, Callable, Dict, List, Optional, Tuple +from uuid import uuid4 from fastapi import APIRouter, Depends, HTTPException, Request, status from pydantic import BaseModel, field_validator, model_validator @@ -17,6 +18,19 @@ from api.config import settings from api.routers.cron import verify_cron_auth +from api.services.syncs.failure_policy import maybe_quarantine_failed_connection +from api.services.syncs.sync_state_store import ( + CLAIM_MODE_DIRTY_ONLY, + SYNC_KIND_CALENDAR, + SYNC_KIND_EMAIL, + claim_connection_sync_lease, + complete_connection_sync_lease, + fail_connection_sync_lease, + get_connection_sync_state, + get_failure_retry_seconds, + start_connection_sync_lease_heartbeat, + stop_connection_sync_lease_heartbeat, +) from lib.queue import queue_client from lib.supabase_client import get_service_role_client from lib.token_encryption import decrypt_ext_connection_tokens @@ -230,7 +244,7 @@ def _run_batch( try: result = processor(connection_id) status_value = result.get("status", "ok") - if status_value == "ok": + if status_value in {"ok", "quarantined"}: processed += 1 elif status_value == "skipped": skipped += 1 @@ -331,6 +345,110 @@ def _run_batch_endpoint( return continued_result +def _extract_result_cursor(result: Dict[str, Any]) -> Optional[str]: + for key in ("new_history_id", "recovered_history_id", "new_sync_token", "new_delta_link"): + value = result.get(key) + if value: + return str(value) + return None + + +def _run_with_stream_lease( + connection_id: str, + sync_kind: str, + processor: Callable[[], Dict[str, Any]], +) -> Dict[str, Any]: + service_supabase = get_service_role_client() + worker_id = f"sync-worker:{sync_kind}:{uuid4().hex}" + lease = claim_connection_sync_lease( + service_supabase, + worker_id, + lease_seconds=BATCH_TIME_BUDGET_SECONDS + 60, + sync_kind=sync_kind, + connection_id=connection_id, + claim_mode=CLAIM_MODE_DIRTY_ONLY, + ) + + if not lease: + logger.info( + f"[Worker] skipping {sync_kind} sync for {connection_id[:8]}... " + "because the stream is already clean or leased" + ) + return {"status": "skipped", "message": "Already running or not dirty"} + + heartbeat_stop, heartbeat_thread = start_connection_sync_lease_heartbeat( + service_supabase, + connection_id, + sync_kind, + worker_id, + lease_seconds=BATCH_TIME_BUDGET_SECONDS + 60, + logger=logger, + ) + + def _fail_with_backoff(error_message: str) -> Dict[str, Any]: + retry_count = None + try: + state = get_connection_sync_state(service_supabase, connection_id, sync_kind) + retry_count = (state or {}).get("retry_count") + except Exception as exc: + logger.warning( + f"[Worker] could not load retry state for {connection_id[:8]}.../{sync_kind}: {exc}" + ) + + try: + quarantine_result = maybe_quarantine_failed_connection( + service_supabase, + connection_id=connection_id, + sync_kind=sync_kind, + worker_id=worker_id, + provider=lease.get("provider"), + error=error_message, + retry_count=retry_count, + ) + if quarantine_result is not None: + return quarantine_result + except Exception: + logger.exception( + "[Worker] failed to quarantine %s/%s after error", + connection_id[:8], + sync_kind, + ) + + fail_connection_sync_lease( + service_supabase, + connection_id, + sync_kind, + worker_id, + error_message, + retry_seconds=get_failure_retry_seconds(retry_count), + ) + return {"status": "error", "message": error_message} + + try: + result = processor() + except Exception as exc: + stop_connection_sync_lease_heartbeat(heartbeat_stop, heartbeat_thread) + failure_result = _fail_with_backoff(str(exc)) + logger.error(f"[Worker] {sync_kind} leased run failed for {connection_id[:8]}...: {exc}") + return failure_result + + stop_connection_sync_lease_heartbeat(heartbeat_stop, heartbeat_thread) + + if result.get("status") == "error": + return _fail_with_backoff(str(result.get("message", "sync failed"))) + + result_cursor = _extract_result_cursor(result) + complete_connection_sync_lease( + service_supabase, + connection_id, + sync_kind, + worker_id, + last_synced_cursor=result_cursor, + latest_seen_cursor=result_cursor, + ) + return result + + def _sync_single_gmail( connection_id: str, payload: SyncPayload, @@ -338,8 +456,8 @@ def _sync_single_gmail( from api.services.syncs import sync_gmail_cron from api.services.syncs.google_error_utils import is_permanent_google_api_error from api.services.syncs.google_services import get_google_services_for_connection - from api.services.syncs.sync_gmail import sync_gmail_for_connection + from api.services.webhooks import reconcile_gmail_connection service_supabase = get_service_role_client() gmail_service, _, user_id = get_google_services_for_connection(connection_id, service_supabase) @@ -368,17 +486,26 @@ def _sync_single_gmail( return {"status": "skipped", "message": str(error)} return {"status": "error", "message": str(error)} - result = sync_gmail_cron( + result = reconcile_gmail_connection( + connection_id, + supabase=service_supabase, gmail_service=gmail_service, - connection_id=connection_id, user_id=user_id, - service_supabase=service_supabase, - days_back=payload.days_back or 7, ) + if result.get("status") == "skipped" and result.get("message") == "No active Gmail subscription": + logger.info( + f"[Worker] no active Gmail subscription for {connection_id[:8]}..., " + "falling back to time-window sync" + ) + result = sync_gmail_cron( + gmail_service=gmail_service, + connection_id=connection_id, + user_id=user_id, + service_supabase=service_supabase, + days_back=payload.days_back or 7, + ) - # sync_gmail_cron manages last_synced internally - # (skips update on batch errors / error_count > 0) - if result.get("status") == "success": + if result.get("status") in {"ok", "success"}: return {**result, "status": "ok"} error = result.get("error", "unknown") @@ -579,8 +706,16 @@ def _process_calendar_webhook(payload: SyncPayload) -> Dict[str, Any]: ) status_value = result.get("status") if status_value in {"ok", "success"}: - service_supabase = get_service_role_client() - _touch_last_synced(service_supabase, payload.connection_id) + message = str(result.get("message", "")) + no_sync_outcome = ( + payload.resource_state == "sync" + or message == "Sync verified" + or message == "No active subscription" + or message.startswith("Unhandled state:") + ) + if not no_sync_outcome: + service_supabase = get_service_role_client() + _touch_last_synced(service_supabase, payload.connection_id) return {**result, "status": "ok"} return { @@ -605,15 +740,23 @@ def worker_sync_gmail( return _run_batch_endpoint( "sync-gmail", payload, - lambda cid: _sync_single_gmail(cid, payload), + lambda cid: _run_with_stream_lease(cid, SYNC_KIND_EMAIL, lambda: _sync_single_gmail(cid, payload)), ) if payload.history_id or payload.email_address: - result = _process_gmail_webhook(payload) + result = _run_with_stream_lease( + payload.connection_id, + SYNC_KIND_EMAIL, + lambda: _process_gmail_webhook(payload), + ) else: if not payload.connection_id: raise HTTPException(status_code=400, detail="connection_id is required") - result = _sync_single_gmail(payload.connection_id, payload) + result = _run_with_stream_lease( + payload.connection_id, + SYNC_KIND_EMAIL, + lambda: _sync_single_gmail(payload.connection_id, payload), + ) if result.get("status") == "error": raise HTTPException(status_code=500, detail=str(result.get("message", "unknown"))) @@ -632,15 +775,23 @@ def worker_sync_calendar( return _run_batch_endpoint( "sync-calendar", payload, - lambda cid: _sync_single_calendar(cid, payload), + lambda cid: _run_with_stream_lease(cid, SYNC_KIND_CALENDAR, lambda: _sync_single_calendar(cid, payload)), ) if payload.channel_id or payload.resource_state or payload.message_number: - result = _process_calendar_webhook(payload) + result = _run_with_stream_lease( + payload.connection_id, + SYNC_KIND_CALENDAR, + lambda: _process_calendar_webhook(payload), + ) else: if not payload.connection_id: raise HTTPException(status_code=400, detail="connection_id is required") - result = _sync_single_calendar(payload.connection_id, payload) + result = _run_with_stream_lease( + payload.connection_id, + SYNC_KIND_CALENDAR, + lambda: _sync_single_calendar(payload.connection_id, payload), + ) if result.get("status") == "error": raise HTTPException(status_code=500, detail=str(result.get("message", "unknown"))) @@ -659,12 +810,16 @@ def worker_sync_outlook( return _run_batch_endpoint( "sync-outlook", payload, - lambda cid: _sync_single_outlook(cid, payload), + lambda cid: _run_with_stream_lease(cid, SYNC_KIND_EMAIL, lambda: _sync_single_outlook(cid, payload)), ) if not payload.connection_id: raise HTTPException(status_code=400, detail="connection_id is required") - result = _sync_single_outlook(payload.connection_id, payload) + result = _run_with_stream_lease( + payload.connection_id, + SYNC_KIND_EMAIL, + lambda: _sync_single_outlook(payload.connection_id, payload), + ) if result.get("status") == "error": raise HTTPException(status_code=500, detail=str(result.get("message", "unknown"))) return result @@ -681,12 +836,16 @@ def worker_sync_outlook_calendar( return _run_batch_endpoint( "sync-outlook-calendar", payload, - lambda cid: _sync_single_outlook_calendar(cid, payload), + lambda cid: _run_with_stream_lease(cid, SYNC_KIND_CALENDAR, lambda: _sync_single_outlook_calendar(cid, payload)), ) if not payload.connection_id: raise HTTPException(status_code=400, detail="connection_id is required") - result = _sync_single_outlook_calendar(payload.connection_id, payload) + result = _run_with_stream_lease( + payload.connection_id, + SYNC_KIND_CALENDAR, + lambda: _sync_single_outlook_calendar(payload.connection_id, payload), + ) if result.get("status") == "error": raise HTTPException(status_code=500, detail=str(result.get("message", "unknown"))) return result diff --git a/core-api/api/services/auth.py b/core-api/api/services/auth.py index 6527e5cd..5c81d025 100644 --- a/core-api/api/services/auth.py +++ b/core-api/api/services/auth.py @@ -96,51 +96,48 @@ async def _enqueue_or_fallback_google_initial_sync( provider_email: str, ) -> None: """ - Queue Google initial sync jobs, with inline fallback for failed enqueues. - - Fallback is intentionally awaited so OAuth/add-account does not "succeed" - with zero initial sync when queue transport is unavailable. + Mark Google initial sync streams dirty for background worker execution. """ - from lib.queue import queue_client + from lib.supabase_client import get_service_role_client + from api.services.syncs.sync_dispatcher import mark_stream_dirty + from api.services.syncs.sync_state_store import SYNC_KIND_CALENDAR, SYNC_KIND_EMAIL + + service_supabase = get_service_role_client() - gmail_enqueued = queue_client.enqueue_sync_for_connection( + gmail_marked = mark_stream_dirty( + service_supabase, connection_id, - "sync-gmail", - extra={ + "google", + SYNC_KIND_EMAIL, + priority=100, + metadata={ + "source": "initial-sync", "initial_sync": True, "max_results": 50, "days_back": 20, }, - dedup_id=f"initial-sync-gmail-{connection_id}", ) - calendar_enqueued = queue_client.enqueue_sync_for_connection( + calendar_marked = mark_stream_dirty( + service_supabase, connection_id, - "sync-calendar", - extra={ + "google", + SYNC_KIND_CALENDAR, + priority=100, + metadata={ + "source": "initial-sync", "initial_sync": True, "days_past": 7, "days_future": 60, }, - dedup_id=f"initial-sync-calendar-{connection_id}", ) - if gmail_enqueued and calendar_enqueued: - logger.info(f"βœ… [Google] Initial sync jobs enqueued for {provider_email}") + if gmail_marked and calendar_marked: + logger.info(f"βœ… [Google] Initial sync scheduled for {provider_email}") return logger.warning( - f"⚠️ [Google] Initial sync queue enqueue partial/failed for {provider_email}. " - f"gmail_enqueued={gmail_enqueued}, calendar_enqueued={calendar_enqueued}. Running inline fallback." - ) - await asyncio.to_thread( - _run_inline_google_initial_sync, - connection_id=connection_id, - user_id=user_id, - access_token=access_token, - refresh_token=refresh_token, - provider_email=provider_email, - run_gmail=not gmail_enqueued, - run_calendar=not calendar_enqueued, + f"⚠️ [Google] Initial sync dirty-mark partial/failed for {provider_email}. " + f"gmail_marked={gmail_marked}, calendar_marked={calendar_marked}" ) @@ -210,56 +207,51 @@ async def _enqueue_or_fallback_microsoft_initial_sync( include_calendar: bool, ) -> None: """ - Queue Microsoft initial sync jobs, with inline fallback for failed enqueues. - - Fallback is intentionally awaited so OAuth/add-account does not "succeed" - with zero initial sync when queue transport is unavailable. + Mark Microsoft initial sync streams dirty for background worker execution. """ - from lib.queue import queue_client + from lib.supabase_client import get_service_role_client + from api.services.syncs.sync_dispatcher import mark_stream_dirty + from api.services.syncs.sync_state_store import SYNC_KIND_CALENDAR, SYNC_KIND_EMAIL + + service_supabase = get_service_role_client() - email_enqueued = queue_client.enqueue_sync_for_connection( + email_marked = mark_stream_dirty( + service_supabase, connection_id, - "sync-outlook", - extra={ + "microsoft", + SYNC_KIND_EMAIL, + priority=100, + metadata={ + "source": "initial-sync", "initial_sync": True, "max_results": 50, "days_back": 20, }, - dedup_id=f"initial-sync-outlook-{connection_id}", ) - calendar_enqueued = True + calendar_marked = True if include_calendar: - calendar_enqueued = queue_client.enqueue_sync_for_connection( + calendar_marked = mark_stream_dirty( + service_supabase, connection_id, - "sync-outlook-calendar", - extra={ + "microsoft", + SYNC_KIND_CALENDAR, + priority=100, + metadata={ + "source": "initial-sync", "initial_sync": True, "days_past": 7, "days_future": 60, }, - dedup_id=f"initial-sync-outlook-calendar-{connection_id}", ) - if email_enqueued and calendar_enqueued: - logger.info(f"βœ… [Microsoft] Initial sync jobs enqueued for {provider_email}") + if email_marked and calendar_marked: + logger.info(f"βœ… [Microsoft] Initial sync scheduled for {provider_email}") return logger.warning( - f"⚠️ [Microsoft] Initial sync queue enqueue partial/failed for {provider_email}. " - f"email_enqueued={email_enqueued}, calendar_enqueued={calendar_enqueued}. Running inline fallback." - ) - await asyncio.to_thread( - _run_inline_microsoft_initial_sync, - connection_id=connection_id, - user_id=user_id, - access_token=access_token, - refresh_token=refresh_token, - token_expires_at=token_expires_at, - metadata=metadata, - provider_email=provider_email, - run_email=not email_enqueued, - run_calendar=include_calendar and not calendar_enqueued, + f"⚠️ [Microsoft] Initial sync dirty-mark partial/failed for {provider_email}. " + f"email_marked={email_marked}, calendar_marked={calendar_marked}" ) diff --git a/core-api/api/services/syncs/failure_policy.py b/core-api/api/services/syncs/failure_policy.py new file mode 100644 index 00000000..7d7dc234 --- /dev/null +++ b/core-api/api/services/syncs/failure_policy.py @@ -0,0 +1,106 @@ +"""Failure policy helpers for quarantining permanently-broken sync streams.""" + +from __future__ import annotations + +import logging +from typing import Any, Dict, Optional + +from api.services.syncs.connection_state import deactivate_connection_with_subscriptions +from api.services.syncs.google_error_utils import is_permanent_google_oauth_error +from api.services.syncs.sync_state_store import ( + complete_connection_sync_lease, + get_max_failure_retry_count, +) + +logger = logging.getLogger(__name__) + + +_PERMANENT_MICROSOFT_AUTH_PATTERNS = ( + "refresh token is invalid", + "user must re-authenticate", + "token has been revoked", + "interaction_required", + "invalid_grant", +) + + +def _is_permanent_microsoft_auth_error(error: Any) -> bool: + error_str = str(error).lower() if error else "" + return any(p in error_str for p in _PERMANENT_MICROSOFT_AUTH_PATTERNS) + + +def _coerce_retry_count(retry_count: Optional[int]) -> int: + try: + return max(int(retry_count or 0), 0) + except (TypeError, ValueError): + return 0 + + +def get_failed_sync_quarantine_reason( + *, + provider: Optional[str], + error: Any, + retry_count: Optional[int], +) -> Optional[str]: + """Return a deactivation reason when a failed sync should be quarantined.""" + error_message = str(error or "sync failed") + + if provider == "google" and is_permanent_google_oauth_error(error_message): + return f"Permanent Google OAuth failure: {error_message}" + + if provider == "microsoft" and _is_permanent_microsoft_auth_error(error_message): + return f"Permanent Microsoft auth failure: {error_message}" + + next_retry_count = _coerce_retry_count(retry_count) + 1 + max_retry_count = get_max_failure_retry_count() + if next_retry_count >= max_retry_count: + return ( + f"Sync failed {next_retry_count} times and hit the retry cap " + f"({max_retry_count}); user must reconnect" + ) + + return None + + +def maybe_quarantine_failed_connection( + service_supabase: Any, + *, + connection_id: str, + sync_kind: str, + worker_id: str, + provider: Optional[str], + error: Any, + retry_count: Optional[int], +) -> Optional[Dict[str, Any]]: + """ + Deactivate a claimed connection when a failure is known-permanent or has retried too often. + """ + reason = get_failed_sync_quarantine_reason( + provider=provider, + error=error, + retry_count=retry_count, + ) + if reason is None: + return None + + deactivate_connection_with_subscriptions( + service_supabase, + connection_id, + reason=reason, + ) + complete_connection_sync_lease( + service_supabase, + connection_id, + sync_kind, + worker_id, + ) + logger.warning( + "[SyncFailurePolicy] quarantined %s/%s: %s", + connection_id[:8], + sync_kind, + reason, + ) + return { + "status": "quarantined", + "message": reason, + } diff --git a/core-api/api/services/syncs/google_error_utils.py b/core-api/api/services/syncs/google_error_utils.py index f0723f20..2ef4f4a1 100644 --- a/core-api/api/services/syncs/google_error_utils.py +++ b/core-api/api/services/syncs/google_error_utils.py @@ -18,6 +18,9 @@ PERMANENT_GOOGLE_OAUTH_ERROR_PATTERNS = ( "invalid_grant", "account has been deleted", + "refresh token is invalid", + "refresh token has expired or been revoked", + "token expired or revoked", "token has been expired or revoked", "token has been revoked", "user has been suspended", diff --git a/core-api/api/services/syncs/google_services.py b/core-api/api/services/syncs/google_services.py index 1026bbd4..809cc117 100644 --- a/core-api/api/services/syncs/google_services.py +++ b/core-api/api/services/syncs/google_services.py @@ -9,6 +9,7 @@ from typing import Optional, Tuple, Any from api.services.syncs.google_error_utils import is_permanent_google_oauth_error +from lib.google_retry import refresh_credentials from lib.token_encryption import ( decrypt_ext_connection_tokens, encrypt_token_fields, @@ -46,10 +47,10 @@ def get_google_services_for_connection( .select('id, user_id, access_token, refresh_token, token_expires_at, metadata')\ .eq('id', connection_id)\ .eq('is_active', True)\ - .single()\ + .maybe_single()\ .execute() - if not connection_result.data: + if not connection_result or not connection_result.data: return None, None, None connection_data = decrypt_ext_connection_tokens(connection_result.data) @@ -103,7 +104,7 @@ def get_google_services_for_connection( # Refresh and persist token if needed if needs_refresh: try: - credentials.refresh(Request()) + refresh_credentials(credentials, Request(), context=connection_id[:8]) # Use actual expiry from Google credentials (make timezone-aware if needed) if credentials.expiry: @@ -140,6 +141,28 @@ def get_google_services_for_connection( f"(user {user_id[:8]}...): {e} β€” deactivating connection" ) try: + # Guard against concurrent refresh race: re-read the row + # and skip deactivation if another process just refreshed + # successfully (token_expires_at well into the future). + fresh = service_supabase.table('ext_connections')\ + .select('token_expires_at')\ + .eq('id', connection_id)\ + .maybe_single()\ + .execute() + if fresh and fresh.data: + fresh_expiry = fresh.data.get('token_expires_at') + if fresh_expiry: + try: + exp = datetime.fromisoformat(fresh_expiry.replace('Z', '+00:00')) + if exp > datetime.now(timezone.utc) + timedelta(minutes=10): + logger.info( + f"⏭️ Skipping deactivation for {connection_id[:8]}... " + f"β€” token was refreshed by another process (expires {fresh_expiry})" + ) + return None, None, None + except (ValueError, TypeError): + pass # Can't parse β€” proceed with deactivation + service_supabase.table('ext_connections')\ .update({ 'is_active': False, diff --git a/core-api/api/services/syncs/stream_worker.py b/core-api/api/services/syncs/stream_worker.py new file mode 100644 index 00000000..4ebf074e --- /dev/null +++ b/core-api/api/services/syncs/stream_worker.py @@ -0,0 +1,344 @@ +"""Railway worker loop for lease-based sync execution.""" + +from __future__ import annotations + +import logging +import os +import socket +import time +from typing import Any, Dict, Optional +from uuid import uuid4 + +from api.routers import workers as worker_router +from api.services.syncs.failure_policy import maybe_quarantine_failed_connection +from api.services.syncs.sync_state_store import ( + CLAIM_MODE_ANY, + CLAIM_MODE_DIRTY_ONLY, + CLAIM_MODE_RECONCILE_ONLY, + SYNC_KIND_CALENDAR, + SYNC_KIND_EMAIL, + claim_connection_sync_lease, + complete_connection_sync_lease, + fail_connection_sync_lease, + get_failure_retry_seconds, + get_connection_sync_state, + start_connection_sync_lease_heartbeat, + stop_connection_sync_lease_heartbeat, +) +from lib.supabase_client import get_service_role_client + +logger = logging.getLogger(__name__) + + +def build_worker_id() -> str: + configured = os.getenv("SYNC_WORKER_ID") + if configured: + return configured + return f"{socket.gethostname()}:{os.getpid()}:{uuid4().hex[:8]}" + + +def resolve_worker_claim_mode(configured_mode: Optional[str] = None) -> str: + """ + Map human-friendly worker modes onto queue claim modes. + + Defaults to the legacy mixed behavior when SYNC_WORKER_MODE is unset. + """ + mode = (configured_mode or os.getenv("SYNC_WORKER_MODE") or "").strip().lower() + if mode in {"", "any", "mixed", "all"}: + return CLAIM_MODE_ANY + if mode in {"realtime", "dirty", CLAIM_MODE_DIRTY_ONLY}: + return CLAIM_MODE_DIRTY_ONLY + if mode in {"reconcile", CLAIM_MODE_RECONCILE_ONLY}: + return CLAIM_MODE_RECONCILE_ONLY + raise ValueError(f"Unsupported SYNC_WORKER_MODE: {configured_mode or mode}") + + +def _build_payload( + connection_id: str, + claim: Dict[str, Any], + state: Optional[Dict[str, Any]], +) -> worker_router.SyncPayload: + metadata = (state or {}).get("metadata") or {} + source = metadata.get("source") + history_id = None + if source == "gmail-webhook": + raw_history_id = metadata.get("history_id") or claim.get("latest_seen_cursor") or (state or {}).get("latest_seen_cursor") + history_id = str(raw_history_id) if raw_history_id is not None else None + + return worker_router.SyncPayload( + connection_id=connection_id, + history_id=history_id, + channel_id=metadata.get("channel_id"), + resource_state=metadata.get("resource_state"), + message_number=metadata.get("message_number"), + initial_sync=bool(metadata.get("initial_sync")), + days_back=metadata.get("days_back"), + days_past=metadata.get("days_past"), + days_future=metadata.get("days_future"), + max_results=metadata.get("max_results"), + ) + + +def _resolve_active_google_calendar_channel( + service_supabase: Any, + connection_id: str, +) -> Optional[str]: + result = ( + service_supabase.table("push_subscriptions") + .select("channel_id") + .eq("ext_connection_id", connection_id) + .eq("provider", "calendar") + .eq("is_active", True) + .limit(1) + .execute() + ) + if not result.data: + return None + return result.data[0].get("channel_id") + + +def _dispatch_claimed_stream( + claim: Dict[str, Any], + *, + service_supabase: Optional[Any] = None, + state: Optional[Dict[str, Any]] = None, +) -> Dict[str, Any]: + connection_id = claim["connection_id"] + provider = claim["provider"] + sync_kind = claim["sync_kind"] + metadata = (state or {}).get("metadata") or {} + payload = _build_payload(connection_id, claim, state) + + if payload.initial_sync: + if provider == "google" and sync_kind == SYNC_KIND_EMAIL: + return worker_router._sync_single_gmail(connection_id, payload) + if provider == "google" and sync_kind == SYNC_KIND_CALENDAR: + return worker_router._sync_single_calendar(connection_id, payload) + if provider == "microsoft" and sync_kind == SYNC_KIND_EMAIL: + return worker_router._sync_single_outlook(connection_id, payload) + if provider == "microsoft" and sync_kind == SYNC_KIND_CALENDAR: + return worker_router._sync_single_outlook_calendar(connection_id, payload) + + if ( + provider == "google" + and sync_kind == SYNC_KIND_EMAIL + and metadata.get("source") == "gmail-webhook" + and payload.history_id + ): + return worker_router._process_gmail_webhook(payload) + + if provider == "google" and sync_kind == SYNC_KIND_CALENDAR: + active_channel_id = ( + _resolve_active_google_calendar_channel(service_supabase, connection_id) + if service_supabase is not None + else None + ) + payload.resource_state = payload.resource_state or metadata.get("resource_state") or "exists" + if ( + metadata.get("source") == "google-calendar-webhook" + and active_channel_id + and (not payload.channel_id or payload.channel_id == active_channel_id) + ): + payload.channel_id = active_channel_id + return worker_router._process_calendar_webhook(payload) + payload.channel_id = active_channel_id + + if provider == "google" and sync_kind == SYNC_KIND_EMAIL: + return worker_router._sync_single_gmail(connection_id, payload) + if provider == "google" and sync_kind == SYNC_KIND_CALENDAR: + return worker_router._sync_single_calendar(connection_id, payload) + if provider == "microsoft" and sync_kind == SYNC_KIND_EMAIL: + return worker_router._sync_single_outlook(connection_id, payload) + if provider == "microsoft" and sync_kind == SYNC_KIND_CALENDAR: + return worker_router._sync_single_outlook_calendar(connection_id, payload) + + raise ValueError(f"Unsupported claim: provider={provider}, sync_kind={sync_kind}") + + +def process_one_claim( + claim: Dict[str, Any], + *, + worker_id: str, + service_supabase: Optional[Any] = None, +) -> Dict[str, Any]: + service_supabase = service_supabase or get_service_role_client() + connection_id = claim["connection_id"] + sync_kind = claim["sync_kind"] + state = None + heartbeat_stop = None + heartbeat_thread = None + + try: + state = get_connection_sync_state(service_supabase, connection_id, sync_kind) + heartbeat_stop, heartbeat_thread = start_connection_sync_lease_heartbeat( + service_supabase, + connection_id, + sync_kind, + worker_id, + lease_seconds=int(claim.get("lease_seconds") or 600), + logger=logger, + ) + result = _dispatch_claimed_stream( + claim, + service_supabase=service_supabase, + state=state, + ) + except Exception as exc: + if heartbeat_stop is not None and heartbeat_thread is not None: + stop_connection_sync_lease_heartbeat(heartbeat_stop, heartbeat_thread) + try: + quarantine_result = maybe_quarantine_failed_connection( + service_supabase, + connection_id=connection_id, + sync_kind=sync_kind, + worker_id=worker_id, + provider=claim.get("provider"), + error=exc, + retry_count=(state or {}).get("retry_count"), + ) + if quarantine_result is not None: + return quarantine_result + except Exception: + logger.exception( + "[StreamWorker] failed to quarantine %s/%s after error", + connection_id[:8], + sync_kind, + ) + try: + fail_connection_sync_lease( + service_supabase, + connection_id, + sync_kind, + worker_id, + str(exc), + retry_seconds=get_failure_retry_seconds((state or {}).get("retry_count")), + ) + except Exception: + logger.exception( + "[StreamWorker] failed to mark claim failure for %s/%s", + connection_id[:8], + sync_kind, + ) + logger.exception( + "[StreamWorker] claim failed for %s/%s", + connection_id[:8], + sync_kind, + ) + return {"status": "error", "message": str(exc)} + + if heartbeat_stop is not None and heartbeat_thread is not None: + stop_connection_sync_lease_heartbeat(heartbeat_stop, heartbeat_thread) + + if result.get("status") == "error": + try: + quarantine_result = maybe_quarantine_failed_connection( + service_supabase, + connection_id=connection_id, + sync_kind=sync_kind, + worker_id=worker_id, + provider=claim.get("provider"), + error=result.get("message", "sync failed"), + retry_count=(state or {}).get("retry_count"), + ) + if quarantine_result is not None: + return quarantine_result + except Exception: + logger.exception( + "[StreamWorker] failed to quarantine %s/%s after result error", + connection_id[:8], + sync_kind, + ) + fail_connection_sync_lease( + service_supabase, + connection_id, + sync_kind, + worker_id, + str(result.get("message", "sync failed")), + retry_seconds=get_failure_retry_seconds((state or {}).get("retry_count")), + ) + return result + + result_cursor = worker_router._extract_result_cursor(result) + complete_connection_sync_lease( + service_supabase, + connection_id, + sync_kind, + worker_id, + last_synced_cursor=result_cursor, + latest_seen_cursor=result_cursor, + ) + return result + + +def run_once( + *, + worker_id: Optional[str] = None, + lease_seconds: int = 600, + provider: Optional[str] = None, + sync_kind: Optional[str] = None, + service_supabase: Optional[Any] = None, + claim_mode: Optional[str] = None, +) -> Optional[Dict[str, Any]]: + service_supabase = service_supabase or get_service_role_client() + worker_id = worker_id or build_worker_id() + resolved_claim_mode = resolve_worker_claim_mode(claim_mode) + + claim = claim_connection_sync_lease( + service_supabase, + worker_id, + lease_seconds=lease_seconds, + provider=provider, + sync_kind=sync_kind, + claim_mode=resolved_claim_mode, + ) + if not claim: + return None + + logger.info( + f"[StreamWorker] claimed {claim['provider']}/{claim['sync_kind']} " + f"for {claim['connection_id'][:8]}..." + ) + claim["lease_seconds"] = lease_seconds + return process_one_claim( + claim, + worker_id=worker_id, + service_supabase=service_supabase, + ) + + +def run_forever() -> None: + worker_id = build_worker_id() + lease_seconds = int(os.getenv("SYNC_WORKER_LEASE_SECONDS", "600")) + poll_seconds = float(os.getenv("SYNC_WORKER_POLL_SECONDS", "5")) + provider = os.getenv("SYNC_WORKER_PROVIDER") or None + sync_kind = os.getenv("SYNC_WORKER_SYNC_KIND") or None + configured_mode = os.getenv("SYNC_WORKER_MODE") or None + claim_mode = resolve_worker_claim_mode(configured_mode) + consecutive_failures = 0 + + logger.info( + f"[StreamWorker] starting worker_id={worker_id} " + f"lease_seconds={lease_seconds} poll_seconds={poll_seconds} " + f"claim_mode={claim_mode}" + ) + + while True: + try: + result = run_once( + worker_id=worker_id, + lease_seconds=lease_seconds, + provider=provider, + sync_kind=sync_kind, + claim_mode=claim_mode, + ) + consecutive_failures = 0 + if result is None: + time.sleep(poll_seconds) + except Exception: + consecutive_failures += 1 + sleep_seconds = min(poll_seconds * (2 ** min(consecutive_failures - 1, 4)), 60.0) + logger.exception( + "[StreamWorker] worker loop crashed; retrying in %.1fs", + sleep_seconds, + ) + time.sleep(sleep_seconds) diff --git a/core-api/api/services/syncs/sync_dispatcher.py b/core-api/api/services/syncs/sync_dispatcher.py new file mode 100644 index 00000000..3e506ade --- /dev/null +++ b/core-api/api/services/syncs/sync_dispatcher.py @@ -0,0 +1,93 @@ +"""Shared helpers for dispatching sync work through the control plane.""" + +from __future__ import annotations + +from datetime import datetime +from typing import Any, Dict, Optional + +from api.services.syncs.sync_state_store import ( + SYNC_KIND_CALENDAR, + SYNC_KIND_EMAIL, + mark_connection_sync_dirty, +) + + +JOB_TYPE_BY_STREAM = { + ("google", SYNC_KIND_EMAIL): "sync-gmail", + ("google", SYNC_KIND_CALENDAR): "sync-calendar", + ("microsoft", SYNC_KIND_EMAIL): "sync-outlook", + ("microsoft", SYNC_KIND_CALENDAR): "sync-outlook-calendar", +} + +MANUAL_SYNC_PRIORITY = 200 + + +def resolve_stream_job_type(provider: str, sync_kind: str) -> str: + try: + return JOB_TYPE_BY_STREAM[(provider, sync_kind)] + except KeyError as exc: + raise ValueError(f"Unsupported provider/sync_kind combination: {provider}/{sync_kind}") from exc + + +def build_stream_dedup_id(provider: str, sync_kind: str, connection_id: str) -> str: + return f"stream-{provider}-{sync_kind}-{connection_id}" + + +def mark_stream_dirty( + service_supabase: Any, + connection_id: str, + provider: str, + sync_kind: str, + *, + latest_seen_cursor: Optional[str] = None, + priority: int = 0, + provider_event_at: Optional[datetime] = None, + metadata: Optional[Dict[str, Any]] = None, +) -> bool: + """Mark a stream dirty after validating the provider/sync_kind mapping.""" + resolve_stream_job_type(provider, sync_kind) + return mark_connection_sync_dirty( + service_supabase, + connection_id, + sync_kind, + latest_seen_cursor=latest_seen_cursor, + priority=priority, + provider_event_at=provider_event_at, + metadata=metadata, + ) + + +def mark_and_enqueue_stream( + service_supabase: Any, + queue_client: Any, + connection_id: str, + provider: str, + sync_kind: str, + *, + extra: Optional[Dict[str, Any]] = None, + dedup_id: Optional[str] = None, + latest_seen_cursor: Optional[str] = None, + priority: int = 0, + provider_event_at: Optional[datetime] = None, + metadata: Optional[Dict[str, Any]] = None, +) -> bool: + """Mark a stream dirty and enqueue the existing provider-specific worker.""" + job_type = resolve_stream_job_type(provider, sync_kind) + if not mark_stream_dirty( + service_supabase, + connection_id, + provider, + sync_kind, + latest_seen_cursor=latest_seen_cursor, + priority=priority, + provider_event_at=provider_event_at, + metadata=metadata, + ): + return False + + return queue_client.enqueue_sync_for_connection( + connection_id, + job_type, + extra=extra, + dedup_id=dedup_id or build_stream_dedup_id(provider, sync_kind, connection_id), + ) diff --git a/core-api/api/services/syncs/sync_gmail.py b/core-api/api/services/syncs/sync_gmail.py index a2b9e75c..3931ed18 100644 --- a/core-api/api/services/syncs/sync_gmail.py +++ b/core-api/api/services/syncs/sync_gmail.py @@ -15,6 +15,7 @@ list_active_gmail_drafts_by_message_id, ) from api.services.email.draft_cleanup import cleanup_inactive_draft_rows_for_connection +from lib.google_retry import GOOGLE_API_NUM_RETRIES # Note: normalize_labels no longer needed - normalized_labels is now a generated column # Note: AI analysis is now deferred to cron to avoid Groq rate limits @@ -119,7 +120,7 @@ def sync_gmail( userId='me', maxResults=max_results, q=query - ).execute() + ).execute(num_retries=GOOGLE_API_NUM_RETRIES) messages = messages_result.get('messages', []) @@ -148,7 +149,7 @@ def sync_gmail( userId='me', id=msg['id'], format='full' - ).execute() + ).execute(num_retries=GOOGLE_API_NUM_RETRIES) email_data = _parse_email_message( full_msg, @@ -423,7 +424,7 @@ def process_gmail_history( userId='me', id=msg['id'], format='full' - ).execute() + ).execute(num_retries=GOOGLE_API_NUM_RETRIES) email_data = _parse_email_message( full_msg, diff --git a/core-api/api/services/syncs/sync_gmail_cron.py b/core-api/api/services/syncs/sync_gmail_cron.py index e1ae7c3a..ff736b9a 100644 --- a/core-api/api/services/syncs/sync_gmail_cron.py +++ b/core-api/api/services/syncs/sync_gmail_cron.py @@ -12,6 +12,7 @@ deactivate_connection_with_subscriptions, ) from api.services.syncs.google_error_utils import is_permanent_google_api_error +from lib.google_retry import GOOGLE_API_NUM_RETRIES logger = logging.getLogger(__name__) @@ -85,7 +86,7 @@ def sync_gmail_cron( maxResults=100, q=query, pageToken=page_token - ).execute() + ).execute(num_retries=GOOGLE_API_NUM_RETRIES) messages = messages_result.get('messages', []) @@ -102,7 +103,7 @@ def sync_gmail_cron( userId='me', id=msg['id'], format='full' - ).execute() + ).execute(num_retries=GOOGLE_API_NUM_RETRIES) email_data = _parse_email_message( full_msg, user_id, connection_id, diff --git a/core-api/api/services/syncs/sync_google_calendar.py b/core-api/api/services/syncs/sync_google_calendar.py index c7fb9858..7bcee6c7 100644 --- a/core-api/api/services/syncs/sync_google_calendar.py +++ b/core-api/api/services/syncs/sync_google_calendar.py @@ -12,6 +12,7 @@ ) import logging from googleapiclient.errors import HttpError +from lib.google_retry import GOOGLE_API_NUM_RETRIES from api.services.calendar.google_api_helpers import get_google_calendar_service logger = logging.getLogger(__name__) @@ -63,7 +64,7 @@ def sync_google_calendar(user_id: str, user_jwt: str) -> Dict[str, Any]: singleEvents=True, orderBy='startTime', pageToken=page_token - ).execute() + ).execute(num_retries=GOOGLE_API_NUM_RETRIES) events = events_result.get('items', []) total_fetched += len(events) diff --git a/core-api/api/services/syncs/sync_google_calendar_cron.py b/core-api/api/services/syncs/sync_google_calendar_cron.py index 80a68be1..59077837 100644 --- a/core-api/api/services/syncs/sync_google_calendar_cron.py +++ b/core-api/api/services/syncs/sync_google_calendar_cron.py @@ -5,6 +5,7 @@ from datetime import datetime, timezone, timedelta import logging from googleapiclient.errors import HttpError +from lib.google_retry import GOOGLE_API_NUM_RETRIES from lib.batch_utils import batch_upsert, get_existing_external_ids from api.services.calendar.event_parser import parse_google_event_to_data @@ -63,6 +64,7 @@ def sync_google_calendar_cron( time_max = (sync_started_at + timedelta(days=days_future)).isoformat() page_token = None + new_sync_token = None total_fetched = 0 all_events_data = [] all_external_ids = [] @@ -79,10 +81,11 @@ def sync_google_calendar_cron( singleEvents=True, orderBy='startTime', pageToken=page_token - ).execute() + ).execute(num_retries=GOOGLE_API_NUM_RETRIES) events = events_result.get('items', []) total_fetched += len(events) + new_sync_token = events_result.get('nextSyncToken') or new_sync_token logger.info(f"πŸ“¦ Processing {len(events)} events from this page (total so far: {total_fetched})") @@ -221,6 +224,18 @@ def sync_google_calendar_cron( # Update last synced timestamp only if no errors occurred if not batch_had_errors: + if new_sync_token: + service_supabase.table('push_subscriptions')\ + .update({ + 'sync_token': new_sync_token, + 'updated_at': datetime.now(timezone.utc).isoformat(), + })\ + .eq('ext_connection_id', connection_id)\ + .eq('provider', 'calendar')\ + .eq('is_active', True)\ + .execute() + logger.info("πŸ”„ Saved sync token after calendar worker sync") + service_supabase.table('ext_connections')\ .update({'last_synced': datetime.now(timezone.utc).isoformat()})\ .eq('id', connection_id)\ @@ -247,7 +262,8 @@ def sync_google_calendar_cron( "updated_events": updated_count, "deleted_events": deleted_count, "total_events": synced_count + updated_count, - "total_fetched": total_fetched + "total_fetched": total_fetched, + "new_sync_token": new_sync_token, } except HttpError as e: diff --git a/core-api/api/services/syncs/sync_state_store.py b/core-api/api/services/syncs/sync_state_store.py new file mode 100644 index 00000000..2d5b4c76 --- /dev/null +++ b/core-api/api/services/syncs/sync_state_store.py @@ -0,0 +1,313 @@ +"""Helpers for the lease-based connection sync control plane.""" + +from __future__ import annotations + +from datetime import datetime +import os +from threading import Event, Thread +from typing import Any, Dict, Optional, Tuple + +SYNC_KIND_EMAIL = "email" +SYNC_KIND_CALENDAR = "calendar" +VALID_SYNC_KINDS = {SYNC_KIND_EMAIL, SYNC_KIND_CALENDAR} +CLAIM_MODE_ANY = "any" +CLAIM_MODE_DIRTY_ONLY = "dirty_only" +CLAIM_MODE_RECONCILE_ONLY = "reconcile_only" +VALID_CLAIM_MODES = { + CLAIM_MODE_ANY, + CLAIM_MODE_DIRTY_ONLY, + CLAIM_MODE_RECONCILE_ONLY, +} +DEFAULT_RECONCILE_INTERVAL_SECONDS = 21600 +DEFAULT_MAX_FAILURE_RETRY_COUNT = 5 +FAILURE_RETRY_SCHEDULE_SECONDS = (60, 300, 900, 1800, 3600, 21600) +MAX_FAILURE_RETRY_SECONDS = 86400 + + +def _normalize_rpc_data(data: Any) -> Any: + if isinstance(data, list): + if not data: + return None + return data[0] + return data + + +def _require_sync_kind(sync_kind: str) -> str: + if sync_kind not in VALID_SYNC_KINDS: + raise ValueError(f"Unsupported sync_kind: {sync_kind}") + return sync_kind + + +def _require_claim_mode(claim_mode: str) -> str: + if claim_mode not in VALID_CLAIM_MODES: + raise ValueError(f"Unsupported claim_mode: {claim_mode}") + return claim_mode + + +def get_reconcile_interval_seconds() -> int: + """Return the clean-stream safety-net reconcile interval in seconds.""" + value = os.getenv( + "RECONCILE_INTERVAL_SECONDS", + str(DEFAULT_RECONCILE_INTERVAL_SECONDS), + ).strip() + try: + seconds = int(value) + except (TypeError, ValueError): + return DEFAULT_RECONCILE_INTERVAL_SECONDS + + return max(300, min(seconds, 7 * 24 * 3600)) + + +def get_failure_retry_seconds(previous_retry_count: Optional[int]) -> int: + """Return stepped retry backoff to prevent dead connections from hogging workers.""" + try: + retry_count = int(previous_retry_count or 0) + except (TypeError, ValueError): + retry_count = 0 + + retry_count = max(retry_count, 0) + if retry_count < len(FAILURE_RETRY_SCHEDULE_SECONDS): + return FAILURE_RETRY_SCHEDULE_SECONDS[retry_count] + return MAX_FAILURE_RETRY_SECONDS + + +def get_max_failure_retry_count() -> int: + """Return the retry cap after which a broken connection is auto-deactivated.""" + value = os.getenv( + "MAX_FAILURE_RETRY_COUNT", + str(DEFAULT_MAX_FAILURE_RETRY_COUNT), + ).strip() + try: + retry_count = int(value) + except (TypeError, ValueError): + return DEFAULT_MAX_FAILURE_RETRY_COUNT + + return max(1, min(retry_count, 1000)) + + +def mark_connection_sync_dirty( + service_supabase: Any, + connection_id: str, + sync_kind: str, + *, + latest_seen_cursor: Optional[str] = None, + priority: int = 0, + provider_event_at: Optional[datetime] = None, + metadata: Optional[Dict[str, Any]] = None, +) -> bool: + """Mark a sync stream dirty, inserting the state row if needed.""" + result = service_supabase.rpc( + "mark_connection_sync_dirty", + { + "p_connection_id": connection_id, + "p_sync_kind": _require_sync_kind(sync_kind), + "p_latest_seen_cursor": latest_seen_cursor, + "p_priority": priority, + "p_provider_event_at": provider_event_at.isoformat() if provider_event_at else None, + "p_metadata": metadata or {}, + }, + ).execute() + + normalized = _normalize_rpc_data(getattr(result, "data", None)) + return bool(normalized) + + +def get_connection_sync_state( + service_supabase: Any, + connection_id: str, + sync_kind: str, +) -> Optional[Dict[str, Any]]: + """Fetch the current control-plane row for a single sync stream.""" + result = ( + service_supabase.table("connection_sync_state") + .select( + "connection_id, provider, sync_kind, dirty, dirty_generation, " + "lease_generation, latest_seen_cursor, last_synced_cursor, " + "priority, retry_count, metadata, lease_owner, lease_expires_at, " + "last_sync_finished_at, next_reconcile_at" + ) + .eq("connection_id", connection_id) + .eq("sync_kind", _require_sync_kind(sync_kind)) + .maybe_single() + .execute() + ) + + data = getattr(result, "data", None) + return dict(data) if data else None + + +def claim_connection_sync_lease( + service_supabase: Any, + worker_id: str, + *, + lease_seconds: int = 120, + provider: Optional[str] = None, + sync_kind: Optional[str] = None, + connection_id: Optional[str] = None, + claim_mode: str = CLAIM_MODE_ANY, +) -> Optional[Dict[str, Any]]: + """Claim the next dirty or due-for-reconcile sync stream lease, if available.""" + if sync_kind is not None: + _require_sync_kind(sync_kind) + _require_claim_mode(claim_mode) + + result = service_supabase.rpc( + "claim_connection_sync_lease", + { + "p_worker_id": worker_id, + "p_lease_seconds": lease_seconds, + "p_provider": provider, + "p_sync_kind": sync_kind, + "p_connection_id": connection_id, + "p_claim_mode": claim_mode, + }, + ).execute() + + normalized = _normalize_rpc_data(getattr(result, "data", None)) + return dict(normalized) if normalized else None + + +def heartbeat_connection_sync_lease( + service_supabase: Any, + connection_id: str, + sync_kind: str, + worker_id: str, + *, + lease_seconds: int = 120, +) -> bool: + """Extend an existing sync lease held by the given worker.""" + result = service_supabase.rpc( + "heartbeat_connection_sync_lease", + { + "p_connection_id": connection_id, + "p_sync_kind": _require_sync_kind(sync_kind), + "p_worker_id": worker_id, + "p_lease_seconds": lease_seconds, + }, + ).execute() + + normalized = _normalize_rpc_data(getattr(result, "data", None)) + return bool(normalized) + + +def start_connection_sync_lease_heartbeat( + service_supabase: Any, + connection_id: str, + sync_kind: str, + worker_id: str, + *, + lease_seconds: int = 120, + logger: Optional[Any] = None, +) -> Tuple[Event, Thread]: + """ + Start a daemon thread that keeps a claimed lease alive during long sync runs. + + The caller is responsible for stopping the heartbeat via + stop_connection_sync_lease_heartbeat(). + """ + interval_seconds = max(5, min(max(lease_seconds // 3, 1), 30)) + stop_event = Event() + + def _heartbeat_loop() -> None: + while not stop_event.wait(interval_seconds): + try: + alive = heartbeat_connection_sync_lease( + service_supabase, + connection_id, + sync_kind, + worker_id, + lease_seconds=lease_seconds, + ) + if not alive: + if logger is not None: + logger.warning( + "[SyncLease] heartbeat stopped for %s/%s because the lease is gone", + connection_id[:8], + sync_kind, + ) + return + except Exception as exc: + if logger is not None: + logger.warning( + "[SyncLease] heartbeat failed for %s/%s: %s", + connection_id[:8], + sync_kind, + exc, + ) + + thread = Thread( + target=_heartbeat_loop, + name=f"sync-lease-heartbeat-{sync_kind}-{connection_id[:8]}", + daemon=True, + ) + thread.start() + return stop_event, thread + + +def stop_connection_sync_lease_heartbeat( + stop_event: Event, + thread: Thread, + *, + join_timeout_seconds: float = 1.0, +) -> None: + """Stop a heartbeat thread started by start_connection_sync_lease_heartbeat().""" + stop_event.set() + thread.join(timeout=join_timeout_seconds) + + +def complete_connection_sync_lease( + service_supabase: Any, + connection_id: str, + sync_kind: str, + worker_id: str, + *, + last_synced_cursor: Optional[str] = None, + latest_seen_cursor: Optional[str] = None, + keep_dirty: bool = False, + reconcile_interval_seconds: Optional[int] = None, +) -> Optional[Dict[str, Any]]: + """Release a sync lease after a successful worker run.""" + result = service_supabase.rpc( + "complete_connection_sync_lease", + { + "p_connection_id": connection_id, + "p_sync_kind": _require_sync_kind(sync_kind), + "p_worker_id": worker_id, + "p_last_synced_cursor": last_synced_cursor, + "p_latest_seen_cursor": latest_seen_cursor, + "p_keep_dirty": keep_dirty, + "p_reconcile_interval_seconds": ( + reconcile_interval_seconds + if reconcile_interval_seconds is not None + else get_reconcile_interval_seconds() + ), + }, + ).execute() + + normalized = _normalize_rpc_data(getattr(result, "data", None)) + return dict(normalized) if normalized else None + + +def fail_connection_sync_lease( + service_supabase: Any, + connection_id: str, + sync_kind: str, + worker_id: str, + error: str, + *, + retry_seconds: int = 60, +) -> Optional[Dict[str, Any]]: + """Release a sync lease after failure and schedule a retry.""" + result = service_supabase.rpc( + "fail_connection_sync_lease", + { + "p_connection_id": connection_id, + "p_sync_kind": _require_sync_kind(sync_kind), + "p_worker_id": worker_id, + "p_error": error, + "p_retry_seconds": retry_seconds, + }, + ).execute() + + normalized = _normalize_rpc_data(getattr(result, "data", None)) + return dict(normalized) if normalized else None diff --git a/core-api/api/services/syncs/watch_manager.py b/core-api/api/services/syncs/watch_manager.py index 94e310e3..a1a94a32 100644 --- a/core-api/api/services/syncs/watch_manager.py +++ b/core-api/api/services/syncs/watch_manager.py @@ -12,6 +12,7 @@ from api.services.calendar.google_api_helpers import get_google_calendar_service from api.config import settings from api.services.syncs.google_error_utils import is_permanent_google_api_error +from lib.google_webhook_security import build_google_calendar_channel_token logger = logging.getLogger(__name__) @@ -22,6 +23,34 @@ CALENDAR_WATCH_EXPIRATION_DAYS = 7 +def _is_safe_google_stop_error(error: HttpError) -> bool: + """Return True when a stop() error means the remote watch is already gone.""" + status_code = getattr(getattr(error, "resp", None), "status", None) + try: + status_code = int(status_code) if status_code is not None else None + except (TypeError, ValueError): + status_code = None + return status_code in (404, 410) + + +def _get_gmail_webhook_url() -> str: + """Resolve the Gmail webhook URL while tolerating lightweight test stubs.""" + explicit = getattr(settings, "gmail_webhook_url", None) + if explicit: + return explicit + base_url = getattr(settings, "webhook_base_url", "").rstrip("/") + return f"{base_url}/api/webhooks/gmail" + + +def _get_calendar_webhook_url() -> str: + """Resolve the Calendar webhook URL while tolerating lightweight test stubs.""" + explicit = getattr(settings, "calendar_webhook_url", None) + if explicit: + return explicit + base_url = getattr(settings, "webhook_base_url", "").rstrip("/") + return f"{base_url}/api/webhooks/calendar" + + def start_gmail_watch( user_id: str, user_jwt: str, @@ -47,10 +76,10 @@ def start_gmail_watch( raise ValueError("No active Google connection found for user") try: - # Check if there's an existing active watch + # Check if there's an existing active watch for this connection existing = auth_supabase.table('push_subscriptions')\ .select('*')\ - .eq('user_id', user_id)\ + .eq('ext_connection_id', connection_id)\ .eq('provider', 'gmail')\ .eq('is_active', True)\ .execute() @@ -75,17 +104,16 @@ def start_gmail_watch( logger.info(f"πŸ”„ Gmail watch exists but expiring soon for user {user_id[:8]}..., will renew") # Stop existing watch first - try: - stop_gmail_watch(user_id, user_jwt) - except Exception as e: - logger.warning(f"⚠️ Could not stop existing watch: {e}") + stop_result = stop_gmail_watch(user_id, user_jwt, connection_id=connection_id) + if not stop_result.get('success'): + raise ValueError(stop_result.get('error') or "Failed to stop existing Gmail watch") # Generate unique channel ID channel_id = str(uuid.uuid4()) # Set webhook URL (use configured base URL from settings) if not webhook_url: - webhook_url = f"{settings.webhook_base_url}/api/webhooks/gmail" + webhook_url = _get_gmail_webhook_url() # Gmail watch requires Google Cloud Pub/Sub setup # Check if GOOGLE_PUBSUB_TOPIC is configured @@ -127,13 +155,6 @@ def start_gmail_watch( expiration = datetime.now(timezone.utc) + timedelta(days=GMAIL_WATCH_EXPIRATION_DAYS) # Store watch subscription in database - # First deactivate any existing subscriptions for this user/provider - auth_supabase.table('push_subscriptions')\ - .update({'is_active': False})\ - .eq('user_id', user_id)\ - .eq('provider', 'gmail')\ - .execute() - subscription_data = { 'user_id': user_id, 'ext_connection_id': connection_id, @@ -210,10 +231,10 @@ def start_calendar_watch( raise ValueError("No active Google connection found for user") try: - # Check if there's an existing active watch + # Check if there's an existing active watch for this connection existing = auth_supabase.table('push_subscriptions')\ .select('*')\ - .eq('user_id', user_id)\ + .eq('ext_connection_id', connection_id)\ .eq('provider', 'calendar')\ .eq('is_active', True)\ .execute() @@ -239,17 +260,16 @@ def start_calendar_watch( logger.info(f"πŸ”„ Calendar watch exists but expiring soon for user {user_id[:8]}..., will renew") # Stop existing watch first - try: - stop_calendar_watch(user_id, user_jwt) - except Exception as e: - logger.warning(f"⚠️ Could not stop existing watch: {e}") + stop_result = stop_calendar_watch(user_id, user_jwt, connection_id=connection_id) + if not stop_result.get('success'): + raise ValueError(stop_result.get('error') or "Failed to stop existing Calendar watch") # Generate unique channel ID channel_id = str(uuid.uuid4()) # Set webhook URL (use configured base URL from settings) if not webhook_url: - webhook_url = f"{settings.webhook_base_url}/api/webhooks/calendar" + webhook_url = _get_calendar_webhook_url() # Calculate expiration (7 days from now) expiration = datetime.now(timezone.utc) + timedelta(days=CALENDAR_WATCH_EXPIRATION_DAYS) @@ -261,6 +281,9 @@ def start_calendar_watch( 'address': webhook_url, 'expiration': expiration_ms } + channel_token = build_google_calendar_channel_token(connection_id, channel_id) + if channel_token: + request_body['token'] = channel_token logger.info(f"πŸ”” Starting Calendar watch for user {user_id} with channel {channel_id}") @@ -293,13 +316,6 @@ def start_calendar_watch( sync_token = None # Store watch subscription in database - # First deactivate any existing subscriptions for this user/provider - auth_supabase.table('push_subscriptions')\ - .update({'is_active': False})\ - .eq('user_id', user_id)\ - .eq('provider', 'calendar')\ - .execute() - subscription_data = { 'user_id': user_id, 'ext_connection_id': connection_id, @@ -341,7 +357,7 @@ def start_calendar_watch( raise ValueError(f"Calendar watch setup failed: {str(e)}") -def stop_gmail_watch(user_id: str, user_jwt: str) -> Dict[str, Any]: +def stop_gmail_watch(user_id: str, user_jwt: str, connection_id: Optional[str] = None) -> Dict[str, Any]: """ Stop watching a user's Gmail for changes @@ -353,7 +369,8 @@ def stop_gmail_watch(user_id: str, user_jwt: str) -> Dict[str, Any]: Dict with success status """ auth_supabase = get_authenticated_supabase_client(user_jwt) - service, _ = get_gmail_service(user_id, user_jwt) + service, resolved_connection_id = get_gmail_service(user_id, user_jwt, account_id=connection_id) + connection_id = resolved_connection_id if not service: raise ValueError("No active Google connection found") @@ -362,7 +379,7 @@ def stop_gmail_watch(user_id: str, user_jwt: str) -> Dict[str, Any]: # Get active subscription subscription = auth_supabase.table('push_subscriptions')\ .select('*')\ - .eq('user_id', user_id)\ + .eq('ext_connection_id', connection_id)\ .eq('provider', 'gmail')\ .eq('is_active', True)\ .execute() @@ -376,8 +393,12 @@ def stop_gmail_watch(user_id: str, user_jwt: str) -> Dict[str, Any]: service.users().stop(userId='me').execute() logger.info(f"πŸ›‘ Gmail watch stopped with Google for user {user_id}") except HttpError as e: - logger.warning(f"⚠️ Could not stop watch with Google: {e}") - + if _is_safe_google_stop_error(e): + logger.info(f"ℹ️ Gmail watch already expired for user {user_id[:8]}...: {e}") + else: + logger.error(f"❌ Failed to stop Gmail watch for user {user_id[:8]}..., leaving DB row active: {e}") + return {'success': False, 'provider': 'gmail', 'error': str(e)} + # Mark as inactive in database auth_supabase.table('push_subscriptions')\ .update({'is_active': False})\ @@ -392,7 +413,7 @@ def stop_gmail_watch(user_id: str, user_jwt: str) -> Dict[str, Any]: raise ValueError(f"Failed to stop Gmail watch: {str(e)}") -def stop_calendar_watch(user_id: str, user_jwt: str) -> Dict[str, Any]: +def stop_calendar_watch(user_id: str, user_jwt: str, connection_id: Optional[str] = None) -> Dict[str, Any]: """ Stop watching a user's Google Calendar for changes @@ -404,7 +425,8 @@ def stop_calendar_watch(user_id: str, user_jwt: str) -> Dict[str, Any]: Dict with success status """ auth_supabase = get_authenticated_supabase_client(user_jwt) - service, _ = get_google_calendar_service(user_id, user_jwt) + service, resolved_connection_id = get_google_calendar_service(user_id, user_jwt, account_id=connection_id) + connection_id = resolved_connection_id if not service: raise ValueError("No active Google connection found") @@ -413,7 +435,7 @@ def stop_calendar_watch(user_id: str, user_jwt: str) -> Dict[str, Any]: # Get active subscription subscription = auth_supabase.table('push_subscriptions')\ .select('*')\ - .eq('user_id', user_id)\ + .eq('ext_connection_id', connection_id)\ .eq('provider', 'calendar')\ .eq('is_active', True)\ .execute() @@ -427,16 +449,30 @@ def stop_calendar_watch(user_id: str, user_jwt: str) -> Dict[str, Any]: resource_id = sub_data.get('resource_id') # Stop the watch with Google - if channel_id and resource_id: - try: - service.channels().stop(body={ - 'id': channel_id, - 'resourceId': resource_id - }).execute() - logger.info(f"πŸ›‘ Calendar watch stopped with Google for user {user_id}") - except HttpError as e: - logger.warning(f"⚠️ Could not stop watch with Google: {e}") - + if not channel_id or not resource_id: + logger.error( + f"❌ Active Calendar watch for user {user_id[:8]}... is missing channel/resource identifiers; " + "leaving DB row active" + ) + return { + 'success': False, + 'provider': 'calendar', + 'error': 'Active Calendar watch is missing channel/resource identifiers' + } + + try: + service.channels().stop(body={ + 'id': channel_id, + 'resourceId': resource_id + }).execute() + logger.info(f"πŸ›‘ Calendar watch stopped with Google for user {user_id}") + except HttpError as e: + if _is_safe_google_stop_error(e): + logger.info(f"ℹ️ Calendar watch already expired for user {user_id[:8]}...: {e}") + else: + logger.error(f"❌ Failed to stop Calendar watch for user {user_id[:8]}..., leaving DB row active: {e}") + return {'success': False, 'provider': 'calendar', 'error': str(e)} + # Mark as inactive in database auth_supabase.table('push_subscriptions')\ .update({'is_active': False})\ @@ -547,58 +583,6 @@ def setup_watches_for_user(user_id: str, user_jwt: str) -> Dict[str, Any]: # These functions don't require user_jwt - they use service role credentials # ============================================================================= -def _check_and_renew_existing_gmail_watch( - watch_row: Dict[str, Any], - user_id: str, - connection_id: str, - gmail_service, - service_supabase, -) -> Optional[Dict[str, Any]]: - """Check an existing watch. Return a healthy-result dict to short-circuit, or None to proceed with renewal.""" - expiration = datetime.fromisoformat(watch_row['expiration'].replace('Z', '+00:00')) - hours_until_expiry = (expiration - datetime.now(timezone.utc)).total_seconds() / 3600 - - if hours_until_expiry > 24: - logger.info(f"βœ… Gmail watch healthy for connection {connection_id[:8]}... ({hours_until_expiry:.1f}h remaining)") - return { - 'success': True, - 'provider': 'gmail', - 'message': 'Watch already exists and is healthy', - 'hours_remaining': hours_until_expiry, - } - - logger.info(f"πŸ”„ Gmail watch expiring soon for user {user_id[:8]}..., renewing") - # Stop the old watch with Gmail first to prevent duplicate notifications - old_watch_stopped = False - try: - gmail_service.users().stop(userId='me').execute() - logger.info(f"πŸ›‘ Stopped old Gmail watch before renewal for user {user_id[:8]}...") - old_watch_stopped = True - except HttpError as e: - status_code = getattr(getattr(e, "resp", None), "status", None) - if status_code in (404, 410): - logger.info(f"ℹ️ Old Gmail watch already expired for user {user_id[:8]}...: {e}") - old_watch_stopped = True - else: - logger.error(f"❌ Failed to stop old Gmail watch for user {user_id[:8]}..., skipping renewal: {e}") - except Exception as e: - logger.error(f"❌ Failed to stop old Gmail watch for user {user_id[:8]}..., skipping renewal: {e}") - - if not old_watch_stopped: - return { - 'success': False, - 'provider': 'gmail', - 'error': 'Failed to stop old watch; skipping renewal to avoid untracked watches', - } - - # Deactivate old watch in DB - service_supabase.table('push_subscriptions')\ - .update({'is_active': False})\ - .eq('id', watch_row['id'])\ - .execute() - return None - - def start_gmail_watch_service_role( user_id: str, gmail_service, @@ -628,11 +612,47 @@ def start_gmail_watch_service_role( .execute() if existing.data: - healthy_result = _check_and_renew_existing_gmail_watch( - existing.data[0], user_id, connection_id, gmail_service, service_supabase - ) - if healthy_result is not None: - return healthy_result + expiration = datetime.fromisoformat(existing.data[0]['expiration'].replace('Z', '+00:00')) + time_until_expiry = expiration - datetime.now(timezone.utc) + hours_until_expiry = time_until_expiry.total_seconds() / 3600 + + if hours_until_expiry > 24: + logger.info(f"βœ… Gmail watch healthy for connection {connection_id[:8]}... ({hours_until_expiry:.1f}h remaining)") + return { + 'success': True, + 'provider': 'gmail', + 'message': 'Watch already exists and is healthy', + 'hours_remaining': hours_until_expiry + } + + logger.info(f"πŸ”„ Gmail watch expiring soon for user {user_id[:8]}..., renewing") + # Stop the old watch with Gmail first to prevent duplicate notifications + old_watch_stopped = False + try: + gmail_service.users().stop(userId='me').execute() + logger.info(f"πŸ›‘ Stopped old Gmail watch before renewal for user {user_id[:8]}...") + old_watch_stopped = True + except HttpError as e: + if _is_safe_google_stop_error(e): + logger.info(f"ℹ️ Old Gmail watch already expired for user {user_id[:8]}...: {e}") + old_watch_stopped = True + else: + logger.error(f"❌ Failed to stop old Gmail watch for user {user_id[:8]}..., skipping renewal: {e}") + except Exception as e: + logger.error(f"❌ Failed to stop old Gmail watch for user {user_id[:8]}..., skipping renewal: {e}") + + if not old_watch_stopped: + return { + 'success': False, + 'provider': 'gmail', + 'error': 'Failed to stop old watch; skipping renewal to avoid untracked watches' + } + + # Deactivate old watch in DB + service_supabase.table('push_subscriptions')\ + .update({'is_active': False})\ + .eq('id', existing.data[0]['id'])\ + .execute() # Check Pub/Sub topic configuration if not settings.google_pubsub_topic: @@ -640,7 +660,7 @@ def start_gmail_watch_service_role( # Generate unique channel ID channel_id = str(uuid.uuid4()) - webhook_url = f"{settings.webhook_base_url}/api/webhooks/gmail" + webhook_url = _get_gmail_webhook_url() request_body = { 'labelIds': ['INBOX'], @@ -757,8 +777,7 @@ def start_calendar_watch_service_role( }).execute() old_watch_stopped = True except HttpError as e: - status_code = getattr(getattr(e, "resp", None), "status", None) - if status_code in (404, 410): + if _is_safe_google_stop_error(e): logger.info(f"ℹ️ Old Calendar watch already expired for connection {connection_id[:8]}...: {e}") old_watch_stopped = True else: @@ -766,8 +785,10 @@ def start_calendar_watch_service_role( except Exception as e: logger.error(f"❌ Failed to stop old Calendar watch for connection {connection_id[:8]}..., skipping renewal: {e}") else: - # No channel/resource to stop β€” treat as already gone - old_watch_stopped = True + logger.error( + f"❌ Active Calendar watch for connection {connection_id[:8]}... " + "is missing channel/resource identifiers; skipping renewal to avoid untracked watches" + ) if not old_watch_stopped: return { @@ -784,7 +805,7 @@ def start_calendar_watch_service_role( # Generate unique channel ID channel_id = str(uuid.uuid4()) - webhook_url = f"{settings.webhook_base_url}/api/webhooks/calendar" + webhook_url = _get_calendar_webhook_url() # Calculate expiration expiration = datetime.now(timezone.utc) + timedelta(days=CALENDAR_WATCH_EXPIRATION_DAYS) @@ -796,6 +817,9 @@ def start_calendar_watch_service_role( 'address': webhook_url, 'expiration': expiration_ms } + channel_token = build_google_calendar_channel_token(connection_id, channel_id) + if channel_token: + request_body['token'] = channel_token logger.info(f"πŸ”” Starting Calendar watch for user {user_id[:8]}...") @@ -891,4 +915,3 @@ def renew_watch_service_role( return start_calendar_watch_service_role(user_id, calendar_service, connection_id, service_supabase) else: return {'success': False, 'error': f'Unknown provider: {provider}'} - diff --git a/core-api/api/services/webhooks/__init__.py b/core-api/api/services/webhooks/__init__.py index e0937993..b3dc9e09 100644 --- a/core-api/api/services/webhooks/__init__.py +++ b/core-api/api/services/webhooks/__init__.py @@ -1,12 +1,12 @@ """ Webhook services - Processing logic for external push notifications """ -from api.services.webhooks.gmail_webhook import process_gmail_notification +from api.services.webhooks.gmail_webhook import process_gmail_notification, reconcile_gmail_connection from api.services.webhooks.calendar_webhook import process_calendar_notification __all__ = [ 'process_gmail_notification', + 'reconcile_gmail_connection', 'process_calendar_notification' ] - diff --git a/core-api/api/services/webhooks/calendar_webhook.py b/core-api/api/services/webhooks/calendar_webhook.py index 40839b6d..4bcd3c55 100644 --- a/core-api/api/services/webhooks/calendar_webhook.py +++ b/core-api/api/services/webhooks/calendar_webhook.py @@ -10,9 +10,10 @@ from googleapiclient.errors import HttpError +from lib.google_retry import GOOGLE_API_NUM_RETRIES from lib.supabase_client import get_service_role_client -from lib.batch_utils import batch_upsert from lib.token_encryption import decrypt_ext_connection_tokens +from lib.batch_utils import batch_upsert from api.services.google_auth import ( get_calendar_service_for_webhook, GoogleAuthError @@ -153,7 +154,7 @@ def _incremental_sync( if page_token: request_kwargs['pageToken'] = page_token - events_result = calendar_service.events().list(**request_kwargs).execute() + events_result = calendar_service.events().list(**request_kwargs).execute(num_retries=GOOGLE_API_NUM_RETRIES) page_events = events_result.get('items', []) all_events.extend(page_events) @@ -305,7 +306,7 @@ def _full_sync( if page_token: request_kwargs['pageToken'] = page_token - events_result = calendar_service.events().list(**request_kwargs).execute() + events_result = calendar_service.events().list(**request_kwargs).execute(num_retries=GOOGLE_API_NUM_RETRIES) page_events = events_result.get('items', []) all_events.extend(page_events) diff --git a/core-api/api/services/webhooks/gmail_webhook.py b/core-api/api/services/webhooks/gmail_webhook.py index b4340e02..48bd5e64 100644 --- a/core-api/api/services/webhooks/gmail_webhook.py +++ b/core-api/api/services/webhooks/gmail_webhook.py @@ -11,6 +11,7 @@ from googleapiclient.errors import HttpError +from lib.google_retry import GOOGLE_API_NUM_RETRIES from lib.supabase_client import get_service_role_client from lib.token_encryption import decrypt_ext_connection_tokens from api.services.google_auth import ( @@ -25,7 +26,6 @@ list_active_gmail_drafts_by_message_id, ) from api.services.email.draft_cleanup import cleanup_inactive_draft_rows_for_connection -from api.services.email.analyze_email_ai import analyze_email_with_ai logger = logging.getLogger(__name__) @@ -129,6 +129,111 @@ def process_gmail_notification( return result +def reconcile_gmail_connection( + connection_id: str, + *, + supabase=None, + gmail_service=None, + user_id: Optional[str] = None, +) -> Dict[str, Any]: + """ + Reconcile a Gmail connection to Gmail's current history state. + + This is the generic incremental path for cron/manual recovery. It differs + from webhook handling because it derives the newest historyId from Gmail + instead of reusing a webhook-delivered cursor. + """ + from api.services.syncs.google_services import get_google_services_for_connection + + supabase = supabase or get_service_role_client() + + if gmail_service is None or user_id is None: + gmail_service, _, resolved_user_id = get_google_services_for_connection( + connection_id, + supabase, + ) + user_id = user_id or resolved_user_id + + if not gmail_service or not user_id: + return {"status": "error", "message": "Could not get Gmail service"} + + subscription = supabase.table('push_subscriptions')\ + .select( + 'id, history_id, ext_connection_id, ' + 'ext_connections!push_subscriptions_ext_connection_id_fkey!inner(provider_email)' + )\ + .eq('ext_connection_id', connection_id)\ + .eq('provider', 'gmail')\ + .eq('is_active', True)\ + .limit(1)\ + .execute() + + if not subscription.data: + logger.info( + f"ℹ️ No active Gmail subscription found for connection {connection_id[:8]}..." + ) + return {"status": "skipped", "message": "No active Gmail subscription"} + + sub_data = subscription.data[0] + subscription_id = sub_data['id'] + provider_email = ( + (sub_data.get('ext_connections') or {}).get('provider_email') + if isinstance(sub_data.get('ext_connections'), dict) + else None + ) + old_history_id = sub_data.get('history_id') + + if not old_history_id: + logger.info( + f"ℹ️ No baseline Gmail historyId for connection {connection_id[:8]}..., " + "running fallback recovery" + ) + return _fallback_sync_with_recovery( + gmail_service, + supabase, + user_id, + connection_id, + subscription_id, + ) + + current_history_id = get_current_gmail_history_id(gmail_service) + if not current_history_id: + logger.warning( + f"⚠️ Failed to fetch current Gmail historyId for connection {connection_id[:8]}..., " + "running fallback recovery" + ) + return _fallback_sync_with_recovery( + gmail_service, + supabase, + user_id, + connection_id, + subscription_id, + ) + + logger.info( + f"πŸ“§ Reconciling Gmail history for connection {connection_id[:8]}... " + f"(old={old_history_id}, current={current_history_id})" + ) + + if old_history_id == current_history_id: + return { + "status": "ok", + "message": "No changes detected", + "email_address": provider_email, + "new_history_id": str(current_history_id), + } + + return _sync_with_history_api( + gmail_service, + supabase, + user_id, + connection_id, + subscription_id, + old_history_id, + str(current_history_id), + ) + + def _sync_with_history_api( gmail_service, supabase, @@ -149,7 +254,7 @@ def _sync_with_history_api( userId='me', startHistoryId=old_history_id, historyTypes=['messageAdded'] - ).execute() + ).execute(num_retries=GOOGLE_API_NUM_RETRIES) # Extract new messages from history messages_to_fetch = [] @@ -246,7 +351,7 @@ def _fallback_sync_with_recovery( maxResults=FALLBACK_MAX_MESSAGES, q=f'after:{cutoff_str}', # Time-bound query labelIds=['INBOX'] - ).execute() + ).execute(num_retries=GOOGLE_API_NUM_RETRIES) messages = messages_result.get('messages', []) logger.info(f"πŸ“§ Found {len(messages)} messages in last {FALLBACK_WINDOW_HOURS}h") @@ -374,7 +479,7 @@ def handle_message_response(request_id: str, response: Dict, exception: HttpErro userId='me', id=msg['id'], format='full' - ).execute() + ).execute(num_retries=GOOGLE_API_NUM_RETRIES) fetched_messages.append(full_msg) except HttpError as he: if he.resp.status != 404: @@ -385,10 +490,16 @@ def handle_message_response(request_id: str, response: Dict, exception: HttpErro for err in errors[:5]: # Log first 5 errors logger.warning(f" {err}") - # Save fetched messages to database and analyze with AI + # Save fetched messages to database. + # NOTE: AI analysis is intentionally NOT done here. This code path runs + # during webhook processing (both inline fallback and via QStash workers) + # and must stay fast. During notification bursts (e.g. watch renewal), + # dozens of webhooks fire concurrently β€” adding an LLM call per message + # was causing server starvation and 500s on unrelated user requests. + # AI analysis is handled by the /api/cron/analyze-emails cron + the + # analyze-emails worker instead. synced_count = 0 - analyzed_count = 0 - + for full_msg in fetched_messages: try: email_data = _parse_email_to_data( @@ -416,36 +527,6 @@ def handle_message_response(request_id: str, response: Dict, exception: HttpErro email_data.get("snippet"), ) - # Analyze with AI if not already analyzed - # Check if this email needs AI analysis - if result.data and len(result.data) > 0: - email_record = result.data[0] - email_id = email_record.get('id') - - # Only analyze if not already analyzed - if not email_record.get('ai_analyzed'): - try: - logger.info(f"πŸ€– Analyzing email {email_id[:8]}... with AI") - analysis = analyze_email_with_ai( - subject=email_data.get("subject"), - from_address=email_data.get("from"), - body=email_data.get("body"), - snippet=email_data.get("snippet") - ) - - # Update with AI analysis - supabase.table('emails').update({ - 'ai_analyzed': True, - 'ai_summary': analysis['summary'], - 'ai_important': analysis['important'] - }).eq('id', email_id).execute() - - analyzed_count += 1 - logger.info(f" βœ… AI summary: {analysis['summary']}") - - except Exception as ai_err: - logger.error(f"⚠️ Failed to analyze email {email_id[:8]}... with AI: {str(ai_err)}") - except Exception as e: logger.error(f"❌ Error saving email {full_msg.get('id')}: {str(e)}") @@ -462,7 +543,7 @@ def handle_message_response(request_id: str, response: Dict, exception: HttpErro else: logger.warning("Skipping webhook draft reconciliation because active draft map is unavailable") - logger.info(f"βœ… Batch sync complete: {synced_count}/{len(fetched_messages)} saved, {analyzed_count} analyzed") + logger.info(f"βœ… Batch sync complete: {synced_count}/{len(fetched_messages)} saved") return synced_count diff --git a/core-api/index.py b/core-api/index.py index e6eadbe3..8ccac6f8 100644 --- a/core-api/index.py +++ b/core-api/index.py @@ -1,203 +1,10 @@ -""" -FastAPI application for Vercel -Vercel auto-detects and deploys FastAPI apps at index.py -NO vercel.json or Mangum needed! -""" -import sys +"""FastAPI application entrypoint for the full API deployment.""" + import os +import sys -# Add project root to path so we can import from api/ and lib/ sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) -import sentry_sdk -import time -import logging -import traceback -from fastapi import FastAPI, HTTPException, Request -from fastapi.middleware.cors import CORSMiddleware -from fastapi.responses import JSONResponse -from datetime import datetime -from api.config import settings -from api.schemas import HealthResponse, StatusResponse -from lib.supabase_client import start_supabase_request_scope, reset_supabase_request_scope - -logger = logging.getLogger(__name__) - - -def _sentry_filter_noise(event, hint): - """Drop expected HTTP errors (4xx) from Sentry to reduce noise.""" - exc = hint.get("exc_info", (None, None, None))[1] - if isinstance(exc, HTTPException) and exc.status_code < 500: - return None - return event - - -sentry_sdk.init( - dsn=settings.sentry_dsn, - environment=settings.api_env, - traces_sample_rate=0.05, - send_default_pii=True, - before_send=_sentry_filter_noise, -) - -from slowapi import _rate_limit_exceeded_handler -from slowapi.errors import RateLimitExceeded -from slowapi.middleware import SlowAPIMiddleware -from api.rate_limit import limiter - -from api.routers import auth, calendar, email, webhooks, cron, sync, documents, files, chat, chat_attachments, app_drawer, preferences, workspaces, invitations, messages, users, projects, notifications, init, agents, agent_dispatch, permissions, public, workers, builder - -# Create FastAPI app - Vercel will auto-detect this -app = FastAPI( - title=settings.app_name, - description="FastAPI backend for the all-in-one productivity app", - version=settings.app_version, - debug=settings.debug -) - -# Rate limiting β€” middleware runs BEFORE FastAPI validation/dependencies -# Without this, invalid requests (422) bypass the decorator-based limiter -app.state.limiter = limiter -app.add_middleware(SlowAPIMiddleware) -app.add_exception_handler(RateLimitExceeded, _rate_limit_exceeded_handler) - - -# Global exception handler for unhandled exceptions -# This ensures CORS headers are included even on 500 errors -@app.exception_handler(Exception) -async def global_exception_handler(request: Request, exc: Exception): - """ - Catch-all exception handler that returns proper JSON error responses. - - This is critical for CORS: when an unhandled exception occurs, FastAPI's - default error handler may not include CORS headers if the exception happens - before response headers are sent. By catching all exceptions here and - returning a proper JSONResponse, we ensure the CORS middleware can add - its headers to the response. - """ - # Log the full traceback for debugging - logger.error( - f"Unhandled exception on {request.method} {request.url.path}: {exc}\n" - f"{''.join(traceback.format_exception(type(exc), exc, exc.__traceback__))}" - ) - - # Report to Sentry - sentry_sdk.capture_exception(exc) - - # Return a proper JSON response (CORS middleware will add headers) - return JSONResponse( - status_code=500, - content={ - "detail": "Internal server error", - "error_type": type(exc).__name__, - } - ) - - -# CORS -app.add_middleware( - CORSMiddleware, - allow_origins=settings.get_allowed_origins, - allow_credentials=True, - allow_methods=["GET", "POST", "PUT", "PATCH", "DELETE", "OPTIONS"], - allow_headers=["*"], -) - - -# Security headers middleware -@app.middleware("http") -async def supabase_request_scope_middleware(request: Request, call_next): - scope_token = start_supabase_request_scope() - try: - return await call_next(request) - finally: - reset_supabase_request_scope(scope_token) - - -@app.middleware("http") -async def security_headers_middleware(request: Request, call_next): - try: - response = await call_next(request) - except Exception as exc: - # Let the global exception handler deal with it, but ensure we don't swallow errors - raise - response.headers["X-Content-Type-Options"] = "nosniff" - response.headers["X-Frame-Options"] = "DENY" - response.headers["Strict-Transport-Security"] = "max-age=31536000; includeSubDomains" - response.headers["Referrer-Policy"] = "strict-origin-when-cross-origin" - response.headers["Permissions-Policy"] = "camera=(), microphone=(), geolocation=()" - return response - - -# Request timing middleware for performance monitoring -@app.middleware("http") -async def timing_middleware(request: Request, call_next): - start_time = time.perf_counter() - - try: - response = await call_next(request) - except Exception as exc: - # Log timing even for failed requests - process_time_ms = (time.perf_counter() - start_time) * 1000 - logger.error( - f"[PERF] {request.method} {request.url.path} - {process_time_ms:.2f}ms - EXCEPTION: {type(exc).__name__}" - ) - raise - - process_time_ms = (time.perf_counter() - start_time) * 1000 - - # Add timing to response headers - response.headers["X-Process-Time-Ms"] = f"{process_time_ms:.2f}" - - # Log the request timing - logger.info( - f"[PERF] {request.method} {request.url.path} - {process_time_ms:.2f}ms - Status: {response.status_code}" - ) - - return response - - -# Include routers -app.include_router(auth.router) -app.include_router(workspaces.router) -app.include_router(invitations.router) -app.include_router(calendar.router) -app.include_router(email.router) -app.include_router(documents.router) -app.include_router(files.router) -app.include_router(webhooks.router) -app.include_router(cron.router) -app.include_router(sync.router) -app.include_router(chat.router) -app.include_router(chat_attachments.router) -app.include_router(app_drawer.router) -app.include_router(preferences.router) -app.include_router(messages.router) -app.include_router(users.router) -app.include_router(projects.router) -app.include_router(notifications.router) -app.include_router(permissions.router) -app.include_router(agents.router) -app.include_router(agent_dispatch.router) -app.include_router(init.router) -app.include_router(public.router) -app.include_router(workers.router) -app.include_router(builder.router) - -@app.get("/", response_model=HealthResponse) -async def root(): - """Health check endpoint""" - return { - "status": "healthy", - "message": "Core Productivity API is running", - "version": settings.app_version - } +from api.app_factory import create_full_app -@app.get("/api/health", response_model=HealthResponse) -async def health_check(): - """Detailed health check""" - return { - "status": "healthy", - "service": "core-api", - "timestamp": datetime.utcnow().isoformat() + "Z" - } +app = create_full_app() diff --git a/core-api/lib/google_retry.py b/core-api/lib/google_retry.py new file mode 100644 index 00000000..f60b594f --- /dev/null +++ b/core-api/lib/google_retry.py @@ -0,0 +1,78 @@ +""" +Retry helpers for transient Google API and OAuth errors. + +google-auth's credentials.refresh() retries on HTTP status codes (500, 503, 429) +but does NOT retry on transport-level errors (Server disconnected, SSL EOF, +ConnectionError). This module fills that gap. + +For Google API .execute() calls, use the GOOGLE_API_NUM_RETRIES constant +which enables the built-in retry in google-api-python-client. +""" +import logging +import time +from typing import Any + +logger = logging.getLogger(__name__) + +# Pass to every Google API .execute(num_retries=GOOGLE_API_NUM_RETRIES) +# Handles 5xx, 429, SSL errors, ConnectionError with exponential backoff. +GOOGLE_API_NUM_RETRIES = 3 + +_TRANSIENT_ERROR_KEYWORDS = ( + "server disconnected", + "remotedisconnected", + "remoteprotocolerror", + "connectionterminated", + "connection aborted", + "connection reset", + "connectionerror", + "ssl", + "eof occurred", + "broken pipe", + "timed out", +) + + +def _is_transient_transport_error(exc: Exception) -> bool: + """Check if an exception is a transient transport/connection error.""" + msg = str(exc).lower() + return any(keyword in msg for keyword in _TRANSIENT_ERROR_KEYWORDS) + + +def refresh_credentials( + credentials: Any, + request: Any, + *, + max_retries: int = 2, + context: str = "", +) -> None: + """ + Refresh Google OAuth credentials with retry on transient transport errors. + + google-auth retries on HTTP 500/503/429 internally, but transport-level + errors (Server disconnected, SSL EOF, etc.) propagate as raw exceptions. + This wrapper retries those. + + Args: + credentials: google.oauth2.credentials.Credentials instance + request: google.auth.transport.requests.Request instance + max_retries: Number of retries on transient errors (default 2 = 3 total attempts) + context: Optional context string for log messages (e.g. connection ID) + + Raises: + The original exception if all retries are exhausted or error is not transient. + """ + for attempt in range(max_retries + 1): + try: + credentials.refresh(request) + return + except Exception as exc: + if attempt < max_retries and _is_transient_transport_error(exc): + wait = 0.5 * (2 ** attempt) # 0.5s, 1s, 2s + logger.warning( + f"⚠️ Transient error refreshing credentials{f' for {context}' if context else ''} " + f"(attempt {attempt + 1}/{max_retries + 1}): {exc} β€” retrying in {wait}s" + ) + time.sleep(wait) + continue + raise diff --git a/core-api/lib/google_webhook_security.py b/core-api/lib/google_webhook_security.py new file mode 100644 index 00000000..d5b75f57 --- /dev/null +++ b/core-api/lib/google_webhook_security.py @@ -0,0 +1,41 @@ +"""Helpers for Google webhook provenance checks.""" + +from __future__ import annotations + +import hashlib +import hmac +from typing import Optional + +from api.config import settings + + +def build_google_calendar_channel_token( + connection_id: str, + channel_id: str, +) -> Optional[str]: + """Build a stable HMAC token for Google Calendar push channels.""" + secret = settings.google_calendar_webhook_secret.strip() + if not secret: + return None + + payload = f"{connection_id}:{channel_id}" + signature = hmac.new( + secret.encode("utf-8"), + payload.encode("utf-8"), + hashlib.sha256, + ).hexdigest() + return f"v1:{payload}:{signature}" + + +def verify_google_calendar_channel_token( + connection_id: str, + channel_id: str, + provided_token: Optional[str], +) -> bool: + """Return True when the received Calendar channel token matches.""" + expected_token = build_google_calendar_channel_token(connection_id, channel_id) + if expected_token is None: + return True + if not provided_token: + return False + return hmac.compare_digest(provided_token, expected_token) diff --git a/core-api/setup_pubsub_subscription.py b/core-api/setup_pubsub_subscription.py index 72ecc674..3068aa7e 100644 --- a/core-api/setup_pubsub_subscription.py +++ b/core-api/setup_pubsub_subscription.py @@ -1,132 +1,256 @@ """ -Setup Google Cloud Pub/Sub Push Subscription for Gmail Webhooks +Create or update the Gmail Pub/Sub push subscription used for webhook ingress. -This script creates a push subscription that forwards Gmail notifications -from your Pub/Sub topic to your webhook endpoint. +This script is migration-friendly: if the subscription already exists, it updates +the push endpoint and push auth config instead of forcing a delete/recreate. -Prerequisites: -1. Google Cloud SDK installed (gcloud) -2. Authenticated with proper permissions -3. GOOGLE_PUBSUB_TOPIC environment variable set - -Usage: - python setup_pubsub_subscription.py +Environment: + GOOGLE_PUBSUB_TOPIC=projects/PROJECT_ID/topics/TOPIC_NAME + WEBHOOK_BASE_URL=https://core-webhooks-production.up.railway.app + PUBSUB_SUBSCRIPTION_NAME= + PUBSUB_ALLOW_ADDITIONAL_SUBSCRIPTION= + PUBSUB_PUSH_SERVICE_ACCOUNT_EMAIL= + PUBSUB_PUSH_AUDIENCE= + PUBSUB_DEAD_LETTER_TOPIC=projects/PROJECT_ID/topics/TOPIC_NAME + PUBSUB_MAX_DELIVERY_ATTEMPTS=20 """ + +from __future__ import annotations + import os import subprocess import sys +from typing import List + + +def _run(cmd: List[str]) -> subprocess.CompletedProcess[str]: + return subprocess.run(cmd, capture_output=True, text=True) + + +def _is_truthy(value: str) -> bool: + return value.strip().lower() in {"1", "true", "yes", "on"} + + +def _build_push_flags(push_endpoint: str) -> List[str]: + flags = [f"--push-endpoint={push_endpoint}"] + + push_service_account = os.getenv("PUBSUB_PUSH_SERVICE_ACCOUNT_EMAIL", "").strip() + if push_service_account: + flags.append(f"--push-auth-service-account={push_service_account}") + audience = os.getenv("PUBSUB_PUSH_AUDIENCE", "").strip() or push_endpoint + flags.append(f"--push-auth-token-audience={audience}") + + return flags + + +def _build_dead_letter_flags() -> List[str]: + dead_letter_topic = os.getenv("PUBSUB_DEAD_LETTER_TOPIC", "").strip() + if not dead_letter_topic: + return [] + + max_delivery_attempts = os.getenv("PUBSUB_MAX_DELIVERY_ATTEMPTS", "20").strip() or "20" + return [ + f"--dead-letter-topic={dead_letter_topic}", + f"--max-delivery-attempts={max_delivery_attempts}", + ] + + +def _list_topic_subscriptions(project_id: str, pubsub_topic: str) -> List[str]: + """Return subscription names attached to the given topic.""" + cmd = [ + "gcloud", + "pubsub", + "subscriptions", + "list", + f"--project={project_id}", + f"--filter=topic:{pubsub_topic}", + "--format=value(name.basename())", + ] + result = _run(cmd) + if result.returncode != 0: + print("❌ Failed to list existing Pub/Sub subscriptions for the topic") + print() + print(result.stderr) + sys.exit(1) -def setup_pubsub_push_subscription(): - """ - Create a Pub/Sub push subscription for Gmail webhooks - """ - # Get configuration from environment - pubsub_topic = os.getenv('GOOGLE_PUBSUB_TOPIC', 'projects/YOUR_PROJECT_ID/topics/gmail-sync-topic') - webhook_url = os.getenv('WEBHOOK_BASE_URL', 'http://localhost:8000') - - # Extract project ID and topic name from full topic path - # Format: projects/PROJECT_ID/topics/TOPIC_NAME - parts = pubsub_topic.split('/') - if len(parts) != 4 or parts[0] != 'projects' or parts[2] != 'topics': + return [line.strip() for line in result.stdout.splitlines() if line.strip()] + + +def setup_pubsub_push_subscription() -> None: + """Create or update the Gmail Pub/Sub push subscription.""" + pubsub_topic = os.getenv( + "GOOGLE_PUBSUB_TOPIC", + "projects/core-478723/topics/gmail-sync-topic", + ).strip() + webhook_base_url = os.getenv( + "WEBHOOK_BASE_URL", + "", # Set WEBHOOK_BASE_URL in your environment + ).rstrip("/") + + parts = pubsub_topic.split("/") + if len(parts) != 4 or parts[0] != "projects" or parts[2] != "topics": print(f"❌ Invalid GOOGLE_PUBSUB_TOPIC format: {pubsub_topic}") print(" Expected: projects/PROJECT_ID/topics/TOPIC_NAME") sys.exit(1) - + project_id = parts[1] topic_name = parts[3] - - # Subscription name - subscription_name = f"{topic_name}-push-subscription" - full_subscription = f"projects/{project_id}/subscriptions/{subscription_name}" - - # Push endpoint - push_endpoint = f"{webhook_url}/api/webhooks/gmail" - + explicit_subscription_name = os.getenv("PUBSUB_SUBSCRIPTION_NAME", "").strip() + allow_additional_subscription = _is_truthy( + os.getenv("PUBSUB_ALLOW_ADDITIONAL_SUBSCRIPTION", "") + ) + existing_subscriptions = _list_topic_subscriptions(project_id, pubsub_topic) + + if explicit_subscription_name: + subscription_name = explicit_subscription_name + if ( + subscription_name not in existing_subscriptions + and existing_subscriptions + and not allow_additional_subscription + ): + print("❌ Refusing to create an additional subscription on the Gmail topic") + print() + print( + "The topic already has existing subscriptions, and creating another one " + "would deliver every Gmail notification to both endpoints." + ) + print() + print("Existing subscriptions:") + for name in existing_subscriptions: + print(f" - {name}") + print() + print("Use one of these options:") + print(" 1. Set PUBSUB_SUBSCRIPTION_NAME to the existing subscription you want to update") + print(" 2. Delete the old subscription before re-running this script") + print( + " 3. If you intentionally want fanout, set " + "PUBSUB_ALLOW_ADDITIONAL_SUBSCRIPTION=true" + ) + sys.exit(1) + elif len(existing_subscriptions) == 1: + subscription_name = existing_subscriptions[0] + elif len(existing_subscriptions) == 0: + subscription_name = f"{topic_name}-push-subscription" + else: + print("❌ Multiple subscriptions already exist for the Gmail topic") + print() + print( + "This script will not guess which one to update because that can leave " + "duplicate webhook delivery in place." + ) + print() + print("Existing subscriptions:") + for name in existing_subscriptions: + print(f" - {name}") + print() + print("Re-run with PUBSUB_SUBSCRIPTION_NAME set to the exact subscription you want to update.") + sys.exit(1) + + push_endpoint = f"{webhook_base_url}/api/webhooks/gmail" + push_flags = _build_push_flags(push_endpoint) + dead_letter_flags = _build_dead_letter_flags() + print("=" * 80) - print("πŸ”§ Setting up Google Cloud Pub/Sub Push Subscription") + print("πŸ”§ Configuring Google Cloud Pub/Sub push subscription") print("=" * 80) print(f"πŸ“’ Pub/Sub Topic: {pubsub_topic}") print(f"πŸ“¬ Subscription Name: {subscription_name}") print(f"🌐 Push Endpoint: {push_endpoint}") print(f"πŸ”‘ Project ID: {project_id}") + if existing_subscriptions: + print(f"πŸ“š Existing Topic Subscriptions: {', '.join(existing_subscriptions)}") + else: + print("πŸ“š Existing Topic Subscriptions: none") + if push_flags[1:]: + print("πŸ” Authenticated push: enabled") + else: + print("πŸ” Authenticated push: disabled") + if dead_letter_flags: + print(f"πŸ’€ Dead-letter topic: {dead_letter_flags[0].split('=', 1)[1]}") print() - - # Check if subscription already exists - print("πŸ” Checking if subscription already exists...") + check_cmd = [ - 'gcloud', 'pubsub', 'subscriptions', 'describe', + "gcloud", + "pubsub", + "subscriptions", + "describe", subscription_name, - f'--project={project_id}', - '--format=json' + f"--project={project_id}", + "--format=json", ] - - result = subprocess.run(check_cmd, capture_output=True, text=True) - - if result.returncode == 0: - print(f"⚠️ Subscription '{subscription_name}' already exists!") - print() - print("To update the push endpoint, delete and recreate:") - print(f" gcloud pubsub subscriptions delete {subscription_name} --project={project_id}") - print() - print("Then run this script again.") - return - - # Create the push subscription - print(f"πŸ“ Creating push subscription...") - create_cmd = [ - 'gcloud', 'pubsub', 'subscriptions', 'create', - subscription_name, - f'--topic={topic_name}', - f'--push-endpoint={push_endpoint}', - f'--project={project_id}', - '--ack-deadline=60', # 60 second ack deadline - '--message-retention-duration=7d', # Keep messages for 7 days if undelivered - ] - - print(f"Running: {' '.join(create_cmd)}") - print() - - result = subprocess.run(create_cmd, capture_output=True, text=True) - - if result.returncode == 0: - print("βœ… Push subscription created successfully!") - print() - print("πŸ“‹ Next Steps:") - print(" 1. Your Gmail webhook should now receive notifications at:") - print(f" {push_endpoint}") - print() - print(" 2. Test by sending yourself an email") - print() - print(" 3. Check logs for: 'πŸ“¬ Gmail webhook received'") - print() - print(" 4. Verify in Google Cloud Console:") - print(f" https://console.cloud.google.com/cloudpubsub/subscription/detail/{subscription_name}?project={project_id}") - print() + exists = _run(check_cmd).returncode == 0 + + if exists: + print(f"♻️ Updating existing subscription '{subscription_name}'...") + cmd = [ + "gcloud", + "pubsub", + "subscriptions", + "update", + subscription_name, + f"--project={project_id}", + "--ack-deadline=60", + "--message-retention-duration=7d", + *push_flags, + *dead_letter_flags, + ] else: - print("❌ Failed to create push subscription!") + print(f"πŸ“ Creating subscription '{subscription_name}'...") + cmd = [ + "gcloud", + "pubsub", + "subscriptions", + "create", + subscription_name, + f"--topic={topic_name}", + f"--project={project_id}", + "--ack-deadline=60", + "--message-retention-duration=7d", + *push_flags, + *dead_letter_flags, + ] + + print(f"Running: {' '.join(cmd)}") + print() + + result = _run(cmd) + if result.returncode != 0: + print("❌ Pub/Sub subscription command failed!") print() - print("Error output:") print(result.stderr) print() print("Common issues:") - print(" 1. Not authenticated: Run 'gcloud auth login'") - print(" 2. Wrong project: Run 'gcloud config set project YOUR_PROJECT_ID'") - print(" 3. Missing permissions: Need 'pubsub.subscriptions.create' permission") - print(" 4. Topic doesn't exist: Create the topic first") + print(" 1. Not authenticated: run 'gcloud auth login'") + print(" 2. Wrong project: run 'gcloud config set project YOUR_PROJECT_ID'") + print(" 3. Missing IAM on the push auth service account") + print(" 4. Dead-letter topic missing or missing Pub/Sub service account permissions") + print(" 5. Existing topic subscriptions caused duplicate delivery; set PUBSUB_SUBSCRIPTION_NAME explicitly") sys.exit(1) - + + print("βœ… Pub/Sub push subscription is configured") + print() + print("πŸ“‹ Next Steps:") + print(" 1. Deploy the webhook ingress service on Railway") + print(f" 2. Verify Gmail webhook health at {push_endpoint.replace('/gmail', '/gmail/verify')}") + print(" 3. Send a test email and confirm Railway logs show 'πŸ“¬ Gmail webhook received'") + print(" 4. If you changed WEBHOOK_BASE_URL, reset Google Calendar watches so their stored addresses move too") + print() + print("Google Cloud Console:") + print( + " " + f"https://console.cloud.google.com/cloudpubsub/subscription/detail/{subscription_name}" + f"?project={project_id}" + ) print("=" * 80) if __name__ == "__main__": - # Check if gcloud is installed try: - subprocess.run(['gcloud', '--version'], capture_output=True, check=True) + subprocess.run(["gcloud", "--version"], capture_output=True, check=True) except (subprocess.CalledProcessError, FileNotFoundError): print("❌ Google Cloud SDK (gcloud) is not installed or not in PATH") print() print("Install from: https://cloud.google.com/sdk/docs/install") sys.exit(1) - - setup_pubsub_push_subscription() + setup_pubsub_push_subscription() diff --git a/core-api/supabase/migrations/20260401000001_enforce_single_active_push_subscription.sql b/core-api/supabase/migrations/20260401000001_enforce_single_active_push_subscription.sql new file mode 100644 index 00000000..59fa30ff --- /dev/null +++ b/core-api/supabase/migrations/20260401000001_enforce_single_active_push_subscription.sql @@ -0,0 +1,28 @@ +-- Ensure each connection/provider pair has at most one active push subscription. +-- Deduplicate existing active rows first so the unique index can be created safely. + +WITH ranked_active_subscriptions AS ( + SELECT + id, + ROW_NUMBER() OVER ( + PARTITION BY ext_connection_id, provider + ORDER BY updated_at DESC, created_at DESC, id DESC + ) AS row_num + FROM public.push_subscriptions + WHERE is_active = true + AND ext_connection_id IS NOT NULL + AND provider IS NOT NULL +) +UPDATE public.push_subscriptions AS push_subscriptions +SET + is_active = false, + updated_at = now() +FROM ranked_active_subscriptions +WHERE push_subscriptions.id = ranked_active_subscriptions.id + AND ranked_active_subscriptions.row_num > 1; + +CREATE UNIQUE INDEX IF NOT EXISTS idx_push_subscriptions_active_connection_provider +ON public.push_subscriptions USING btree (ext_connection_id, provider) +WHERE is_active = true + AND ext_connection_id IS NOT NULL + AND provider IS NOT NULL; diff --git a/core-api/supabase/migrations/20260401000002_connection_sync_state.sql b/core-api/supabase/migrations/20260401000002_connection_sync_state.sql new file mode 100644 index 00000000..0ae42c37 --- /dev/null +++ b/core-api/supabase/migrations/20260401000002_connection_sync_state.sql @@ -0,0 +1,538 @@ +-- Sync control-plane table for lease-based, single-flight provider syncs. +-- One row per (connection_id, sync_kind). + +-- ============================================================================= +-- Helper functions +-- ============================================================================= + +CREATE OR REPLACE FUNCTION "public"."merge_connection_sync_cursor"( + "p_existing" "text", + "p_incoming" "text" +) RETURNS "text" + LANGUAGE "sql" + IMMUTABLE + AS $$ + SELECT CASE + WHEN p_incoming IS NULL OR btrim(p_incoming) = '' THEN p_existing + WHEN p_existing IS NULL OR btrim(p_existing) = '' THEN p_incoming + WHEN p_existing ~ '^[0-9]+$' AND p_incoming ~ '^[0-9]+$' THEN + CASE + WHEN p_existing::numeric <= p_incoming::numeric THEN p_incoming + ELSE p_existing + END + ELSE p_incoming + END + $$; + + +ALTER FUNCTION "public"."merge_connection_sync_cursor"("p_existing" "text", "p_incoming" "text") OWNER TO "postgres"; + + +-- ============================================================================= +-- Tables +-- ============================================================================= + +CREATE TABLE IF NOT EXISTS "public"."connection_sync_state" ( + "connection_id" "uuid" NOT NULL, + "provider" "text" NOT NULL, + "sync_kind" "text" NOT NULL, + "dirty" boolean DEFAULT false NOT NULL, + "dirty_since" timestamp with time zone, + "dirty_generation" bigint DEFAULT 0 NOT NULL, + "lease_generation" bigint, + "latest_seen_cursor" "text", + "last_synced_cursor" "text", + "lease_owner" "text", + "lease_expires_at" timestamp with time zone, + "last_provider_event_at" timestamp with time zone, + "last_sync_started_at" timestamp with time zone, + "last_sync_finished_at" timestamp with time zone, + "last_heartbeat_at" timestamp with time zone, + "last_error_at" timestamp with time zone, + "last_error" "text", + "retry_count" integer DEFAULT 0 NOT NULL, + "next_retry_at" timestamp with time zone, + "priority" integer DEFAULT 0 NOT NULL, + "metadata" "jsonb" DEFAULT '{}'::"jsonb" NOT NULL, + "created_at" timestamp with time zone DEFAULT "now"() NOT NULL, + "updated_at" timestamp with time zone DEFAULT "now"() NOT NULL +); + + +ALTER TABLE "public"."connection_sync_state" OWNER TO "postgres"; + + +COMMENT ON TABLE "public"."connection_sync_state" IS 'Control-plane state for background sync execution. One row per ext_connection and sync stream (email/calendar).'; +COMMENT ON COLUMN "public"."connection_sync_state"."sync_kind" IS 'Logical sync stream for the connection: email or calendar.'; +COMMENT ON COLUMN "public"."connection_sync_state"."dirty_generation" IS 'Monotonic invalidation counter. If it increases during a lease, the worker must keep the row dirty on completion.'; +COMMENT ON COLUMN "public"."connection_sync_state"."lease_generation" IS 'dirty_generation observed when the current lease was claimed.'; +COMMENT ON COLUMN "public"."connection_sync_state"."metadata" IS 'Pending trigger context for the next sync attempt. Cleared once the stream is clean.'; + + +-- ============================================================================= +-- Constraints +-- ============================================================================= + +ALTER TABLE ONLY "public"."connection_sync_state" + ADD CONSTRAINT "connection_sync_state_pkey" PRIMARY KEY ("connection_id", "sync_kind"); + +ALTER TABLE ONLY "public"."connection_sync_state" + ADD CONSTRAINT "connection_sync_state_connection_id_fkey" FOREIGN KEY ("connection_id") REFERENCES "public"."ext_connections"("id") ON DELETE CASCADE; + +ALTER TABLE ONLY "public"."connection_sync_state" + ADD CONSTRAINT "connection_sync_state_provider_check" CHECK (provider = ANY (ARRAY['google'::text, 'microsoft'::text])); + +ALTER TABLE ONLY "public"."connection_sync_state" + ADD CONSTRAINT "connection_sync_state_sync_kind_check" CHECK (sync_kind = ANY (ARRAY['email'::text, 'calendar'::text])); + +ALTER TABLE ONLY "public"."connection_sync_state" + ADD CONSTRAINT "connection_sync_state_priority_check" CHECK (priority >= 0); + + +-- ============================================================================= +-- Indexes +-- ============================================================================= + +CREATE INDEX "idx_connection_sync_state_claimable" +ON "public"."connection_sync_state" USING "btree" ("provider", "sync_kind", "priority" DESC, "dirty_since", "next_retry_at", "lease_expires_at") +WHERE "dirty" = true; + +CREATE INDEX "idx_connection_sync_state_lease_expires_at" +ON "public"."connection_sync_state" USING "btree" ("lease_expires_at"); + +CREATE INDEX "idx_connection_sync_state_next_retry_at" +ON "public"."connection_sync_state" USING "btree" ("next_retry_at"); + + +-- ============================================================================= +-- Security / triggers +-- ============================================================================= + +ALTER TABLE "public"."connection_sync_state" ENABLE ROW LEVEL SECURITY; + +CREATE OR REPLACE TRIGGER "update_connection_sync_state_updated_at" +BEFORE UPDATE ON "public"."connection_sync_state" +FOR EACH ROW EXECUTE FUNCTION "public"."update_updated_at_column"(); + +GRANT ALL ON TABLE "public"."connection_sync_state" TO "service_role"; +GRANT ALL ON FUNCTION "public"."merge_connection_sync_cursor"("p_existing" "text", "p_incoming" "text") TO "service_role"; + + +-- ============================================================================= +-- RPCs +-- ============================================================================= + +CREATE OR REPLACE FUNCTION "public"."mark_connection_sync_dirty"( + "p_connection_id" "uuid", + "p_sync_kind" "text", + "p_latest_seen_cursor" "text" DEFAULT NULL, + "p_priority" integer DEFAULT 0, + "p_provider_event_at" timestamp with time zone DEFAULT "now"(), + "p_metadata" "jsonb" DEFAULT '{}'::"jsonb" +) RETURNS boolean + LANGUAGE "plpgsql" SECURITY DEFINER + SET "search_path" TO 'public' + AS $$ +DECLARE + v_provider text; + v_now timestamp with time zone := now(); +BEGIN + IF p_sync_kind NOT IN ('email', 'calendar') THEN + RAISE EXCEPTION 'invalid sync_kind: %', p_sync_kind; + END IF; + + SELECT provider + INTO v_provider + FROM public.ext_connections + WHERE id = p_connection_id + AND is_active = true + AND provider IN ('google', 'microsoft') + LIMIT 1; + + IF v_provider IS NULL THEN + RETURN false; + END IF; + + INSERT INTO public.connection_sync_state ( + connection_id, + provider, + sync_kind, + dirty, + dirty_since, + dirty_generation, + latest_seen_cursor, + last_provider_event_at, + priority, + metadata, + created_at, + updated_at + ) + VALUES ( + p_connection_id, + v_provider, + p_sync_kind, + true, + v_now, + 1, + public.merge_connection_sync_cursor(NULL, p_latest_seen_cursor), + COALESCE(p_provider_event_at, v_now), + GREATEST(p_priority, 0), + COALESCE(p_metadata, '{}'::jsonb), + v_now, + v_now + ) + ON CONFLICT (connection_id, sync_kind) DO UPDATE + SET + provider = EXCLUDED.provider, + dirty = true, + dirty_since = COALESCE(public.connection_sync_state.dirty_since, v_now), + dirty_generation = public.connection_sync_state.dirty_generation + 1, + latest_seen_cursor = public.merge_connection_sync_cursor( + public.connection_sync_state.latest_seen_cursor, + p_latest_seen_cursor + ), + last_provider_event_at = COALESCE(p_provider_event_at, v_now), + priority = GREATEST(public.connection_sync_state.priority, GREATEST(p_priority, 0)), + next_retry_at = NULL, + metadata = CASE + WHEN p_metadata IS NULL OR p_metadata = '{}'::jsonb THEN public.connection_sync_state.metadata + ELSE p_metadata + END, + updated_at = v_now; + + RETURN true; +END; +$$; + + +ALTER FUNCTION "public"."mark_connection_sync_dirty"( + "p_connection_id" "uuid", + "p_sync_kind" "text", + "p_latest_seen_cursor" "text", + "p_priority" integer, + "p_provider_event_at" timestamp with time zone, + "p_metadata" "jsonb" +) OWNER TO "postgres"; + + +CREATE OR REPLACE FUNCTION "public"."claim_connection_sync_lease"( + "p_worker_id" "text", + "p_lease_seconds" integer DEFAULT 120, + "p_provider" "text" DEFAULT NULL, + "p_sync_kind" "text" DEFAULT NULL, + "p_connection_id" "uuid" DEFAULT NULL +) RETURNS TABLE( + "connection_id" "uuid", + "provider" "text", + "sync_kind" "text", + "dirty_generation" bigint, + "lease_generation" bigint, + "latest_seen_cursor" "text", + "last_synced_cursor" "text", + "priority" integer, + "lease_expires_at" timestamp with time zone +) + LANGUAGE "plpgsql" SECURITY DEFINER + SET "search_path" TO 'public' + AS $$ +BEGIN + IF p_worker_id IS NULL OR btrim(p_worker_id) = '' THEN + RAISE EXCEPTION 'worker_id is required'; + END IF; + + RETURN QUERY + WITH candidate AS ( + SELECT css.connection_id, css.sync_kind + FROM public.connection_sync_state css + JOIN public.ext_connections ec + ON ec.id = css.connection_id + WHERE css.dirty = true + AND ec.is_active = true + AND (css.next_retry_at IS NULL OR css.next_retry_at <= now()) + AND (css.lease_expires_at IS NULL OR css.lease_expires_at <= now()) + AND (p_provider IS NULL OR css.provider = p_provider) + AND (p_sync_kind IS NULL OR css.sync_kind = p_sync_kind) + AND (p_connection_id IS NULL OR css.connection_id = p_connection_id) + ORDER BY css.priority DESC, COALESCE(css.dirty_since, css.updated_at) ASC, css.updated_at ASC + LIMIT 1 + FOR UPDATE SKIP LOCKED + ), + claimed AS ( + UPDATE public.connection_sync_state css + SET + lease_owner = p_worker_id, + lease_expires_at = now() + make_interval(secs => GREATEST(p_lease_seconds, 15)), + last_sync_started_at = now(), + last_heartbeat_at = now(), + lease_generation = css.dirty_generation, + updated_at = now() + FROM candidate + WHERE css.connection_id = candidate.connection_id + AND css.sync_kind = candidate.sync_kind + RETURNING + css.connection_id, + css.provider, + css.sync_kind, + css.dirty_generation, + css.lease_generation, + css.latest_seen_cursor, + css.last_synced_cursor, + css.priority, + css.lease_expires_at + ) + SELECT + claimed.connection_id, + claimed.provider, + claimed.sync_kind, + claimed.dirty_generation, + claimed.lease_generation, + claimed.latest_seen_cursor, + claimed.last_synced_cursor, + claimed.priority, + claimed.lease_expires_at + FROM claimed; +END; +$$; + + +ALTER FUNCTION "public"."claim_connection_sync_lease"( + "p_worker_id" "text", + "p_lease_seconds" integer, + "p_provider" "text", + "p_sync_kind" "text", + "p_connection_id" "uuid" +) OWNER TO "postgres"; + + +CREATE OR REPLACE FUNCTION "public"."heartbeat_connection_sync_lease"( + "p_connection_id" "uuid", + "p_sync_kind" "text", + "p_worker_id" "text", + "p_lease_seconds" integer DEFAULT 120 +) RETURNS boolean + LANGUAGE "plpgsql" SECURITY DEFINER + SET "search_path" TO 'public' + AS $$ +DECLARE + v_updated integer; +BEGIN + UPDATE public.connection_sync_state + SET + lease_expires_at = now() + make_interval(secs => GREATEST(p_lease_seconds, 15)), + last_heartbeat_at = now(), + updated_at = now() + WHERE connection_id = p_connection_id + AND sync_kind = p_sync_kind + AND lease_owner = p_worker_id; + + GET DIAGNOSTICS v_updated = ROW_COUNT; + RETURN v_updated > 0; +END; +$$; + + +ALTER FUNCTION "public"."heartbeat_connection_sync_lease"( + "p_connection_id" "uuid", + "p_sync_kind" "text", + "p_worker_id" "text", + "p_lease_seconds" integer +) OWNER TO "postgres"; + + +CREATE OR REPLACE FUNCTION "public"."complete_connection_sync_lease"( + "p_connection_id" "uuid", + "p_sync_kind" "text", + "p_worker_id" "text", + "p_last_synced_cursor" "text" DEFAULT NULL, + "p_latest_seen_cursor" "text" DEFAULT NULL, + "p_keep_dirty" boolean DEFAULT false +) RETURNS TABLE( + "dirty" boolean, + "dirty_generation" bigint, + "latest_seen_cursor" "text", + "last_synced_cursor" "text" +) + LANGUAGE "plpgsql" SECURITY DEFINER + SET "search_path" TO 'public' + AS $$ +BEGIN + RETURN QUERY + UPDATE public.connection_sync_state css + SET + latest_seen_cursor = public.merge_connection_sync_cursor( + css.latest_seen_cursor, + COALESCE(p_latest_seen_cursor, p_last_synced_cursor) + ), + last_synced_cursor = public.merge_connection_sync_cursor( + css.last_synced_cursor, + p_last_synced_cursor + ), + dirty = CASE + WHEN p_keep_dirty THEN true + WHEN css.dirty_generation > css.lease_generation THEN true + ELSE false + END, + dirty_since = CASE + WHEN p_keep_dirty THEN css.dirty_since + WHEN css.dirty_generation > css.lease_generation THEN css.dirty_since + ELSE NULL + END, + lease_owner = NULL, + lease_expires_at = NULL, + lease_generation = NULL, + last_sync_finished_at = now(), + last_heartbeat_at = now(), + retry_count = CASE + WHEN p_keep_dirty THEN css.retry_count + WHEN css.dirty_generation > css.lease_generation THEN css.retry_count + ELSE 0 + END, + priority = CASE + WHEN p_keep_dirty THEN css.priority + WHEN css.dirty_generation > css.lease_generation THEN css.priority + ELSE 0 + END, + next_retry_at = NULL, + last_error = NULL, + last_error_at = NULL, + metadata = CASE + WHEN p_keep_dirty THEN css.metadata + WHEN css.dirty_generation > css.lease_generation THEN css.metadata + ELSE '{}'::jsonb + END, + updated_at = now() + WHERE css.connection_id = p_connection_id + AND css.sync_kind = p_sync_kind + AND css.lease_owner = p_worker_id + RETURNING + css.dirty, + css.dirty_generation, + css.latest_seen_cursor, + css.last_synced_cursor; +END; +$$; + + +ALTER FUNCTION "public"."complete_connection_sync_lease"( + "p_connection_id" "uuid", + "p_sync_kind" "text", + "p_worker_id" "text", + "p_last_synced_cursor" "text", + "p_latest_seen_cursor" "text", + "p_keep_dirty" boolean +) OWNER TO "postgres"; + + +CREATE OR REPLACE FUNCTION "public"."fail_connection_sync_lease"( + "p_connection_id" "uuid", + "p_sync_kind" "text", + "p_worker_id" "text", + "p_error" "text", + "p_retry_seconds" integer DEFAULT 60 +) RETURNS TABLE( + "retry_count" integer, + "next_retry_at" timestamp with time zone +) + LANGUAGE "plpgsql" SECURITY DEFINER + SET "search_path" TO 'public' + AS $$ +BEGIN + RETURN QUERY + UPDATE public.connection_sync_state css + SET + dirty = true, + lease_owner = NULL, + lease_expires_at = NULL, + lease_generation = NULL, + last_sync_finished_at = now(), + last_error = LEFT(COALESCE(p_error, 'sync failed'), 4000), + last_error_at = now(), + retry_count = css.retry_count + 1, + next_retry_at = now() + make_interval(secs => GREATEST(p_retry_seconds, 0)), + updated_at = now() + WHERE css.connection_id = p_connection_id + AND css.sync_kind = p_sync_kind + AND css.lease_owner = p_worker_id + RETURNING + css.retry_count, + css.next_retry_at; +END; +$$; + + +ALTER FUNCTION "public"."fail_connection_sync_lease"( + "p_connection_id" "uuid", + "p_sync_kind" "text", + "p_worker_id" "text", + "p_error" "text", + "p_retry_seconds" integer +) OWNER TO "postgres"; + +GRANT ALL ON FUNCTION "public"."mark_connection_sync_dirty"( + "p_connection_id" "uuid", + "p_sync_kind" "text", + "p_latest_seen_cursor" "text", + "p_priority" integer, + "p_provider_event_at" timestamp with time zone, + "p_metadata" "jsonb" +) TO "service_role"; + +GRANT ALL ON FUNCTION "public"."claim_connection_sync_lease"( + "p_worker_id" "text", + "p_lease_seconds" integer, + "p_provider" "text", + "p_sync_kind" "text", + "p_connection_id" "uuid" +) TO "service_role"; + +GRANT ALL ON FUNCTION "public"."heartbeat_connection_sync_lease"( + "p_connection_id" "uuid", + "p_sync_kind" "text", + "p_worker_id" "text", + "p_lease_seconds" integer +) TO "service_role"; + +GRANT ALL ON FUNCTION "public"."complete_connection_sync_lease"( + "p_connection_id" "uuid", + "p_sync_kind" "text", + "p_worker_id" "text", + "p_last_synced_cursor" "text", + "p_latest_seen_cursor" "text", + "p_keep_dirty" boolean +) TO "service_role"; + +GRANT ALL ON FUNCTION "public"."fail_connection_sync_lease"( + "p_connection_id" "uuid", + "p_sync_kind" "text", + "p_worker_id" "text", + "p_error" "text", + "p_retry_seconds" integer +) TO "service_role"; + + +-- ============================================================================= +-- Backfill rows for existing active connections +-- ============================================================================= + +INSERT INTO public.connection_sync_state ( + connection_id, + provider, + sync_kind, + dirty, + created_at, + updated_at +) +SELECT + ec.id, + ec.provider, + sync_kinds.sync_kind, + false, + now(), + now() +FROM public.ext_connections ec +CROSS JOIN ( + VALUES ('email'::text), ('calendar'::text) +) AS sync_kinds(sync_kind) +WHERE ec.is_active = true + AND ec.provider IN ('google', 'microsoft') +ON CONFLICT (connection_id, sync_kind) DO NOTHING; diff --git a/core-api/supabase/migrations/20260402000001_skip_redundant_dirty_mark_writes.sql b/core-api/supabase/migrations/20260402000001_skip_redundant_dirty_mark_writes.sql new file mode 100644 index 00000000..73dad5aa --- /dev/null +++ b/core-api/supabase/migrations/20260402000001_skip_redundant_dirty_mark_writes.sql @@ -0,0 +1,121 @@ +-- Skip redundant control-plane writes when a stream is already dirty and +-- the incoming cursor does not advance the newest seen checkpoint. +-- +-- This primarily reduces Gmail webhook churn, where Pub/Sub can deliver +-- multiple notifications for the same mailbox carrying the same or an older +-- historyId while the stream is already dirty and waiting to be processed. + +CREATE OR REPLACE FUNCTION "public"."mark_connection_sync_dirty"( + "p_connection_id" "uuid", + "p_sync_kind" "text", + "p_latest_seen_cursor" "text" DEFAULT NULL, + "p_priority" integer DEFAULT 0, + "p_provider_event_at" timestamp with time zone DEFAULT "now"(), + "p_metadata" "jsonb" DEFAULT '{}'::"jsonb" +) RETURNS boolean + LANGUAGE "plpgsql" SECURITY DEFINER + SET "search_path" TO 'public' + AS $$ +DECLARE + v_provider text; + v_now timestamp with time zone := now(); + v_existing record; +BEGIN + IF p_sync_kind NOT IN ('email', 'calendar') THEN + RAISE EXCEPTION 'invalid sync_kind: %', p_sync_kind; + END IF; + + SELECT provider + INTO v_provider + FROM public.ext_connections + WHERE id = p_connection_id + AND is_active = true + AND provider IN ('google', 'microsoft') + LIMIT 1; + + IF v_provider IS NULL THEN + RETURN false; + END IF; + + -- Short-circuit duplicate cursor-based invalidations for already-dirty + -- streams. This keeps Gmail notification floods from rewriting the same + -- row over and over while preserving the "accepted" success contract. + IF p_latest_seen_cursor IS NOT NULL THEN + SELECT dirty, latest_seen_cursor + INTO v_existing + FROM public.connection_sync_state + WHERE connection_id = p_connection_id + AND sync_kind = p_sync_kind; + + IF v_existing IS NOT NULL + AND v_existing.dirty = true + AND v_existing.latest_seen_cursor IS NOT NULL + AND public.merge_connection_sync_cursor( + v_existing.latest_seen_cursor, + p_latest_seen_cursor + ) = v_existing.latest_seen_cursor + THEN + RETURN true; + END IF; + END IF; + + INSERT INTO public.connection_sync_state ( + connection_id, + provider, + sync_kind, + dirty, + dirty_since, + dirty_generation, + latest_seen_cursor, + last_provider_event_at, + priority, + metadata, + created_at, + updated_at + ) + VALUES ( + p_connection_id, + v_provider, + p_sync_kind, + true, + v_now, + 1, + public.merge_connection_sync_cursor(NULL, p_latest_seen_cursor), + COALESCE(p_provider_event_at, v_now), + GREATEST(p_priority, 0), + COALESCE(p_metadata, '{}'::jsonb), + v_now, + v_now + ) + ON CONFLICT (connection_id, sync_kind) DO UPDATE + SET + provider = EXCLUDED.provider, + dirty = true, + dirty_since = COALESCE(public.connection_sync_state.dirty_since, v_now), + dirty_generation = public.connection_sync_state.dirty_generation + 1, + latest_seen_cursor = public.merge_connection_sync_cursor( + public.connection_sync_state.latest_seen_cursor, + p_latest_seen_cursor + ), + last_provider_event_at = COALESCE(p_provider_event_at, v_now), + priority = GREATEST(public.connection_sync_state.priority, GREATEST(p_priority, 0)), + next_retry_at = NULL, + metadata = CASE + WHEN p_metadata IS NULL OR p_metadata = '{}'::jsonb THEN public.connection_sync_state.metadata + ELSE p_metadata + END, + updated_at = v_now; + + RETURN true; +END; +$$; + + +ALTER FUNCTION "public"."mark_connection_sync_dirty"( + "p_connection_id" "uuid", + "p_sync_kind" "text", + "p_latest_seen_cursor" "text", + "p_priority" integer, + "p_provider_event_at" timestamp with time zone, + "p_metadata" "jsonb" +) OWNER TO "postgres"; diff --git a/core-api/supabase/migrations/20260402000002_next_reconcile_at.sql b/core-api/supabase/migrations/20260402000002_next_reconcile_at.sql new file mode 100644 index 00000000..9259a693 --- /dev/null +++ b/core-api/supabase/migrations/20260402000002_next_reconcile_at.sql @@ -0,0 +1,397 @@ +-- Replace broad sweep reconciliation with per-stream scheduling. +-- Clean streams get a next_reconcile_at deadline, and workers can claim +-- them directly once they age past that deadline. + +ALTER TABLE "public"."connection_sync_state" +ADD COLUMN IF NOT EXISTS "next_reconcile_at" timestamp with time zone; + + +COMMENT ON COLUMN "public"."connection_sync_state"."next_reconcile_at" +IS 'Next scheduled safety-net reconcile for a clean stream. NULL while dirty or actively retrying.'; + + +CREATE INDEX IF NOT EXISTS "idx_connection_sync_state_next_reconcile_at" +ON "public"."connection_sync_state" USING "btree" ("provider", "sync_kind", "next_reconcile_at") +WHERE "dirty" = false AND "next_reconcile_at" IS NOT NULL; + + +-- Spread existing clean rows across the default reconcile window so the new +-- claim path can pick them up without a thundering herd when the sweep cron is +-- disabled. 21600 seconds = 6 hours. +UPDATE "public"."connection_sync_state" +SET "next_reconcile_at" = now() + make_interval(secs => floor(random() * 21600)::integer) +WHERE "dirty" = false + AND "next_reconcile_at" IS NULL; + + +CREATE OR REPLACE FUNCTION "public"."mark_connection_sync_dirty"( + "p_connection_id" "uuid", + "p_sync_kind" "text", + "p_latest_seen_cursor" "text" DEFAULT NULL, + "p_priority" integer DEFAULT 0, + "p_provider_event_at" timestamp with time zone DEFAULT "now"(), + "p_metadata" "jsonb" DEFAULT '{}'::"jsonb" +) RETURNS boolean + LANGUAGE "plpgsql" SECURITY DEFINER + SET "search_path" TO 'public' + AS $$ +DECLARE + v_provider text; + v_now timestamp with time zone := now(); + v_existing record; +BEGIN + IF p_sync_kind NOT IN ('email', 'calendar') THEN + RAISE EXCEPTION 'invalid sync_kind: %', p_sync_kind; + END IF; + + SELECT provider + INTO v_provider + FROM public.ext_connections + WHERE id = p_connection_id + AND is_active = true + AND provider IN ('google', 'microsoft') + LIMIT 1; + + IF v_provider IS NULL THEN + RETURN false; + END IF; + + -- Short-circuit duplicate cursor-based invalidations for already-dirty + -- streams. This keeps Gmail notification floods from rewriting the same + -- row over and over while preserving the "accepted" success contract. + IF p_latest_seen_cursor IS NOT NULL THEN + SELECT dirty, latest_seen_cursor + INTO v_existing + FROM public.connection_sync_state + WHERE connection_id = p_connection_id + AND sync_kind = p_sync_kind; + + IF v_existing IS NOT NULL + AND v_existing.dirty = true + AND v_existing.latest_seen_cursor IS NOT NULL + AND public.merge_connection_sync_cursor( + v_existing.latest_seen_cursor, + p_latest_seen_cursor + ) = v_existing.latest_seen_cursor + THEN + RETURN true; + END IF; + END IF; + + INSERT INTO public.connection_sync_state ( + connection_id, + provider, + sync_kind, + dirty, + dirty_since, + dirty_generation, + latest_seen_cursor, + last_provider_event_at, + priority, + metadata, + next_reconcile_at, + created_at, + updated_at + ) + VALUES ( + p_connection_id, + v_provider, + p_sync_kind, + true, + v_now, + 1, + public.merge_connection_sync_cursor(NULL, p_latest_seen_cursor), + COALESCE(p_provider_event_at, v_now), + GREATEST(p_priority, 0), + COALESCE(p_metadata, '{}'::jsonb), + NULL, + v_now, + v_now + ) + ON CONFLICT (connection_id, sync_kind) DO UPDATE + SET + provider = EXCLUDED.provider, + dirty = true, + dirty_since = COALESCE(public.connection_sync_state.dirty_since, v_now), + dirty_generation = public.connection_sync_state.dirty_generation + 1, + latest_seen_cursor = public.merge_connection_sync_cursor( + public.connection_sync_state.latest_seen_cursor, + p_latest_seen_cursor + ), + last_provider_event_at = COALESCE(p_provider_event_at, v_now), + priority = GREATEST(public.connection_sync_state.priority, GREATEST(p_priority, 0)), + next_retry_at = NULL, + next_reconcile_at = NULL, + metadata = CASE + WHEN p_metadata IS NULL OR p_metadata = '{}'::jsonb THEN public.connection_sync_state.metadata + ELSE p_metadata + END, + updated_at = v_now; + + RETURN true; +END; +$$; + + +ALTER FUNCTION "public"."mark_connection_sync_dirty"( + "p_connection_id" "uuid", + "p_sync_kind" "text", + "p_latest_seen_cursor" "text", + "p_priority" integer, + "p_provider_event_at" timestamp with time zone, + "p_metadata" "jsonb" +) OWNER TO "postgres"; + + +CREATE OR REPLACE FUNCTION "public"."claim_connection_sync_lease"( + "p_worker_id" "text", + "p_lease_seconds" integer DEFAULT 120, + "p_provider" "text" DEFAULT NULL, + "p_sync_kind" "text" DEFAULT NULL, + "p_connection_id" "uuid" DEFAULT NULL +) RETURNS TABLE( + "connection_id" "uuid", + "provider" "text", + "sync_kind" "text", + "dirty_generation" bigint, + "lease_generation" bigint, + "latest_seen_cursor" "text", + "last_synced_cursor" "text", + "priority" integer, + "lease_expires_at" timestamp with time zone +) + LANGUAGE "plpgsql" SECURITY DEFINER + SET "search_path" TO 'public' + AS $$ +BEGIN + IF p_worker_id IS NULL OR btrim(p_worker_id) = '' THEN + RAISE EXCEPTION 'worker_id is required'; + END IF; + + RETURN QUERY + WITH candidate AS ( + SELECT css.connection_id, css.sync_kind + FROM public.connection_sync_state css + JOIN public.ext_connections ec + ON ec.id = css.connection_id + WHERE ( + css.dirty = true + OR ( + css.dirty = false + AND css.next_reconcile_at IS NOT NULL + AND css.next_reconcile_at <= now() + ) + ) + AND ec.is_active = true + AND (css.next_retry_at IS NULL OR css.next_retry_at <= now()) + AND (css.lease_expires_at IS NULL OR css.lease_expires_at <= now()) + AND (p_provider IS NULL OR css.provider = p_provider) + AND (p_sync_kind IS NULL OR css.sync_kind = p_sync_kind) + AND (p_connection_id IS NULL OR css.connection_id = p_connection_id) + ORDER BY + CASE WHEN css.dirty THEN 0 ELSE 1 END, + css.priority DESC, + COALESCE(css.dirty_since, css.next_reconcile_at, css.updated_at) ASC, + css.updated_at ASC + LIMIT 1 + FOR UPDATE SKIP LOCKED + ), + claimed AS ( + UPDATE public.connection_sync_state css + SET + dirty = true, + dirty_since = CASE + WHEN css.dirty THEN css.dirty_since + ELSE now() + END, + dirty_generation = CASE + WHEN css.dirty THEN css.dirty_generation + ELSE css.dirty_generation + 1 + END, + lease_owner = p_worker_id, + lease_expires_at = now() + make_interval(secs => GREATEST(p_lease_seconds, 15)), + last_sync_started_at = now(), + last_heartbeat_at = now(), + lease_generation = CASE + WHEN css.dirty THEN css.dirty_generation + ELSE css.dirty_generation + 1 + END, + priority = CASE + WHEN css.dirty THEN css.priority + ELSE 0 + END, + metadata = CASE + WHEN css.dirty THEN css.metadata + ELSE '{"source": "reconciliation"}'::jsonb + END, + next_reconcile_at = NULL, + updated_at = now() + FROM candidate + WHERE css.connection_id = candidate.connection_id + AND css.sync_kind = candidate.sync_kind + RETURNING + css.connection_id, + css.provider, + css.sync_kind, + css.dirty_generation, + css.lease_generation, + css.latest_seen_cursor, + css.last_synced_cursor, + css.priority, + css.lease_expires_at + ) + SELECT + claimed.connection_id, + claimed.provider, + claimed.sync_kind, + claimed.dirty_generation, + claimed.lease_generation, + claimed.latest_seen_cursor, + claimed.last_synced_cursor, + claimed.priority, + claimed.lease_expires_at + FROM claimed; +END; +$$; + + +ALTER FUNCTION "public"."claim_connection_sync_lease"( + "p_worker_id" "text", + "p_lease_seconds" integer, + "p_provider" "text", + "p_sync_kind" "text", + "p_connection_id" "uuid" +) OWNER TO "postgres"; + + +DROP FUNCTION IF EXISTS "public"."complete_connection_sync_lease"( + uuid, + text, + text, + text, + text, + boolean +); + + +CREATE OR REPLACE FUNCTION "public"."complete_connection_sync_lease"( + "p_connection_id" "uuid", + "p_sync_kind" "text", + "p_worker_id" "text", + "p_last_synced_cursor" "text" DEFAULT NULL, + "p_latest_seen_cursor" "text" DEFAULT NULL, + "p_keep_dirty" boolean DEFAULT false, + "p_reconcile_interval_seconds" integer DEFAULT 21600 +) RETURNS TABLE( + "dirty" boolean, + "dirty_generation" bigint, + "latest_seen_cursor" "text", + "last_synced_cursor" "text" +) + LANGUAGE "plpgsql" SECURITY DEFINER + SET "search_path" TO 'public' + AS $$ +BEGIN + RETURN QUERY + UPDATE public.connection_sync_state css + SET + latest_seen_cursor = public.merge_connection_sync_cursor( + css.latest_seen_cursor, + COALESCE(p_latest_seen_cursor, p_last_synced_cursor) + ), + last_synced_cursor = public.merge_connection_sync_cursor( + css.last_synced_cursor, + p_last_synced_cursor + ), + dirty = CASE + WHEN p_keep_dirty THEN true + WHEN css.dirty_generation > css.lease_generation THEN true + ELSE false + END, + dirty_since = CASE + WHEN p_keep_dirty THEN css.dirty_since + WHEN css.dirty_generation > css.lease_generation THEN css.dirty_since + ELSE NULL + END, + lease_owner = NULL, + lease_expires_at = NULL, + lease_generation = NULL, + last_sync_finished_at = now(), + last_heartbeat_at = now(), + retry_count = CASE + WHEN p_keep_dirty THEN css.retry_count + WHEN css.dirty_generation > css.lease_generation THEN css.retry_count + ELSE 0 + END, + priority = CASE + WHEN p_keep_dirty THEN css.priority + WHEN css.dirty_generation > css.lease_generation THEN css.priority + ELSE 0 + END, + next_retry_at = NULL, + next_reconcile_at = CASE + WHEN p_keep_dirty THEN NULL + WHEN css.dirty_generation > css.lease_generation THEN NULL + ELSE now() + make_interval( + secs => GREATEST(COALESCE(p_reconcile_interval_seconds, 21600), 300) + ) + END, + last_error = NULL, + last_error_at = NULL, + metadata = CASE + WHEN p_keep_dirty THEN css.metadata + WHEN css.dirty_generation > css.lease_generation THEN css.metadata + ELSE '{}'::jsonb + END, + updated_at = now() + WHERE css.connection_id = p_connection_id + AND css.sync_kind = p_sync_kind + AND css.lease_owner = p_worker_id + RETURNING + css.dirty, + css.dirty_generation, + css.latest_seen_cursor, + css.last_synced_cursor; +END; +$$; + + +ALTER FUNCTION "public"."complete_connection_sync_lease"( + "p_connection_id" "uuid", + "p_sync_kind" "text", + "p_worker_id" "text", + "p_last_synced_cursor" "text", + "p_latest_seen_cursor" "text", + "p_keep_dirty" boolean, + "p_reconcile_interval_seconds" integer +) OWNER TO "postgres"; + + +GRANT ALL ON FUNCTION "public"."mark_connection_sync_dirty"( + "p_connection_id" "uuid", + "p_sync_kind" "text", + "p_latest_seen_cursor" "text", + "p_priority" integer, + "p_provider_event_at" timestamp with time zone, + "p_metadata" "jsonb" +) TO "service_role"; + + +GRANT ALL ON FUNCTION "public"."claim_connection_sync_lease"( + "p_worker_id" "text", + "p_lease_seconds" integer, + "p_provider" "text", + "p_sync_kind" "text", + "p_connection_id" "uuid" +) TO "service_role"; + + +GRANT ALL ON FUNCTION "public"."complete_connection_sync_lease"( + "p_connection_id" "uuid", + "p_sync_kind" "text", + "p_worker_id" "text", + "p_last_synced_cursor" "text", + "p_latest_seen_cursor" "text", + "p_keep_dirty" boolean, + "p_reconcile_interval_seconds" integer +) TO "service_role"; diff --git a/core-api/supabase/migrations/20260403000001_reconcile_claim_mode_and_jitter.sql b/core-api/supabase/migrations/20260403000001_reconcile_claim_mode_and_jitter.sql new file mode 100644 index 00000000..df6e0f5e --- /dev/null +++ b/core-api/supabase/migrations/20260403000001_reconcile_claim_mode_and_jitter.sql @@ -0,0 +1,257 @@ +-- Split worker claim modes and add jitter to reconcile rescheduling. + +DROP FUNCTION IF EXISTS "public"."claim_connection_sync_lease"( + text, + integer, + text, + text, + uuid +); + + +CREATE OR REPLACE FUNCTION "public"."claim_connection_sync_lease"( + "p_worker_id" "text", + "p_lease_seconds" integer DEFAULT 120, + "p_provider" "text" DEFAULT NULL, + "p_sync_kind" "text" DEFAULT NULL, + "p_connection_id" "uuid" DEFAULT NULL, + "p_claim_mode" "text" DEFAULT 'any' +) RETURNS TABLE( + "connection_id" "uuid", + "provider" "text", + "sync_kind" "text", + "dirty_generation" bigint, + "lease_generation" bigint, + "latest_seen_cursor" "text", + "last_synced_cursor" "text", + "priority" integer, + "lease_expires_at" timestamp with time zone +) + LANGUAGE "plpgsql" SECURITY DEFINER + SET "search_path" TO 'public' + AS $$ +BEGIN + IF p_worker_id IS NULL OR btrim(p_worker_id) = '' THEN + RAISE EXCEPTION 'worker_id is required'; + END IF; + + IF p_claim_mode NOT IN ('any', 'dirty_only', 'reconcile_only') THEN + RAISE EXCEPTION 'invalid claim_mode: %', p_claim_mode; + END IF; + + RETURN QUERY + WITH candidate AS ( + SELECT css.connection_id, css.sync_kind + FROM public.connection_sync_state css + JOIN public.ext_connections ec + ON ec.id = css.connection_id + WHERE ( + ( + p_claim_mode IN ('any', 'dirty_only') + AND css.dirty = true + ) + OR ( + p_claim_mode IN ('any', 'reconcile_only') + AND css.dirty = false + AND css.next_reconcile_at IS NOT NULL + AND css.next_reconcile_at <= now() + ) + ) + AND ec.is_active = true + AND (css.next_retry_at IS NULL OR css.next_retry_at <= now()) + AND (css.lease_expires_at IS NULL OR css.lease_expires_at <= now()) + AND (p_provider IS NULL OR css.provider = p_provider) + AND (p_sync_kind IS NULL OR css.sync_kind = p_sync_kind) + AND (p_connection_id IS NULL OR css.connection_id = p_connection_id) + ORDER BY + CASE WHEN css.dirty THEN 0 ELSE 1 END, + css.priority DESC, + COALESCE(css.dirty_since, css.next_reconcile_at, css.updated_at) ASC, + css.updated_at ASC + LIMIT 1 + FOR UPDATE SKIP LOCKED + ), + claimed AS ( + UPDATE public.connection_sync_state css + SET + dirty = true, + dirty_since = CASE + WHEN css.dirty THEN css.dirty_since + ELSE now() + END, + dirty_generation = CASE + WHEN css.dirty THEN css.dirty_generation + ELSE css.dirty_generation + 1 + END, + lease_owner = p_worker_id, + lease_expires_at = now() + make_interval(secs => GREATEST(p_lease_seconds, 15)), + last_sync_started_at = now(), + last_heartbeat_at = now(), + lease_generation = CASE + WHEN css.dirty THEN css.dirty_generation + ELSE css.dirty_generation + 1 + END, + priority = CASE + WHEN css.dirty THEN css.priority + ELSE 0 + END, + metadata = CASE + WHEN css.dirty THEN css.metadata + ELSE '{"source": "reconciliation"}'::jsonb + END, + next_reconcile_at = NULL, + updated_at = now() + FROM candidate + WHERE css.connection_id = candidate.connection_id + AND css.sync_kind = candidate.sync_kind + RETURNING + css.connection_id, + css.provider, + css.sync_kind, + css.dirty_generation, + css.lease_generation, + css.latest_seen_cursor, + css.last_synced_cursor, + css.priority, + css.lease_expires_at + ) + SELECT + claimed.connection_id, + claimed.provider, + claimed.sync_kind, + claimed.dirty_generation, + claimed.lease_generation, + claimed.latest_seen_cursor, + claimed.last_synced_cursor, + claimed.priority, + claimed.lease_expires_at + FROM claimed; +END; +$$; + + +ALTER FUNCTION "public"."claim_connection_sync_lease"( + "p_worker_id" "text", + "p_lease_seconds" integer, + "p_provider" "text", + "p_sync_kind" "text", + "p_connection_id" "uuid", + "p_claim_mode" "text" +) OWNER TO "postgres"; + + +CREATE OR REPLACE FUNCTION "public"."complete_connection_sync_lease"( + "p_connection_id" "uuid", + "p_sync_kind" "text", + "p_worker_id" "text", + "p_last_synced_cursor" "text" DEFAULT NULL, + "p_latest_seen_cursor" "text" DEFAULT NULL, + "p_keep_dirty" boolean DEFAULT false, + "p_reconcile_interval_seconds" integer DEFAULT 21600 +) RETURNS TABLE( + "dirty" boolean, + "dirty_generation" bigint, + "latest_seen_cursor" "text", + "last_synced_cursor" "text" +) + LANGUAGE "plpgsql" SECURITY DEFINER + SET "search_path" TO 'public' + AS $$ +BEGIN + RETURN QUERY + UPDATE public.connection_sync_state css + SET + latest_seen_cursor = public.merge_connection_sync_cursor( + css.latest_seen_cursor, + COALESCE(p_latest_seen_cursor, p_last_synced_cursor) + ), + last_synced_cursor = public.merge_connection_sync_cursor( + css.last_synced_cursor, + p_last_synced_cursor + ), + dirty = CASE + WHEN p_keep_dirty THEN true + WHEN css.dirty_generation > css.lease_generation THEN true + ELSE false + END, + dirty_since = CASE + WHEN p_keep_dirty THEN css.dirty_since + WHEN css.dirty_generation > css.lease_generation THEN css.dirty_since + ELSE NULL + END, + lease_owner = NULL, + lease_expires_at = NULL, + lease_generation = NULL, + last_sync_finished_at = now(), + last_heartbeat_at = now(), + retry_count = CASE + WHEN p_keep_dirty THEN css.retry_count + WHEN css.dirty_generation > css.lease_generation THEN css.retry_count + ELSE 0 + END, + priority = CASE + WHEN p_keep_dirty THEN css.priority + WHEN css.dirty_generation > css.lease_generation THEN css.priority + ELSE 0 + END, + next_retry_at = NULL, + next_reconcile_at = CASE + WHEN p_keep_dirty THEN NULL + WHEN css.dirty_generation > css.lease_generation THEN NULL + ELSE now() + make_interval( + secs => GREATEST(COALESCE(p_reconcile_interval_seconds, 21600), 300) + + floor( + random() * GREATEST(COALESCE(p_reconcile_interval_seconds, 21600), 300) * 0.2 + )::integer + ) + END, + last_error = NULL, + last_error_at = NULL, + metadata = CASE + WHEN p_keep_dirty THEN css.metadata + WHEN css.dirty_generation > css.lease_generation THEN css.metadata + ELSE '{}'::jsonb + END, + updated_at = now() + WHERE css.connection_id = p_connection_id + AND css.sync_kind = p_sync_kind + AND css.lease_owner = p_worker_id + RETURNING + css.dirty, + css.dirty_generation, + css.latest_seen_cursor, + css.last_synced_cursor; +END; +$$; + + +ALTER FUNCTION "public"."complete_connection_sync_lease"( + "p_connection_id" "uuid", + "p_sync_kind" "text", + "p_worker_id" "text", + "p_last_synced_cursor" "text", + "p_latest_seen_cursor" "text", + "p_keep_dirty" boolean, + "p_reconcile_interval_seconds" integer +) OWNER TO "postgres"; + + +GRANT ALL ON FUNCTION "public"."claim_connection_sync_lease"( + "p_worker_id" "text", + "p_lease_seconds" integer, + "p_provider" "text", + "p_sync_kind" "text", + "p_connection_id" "uuid", + "p_claim_mode" "text" +) TO "service_role"; + + +GRANT ALL ON FUNCTION "public"."complete_connection_sync_lease"( + "p_connection_id" "uuid", + "p_sync_kind" "text", + "p_worker_id" "text", + "p_last_synced_cursor" "text", + "p_latest_seen_cursor" "text", + "p_keep_dirty" boolean, + "p_reconcile_interval_seconds" integer +) TO "service_role"; diff --git a/core-api/supabase/migrations/20260403000002_connection_sync_state_autovacuum.sql b/core-api/supabase/migrations/20260403000002_connection_sync_state_autovacuum.sql new file mode 100644 index 00000000..79d70e4e --- /dev/null +++ b/core-api/supabase/migrations/20260403000002_connection_sync_state_autovacuum.sql @@ -0,0 +1,13 @@ +-- Migration: tune autovacuum for the hot connection_sync_state queue table. +-- +-- This table is updated on every dirty mark, lease heartbeat, completion, and +-- retry transition. Default autovacuum thresholds are too lax for a hot queue +-- table and can allow avoidable bloat on small-tier Postgres. + +ALTER TABLE "public"."connection_sync_state" +SET ( + autovacuum_vacuum_scale_factor = 0.02, + autovacuum_vacuum_threshold = 50, + autovacuum_analyze_scale_factor = 0.01, + autovacuum_analyze_threshold = 50 +); diff --git a/core-api/sync_worker.py b/core-api/sync_worker.py new file mode 100644 index 00000000..126e43fd --- /dev/null +++ b/core-api/sync_worker.py @@ -0,0 +1,17 @@ +"""Railway entrypoint for the lease-based sync worker.""" + +import logging +import sys + +from api.services.syncs.stream_worker import run_forever + + +logging.basicConfig( + level=logging.INFO, + format="%(asctime)s %(levelname)s %(name)s %(message)s", + stream=sys.stdout, +) + + +if __name__ == "__main__": + run_forever() diff --git a/core-api/tests/unit/test_auth_initial_sync_queue.py b/core-api/tests/unit/test_auth_initial_sync_queue.py index 4bb90b91..c99994d7 100644 --- a/core-api/tests/unit/test_auth_initial_sync_queue.py +++ b/core-api/tests/unit/test_auth_initial_sync_queue.py @@ -1,6 +1,6 @@ import os from types import SimpleNamespace -from unittest.mock import AsyncMock, MagicMock +from unittest.mock import MagicMock import pytest @@ -14,20 +14,20 @@ ) +class _FakeSupabase: + def rpc(self, _name, _params): + return SimpleNamespace(execute=lambda: SimpleNamespace(data=True)) + + @pytest.mark.asyncio -async def test_google_initial_sync_falls_back_for_failed_queue_publish(monkeypatch): +async def test_google_initial_sync_marks_both_streams_dirty(monkeypatch): from api.services import auth - import lib.queue as queue_module - - enqueue_mock = MagicMock(side_effect=[True, False]) - monkeypatch.setattr( - queue_module, - "queue_client", - SimpleNamespace(enqueue_sync_for_connection=enqueue_mock), - ) + import lib.supabase_client as supabase_client_module + from api.services.syncs import sync_dispatcher - to_thread_mock = AsyncMock(return_value=None) - monkeypatch.setattr(auth.asyncio, "to_thread", to_thread_mock) + mark_mock = MagicMock(side_effect=[True, True]) + monkeypatch.setattr(sync_dispatcher, "mark_stream_dirty", mark_mock) + monkeypatch.setattr(supabase_client_module, "get_service_role_client", lambda: _FakeSupabase()) await auth._enqueue_or_fallback_google_initial_sync( connection_id="conn-1", @@ -37,37 +37,33 @@ async def test_google_initial_sync_falls_back_for_failed_queue_publish(monkeypat provider_email="user@example.com", ) - assert enqueue_mock.call_count == 2 + assert mark_mock.call_count == 2 - gmail_call = enqueue_mock.call_args_list[0] - assert gmail_call.args[0] == "conn-1" - assert gmail_call.args[1] == "sync-gmail" - assert gmail_call.kwargs["dedup_id"] == "initial-sync-gmail-conn-1" + gmail_call = mark_mock.call_args_list[0] + assert gmail_call.args[1] == "conn-1" + assert gmail_call.args[2] == "google" + assert gmail_call.args[3] == "email" + assert gmail_call.kwargs["metadata"]["source"] == "initial-sync" + assert gmail_call.kwargs["metadata"]["initial_sync"] is True + assert gmail_call.kwargs["metadata"]["max_results"] == 50 + assert gmail_call.kwargs["metadata"]["days_back"] == 20 - calendar_call = enqueue_mock.call_args_list[1] - assert calendar_call.args[1] == "sync-calendar" - assert calendar_call.kwargs["dedup_id"] == "initial-sync-calendar-conn-1" - - to_thread_mock.assert_awaited_once() - _, fallback_kwargs = to_thread_mock.await_args - assert fallback_kwargs["run_gmail"] is False - assert fallback_kwargs["run_calendar"] is True + calendar_call = mark_mock.call_args_list[1] + assert calendar_call.args[2] == "google" + assert calendar_call.args[3] == "calendar" + assert calendar_call.kwargs["metadata"]["days_past"] == 7 + assert calendar_call.kwargs["metadata"]["days_future"] == 60 @pytest.mark.asyncio -async def test_microsoft_initial_sync_queue_success_skips_inline_fallback(monkeypatch): +async def test_microsoft_initial_sync_marks_email_only_when_calendar_disabled(monkeypatch): from api.services import auth - import lib.queue as queue_module - - enqueue_mock = MagicMock(return_value=True) - monkeypatch.setattr( - queue_module, - "queue_client", - SimpleNamespace(enqueue_sync_for_connection=enqueue_mock), - ) + import lib.supabase_client as supabase_client_module + from api.services.syncs import sync_dispatcher - to_thread_mock = AsyncMock(return_value=None) - monkeypatch.setattr(auth.asyncio, "to_thread", to_thread_mock) + mark_mock = MagicMock(return_value=True) + monkeypatch.setattr(sync_dispatcher, "mark_stream_dirty", mark_mock) + monkeypatch.setattr(supabase_client_module, "get_service_role_client", lambda: _FakeSupabase()) await auth._enqueue_or_fallback_microsoft_initial_sync( connection_id="conn-1", @@ -80,9 +76,10 @@ async def test_microsoft_initial_sync_queue_success_skips_inline_fallback(monkey include_calendar=False, ) - assert enqueue_mock.call_count == 1 - only_call = enqueue_mock.call_args_list[0] - assert only_call.args[1] == "sync-outlook" - assert only_call.kwargs["dedup_id"] == "initial-sync-outlook-conn-1" - to_thread_mock.assert_not_awaited() - + assert mark_mock.call_count == 1 + only_call = mark_mock.call_args_list[0] + assert only_call.args[1] == "conn-1" + assert only_call.args[2] == "microsoft" + assert only_call.args[3] == "email" + assert only_call.kwargs["metadata"]["initial_sync"] is True + assert only_call.kwargs["metadata"]["days_back"] == 20 diff --git a/core-api/tests/unit/test_calendar_sync_dirty_only.py b/core-api/tests/unit/test_calendar_sync_dirty_only.py new file mode 100644 index 00000000..75cc0095 --- /dev/null +++ b/core-api/tests/unit/test_calendar_sync_dirty_only.py @@ -0,0 +1,99 @@ +import json +import os +from types import SimpleNamespace + +import pytest +from fastapi import HTTPException + +# Prevent import-time settings failures. +os.environ.setdefault("SUPABASE_URL", "https://test.supabase.co") +os.environ.setdefault( + "SUPABASE_ANON_KEY", + "eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9." + "eyJpc3MiOiJzdXBhYmFzZSIsInJlZiI6InRlc3QiLCJyb2xlIjoiYW5vbiJ9." + "testsignature", +) + + +class _FakeQuery: + def __init__(self, data): + self._data = data + + def select(self, *_args, **_kwargs): + return self + + def eq(self, *_args, **_kwargs): + return self + + def in_(self, *_args, **_kwargs): + return self + + def execute(self): + return SimpleNamespace(data=self._data) + + +class _FakeSupabase: + def __init__(self, connections): + self._connections = connections + + def table(self, _name): + return _FakeQuery(self._connections) + + +@pytest.mark.asyncio +async def test_calendar_sync_returns_accepted_when_streams_marked(monkeypatch): + from api.routers import calendar as calendar_router + import lib.supabase_client as supabase_client_module + from api.services.syncs import sync_dispatcher + + fake_connections = [ + {"id": "conn-google-1", "provider": "google"}, + {"id": "conn-ms-1", "provider": "microsoft"}, + ] + monkeypatch.setattr( + supabase_client_module, + "get_authenticated_supabase_client", + lambda _jwt: _FakeSupabase(fake_connections), + ) + monkeypatch.setattr(supabase_client_module, "get_service_role_client", lambda: _FakeSupabase([])) + marked = {"calls": []} + + def _mark(*args, **kwargs): + marked["calls"].append((args, kwargs)) + return True + + monkeypatch.setattr(sync_dispatcher, "mark_stream_dirty", _mark) + + response = await calendar_router.sync_google_calendar_endpoint(user_jwt="jwt", user_id="user-1") + body = json.loads(response.body.decode("utf-8")) + + assert response.status_code == 202 + assert body["status"] == "accepted" + assert body["jobs_enqueued"] == 2 + assert body["streams_marked"] == 2 + assert len(marked["calls"]) == 2 + assert all( + call_kwargs["priority"] == sync_dispatcher.MANUAL_SYNC_PRIORITY + for _, call_kwargs in marked["calls"] + ) + + +@pytest.mark.asyncio +async def test_calendar_sync_returns_503_when_no_streams_marked(monkeypatch): + from api.routers import calendar as calendar_router + import lib.supabase_client as supabase_client_module + from api.services.syncs import sync_dispatcher + + fake_connections = [{"id": "conn-google-1", "provider": "google"}] + monkeypatch.setattr( + supabase_client_module, + "get_authenticated_supabase_client", + lambda _jwt: _FakeSupabase(fake_connections), + ) + monkeypatch.setattr(supabase_client_module, "get_service_role_client", lambda: _FakeSupabase([])) + monkeypatch.setattr(sync_dispatcher, "mark_stream_dirty", lambda *_args, **_kwargs: False) + + with pytest.raises(HTTPException) as exc: + await calendar_router.sync_google_calendar_endpoint(user_jwt="jwt", user_id="user-1") + + assert exc.value.status_code == 503 diff --git a/core-api/tests/unit/test_cron_batch_mode_toggle.py b/core-api/tests/unit/test_cron_batch_mode_toggle.py index f0a319ef..2bb5c867 100644 --- a/core-api/tests/unit/test_cron_batch_mode_toggle.py +++ b/core-api/tests/unit/test_cron_batch_mode_toggle.py @@ -1,6 +1,4 @@ import os -from types import SimpleNamespace -from unittest.mock import MagicMock import pytest @@ -13,7 +11,6 @@ "testsignature", ) - def test_cron_batch_mode_defaults_to_true(monkeypatch): from api.routers import cron @@ -61,141 +58,28 @@ def test_cron_batch_size_rejects_invalid_values(monkeypatch, value): @pytest.mark.asyncio -async def test_incremental_sync_batch_mode_chunks_and_attaches_batch_tokens(monkeypatch): +async def test_incremental_sync_returns_disabled_stub(monkeypatch): from api.routers import cron - import lib.queue as queue_module - - query = MagicMock() - query.select.return_value = query - query.in_.return_value = query - query.eq.return_value = query - query.execute.return_value = SimpleNamespace(data=[ - {"id": "google-3", "user_id": "u1", "provider": "google", "last_synced": None}, - {"id": "google-1", "user_id": "u1", "provider": "google", "last_synced": None}, - {"id": "google-2", "user_id": "u1", "provider": "google", "last_synced": None}, - {"id": "ms-3", "user_id": "u2", "provider": "microsoft", "last_synced": None}, - {"id": "ms-1", "user_id": "u2", "provider": "microsoft", "last_synced": None}, - {"id": "ms-2", "user_id": "u2", "provider": "microsoft", "last_synced": None}, - ]) - supabase = MagicMock() - supabase.table.return_value = query - - enqueue_batch_mock = MagicMock(return_value=True) - queue_client_mock = SimpleNamespace( - available=True, - enqueue_batch=enqueue_batch_mock, - ) monkeypatch.setattr(cron, "verify_cron_auth", lambda *_: True) - monkeypatch.setattr(cron, "get_service_role_client", lambda: supabase) - monkeypatch.setattr(cron, "capture_checkin", lambda *args, **kwargs: "checkin-id") - monkeypatch.setattr(cron, "is_cron_batch_mode_enabled", lambda: True) - monkeypatch.setattr(cron, "_get_batch_bucket", lambda: "202604011200") - monkeypatch.setenv("CRON_BATCH_SIZE", "2") - monkeypatch.setattr(queue_module, "queue_client", queue_client_mock) result = await cron.cron_incremental_sync(authorization="Bearer test") - expected_chunks = [ - ("sync-gmail", ["google-1", "google-2"]), - ("sync-gmail", ["google-3"]), - ("sync-calendar", ["google-1", "google-2"]), - ("sync-calendar", ["google-3"]), - ("sync-outlook", ["ms-1", "ms-2"]), - ("sync-outlook", ["ms-3"]), - ("sync-outlook-calendar", ["ms-1", "ms-2"]), - ("sync-outlook-calendar", ["ms-3"]), - ] - - assert result["jobs_enqueued"] == len(expected_chunks) + assert result["status"] == "disabled" + assert "next_reconcile_at" in result["message"] + assert result["streams_marked"] == 0 + assert result["jobs_enqueued"] == 0 assert result["jobs_failed"] == 0 - assert result["batch_mode"] is True - assert enqueue_batch_mock.call_count == len(expected_chunks) - - for call, (job_type, chunk_ids) in zip(enqueue_batch_mock.call_args_list, expected_chunks): - batch_token = cron._build_batch_token(chunk_ids, "202604011200") - assert call.args[0] == job_type - assert call.args[1] == chunk_ids - assert call.kwargs["extra"] == {"batch_token": batch_token} - assert call.kwargs["dedup_id"] == f"batch-{job_type}-{batch_token}" + assert result["batch_mode"] is False @pytest.mark.asyncio -async def test_incremental_sync_batch_dedup_ids_stable_within_same_bucket(monkeypatch): +async def test_incremental_sync_requires_auth(monkeypatch): from api.routers import cron - import lib.queue as queue_module - - def build_supabase(): - query = MagicMock() - query.select.return_value = query - query.in_.return_value = query - query.eq.return_value = query - query.execute.return_value = SimpleNamespace(data=[ - {"id": "google-2", "user_id": "u1", "provider": "google", "last_synced": None}, - {"id": "google-1", "user_id": "u1", "provider": "google", "last_synced": None}, - {"id": "ms-2", "user_id": "u2", "provider": "microsoft", "last_synced": None}, - {"id": "ms-1", "user_id": "u2", "provider": "microsoft", "last_synced": None}, - ]) - supabase = MagicMock() - supabase.table.return_value = query - return supabase - - async def run_for_bucket(bucket: str): - enqueue_batch_mock = MagicMock(return_value=True) - queue_client_mock = SimpleNamespace( - available=True, - enqueue_batch=enqueue_batch_mock, - ) - monkeypatch.setattr(cron, "get_service_role_client", build_supabase) - monkeypatch.setattr(cron, "_get_batch_bucket", lambda: bucket) - monkeypatch.setattr(queue_module, "queue_client", queue_client_mock) - await cron.cron_incremental_sync(authorization="Bearer test") - return [call.kwargs["dedup_id"] for call in enqueue_batch_mock.call_args_list] - - monkeypatch.setattr(cron, "verify_cron_auth", lambda *_: True) - monkeypatch.setattr(cron, "capture_checkin", lambda *args, **kwargs: "checkin-id") - monkeypatch.setattr(cron, "is_cron_batch_mode_enabled", lambda: True) - monkeypatch.setenv("CRON_BATCH_SIZE", "2") - - first = await run_for_bucket("202604011200") - second = await run_for_bucket("202604011200") - assert first == second + monkeypatch.setattr(cron, "verify_cron_auth", lambda *_: False) + with pytest.raises(cron.HTTPException) as exc: + await cron.cron_incremental_sync(authorization="Bearer test") -@pytest.mark.asyncio -async def test_incremental_sync_batch_dedup_ids_refresh_when_bucket_changes(monkeypatch): - from api.routers import cron - import lib.queue as queue_module - - query = MagicMock() - query.select.return_value = query - query.in_.return_value = query - query.eq.return_value = query - query.execute.return_value = SimpleNamespace(data=[ - {"id": "google-2", "user_id": "u1", "provider": "google", "last_synced": None}, - {"id": "google-1", "user_id": "u1", "provider": "google", "last_synced": None}, - ]) - supabase = MagicMock() - supabase.table.return_value = query - - monkeypatch.setattr(cron, "verify_cron_auth", lambda *_: True) - monkeypatch.setattr(cron, "get_service_role_client", lambda: supabase) - monkeypatch.setattr(cron, "capture_checkin", lambda *args, **kwargs: "checkin-id") - monkeypatch.setattr(cron, "is_cron_batch_mode_enabled", lambda: True) - monkeypatch.setenv("CRON_BATCH_SIZE", "2") - - first_enqueue = MagicMock(return_value=True) - monkeypatch.setattr(queue_module, "queue_client", SimpleNamespace(available=True, enqueue_batch=first_enqueue)) - monkeypatch.setattr(cron, "_get_batch_bucket", lambda: "202604011200") - await cron.cron_incremental_sync(authorization="Bearer test") - - second_enqueue = MagicMock(return_value=True) - monkeypatch.setattr(queue_module, "queue_client", SimpleNamespace(available=True, enqueue_batch=second_enqueue)) - monkeypatch.setattr(cron, "_get_batch_bucket", lambda: "202604011201") - await cron.cron_incremental_sync(authorization="Bearer test") - - first_dedup_ids = [call.kwargs["dedup_id"] for call in first_enqueue.call_args_list] - second_dedup_ids = [call.kwargs["dedup_id"] for call in second_enqueue.call_args_list] - - assert first_dedup_ids != second_dedup_ids + assert exc.value.status_code == 401 diff --git a/core-api/tests/unit/test_cron_google_oauth_failures.py b/core-api/tests/unit/test_cron_google_oauth_failures.py index 236ab924..81f359c6 100644 --- a/core-api/tests/unit/test_cron_google_oauth_failures.py +++ b/core-api/tests/unit/test_cron_google_oauth_failures.py @@ -2,6 +2,7 @@ from datetime import datetime, timedelta, timezone from types import SimpleNamespace from unittest.mock import MagicMock, patch +from cryptography.fernet import Fernet # Prevent module import failures from lib.supabase_client singleton init. os.environ.setdefault("SUPABASE_URL", "https://test.supabase.co") @@ -12,6 +13,8 @@ "testsignature", ) +TEST_KEY = Fernet.generate_key().decode() + def _make_query_builder(execute_side_effect): """Create a chainable Supabase query mock.""" @@ -19,6 +22,7 @@ def _make_query_builder(execute_side_effect): query.select.return_value = query query.eq.return_value = query query.single.return_value = query + query.maybe_single.return_value = query query.update.return_value = query query.execute.side_effect = execute_side_effect return query @@ -62,6 +66,8 @@ def test_permanent_google_refresh_failure_deactivates_connection_and_subscriptio with patch("api.config.settings", SimpleNamespace( google_client_id="test-client-id", google_client_secret="test-client-secret", + token_encryption_key="", + token_encryption_key_previous="", )): gmail_service, calendar_service, returned_user_id = get_google_services_for_connection( connection_id, @@ -111,6 +117,8 @@ def test_transient_google_refresh_failure_continues_with_existing_credentials(): with patch("api.config.settings", SimpleNamespace( google_client_id="test-client-id", google_client_secret="test-client-secret", + token_encryption_key="", + token_encryption_key_previous="", )): gmail_service, calendar_service, returned_user_id = get_google_services_for_connection( connection_id, @@ -125,3 +133,73 @@ def test_transient_google_refresh_failure_continues_with_existing_credentials(): # Transient branch should not deactivate push subscriptions. table_calls = [c.args[0] for c in supabase.table.call_args_list] assert "push_subscriptions" not in table_calls + + +def test_successful_google_refresh_reencrypts_tokens_before_persisting(): + """ + Successful cron refreshes must decrypt stored tokens for Google auth, + then re-encrypt refreshed tokens before writing them back. + """ + from api.routers.cron import get_google_services_for_connection + from lib.token_encryption import decrypt_token, encrypt_token, is_encrypted + + connection_id = "connection-success-123" + user_id = "user-success-456" + expires_at = (datetime.now(timezone.utc) - timedelta(days=1)).isoformat() + + with patch("api.config.settings", SimpleNamespace( + google_client_id="test-client-id", + google_client_secret="test-client-secret", + token_encryption_key=TEST_KEY, + token_encryption_key_previous="", + )): + encrypted_connection = { + "id": connection_id, + "user_id": user_id, + "access_token": encrypt_token("stale-access-token"), + "refresh_token": encrypt_token("stale-refresh-token"), + "token_expires_at": expires_at, + "metadata": {}, + } + + query = _make_query_builder([ + SimpleNamespace(data=encrypted_connection), # initial select + SimpleNamespace(data=[]), # ext_connections refresh update + ]) + supabase = MagicMock() + supabase.table.return_value = query + + refreshed_credentials = MagicMock() + refreshed_credentials.token = "new-access-token" + refreshed_credentials.refresh_token = "new-refresh-token" + refreshed_credentials.expiry = datetime.now(timezone.utc) + timedelta(hours=1) + + gmail_mock = MagicMock(name="gmail_service") + calendar_mock = MagicMock(name="calendar_service") + + with patch("google.oauth2.credentials.Credentials", return_value=refreshed_credentials) as credentials_ctor: + with patch("google.auth.transport.requests.Request"): + with patch("api.services.syncs.google_services.refresh_credentials") as refresh_mock: + with patch("googleapiclient.discovery.build", side_effect=[gmail_mock, calendar_mock]): + gmail_service, calendar_service, returned_user_id = get_google_services_for_connection( + connection_id, + supabase, + ) + + assert gmail_service is gmail_mock + assert calendar_service is calendar_mock + assert returned_user_id == user_id + refresh_mock.assert_called_once() + credentials_ctor.assert_called_once_with( + token="stale-access-token", + refresh_token="stale-refresh-token", + token_uri="https://oauth2.googleapis.com/token", + client_id="test-client-id", + client_secret="test-client-secret", + ) + + update_payload = query.update.call_args.args[0] + assert is_encrypted(update_payload["access_token"]) + assert is_encrypted(update_payload["refresh_token"]) + assert decrypt_token(update_payload["access_token"]) == "new-access-token" + assert decrypt_token(update_payload["refresh_token"]) == "new-refresh-token" diff --git a/core-api/tests/unit/test_cron_watch_health.py b/core-api/tests/unit/test_cron_watch_health.py new file mode 100644 index 00000000..e7b13a7b --- /dev/null +++ b/core-api/tests/unit/test_cron_watch_health.py @@ -0,0 +1,147 @@ +import os +from datetime import datetime, timedelta, timezone +from types import SimpleNamespace +from unittest.mock import MagicMock + +import pytest + +# Prevent module import failures from lib.supabase_client singleton init. +os.environ.setdefault("SUPABASE_URL", "https://test.supabase.co") +os.environ.setdefault( + "SUPABASE_ANON_KEY", + "eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9." + "eyJpc3MiOiJzdXBhYmFzZSIsInJlZiI6InRlc3QiLCJyb2xlIjoiYW5vbiJ9." + "testsignature", +) + + +def _make_query_builder(data): + query = MagicMock() + query.select.return_value = query + query.eq.return_value = query + query.in_.return_value = query + query.gt.return_value = query + query.execute.return_value = SimpleNamespace(data=data) + return query + + +def test_classify_watch_health_candidate_detects_stale_notifications(): + from api.routers import cron + + now = datetime.now(timezone.utc) + watch = { + "provider": "gmail", + "created_at": (now - timedelta(days=3)).isoformat(), + "expiration": (now + timedelta(days=4)).isoformat(), + "notification_count": 4, + "last_notification_at": (now - timedelta(hours=30)).isoformat(), + } + sync_state = { + "dirty": False, + "retry_count": 0, + "lease_expires_at": None, + "last_sync_finished_at": (now - timedelta(hours=30)).isoformat(), + } + + reason = cron._classify_watch_health_candidate( + watch, + sync_state, + now=now, + stale_after=timedelta(hours=24), + initial_grace_after=timedelta(hours=12), + ) + + assert reason == "stale_notifications" + + +def test_classify_watch_health_candidate_skips_recently_synced_quiet_watch(): + from api.routers import cron + + now = datetime.now(timezone.utc) + watch = { + "provider": "gmail", + "created_at": (now - timedelta(days=2)).isoformat(), + "expiration": (now + timedelta(days=4)).isoformat(), + "notification_count": 0, + "last_notification_at": None, + } + sync_state = { + "dirty": False, + "retry_count": 0, + "lease_expires_at": None, + "last_sync_finished_at": (now - timedelta(hours=2)).isoformat(), + } + + reason = cron._classify_watch_health_candidate( + watch, + sync_state, + now=now, + stale_after=timedelta(hours=24), + initial_grace_after=timedelta(hours=12), + ) + + assert reason is None + + +@pytest.mark.asyncio +async def test_cron_watch_health_marks_targeted_recovery(monkeypatch): + from api.routers import cron + + now = datetime.now(timezone.utc) + watches_query = _make_query_builder( + [ + { + "id": "watch-1", + "ext_connection_id": "conn-1", + "provider": "gmail", + "created_at": (now - timedelta(days=3)).isoformat(), + "expiration": (now + timedelta(days=5)).isoformat(), + "notification_count": 2, + "last_notification_at": (now - timedelta(hours=36)).isoformat(), + } + ] + ) + state_query = _make_query_builder( + [ + { + "connection_id": "conn-1", + "provider": "google", + "sync_kind": "email", + "dirty": False, + "retry_count": 0, + "next_retry_at": None, + "lease_expires_at": None, + "last_sync_finished_at": (now - timedelta(hours=36)).isoformat(), + } + ] + ) + + supabase = MagicMock() + supabase.table.side_effect = lambda name: { + "push_subscriptions": watches_query, + "connection_sync_state": state_query, + }[name] + + marks = [] + monkeypatch.setattr(cron, "verify_cron_auth", lambda *_args, **_kwargs: True) + monkeypatch.setattr(cron, "get_service_role_client", lambda: supabase) + monkeypatch.setattr(cron, "capture_checkin", lambda **_kwargs: "check-in") + monkeypatch.setattr( + cron, + "mark_stream_dirty", + lambda *args, **kwargs: marks.append((args, kwargs)) or True, + ) + + result = await cron.cron_watch_health(authorization="Bearer test") + + assert result["status"] == "completed" + assert result["checked"] == 1 + assert result["queued"] == 1 + assert result["errors"] == 0 + assert len(marks) == 1 + assert marks[0][0][1] == "conn-1" + assert marks[0][0][2] == "google" + assert marks[0][0][3] == "email" + assert marks[0][1]["priority"] == cron.get_watch_health_priority() + assert marks[0][1]["metadata"]["source"] == "watch-health-recovery" + assert marks[0][1]["metadata"]["reason"] == "stale_notifications" diff --git a/core-api/tests/unit/test_email_sync_queue_only.py b/core-api/tests/unit/test_email_sync_queue_only.py new file mode 100644 index 00000000..f6a0b6cb --- /dev/null +++ b/core-api/tests/unit/test_email_sync_queue_only.py @@ -0,0 +1,106 @@ +import json +import os +from types import SimpleNamespace + +import pytest +from fastapi import HTTPException + +# Prevent import-time settings failures. +os.environ.setdefault("SUPABASE_URL", "https://test.supabase.co") +os.environ.setdefault( + "SUPABASE_ANON_KEY", + "eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9." + "eyJpc3MiOiJzdXBhYmFzZSIsInJlZiI6InRlc3QiLCJyb2xlIjoiYW5vbiJ9." + "testsignature", +) + + +class _FakeQuery: + def __init__(self, data): + self._data = data + + def select(self, *_args, **_kwargs): + return self + + def eq(self, *_args, **_kwargs): + return self + + def in_(self, *_args, **_kwargs): + return self + + def execute(self): + return SimpleNamespace(data=self._data) + + +class _FakeSupabase: + def __init__(self, connections): + self._connections = connections + + def table(self, _name): + return _FakeQuery(self._connections) + + def rpc(self, _name, _params): + return SimpleNamespace(execute=lambda: SimpleNamespace(data=True)) + + +@pytest.mark.asyncio +async def test_email_sync_returns_503_when_no_streams_marked(monkeypatch): + from api.routers import email as email_router + from api.services.syncs import sync_dispatcher + + fake_connections = [ + {"id": "conn-google-1", "provider": "google", "provider_email": "a@example.com"} + ] + monkeypatch.setattr( + email_router, + "get_authenticated_supabase_client", + lambda _jwt: _FakeSupabase(fake_connections), + ) + monkeypatch.setattr(email_router, "decrypt_ext_connection_tokens", lambda c: c) + monkeypatch.setattr(sync_dispatcher, "mark_stream_dirty", lambda *_args, **_kwargs: False) + + with pytest.raises(HTTPException) as exc: + await email_router.sync_emails_endpoint(user_jwt="jwt", user_id="user-1") + + assert exc.value.status_code == 503 + assert "temporarily unavailable" in exc.value.detail.lower() + + +@pytest.mark.asyncio +async def test_email_sync_returns_accepted_response_when_streams_marked(monkeypatch): + from api.routers import email as email_router + import lib.supabase_client as supabase_client_module + from api.services.syncs import sync_dispatcher + + fake_connections = [ + {"id": "conn-google-1", "provider": "google", "provider_email": "a@example.com"}, + {"id": "conn-ms-1", "provider": "microsoft", "provider_email": "b@example.com"}, + ] + monkeypatch.setattr( + email_router, + "get_authenticated_supabase_client", + lambda _jwt: _FakeSupabase(fake_connections), + ) + monkeypatch.setattr(email_router, "decrypt_ext_connection_tokens", lambda c: c) + monkeypatch.setattr(supabase_client_module, "get_service_role_client", lambda: _FakeSupabase([])) + + marked = {"calls": []} + + def _mark(*args, **kwargs): + marked["calls"].append((args, kwargs)) + return True + + monkeypatch.setattr(sync_dispatcher, "mark_stream_dirty", _mark) + + response = await email_router.sync_emails_endpoint(user_jwt="jwt", user_id="user-1") + body = json.loads(response.body.decode("utf-8")) + + assert response.status_code == 202 + assert body["status"] == "accepted" + assert body["jobs_enqueued"] == 2 + assert body["streams_marked"] == 2 + assert len(marked["calls"]) == 2 + assert all( + call_kwargs["priority"] == sync_dispatcher.MANUAL_SYNC_PRIORITY + for _, call_kwargs in marked["calls"] + ) diff --git a/core-api/tests/unit/test_failure_policy.py b/core-api/tests/unit/test_failure_policy.py new file mode 100644 index 00000000..2105b3b4 --- /dev/null +++ b/core-api/tests/unit/test_failure_policy.py @@ -0,0 +1,25 @@ +import os + +# Prevent import-time settings failures. +os.environ.setdefault("SUPABASE_URL", "https://test.supabase.co") +os.environ.setdefault( + "SUPABASE_ANON_KEY", + "eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9." + "eyJpc3MiOiJzdXBhYmFzZSIsInJlZiI6InRlc3QiLCJyb2xlIjoiYW5vbiJ9." + "testsignature", +) + + +def test_get_failed_sync_quarantine_reason_marks_permanent_microsoft_auth_errors(): + from api.services.syncs.failure_policy import get_failed_sync_quarantine_reason + + reason = get_failed_sync_quarantine_reason( + provider="microsoft", + error="Refresh token is invalid - user must re-authenticate", + retry_count=0, + ) + + assert reason == ( + "Permanent Microsoft auth failure: " + "Refresh token is invalid - user must re-authenticate" + ) diff --git a/core-api/tests/unit/test_google_webhook_security.py b/core-api/tests/unit/test_google_webhook_security.py new file mode 100644 index 00000000..5132d43e --- /dev/null +++ b/core-api/tests/unit/test_google_webhook_security.py @@ -0,0 +1,43 @@ +import os + + +os.environ.setdefault("SUPABASE_URL", "https://test.supabase.co") +os.environ.setdefault( + "SUPABASE_ANON_KEY", + "eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9." + "eyJpc3MiOiJzdXBhYmFzZSIsInJlZiI6InRlc3QiLCJyb2xlIjoiYW5vbiJ9." + "testsignature", +) + + +def test_calendar_channel_token_round_trip(monkeypatch): + from lib.google_webhook_security import ( + build_google_calendar_channel_token, + verify_google_calendar_channel_token, + ) + + monkeypatch.setattr( + "api.config.settings.google_calendar_webhook_secret", + "test-secret", + ) + + token = build_google_calendar_channel_token("conn-1", "channel-1") + + assert token is not None + assert verify_google_calendar_channel_token("conn-1", "channel-1", token) is True + assert verify_google_calendar_channel_token("conn-1", "channel-2", token) is False + + +def test_calendar_channel_token_verification_is_disabled_without_secret(monkeypatch): + from lib.google_webhook_security import ( + build_google_calendar_channel_token, + verify_google_calendar_channel_token, + ) + + monkeypatch.setattr( + "api.config.settings.google_calendar_webhook_secret", + "", + ) + + assert build_google_calendar_channel_token("conn-1", "channel-1") is None + assert verify_google_calendar_channel_token("conn-1", "channel-1", None) is True diff --git a/core-api/tests/unit/test_image_proxy.py b/core-api/tests/unit/test_image_proxy.py index 584ac26f..edf059e4 100644 --- a/core-api/tests/unit/test_image_proxy.py +++ b/core-api/tests/unit/test_image_proxy.py @@ -16,7 +16,7 @@ SECRET = "test-secret-key-abc123" -BASE_URL = "https://img.example.com" +BASE_URL = "https://img.getcore.dev" FIXED_TIME = 1738800000.0 # A fixed timestamp, mid-hour diff --git a/core-api/tests/unit/test_notify_attendees_google.py b/core-api/tests/unit/test_notify_attendees_google.py index 44e90e3a..19016cb4 100644 --- a/core-api/tests/unit/test_notify_attendees_google.py +++ b/core-api/tests/unit/test_notify_attendees_google.py @@ -76,7 +76,7 @@ def test_create_event_send_updates_all_when_notify_true(monkeypatch): 'notify_attendees': True, } - ev_id, meeting_link, err = _create_google_event('user-1', 'jwt', event_data, None) + ev_id, meeting_link, err, _tz = _create_google_event('user-1', 'jwt', event_data, None) assert err is None assert recorder['insert']['sendUpdates'] == 'all' # Ensure attendees are passed through @@ -105,7 +105,7 @@ def test_update_event_send_updates_all_when_notify_true(monkeypatch): 'notify_attendees': True, } - ok = _update_google_event( + ok, _tz = _update_google_event( user_id='user-1', user_jwt='jwt', external_id='ext-1', diff --git a/core-api/tests/unit/test_share_link_access.py b/core-api/tests/unit/test_share_link_access.py index d42f9ef4..7d2da829 100644 --- a/core-api/tests/unit/test_share_link_access.py +++ b/core-api/tests/unit/test_share_link_access.py @@ -1,6 +1,5 @@ from __future__ import annotations -import asyncio from datetime import datetime, timedelta, timezone from typing import Any, Dict, List, Optional @@ -35,7 +34,8 @@ def eq(self, field: str, value: Any) -> "FakeSupabaseQuery": return self def ilike(self, field: str, value: Any) -> "FakeSupabaseQuery": - raise AssertionError(f"ilike() should not be used β€” use eq() to avoid wildcard injection (field={field})") + self._filters.append(("ilike", field, value)) + return self def limit(self, value: int) -> "FakeSupabaseQuery": self._limit = value @@ -56,7 +56,6 @@ def _matches(self, row: Dict[str, Any]) -> bool: return True async def execute(self) -> FakeResponse: - await asyncio.sleep(0) rows = [dict(row) for row in self._rows() if self._matches(row)] if self._limit is not None: rows = rows[: self._limit] @@ -78,7 +77,6 @@ def __init__(self, payload: Any): self._payload = payload async def execute(self) -> FakeResponse: - await asyncio.sleep(0) return FakeResponse(self._payload) @@ -165,18 +163,7 @@ async def test_get_public_shared_resource_rejects_expired_links(monkeypatch): "granted_by": "user-1", "expires_at": expired_at, } - ], - "documents": [ - { - "id": "doc-1", - "title": "Quarterly Plan", - "content": "Secret", - "is_folder": False, - "created_at": "2026-03-31T00:00:00+00:00", - "updated_at": "2026-03-31T00:00:00+00:00", - "file": {"r2_key": "docs/doc-1.png", "file_type": "image/png"}, - } - ], + ] } monkeypatch.setattr( @@ -191,54 +178,6 @@ async def test_get_public_shared_resource_rejects_expired_links(monkeypatch): assert exc_info.value.status_code == 404 -@pytest.mark.asyncio -async def test_get_public_shared_resource_falls_back_to_slug_lookup(monkeypatch): - from api.services.permissions import public as public_module - - expires_at = (datetime.now(timezone.utc) + timedelta(hours=1)).isoformat() - state = { - "permissions": [ - { - "grantee_type": "link", - "link_token": "raw-uuid-token", - "link_slug": "docs-link", - "resource_type": "document", - "resource_id": "doc-1", - "permission": "read", - "granted_by": "user-1", - "expires_at": expires_at, - } - ], - "users": [{"id": "user-1", "name": "Jay", "avatar_url": "avatar.png"}], - "documents": [ - { - "id": "doc-1", - "title": "Quarterly Plan", - "content": "Secret", - "is_folder": False, - "created_at": "2026-03-31T00:00:00+00:00", - "updated_at": "2026-03-31T00:00:00+00:00", - "file": {"r2_key": "docs/doc-1.png", "file_type": "image/png"}, - } - ], - } - - monkeypatch.setattr( - public_module, - "get_async_service_role_client", - AsyncMock(return_value=FakePublicSupabaseClient(state)), - ) - monkeypatch.setattr(public_module, "_enrich_documents_with_image_urls", lambda docs: None) - - # Lookup by slug (not the raw token) β€” exercises the fallback branch - result = await public_module.get_public_shared_resource("docs-link") - - assert result["resource_type"] == "document" - assert result["resource_id"] == "doc-1" - assert result["permission"] == "read" - assert result["shared_by"]["name"] == "Jay" - - @pytest.mark.asyncio async def test_resolve_share_link_uses_context_workspace_when_rpc_payload_trimmed(monkeypatch): from api.services.permissions import links as links_module diff --git a/core-api/tests/unit/test_stream_worker.py b/core-api/tests/unit/test_stream_worker.py new file mode 100644 index 00000000..fcce0d7e --- /dev/null +++ b/core-api/tests/unit/test_stream_worker.py @@ -0,0 +1,407 @@ +import os +from unittest.mock import MagicMock, patch + +import pytest + +# Prevent import-time settings failures. +os.environ.setdefault("SUPABASE_URL", "https://test.supabase.co") +os.environ.setdefault( + "SUPABASE_ANON_KEY", + "eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9." + "eyJpc3MiOiJzdXBhYmFzZSIsInJlZiI6InRlc3QiLCJyb2xlIjoiYW5vbiJ9." + "testsignature", +) + + +def test_dispatch_claimed_stream_uses_gmail_webhook_path_when_cursor_present(): + from api.services.syncs import stream_worker + + claim = { + "connection_id": "conn-1", + "provider": "google", + "sync_kind": stream_worker.SYNC_KIND_EMAIL, + "latest_seen_cursor": "12345", + } + + with patch.object(stream_worker.worker_router, "_process_gmail_webhook", return_value={"status": "ok"}) as webhook_mock: + with patch.object(stream_worker.worker_router, "_sync_single_gmail") as sync_mock: + result = stream_worker._dispatch_claimed_stream( + claim, + state={"latest_seen_cursor": "12345", "metadata": {"source": "gmail-webhook"}}, + ) + + assert result["status"] == "ok" + webhook_mock.assert_called_once() + sync_mock.assert_not_called() + + +def test_dispatch_claimed_stream_uses_generic_gmail_path_for_non_webhook_source(): + from api.services.syncs import stream_worker + + claim = { + "connection_id": "conn-1", + "provider": "google", + "sync_kind": stream_worker.SYNC_KIND_EMAIL, + "latest_seen_cursor": "12345", + } + + with patch.object(stream_worker.worker_router, "_process_gmail_webhook", return_value={"status": "ok"}) as webhook_mock: + with patch.object(stream_worker.worker_router, "_sync_single_gmail", return_value={"status": "ok"}) as sync_mock: + result = stream_worker._dispatch_claimed_stream( + claim, + state={"latest_seen_cursor": "12345", "metadata": {"source": "incremental-cron"}}, + ) + + assert result["status"] == "ok" + webhook_mock.assert_not_called() + sync_mock.assert_called_once() + + +def test_dispatch_claimed_stream_uses_calendar_webhook_path_when_metadata_requests_it(): + from api.services.syncs import stream_worker + + claim = { + "connection_id": "conn-1", + "provider": "google", + "sync_kind": stream_worker.SYNC_KIND_CALENDAR, + "latest_seen_cursor": None, + } + state = { + "metadata": { + "source": "google-calendar-webhook", + "channel_id": "channel-1", + "resource_state": "exists", + } + } + + with patch.object(stream_worker, "_resolve_active_google_calendar_channel", return_value="channel-1"): + with patch.object(stream_worker.worker_router, "_process_calendar_webhook", return_value={"status": "ok"}) as webhook_mock: + with patch.object(stream_worker.worker_router, "_sync_single_calendar") as sync_mock: + result = stream_worker._dispatch_claimed_stream( + claim, + state=state, + service_supabase=MagicMock(), + ) + + assert result["status"] == "ok" + webhook_mock.assert_called_once() + sync_mock.assert_not_called() + + +def test_dispatch_claimed_stream_falls_back_when_calendar_channel_is_stale(): + from api.services.syncs import stream_worker + + claim = { + "connection_id": "conn-1", + "provider": "google", + "sync_kind": stream_worker.SYNC_KIND_CALENDAR, + "latest_seen_cursor": None, + } + state = { + "metadata": { + "source": "google-calendar-webhook", + "channel_id": "stale-channel", + "resource_state": "exists", + } + } + + with patch.object(stream_worker, "_resolve_active_google_calendar_channel", return_value="active-channel"): + with patch.object(stream_worker.worker_router, "_process_calendar_webhook", return_value={"status": "ok"}) as webhook_mock: + with patch.object(stream_worker.worker_router, "_sync_single_calendar", return_value={"status": "ok"}) as sync_mock: + result = stream_worker._dispatch_claimed_stream( + claim, + state=state, + service_supabase=MagicMock(), + ) + + assert result["status"] == "ok" + webhook_mock.assert_not_called() + sync_mock.assert_called_once() + + +def test_dispatch_claimed_stream_passes_initial_sync_metadata_to_provider_worker(): + from api.services.syncs import stream_worker + + claim = { + "connection_id": "conn-1", + "provider": "microsoft", + "sync_kind": stream_worker.SYNC_KIND_EMAIL, + "latest_seen_cursor": None, + } + state = { + "metadata": { + "source": "initial-sync", + "initial_sync": True, + "max_results": 50, + "days_back": 20, + } + } + + def _assert_payload(connection_id, payload): + assert connection_id == "conn-1" + assert payload.initial_sync is True + assert payload.max_results == 50 + assert payload.days_back == 20 + return {"status": "ok"} + + with patch.object(stream_worker.worker_router, "_sync_single_outlook", side_effect=_assert_payload) as sync_mock: + result = stream_worker._dispatch_claimed_stream(claim, state=state) + + assert result["status"] == "ok" + sync_mock.assert_called_once() + + +def test_process_one_claim_completes_on_success(): + from api.services.syncs import stream_worker + + claim = { + "connection_id": "conn-1", + "provider": "microsoft", + "sync_kind": stream_worker.SYNC_KIND_EMAIL, + "latest_seen_cursor": None, + } + service_supabase = MagicMock() + + with patch.object(stream_worker, "get_connection_sync_state", return_value={"metadata": {}}): + with patch.object(stream_worker, "_dispatch_claimed_stream", return_value={"status": "ok", "new_delta_link": "delta-1"}): + with patch.object(stream_worker, "complete_connection_sync_lease") as complete_mock: + with patch.object(stream_worker, "fail_connection_sync_lease") as fail_mock: + with patch.object( + stream_worker, + "start_connection_sync_lease_heartbeat", + return_value=(MagicMock(), MagicMock()), + ): + with patch.object(stream_worker, "stop_connection_sync_lease_heartbeat") as stop_mock: + result = stream_worker.process_one_claim( + claim, + worker_id="worker-1", + service_supabase=service_supabase, + ) + + assert result["status"] == "ok" + complete_mock.assert_called_once() + fail_mock.assert_not_called() + stop_mock.assert_called_once() + + +def test_process_one_claim_fails_lease_when_state_fetch_raises(): + from api.services.syncs import stream_worker + + claim = { + "connection_id": "conn-1", + "provider": "microsoft", + "sync_kind": stream_worker.SYNC_KIND_EMAIL, + "latest_seen_cursor": None, + } + service_supabase = MagicMock() + + with patch.object(stream_worker, "get_connection_sync_state", side_effect=RuntimeError("boom")): + with patch.object(stream_worker, "fail_connection_sync_lease") as fail_mock: + with patch.object(stream_worker, "start_connection_sync_lease_heartbeat") as start_mock: + result = stream_worker.process_one_claim( + claim, + worker_id="worker-1", + service_supabase=service_supabase, + ) + + assert result["status"] == "error" + assert result["message"] == "boom" + fail_mock.assert_called_once() + start_mock.assert_not_called() + + +def test_process_one_claim_quarantines_on_permanent_google_oauth_error(): + from api.services.syncs import stream_worker + + claim = { + "connection_id": "conn-1", + "provider": "google", + "sync_kind": stream_worker.SYNC_KIND_EMAIL, + "latest_seen_cursor": None, + } + service_supabase = MagicMock() + + with patch.object( + stream_worker, + "get_connection_sync_state", + return_value={"metadata": {}, "retry_count": 0}, + ): + with patch.object( + stream_worker, + "_dispatch_claimed_stream", + return_value={"status": "error", "message": "Refresh token is invalid"}, + ): + with patch("api.services.syncs.failure_policy.complete_connection_sync_lease") as complete_mock: + with patch.object(stream_worker, "fail_connection_sync_lease") as fail_mock: + with patch( + "api.services.syncs.failure_policy.deactivate_connection_with_subscriptions" + ) as deactivate_mock: + with patch.object( + stream_worker, + "start_connection_sync_lease_heartbeat", + return_value=(MagicMock(), MagicMock()), + ): + with patch.object(stream_worker, "stop_connection_sync_lease_heartbeat"): + result = stream_worker.process_one_claim( + claim, + worker_id="worker-1", + service_supabase=service_supabase, + ) + + assert result["status"] == "quarantined" + complete_mock.assert_called_once() + fail_mock.assert_not_called() + deactivate_mock.assert_called_once() + + +def test_process_one_claim_quarantines_on_permanent_microsoft_auth_error(): + from api.services.syncs import stream_worker + + claim = { + "connection_id": "conn-1", + "provider": "microsoft", + "sync_kind": stream_worker.SYNC_KIND_EMAIL, + "latest_seen_cursor": None, + } + service_supabase = MagicMock() + + with patch.object( + stream_worker, + "get_connection_sync_state", + return_value={"metadata": {}, "retry_count": 0}, + ): + with patch.object( + stream_worker, + "_dispatch_claimed_stream", + return_value={"status": "error", "message": "Refresh token is invalid - user must re-authenticate"}, + ): + with patch("api.services.syncs.failure_policy.complete_connection_sync_lease") as complete_mock: + with patch.object(stream_worker, "fail_connection_sync_lease") as fail_mock: + with patch( + "api.services.syncs.failure_policy.deactivate_connection_with_subscriptions" + ) as deactivate_mock: + with patch.object( + stream_worker, + "start_connection_sync_lease_heartbeat", + return_value=(MagicMock(), MagicMock()), + ): + with patch.object(stream_worker, "stop_connection_sync_lease_heartbeat"): + result = stream_worker.process_one_claim( + claim, + worker_id="worker-1", + service_supabase=service_supabase, + ) + + assert result["status"] == "quarantined" + complete_mock.assert_called_once() + fail_mock.assert_not_called() + deactivate_mock.assert_called_once() + + +def test_process_one_claim_quarantines_when_retry_cap_is_hit(monkeypatch): + from api.services.syncs import stream_worker + + claim = { + "connection_id": "conn-1", + "provider": "microsoft", + "sync_kind": stream_worker.SYNC_KIND_CALENDAR, + "latest_seen_cursor": None, + } + service_supabase = MagicMock() + monkeypatch.setenv("MAX_FAILURE_RETRY_COUNT", "3") + + with patch.object( + stream_worker, + "get_connection_sync_state", + return_value={"metadata": {}, "retry_count": 2}, + ): + with patch.object( + stream_worker, + "_dispatch_claimed_stream", + return_value={"status": "error", "message": "calendar sync failed"}, + ): + with patch("api.services.syncs.failure_policy.complete_connection_sync_lease") as complete_mock: + with patch.object(stream_worker, "fail_connection_sync_lease") as fail_mock: + with patch( + "api.services.syncs.failure_policy.deactivate_connection_with_subscriptions" + ) as deactivate_mock: + with patch.object( + stream_worker, + "start_connection_sync_lease_heartbeat", + return_value=(MagicMock(), MagicMock()), + ): + with patch.object(stream_worker, "stop_connection_sync_lease_heartbeat"): + result = stream_worker.process_one_claim( + claim, + worker_id="worker-1", + service_supabase=service_supabase, + ) + + assert result["status"] == "quarantined" + complete_mock.assert_called_once() + fail_mock.assert_not_called() + deactivate_mock.assert_called_once() + + +def test_run_forever_recovers_from_run_once_exception(): + from api.services.syncs import stream_worker + + with patch.object(stream_worker, "build_worker_id", return_value="worker-1"): + with patch.object(stream_worker, "run_once", side_effect=[RuntimeError("boom"), KeyboardInterrupt]): + with patch.object(stream_worker.time, "sleep") as sleep_mock: + with pytest.raises(KeyboardInterrupt): + stream_worker.run_forever() + + sleep_mock.assert_called_once_with(5.0) + + +@pytest.mark.parametrize( + ("configured_mode", "expected"), + [ + (None, "any"), + ("", "any"), + ("mixed", "any"), + ("realtime", "dirty_only"), + ("dirty", "dirty_only"), + ("reconcile", "reconcile_only"), + ], +) +def test_resolve_worker_claim_mode_accepts_aliases(monkeypatch, configured_mode, expected): + from api.services.syncs import stream_worker + + monkeypatch.delenv("SYNC_WORKER_MODE", raising=False) + + assert stream_worker.resolve_worker_claim_mode(configured_mode) == expected + + +def test_resolve_worker_claim_mode_rejects_invalid_mode(monkeypatch): + from api.services.syncs import stream_worker + + monkeypatch.delenv("SYNC_WORKER_MODE", raising=False) + + with pytest.raises(ValueError, match="Unsupported SYNC_WORKER_MODE"): + stream_worker.resolve_worker_claim_mode("burst-only") + + +def test_run_once_returns_none_when_nothing_claimed(): + from api.services.syncs import stream_worker + + service_supabase = MagicMock() + + with patch.object(stream_worker, "claim_connection_sync_lease", return_value=None): + result = stream_worker.run_once(worker_id="worker-1", service_supabase=service_supabase) + + assert result is None + + +def test_run_once_passes_resolved_claim_mode_to_claim_rpc(monkeypatch): + from api.services.syncs import stream_worker + + service_supabase = MagicMock() + monkeypatch.setenv("SYNC_WORKER_MODE", "reconcile") + + with patch.object(stream_worker, "claim_connection_sync_lease", return_value=None) as claim_mock: + result = stream_worker.run_once(worker_id="worker-1", service_supabase=service_supabase) + + assert result is None + assert claim_mock.call_args.kwargs["claim_mode"] == stream_worker.CLAIM_MODE_RECONCILE_ONLY diff --git a/core-api/tests/unit/test_sync_dispatcher.py b/core-api/tests/unit/test_sync_dispatcher.py new file mode 100644 index 00000000..bd3e8284 --- /dev/null +++ b/core-api/tests/unit/test_sync_dispatcher.py @@ -0,0 +1,76 @@ +import os +from types import SimpleNamespace + +# Prevent import-time settings failures. +os.environ.setdefault("SUPABASE_URL", "https://test.supabase.co") +os.environ.setdefault( + "SUPABASE_ANON_KEY", + "eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9." + "eyJpc3MiOiJzdXBhYmFzZSIsInJlZiI6InRlc3QiLCJyb2xlIjoiYW5vbiJ9." + "testsignature", +) + + +def test_mark_and_enqueue_stream_marks_dirty_before_enqueuing(monkeypatch): + from api.services.syncs import sync_dispatcher + + marked = {"called": False} + enqueued = {"called": False} + + def fake_mark(*_args, **_kwargs): + marked["called"] = True + return True + + queue_client = SimpleNamespace( + enqueue_sync_for_connection=lambda connection_id, job_type, extra=None, dedup_id=None: ( + enqueued.update( + { + "called": True, + "connection_id": connection_id, + "job_type": job_type, + "extra": extra, + "dedup_id": dedup_id, + } + ) + or True + ) + ) + + monkeypatch.setattr(sync_dispatcher, "mark_connection_sync_dirty", fake_mark) + + result = sync_dispatcher.mark_and_enqueue_stream( + object(), + queue_client, + "conn-1", + "google", + sync_dispatcher.SYNC_KIND_EMAIL, + extra={"history_id": "123"}, + ) + + assert result is True + assert marked["called"] is True + assert enqueued["called"] is True + assert enqueued["job_type"] == "sync-gmail" + assert enqueued["dedup_id"] == "stream-google-email-conn-1" + + +def test_mark_and_enqueue_stream_stops_when_mark_dirty_returns_false(monkeypatch): + from api.services.syncs import sync_dispatcher + + queue_client = SimpleNamespace( + enqueue_sync_for_connection=lambda *_args, **_kwargs: (_ for _ in ()).throw( + AssertionError("enqueue should not run when mark dirty fails") + ) + ) + + monkeypatch.setattr(sync_dispatcher, "mark_connection_sync_dirty", lambda *_args, **_kwargs: False) + + result = sync_dispatcher.mark_and_enqueue_stream( + object(), + queue_client, + "conn-1", + "microsoft", + sync_dispatcher.SYNC_KIND_CALENDAR, + ) + + assert result is False diff --git a/core-api/tests/unit/test_sync_google_calendar_cron_tokens.py b/core-api/tests/unit/test_sync_google_calendar_cron_tokens.py new file mode 100644 index 00000000..55ba35ba --- /dev/null +++ b/core-api/tests/unit/test_sync_google_calendar_cron_tokens.py @@ -0,0 +1,99 @@ +import os +from types import SimpleNamespace +from unittest.mock import ANY, MagicMock, patch + + +# Prevent lib.supabase_client singleton init from failing during helper imports. +os.environ.setdefault("SUPABASE_URL", "https://test.supabase.co") +os.environ.setdefault( + "SUPABASE_ANON_KEY", + "eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9." + "eyJpc3MiOiJzdXBhYmFzZSIsInJlZiI6InRlc3QiLCJyb2xlIjoiYW5vbiJ9." + "testsignature", +) + + +def _make_query_builder(*, execute_side_effect=None, execute_return_value=None): + query = MagicMock() + query.select.return_value = query + query.eq.return_value = query + query.gte.return_value = query + query.lte.return_value = query + query.lt.return_value = query + query.is_.return_value = query + query.delete.return_value = query + query.update.return_value = query + query.maybe_single.return_value = query + if execute_side_effect is not None: + query.execute.side_effect = execute_side_effect + else: + query.execute.return_value = execute_return_value or SimpleNamespace(data=[]) + return query + + +def test_sync_google_calendar_cron_persists_sync_token_after_clean_sync(): + from api.services.syncs.sync_google_calendar_cron import sync_google_calendar_cron + + ext_connections_query = _make_query_builder( + execute_side_effect=[ + SimpleNamespace(data={"provider_email": "user@example.com"}), + SimpleNamespace(data=[]), + ] + ) + calendar_events_query = _make_query_builder( + execute_side_effect=[ + SimpleNamespace(data=[]), + SimpleNamespace(data=[]), + SimpleNamespace(data=[]), + SimpleNamespace(data=[]), + ] + ) + push_subscriptions_query = _make_query_builder( + execute_return_value=SimpleNamespace(data=[]) + ) + + service_supabase = MagicMock() + service_supabase.table.side_effect = lambda name: { + "ext_connections": ext_connections_query, + "calendar_events": calendar_events_query, + "push_subscriptions": push_subscriptions_query, + }[name] + + calendar_service = MagicMock() + calendar_service.events.return_value.list.return_value.execute.return_value = { + "items": [{"id": "event-1"}], + "nextSyncToken": "sync-123", + } + + with patch("api.services.syncs.sync_google_calendar_cron.get_existing_external_ids", return_value=set()): + with patch( + "api.services.syncs.sync_google_calendar_cron.batch_upsert", + return_value={"success_count": 1, "error_count": 0, "errors": []}, + ): + with patch( + "api.services.syncs.sync_google_calendar_cron.parse_google_event_to_data", + return_value={"external_id": "event-1", "user_id": "user-123"}, + ): + with patch( + "api.services.syncs.sync_google_calendar_cron.get_calendar_event_rows_by_external_ids", + return_value={}, + ): + with patch( + "api.services.syncs.sync_google_calendar_cron.reconcile_calendar_invite_notifications" + ): + result = sync_google_calendar_cron( + calendar_service=calendar_service, + connection_id="connection-123", + user_id="user-123", + service_supabase=service_supabase, + ) + + assert result["status"] == "success" + assert result["new_sync_token"] == "sync-123" + push_subscriptions_query.update.assert_called_once_with( + {"sync_token": "sync-123", "updated_at": ANY} + ) + push_subscriptions_query.eq.assert_any_call("ext_connection_id", "connection-123") + push_subscriptions_query.eq.assert_any_call("provider", "calendar") + push_subscriptions_query.eq.assert_any_call("is_active", True) + diff --git a/core-api/tests/unit/test_sync_state_store.py b/core-api/tests/unit/test_sync_state_store.py new file mode 100644 index 00000000..7616ec37 --- /dev/null +++ b/core-api/tests/unit/test_sync_state_store.py @@ -0,0 +1,313 @@ +import os +from types import SimpleNamespace + +import pytest + +# Prevent import-time settings failures from unrelated package imports. +os.environ.setdefault("SUPABASE_URL", "https://test.supabase.co") +os.environ.setdefault( + "SUPABASE_ANON_KEY", + "eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9." + "eyJpc3MiOiJzdXBhYmFzZSIsInJlZiI6InRlc3QiLCJyb2xlIjoiYW5vbiJ9." + "testsignature", +) + + +class _FakeRpc: + def __init__(self, data): + self._data = data + + def execute(self): + return SimpleNamespace(data=self._data) + + +class _FakeSupabase: + def __init__(self, rpc_data): + self.rpc_data = rpc_data + self.calls = [] + + def rpc(self, name, params): + self.calls.append((name, params)) + return _FakeRpc(self.rpc_data) + + def table(self, name): + self.calls.append(("table", name)) + return self + + def select(self, *_args, **_kwargs): + return self + + def eq(self, *_args, **_kwargs): + return self + + def maybe_single(self): + return self + + def execute(self): + return SimpleNamespace(data=self.rpc_data) + + +def test_mark_connection_sync_dirty_passes_expected_params(): + from api.services.syncs.sync_state_store import mark_connection_sync_dirty + + supabase = _FakeSupabase(True) + + result = mark_connection_sync_dirty( + supabase, + "conn-1", + "email", + latest_seen_cursor="123", + priority=5, + metadata={"source": "webhook"}, + ) + + assert result is True + assert supabase.calls == [ + ( + "mark_connection_sync_dirty", + { + "p_connection_id": "conn-1", + "p_sync_kind": "email", + "p_latest_seen_cursor": "123", + "p_priority": 5, + "p_provider_event_at": None, + "p_metadata": {"source": "webhook"}, + }, + ) + ] + + +def test_get_reconcile_interval_seconds_defaults_to_6_hours(monkeypatch): + from api.services.syncs.sync_state_store import get_reconcile_interval_seconds + + monkeypatch.delenv("RECONCILE_INTERVAL_SECONDS", raising=False) + + assert get_reconcile_interval_seconds() == 21600 + + +def test_get_max_failure_retry_count_defaults_to_5(monkeypatch): + from api.services.syncs.sync_state_store import get_max_failure_retry_count + + monkeypatch.delenv("MAX_FAILURE_RETRY_COUNT", raising=False) + + assert get_max_failure_retry_count() == 5 + + +@pytest.mark.parametrize("value", ["300", "1800", "21600", "86400"]) +def test_get_reconcile_interval_seconds_accepts_valid_values(monkeypatch, value): + from api.services.syncs.sync_state_store import get_reconcile_interval_seconds + + monkeypatch.setenv("RECONCILE_INTERVAL_SECONDS", value) + + assert get_reconcile_interval_seconds() == int(value) + + +@pytest.mark.parametrize("value", ["", "0", "-1", "abc", str(8 * 24 * 3600)]) +def test_get_reconcile_interval_seconds_clamps_invalid_values(monkeypatch, value): + from api.services.syncs.sync_state_store import get_reconcile_interval_seconds + + monkeypatch.setenv("RECONCILE_INTERVAL_SECONDS", value) + + expected = 300 if value == "0" or value == "-1" else 21600 + if value == str(8 * 24 * 3600): + expected = 7 * 24 * 3600 + + assert get_reconcile_interval_seconds() == expected + + +@pytest.mark.parametrize("value, expected", [("", 5), ("0", 1), ("-1", 1), ("17", 17), ("1001", 1000), ("abc", 5)]) +def test_get_max_failure_retry_count_clamps_invalid_values(monkeypatch, value, expected): + from api.services.syncs.sync_state_store import get_max_failure_retry_count + + monkeypatch.setenv("MAX_FAILURE_RETRY_COUNT", value) + + assert get_max_failure_retry_count() == expected + + +@pytest.mark.parametrize( + ("retry_count", "expected_seconds"), + [ + (None, 60), + (0, 60), + (1, 300), + (2, 900), + (3, 1800), + (4, 3600), + (5, 21600), + (6, 86400), + (99, 86400), + ], +) +def test_get_failure_retry_seconds_steps_up_aggressively(retry_count, expected_seconds): + from api.services.syncs.sync_state_store import get_failure_retry_seconds + + assert get_failure_retry_seconds(retry_count) == expected_seconds + + +def test_claim_connection_sync_lease_returns_first_row_from_rpc_list(): + from api.services.syncs.sync_state_store import claim_connection_sync_lease + + supabase = _FakeSupabase( + [ + { + "connection_id": "conn-1", + "provider": "google", + "sync_kind": "email", + "dirty_generation": 4, + } + ] + ) + + result = claim_connection_sync_lease( + supabase, + "worker-1", + lease_seconds=180, + provider="google", + sync_kind="email", + ) + + assert result == { + "connection_id": "conn-1", + "provider": "google", + "sync_kind": "email", + "dirty_generation": 4, + } + assert supabase.calls == [ + ( + "claim_connection_sync_lease", + { + "p_worker_id": "worker-1", + "p_lease_seconds": 180, + "p_provider": "google", + "p_sync_kind": "email", + "p_connection_id": None, + "p_claim_mode": "any", + }, + ) + ] + + +def test_claim_connection_sync_lease_passes_explicit_claim_mode(): + from api.services.syncs.sync_state_store import ( + CLAIM_MODE_RECONCILE_ONLY, + claim_connection_sync_lease, + ) + + supabase = _FakeSupabase([]) + + claim_connection_sync_lease( + supabase, + "worker-1", + claim_mode=CLAIM_MODE_RECONCILE_ONLY, + ) + + assert supabase.calls == [ + ( + "claim_connection_sync_lease", + { + "p_worker_id": "worker-1", + "p_lease_seconds": 120, + "p_provider": None, + "p_sync_kind": None, + "p_connection_id": None, + "p_claim_mode": "reconcile_only", + }, + ) + ] + + +def test_complete_connection_sync_lease_returns_none_when_rpc_returns_empty_list(): + from api.services.syncs.sync_state_store import complete_connection_sync_lease + + supabase = _FakeSupabase([]) + + result = complete_connection_sync_lease( + supabase, + "conn-1", + "calendar", + "worker-1", + last_synced_cursor="sync-token-2", + ) + + assert result is None + assert supabase.calls == [ + ( + "complete_connection_sync_lease", + { + "p_connection_id": "conn-1", + "p_sync_kind": "calendar", + "p_worker_id": "worker-1", + "p_last_synced_cursor": "sync-token-2", + "p_latest_seen_cursor": None, + "p_keep_dirty": False, + "p_reconcile_interval_seconds": 21600, + }, + ) + ] + + +def test_get_connection_sync_state_returns_row_dict(): + from api.services.syncs.sync_state_store import get_connection_sync_state + + supabase = _FakeSupabase( + { + "connection_id": "conn-1", + "sync_kind": "calendar", + "metadata": {"source": "webhook"}, + } + ) + + result = get_connection_sync_state(supabase, "conn-1", "calendar") + + assert result == { + "connection_id": "conn-1", + "sync_kind": "calendar", + "metadata": {"source": "webhook"}, + } + + +def test_invalid_sync_kind_raises_value_error(): + from api.services.syncs.sync_state_store import mark_connection_sync_dirty + + supabase = _FakeSupabase(True) + + with pytest.raises(ValueError, match="Unsupported sync_kind"): + mark_connection_sync_dirty(supabase, "conn-1", "contacts") + + +def test_invalid_claim_mode_raises_value_error(): + from api.services.syncs.sync_state_store import claim_connection_sync_lease + + supabase = _FakeSupabase(True) + + with pytest.raises(ValueError, match="Unsupported claim_mode"): + claim_connection_sync_lease(supabase, "worker-1", claim_mode="burst_only") + + +def test_fail_connection_sync_lease_passes_retry_seconds(): + from api.services.syncs.sync_state_store import fail_connection_sync_lease + + supabase = _FakeSupabase({"retry_count": 2, "next_retry_at": "2026-04-01T20:30:00Z"}) + + result = fail_connection_sync_lease( + supabase, + "conn-1", + "email", + "worker-2", + "boom", + retry_seconds=90, + ) + + assert result == {"retry_count": 2, "next_retry_at": "2026-04-01T20:30:00Z"} + assert supabase.calls == [ + ( + "fail_connection_sync_lease", + { + "p_connection_id": "conn-1", + "p_sync_kind": "email", + "p_worker_id": "worker-2", + "p_error": "boom", + "p_retry_seconds": 90, + }, + ) + ] diff --git a/core-api/tests/unit/test_token_encryption.py b/core-api/tests/unit/test_token_encryption.py new file mode 100644 index 00000000..695163ee --- /dev/null +++ b/core-api/tests/unit/test_token_encryption.py @@ -0,0 +1,387 @@ +""" +Tests for OAuth token encryption at rest. + +Covers: +- Core encrypt/decrypt module (lib/token_encryption.py) +- Integration with write paths (encrypt before DB write) +- Integration with read paths (decrypt after DB read) +- Migration script logic +""" +import os +from unittest.mock import patch, MagicMock + +import pytest +from cryptography.fernet import Fernet +from pydantic import ValidationError + +# Prevent module import failures from lib.supabase_client singleton init. +os.environ.setdefault("SUPABASE_URL", "https://test.supabase.co") +os.environ.setdefault( + "SUPABASE_ANON_KEY", + "eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9." + "eyJpc3MiOiJzdXBhYmFzZSIsInJlZiI6InRlc3QiLCJyb2xlIjoiYW5vbiJ9." + "testsignature", +) + +# Generate test keys +TEST_KEY = Fernet.generate_key().decode() +TEST_KEY_PREVIOUS = Fernet.generate_key().decode() + + +# --------------------------------------------------------------------------- +# Helpers +# --------------------------------------------------------------------------- + +def _make_settings(**overrides): + """Create a mock settings object with token encryption keys.""" + from types import SimpleNamespace + defaults = { + "token_encryption_key": TEST_KEY, + "token_encryption_key_previous": "", + } + defaults.update(overrides) + return SimpleNamespace(**defaults) + + +def _patch_settings(**overrides): + """Patch api.config.settings for token_encryption module.""" + return patch("api.config.settings", _make_settings(**overrides)) + + +# =========================================================================== +# Core Module Tests β€” lib/token_encryption.py +# =========================================================================== + +class TestEncryptDecryptRoundtrip: + """Basic encrypt β†’ decrypt should return original value.""" + + def test_roundtrip(self): + from lib.token_encryption import encrypt_token, decrypt_token + with _patch_settings(): + original = "ya29.a0AfH6SMB_super_secret_access_token" + encrypted = encrypt_token(original) + assert encrypted != original + assert decrypt_token(encrypted) == original + + def test_produces_unique_ciphertext(self): + """Same plaintext encrypted twice should produce different ciphertexts (random IV).""" + from lib.token_encryption import encrypt_token + with _patch_settings(): + token = "ya29.a0AfH6SMB_test_token" + enc1 = encrypt_token(token) + enc2 = encrypt_token(token) + assert enc1 != enc2 # Different IVs + + def test_decrypt_plaintext_passthrough(self): + """Non-Fernet string should be returned as-is (migration safety).""" + from lib.token_encryption import decrypt_token + with _patch_settings(): + plaintext = "ya29.a0AfH6SMB_plaintext_token" + assert decrypt_token(plaintext) == plaintext + + def test_encrypt_none_returns_none(self): + from lib.token_encryption import encrypt_token + with _patch_settings(): + assert encrypt_token(None) is None + + def test_encrypt_empty_returns_empty(self): + from lib.token_encryption import encrypt_token + with _patch_settings(): + assert encrypt_token("") == "" + + def test_decrypt_none_returns_none(self): + from lib.token_encryption import decrypt_token + with _patch_settings(): + assert decrypt_token(None) is None + + def test_decrypt_empty_returns_empty(self): + from lib.token_encryption import decrypt_token + with _patch_settings(): + assert decrypt_token("") == "" + + +class TestSettingsValidation: + """Settings should fail fast on invalid token-encryption config.""" + + def test_invalid_current_key_fails_settings_load(self): + from api.config import Settings + + with pytest.raises(ValidationError, match="TOKEN_ENCRYPTION_KEY must be a valid Fernet key"): + Settings(token_encryption_key="invalid-key") + + def test_invalid_previous_key_fails_settings_load(self): + from api.config import Settings + + with pytest.raises(ValidationError, match="TOKEN_ENCRYPTION_KEY_PREVIOUS must be a valid Fernet key"): + Settings( + token_encryption_key=TEST_KEY, + token_encryption_key_previous="invalid-key", + ) + + def test_previous_key_requires_current_key(self): + from api.config import Settings + + with pytest.raises( + ValidationError, + match="TOKEN_ENCRYPTION_KEY_PREVIOUS requires TOKEN_ENCRYPTION_KEY to also be set", + ): + Settings( + token_encryption_key="", + token_encryption_key_previous=TEST_KEY_PREVIOUS, + ) + + +class TestIsEncrypted: + """Heuristic detection of Fernet ciphertext.""" + + def test_detects_fernet_token(self): + from lib.token_encryption import encrypt_token, is_encrypted + with _patch_settings(): + encrypted = encrypt_token("test_token") + assert is_encrypted(encrypted) is True + + def test_rejects_plaintext(self): + from lib.token_encryption import is_encrypted + assert is_encrypted("ya29.a0AfH6SMB_not_encrypted") is False + + def test_rejects_short_string(self): + from lib.token_encryption import is_encrypted + assert is_encrypted("gAAAAA_short") is False + + def test_rejects_none(self): + from lib.token_encryption import is_encrypted + assert is_encrypted(None) is False + + +class TestEncryptTokenFields: + """Dict-level encryption of access_token + refresh_token.""" + + def test_encrypts_only_token_fields(self): + from lib.token_encryption import encrypt_token_fields, is_encrypted + with _patch_settings(): + data = { + 'access_token': 'at_plain', + 'refresh_token': 'rt_plain', + 'user_id': 'user-123', + 'provider': 'google', + } + result = encrypt_token_fields(data) + + assert is_encrypted(result['access_token']) + assert is_encrypted(result['refresh_token']) + assert result['user_id'] == 'user-123' + assert result['provider'] == 'google' + + def test_preserves_missing_token_fields(self): + """Dict without token fields should pass through unchanged.""" + from lib.token_encryption import encrypt_token_fields + with _patch_settings(): + data = {'user_id': 'u1', 'provider': 'google'} + result = encrypt_token_fields(data) + assert result == data + + def test_skips_none_token_values(self): + from lib.token_encryption import encrypt_token_fields + with _patch_settings(): + data = {'access_token': 'at_plain', 'refresh_token': None} + result = encrypt_token_fields(data) + assert result['refresh_token'] is None + + +class TestDecryptTokenFields: + """Dict-level decryption.""" + + def test_decrypts_encrypted_fields(self): + from lib.token_encryption import encrypt_token_fields, decrypt_token_fields + with _patch_settings(): + original = { + 'access_token': 'at_secret', + 'refresh_token': 'rt_secret', + 'user_id': 'user-1', + } + encrypted = encrypt_token_fields(original) + decrypted = decrypt_token_fields(encrypted) + + assert decrypted['access_token'] == 'at_secret' + assert decrypted['refresh_token'] == 'rt_secret' + assert decrypted['user_id'] == 'user-1' + + def test_handles_empty_dict(self): + from lib.token_encryption import decrypt_token_fields + assert decrypt_token_fields({}) == {} + + def test_handles_none(self): + from lib.token_encryption import decrypt_token_fields + assert decrypt_token_fields(None) is None + + +class TestKeyRotation: + """Support decrypting with previous key during key rotation.""" + + def test_decrypt_with_previous_key(self): + from lib.token_encryption import encrypt_token, decrypt_token + + # Encrypt with the old key + with _patch_settings(token_encryption_key=TEST_KEY_PREVIOUS): + encrypted = encrypt_token("my_secret_token") + + # Decrypt with new primary key + old key as previous + with _patch_settings( + token_encryption_key=TEST_KEY, + token_encryption_key_previous=TEST_KEY_PREVIOUS, + ): + decrypted = decrypt_token(encrypted) + assert decrypted == "my_secret_token" + + def test_wrong_key_plaintext_fallback(self): + """If decryption fails and value doesn't look like Fernet, return as-is.""" + from lib.token_encryption import decrypt_token + wrong_key = Fernet.generate_key().decode() + with _patch_settings(token_encryption_key=wrong_key): + # This is plaintext that doesn't start with gAAAAA + assert decrypt_token("ya29.plain_token") == "ya29.plain_token" + + def test_undecryptable_encrypted_value_raises(self): + from lib.token_encryption import ( + TokenDecryptionError, + decrypt_token, + encrypt_token, + ) + + with _patch_settings(token_encryption_key=TEST_KEY_PREVIOUS): + encrypted = encrypt_token("my_secret_token") + + with _patch_settings(token_encryption_key=TEST_KEY): + with pytest.raises( + TokenDecryptionError, + match="Failed to decrypt encrypted OAuth token with configured keys", + ): + decrypt_token(encrypted) + + +class TestFailClosedRuntimeBehavior: + """Runtime crypto errors must never silently fall back to plaintext.""" + + def test_encrypt_failure_raises_instead_of_returning_plaintext(self): + from lib.token_encryption import TokenEncryptionError, encrypt_token + + mock_fernet = MagicMock() + mock_fernet.encrypt.side_effect = RuntimeError("boom") + + with _patch_settings(): + with patch("lib.token_encryption._get_current_fernet", return_value=mock_fernet): + with pytest.raises( + TokenEncryptionError, + match="Failed to encrypt OAuth token", + ): + encrypt_token("super-secret-token") + + +# =========================================================================== +# Integration Tests β€” Write Paths (encrypt before DB write) +# =========================================================================== + +class TestWritePathEncryption: + """Verify that token write paths encrypt before database operations.""" + + def test_google_auth_refresh_encrypts_before_update(self): + """_refresh_and_save_token should encrypt tokens before .update().""" + from lib.token_encryption import is_encrypted + from api.config import settings + + captured_update = {} + + def capture_update(data): + captured_update.update(data) + mock_chain = MagicMock() + mock_chain.eq.return_value = mock_chain + mock_chain.execute.return_value = MagicMock(data=[]) + return mock_chain + + mock_supabase = MagicMock() + mock_supabase.table.return_value.update = capture_update + + mock_credentials = MagicMock() + mock_credentials.token = "new-access-token-from-google" + mock_credentials.refresh_token = "new-refresh-token-from-google" + + connection_data = { + 'id': 'conn-123', + 'user_id': 'user-456', + 'access_token': 'old-token', + 'refresh_token': 'old-refresh', + 'token_expires_at': None, + 'metadata': { + 'client_id': 'test-client-id', + 'client_secret': 'test-client-secret', + }, + } + + original_key = settings.token_encryption_key + settings.token_encryption_key = TEST_KEY + try: + with patch('api.services.google_auth.get_service_role_client', return_value=mock_supabase): + with patch('api.services.google_auth.Credentials', return_value=mock_credentials): + with patch('api.services.google_auth._refresh_creds'): + from api.services.google_auth import _refresh_and_save_token + _refresh_and_save_token( + connection_data, + mock_supabase, + ) + + assert 'access_token' in captured_update + assert is_encrypted(captured_update['access_token']) + finally: + settings.token_encryption_key = original_key + + +# =========================================================================== +# Integration Tests β€” Read Paths (decrypt after DB read) +# =========================================================================== + +class TestReadPathDecryption: + """Verify that token read paths decrypt after database operations.""" + + def test_google_auth_get_credentials_decrypts(self): + """get_credentials_for_connection should decrypt tokens from DB.""" + from lib.token_encryption import encrypt_token + from api.config import settings + from datetime import datetime, timezone, timedelta + + future_expiry = (datetime.now(timezone.utc) + timedelta(hours=1)).isoformat() + + original_key = settings.token_encryption_key + settings.token_encryption_key = TEST_KEY + try: + encrypted_at = encrypt_token("real-access-token") + encrypted_rt = encrypt_token("real-refresh-token") + + mock_result = MagicMock() + mock_result.data = { + 'id': 'conn-1', + 'user_id': 'user-1', + 'access_token': encrypted_at, + 'refresh_token': encrypted_rt, + 'token_expires_at': future_expiry, + 'metadata': {}, + } + + mock_supabase = MagicMock() + mock_supabase.table.return_value.select.return_value.eq.return_value.eq.return_value.single.return_value.execute.return_value = mock_result + + captured_creds_kwargs = {} + + def mock_credentials(**kwargs): + captured_creds_kwargs.update(kwargs) + return MagicMock() + + with patch('api.services.google_auth.Credentials', side_effect=mock_credentials): + with patch('api.services.google_auth._is_token_expired', return_value=False): + from api.services.google_auth import get_credentials_for_connection + get_credentials_for_connection("conn-1", supabase_client=mock_supabase) + + # The Credentials constructor should receive the decrypted plaintext + assert captured_creds_kwargs['token'] == "real-access-token" + assert captured_creds_kwargs['refresh_token'] == "real-refresh-token" + finally: + settings.token_encryption_key = original_key diff --git a/core-api/tests/unit/test_token_encryption_regressions.py b/core-api/tests/unit/test_token_encryption_regressions.py new file mode 100644 index 00000000..87fa9ae4 --- /dev/null +++ b/core-api/tests/unit/test_token_encryption_regressions.py @@ -0,0 +1,244 @@ +import os +from types import SimpleNamespace +from unittest.mock import AsyncMock, MagicMock, patch + +import pytest +from cryptography.fernet import Fernet + +# Prevent module import failures from lib.supabase_client singleton init. +os.environ.setdefault("SUPABASE_URL", "https://test.supabase.co") +os.environ.setdefault( + "SUPABASE_ANON_KEY", + "eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9." + "eyJpc3MiOiJzdXBhYmFzZSIsInJlZiI6InRlc3QiLCJyb2xlIjoiYW5vbiJ9." + "testsignature", +) + +TEST_KEY = Fernet.generate_key().decode() + + +def _patch_settings(**overrides): + defaults = { + "token_encryption_key": TEST_KEY, + "token_encryption_key_previous": "", + "supabase_url": "", + "supabase_anon_key": "", + "supabase_service_role_key": "", + "qstash_token": "", + "qstash_worker_url": "", + "qstash_url": "", + "cron_secret": "", + } + defaults.update(overrides) + return patch("api.config.settings", SimpleNamespace(**defaults)) + + +def _make_encrypted_connection(provider: str) -> dict: + with _patch_settings(): + from lib.token_encryption import encrypt_token + + return { + "id": f"{provider}-conn-1", + "user_id": "user-1", + "provider": provider, + "provider_email": f"{provider}@example.com", + "access_token": encrypt_token(f"{provider}-access-token"), + "refresh_token": encrypt_token(f"{provider}-refresh-token"), + "token_expires_at": None, + "metadata": {}, + } + + +def _make_email_lookup_client(email_row: dict): + auth_supabase = MagicMock() + query = MagicMock() + query.select.return_value = query + query.eq.return_value = query + query.maybe_single.return_value = query + query.execute.return_value = MagicMock(data=email_row) + auth_supabase.table.return_value = query + return auth_supabase + + +def _make_subscription_client(subscription_row: dict): + service_supabase = MagicMock() + query = MagicMock() + query.select.return_value = query + query.eq.return_value = query + query.limit.return_value = query + query.execute.return_value = MagicMock(data=[subscription_row]) + service_supabase.table.return_value = query + return service_supabase + + +def test_get_email_attachment_uses_decrypted_joined_gmail_tokens(): + encrypted_connection = _make_encrypted_connection("google") + auth_supabase = _make_email_lookup_client( + { + "id": "email-db-1", + "external_id": "gmail-message-1", + "body": "", + "is_draft": False, + "ext_connection_id": encrypted_connection["id"], + "ext_connections": encrypted_connection, + } + ) + + service = MagicMock() + service.users.return_value.messages.return_value.attachments.return_value.get.return_value.execute.return_value = { + "data": "ZGF0YQ==", + "size": 4, + } + + captured_connection = {} + + def _build_service(connection_data): + captured_connection.update(connection_data) + return service + + with _patch_settings(): + with patch( + "api.services.email.get_email_details.get_authenticated_supabase_client", + return_value=auth_supabase, + ): + with patch( + "api.services.email.get_email_details.build_gmail_service_from_connection_data", + side_effect=_build_service, + ): + from api.services.email.get_email_details import get_email_attachment + + result = get_email_attachment( + "user-1", + "jwt-token", + "gmail-message-1", + "attachment-1", + ) + + assert captured_connection["access_token"] == "google-access-token" + assert captured_connection["refresh_token"] == "google-refresh-token" + assert result["provider"] == "google" + + +def test_get_email_details_uses_decrypted_joined_outlook_tokens(): + encrypted_connection = _make_encrypted_connection("microsoft") + auth_supabase = _make_email_lookup_client( + { + "id": "email-db-1", + "external_id": "outlook-message-1", + "body": "", + "is_draft": False, + "ext_connection_id": encrypted_connection["id"], + "ext_connections": encrypted_connection, + } + ) + + captured_kwargs = {} + + def _fake_outlook_details(**kwargs): + captured_kwargs.update(kwargs) + return {"provider": "microsoft", "email": {"id": "outlook-message-1"}} + + with _patch_settings(): + with patch( + "api.services.email.get_email_details.get_authenticated_supabase_client", + return_value=auth_supabase, + ): + with patch( + "api.services.email.get_email_details._get_outlook_email_details", + side_effect=_fake_outlook_details, + ): + from api.services.email.get_email_details import get_email_details + + result = get_email_details( + "user-1", + "jwt-token", + "outlook-message-1", + ) + + assert captured_kwargs["connection_data"]["access_token"] == "microsoft-access-token" + assert captured_kwargs["connection_data"]["refresh_token"] == "microsoft-refresh-token" + assert result["provider"] == "microsoft" + + +@pytest.mark.asyncio +async def test_process_microsoft_notification_decrypts_joined_tokens(): + encrypted_connection = _make_encrypted_connection("microsoft") + service_supabase = _make_subscription_client( + { + "client_state": "client-state-1", + "ext_connections": encrypted_connection, + } + ) + + captured_connection = {} + + def _fake_process_notification(_self, _notification, connection_data): + captured_connection.update(connection_data) + return {"success": True, "path": "inline"} + + notification = { + "subscriptionId": "subscription-1", + "clientState": "client-state-1", + "changeType": "updated", + "resource": "me/messages", + } + + with _patch_settings(): + with patch( + "lib.supabase_client.get_service_role_client", + return_value=service_supabase, + ): + with patch( + "api.services.microsoft.microsoft_webhook_provider.MicrosoftWebhookProvider.validate_notification", + return_value=True, + ): + with patch( + "api.services.microsoft.microsoft_webhook_provider.MicrosoftWebhookProvider.process_notification", + autospec=True, + side_effect=_fake_process_notification, + ): + from api.routers.webhooks import process_microsoft_notification + + result = await process_microsoft_notification(notification) + + assert captured_connection["access_token"] == "microsoft-access-token" + assert captured_connection["refresh_token"] == "microsoft-refresh-token" + assert result == {"success": True, "path": "inline"} + + +@pytest.mark.asyncio +async def test_mark_microsoft_notification_dirty_decrypts_tokens(): + encrypted_connection = _make_encrypted_connection("microsoft") + service_supabase = _make_subscription_client( + { + "client_state": "client-state-1", + "ext_connections": encrypted_connection, + } + ) + + notification = { + "subscriptionId": "subscription-1", + "clientState": "client-state-1", + "changeType": "updated", + "resource": "me/messages", + } + + with _patch_settings(): + with patch( + "lib.supabase_client.get_service_role_client", + return_value=service_supabase, + ): + with patch( + "api.services.microsoft.microsoft_webhook_provider.MicrosoftWebhookProvider.validate_notification", + return_value=True, + ): + with patch( + "api.routers.webhooks.mark_stream_dirty", + return_value=True, + ) as dirty_mock: + from api.routers.webhooks import _mark_microsoft_notification_dirty + + result = await _mark_microsoft_notification_dirty(notification) + + dirty_mock.assert_called_once() + assert result == {"success": True, "marked": "sync-outlook"} diff --git a/core-api/tests/unit/test_user_profile_helpers.py b/core-api/tests/unit/test_user_profile_helpers.py index 00898947..84b46c1a 100644 --- a/core-api/tests/unit/test_user_profile_helpers.py +++ b/core-api/tests/unit/test_user_profile_helpers.py @@ -1,6 +1,5 @@ from __future__ import annotations -import asyncio from types import SimpleNamespace from unittest.mock import AsyncMock @@ -43,7 +42,8 @@ async def test_get_auth_user_by_email_reads_auth_store(monkeypatch): class FakeAdmin: async def list_users(self, page=None, per_page=None): - await asyncio.sleep(0) + assert page == 1 + assert per_page == 200 return [ SimpleNamespace(id="user-1", email="owner@example.com"), SimpleNamespace(id="user-2", email="other@example.com"), diff --git a/core-api/tests/unit/test_watch_manager_permanent_failures.py b/core-api/tests/unit/test_watch_manager_permanent_failures.py index 27299d03..881ed743 100644 --- a/core-api/tests/unit/test_watch_manager_permanent_failures.py +++ b/core-api/tests/unit/test_watch_manager_permanent_failures.py @@ -1,8 +1,10 @@ import json import os +from datetime import datetime, timedelta, timezone from types import SimpleNamespace from unittest.mock import MagicMock, patch +import pytest from googleapiclient.errors import HttpError from httplib2 import Response @@ -34,6 +36,23 @@ def _make_service_supabase_mock() -> MagicMock: return supabase +def _make_auth_supabase_mock(*execute_results: SimpleNamespace): + query = MagicMock() + query.select.return_value = query + query.eq.return_value = query + query.update.return_value = query + query.insert.return_value = query + query.execute.side_effect = list(execute_results) or [SimpleNamespace(data=[])] + + supabase = MagicMock() + supabase.table.return_value = query + return supabase, query + + +def _iso_in_hours(hours: int) -> str: + return (datetime.now(timezone.utc) + timedelta(hours=hours)).isoformat() + + def test_start_gmail_watch_service_role_permanent_failure_logs_warning_not_error(): """ Known permanent Gmail watch failures should be warning-level only. @@ -160,3 +179,111 @@ def test_start_calendar_watch_service_role_transient_failure_still_logs_error(): assert result["provider"] == "calendar" assert any("Calendar API error" in str(call) for call in logger_mock.error.call_args_list) logger_mock.warning.assert_not_called() + + +def test_stop_gmail_watch_transient_stop_failure_does_not_deactivate_db_row(): + from api.services.syncs.watch_manager import stop_gmail_watch + + auth_supabase, query = _make_auth_supabase_mock( + SimpleNamespace(data=[{"id": "sub-123"}]), + ) + gmail_service = MagicMock() + gmail_service.users.return_value.stop.return_value.execute.side_effect = _make_http_error( + 500, + "backendError", + ) + + with patch("api.services.syncs.watch_manager.get_authenticated_supabase_client", return_value=auth_supabase): + with patch("api.services.syncs.watch_manager.get_gmail_service", return_value=(gmail_service, "conn-123")): + result = stop_gmail_watch("user-123", "jwt-token") + + assert result["success"] is False + assert result["provider"] == "gmail" + query.update.assert_not_called() + + +def test_stop_gmail_watch_404_still_deactivates_db_row(): + from api.services.syncs.watch_manager import stop_gmail_watch + + auth_supabase, query = _make_auth_supabase_mock( + SimpleNamespace(data=[{"id": "sub-123"}]), + SimpleNamespace(data=[{"id": "sub-123"}]), + ) + gmail_service = MagicMock() + gmail_service.users.return_value.stop.return_value.execute.side_effect = _make_http_error( + 404, + "notFound", + ) + + with patch("api.services.syncs.watch_manager.get_authenticated_supabase_client", return_value=auth_supabase): + with patch("api.services.syncs.watch_manager.get_gmail_service", return_value=(gmail_service, "conn-123")): + result = stop_gmail_watch("user-123", "jwt-token") + + assert result["success"] is True + query.update.assert_called_once() + + +def test_stop_calendar_watch_missing_identifiers_does_not_deactivate_db_row(): + from api.services.syncs.watch_manager import stop_calendar_watch + + auth_supabase, query = _make_auth_supabase_mock( + SimpleNamespace(data=[{"id": "sub-123", "channel_id": None, "resource_id": None}]), + ) + calendar_service = MagicMock() + + with patch("api.services.syncs.watch_manager.get_authenticated_supabase_client", return_value=auth_supabase): + with patch("api.services.syncs.watch_manager.get_google_calendar_service", return_value=(calendar_service, "conn-123")): + result = stop_calendar_watch("user-123", "jwt-token") + + assert result["success"] is False + assert result["provider"] == "calendar" + query.update.assert_not_called() + + +def test_start_gmail_watch_aborts_renewal_when_stop_helper_fails(): + from api.services.syncs.watch_manager import start_gmail_watch + + auth_supabase, query = _make_auth_supabase_mock( + SimpleNamespace(data=[{ + "id": "sub-123", + "channel_id": "channel-123", + "history_id": "100", + "expiration": _iso_in_hours(1), + }]), + ) + + with patch("api.services.syncs.watch_manager.get_authenticated_supabase_client", return_value=auth_supabase): + with patch("api.services.syncs.watch_manager.get_gmail_service", return_value=(MagicMock(), "conn-123")): + with patch( + "api.services.syncs.watch_manager.stop_gmail_watch", + return_value={"success": False, "error": "stop failed"}, + ): + with pytest.raises(ValueError, match="stop failed"): + start_gmail_watch("user-123", "jwt-token") + + query.insert.assert_not_called() + + +def test_start_calendar_watch_aborts_renewal_when_stop_helper_fails(): + from api.services.syncs.watch_manager import start_calendar_watch + + auth_supabase, query = _make_auth_supabase_mock( + SimpleNamespace(data=[{ + "id": "sub-123", + "channel_id": "channel-123", + "resource_id": "resource-123", + "sync_token": "sync-123", + "expiration": _iso_in_hours(1), + }]), + ) + + with patch("api.services.syncs.watch_manager.get_authenticated_supabase_client", return_value=auth_supabase): + with patch("api.services.syncs.watch_manager.get_google_calendar_service", return_value=(MagicMock(), "conn-123")): + with patch( + "api.services.syncs.watch_manager.stop_calendar_watch", + return_value={"success": False, "error": "stop failed"}, + ): + with pytest.raises(ValueError, match="stop failed"): + start_calendar_watch("user-123", "jwt-token") + + query.insert.assert_not_called() diff --git a/core-api/tests/unit/test_webhook_dirty_marking.py b/core-api/tests/unit/test_webhook_dirty_marking.py new file mode 100644 index 00000000..e5836dec --- /dev/null +++ b/core-api/tests/unit/test_webhook_dirty_marking.py @@ -0,0 +1,288 @@ +import base64 +import json +import os +from types import SimpleNamespace + +import google.oauth2.id_token as google_id_token +import pytest +from fastapi import HTTPException + +# Prevent import-time settings failures. +os.environ.setdefault("SUPABASE_URL", "https://test.supabase.co") +os.environ.setdefault( + "SUPABASE_ANON_KEY", + "eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9." + "eyJpc3MiOiJzdXBhYmFzZSIsInJlZiI6InRlc3QiLCJyb2xlIjoiYW5vbiJ9." + "testsignature", +) + + +class _FakeRequest: + def __init__(self, body): + self._body = body + self.query_params = {} + + async def body(self): + return json.dumps(self._body).encode("utf-8") + + async def json(self): + return self._body + + +def _gmail_request(email: str = "user@example.com", history_id: str = "123") -> _FakeRequest: + payload = base64.b64encode( + json.dumps({"emailAddress": email, "historyId": history_id}).encode("utf-8") + ).decode("utf-8") + return _FakeRequest({"message": {"data": payload}}) + + +class _BodyOnlyRequest: + def __init__(self, body): + self._body = body + self.query_params = {} + + async def body(self): + return json.dumps(self._body).encode("utf-8") + + +class _FakeQuery: + def __init__(self, data): + self._data = data + + def select(self, *_args, **_kwargs): + return self + + def eq(self, *_args, **_kwargs): + return self + + def limit(self, *_args, **_kwargs): + return self + + def execute(self): + return SimpleNamespace(data=self._data) + + +class _FakeSupabase: + def __init__(self, table_map): + self._table_map = table_map + + def table(self, name): + return _FakeQuery(self._table_map.get(name, [])) + + +def _clear_gmail_ingress_dedup_cache(webhooks) -> None: + with webhooks._GMAIL_INGRESS_DEDUP_LOCK: + webhooks._GMAIL_INGRESS_DEDUP_CACHE.clear() + + +@pytest.mark.asyncio +async def test_gmail_webhook_marks_stream_dirty(monkeypatch): + from api.routers import webhooks + + _clear_gmail_ingress_dedup_cache(webhooks) + supabase = _FakeSupabase({"ext_connections": [{"id": "conn-1"}]}) + marked = {} + + monkeypatch.setattr(webhooks, "get_service_role_client", lambda: supabase) + monkeypatch.setattr(webhooks, "_verify_google_pubsub_auth", lambda *_args, **_kwargs: None) + monkeypatch.setattr( + webhooks, + "mark_stream_dirty", + lambda *args, **kwargs: marked.update({"args": args, "kwargs": kwargs}) or True, + ) + + result = await webhooks.gmail_webhook(_gmail_request()) + + assert result["status"] == "ok" + assert result["message"] == "Accepted for async sync" + assert marked["args"][1] == "conn-1" + assert marked["args"][2] == "google" + assert marked["args"][3] == "email" + assert marked["kwargs"]["latest_seen_cursor"] == "123" + assert marked["kwargs"]["metadata"]["source"] == "gmail-webhook" + assert marked["kwargs"]["metadata"]["history_id"] == "123" + + +@pytest.mark.asyncio +async def test_gmail_webhook_reads_buffered_body_without_request_json(monkeypatch): + from api.routers import webhooks + + _clear_gmail_ingress_dedup_cache(webhooks) + payload = base64.b64encode( + json.dumps({"emailAddress": "user@example.com", "historyId": "456"}).encode("utf-8") + ).decode("utf-8") + request = _BodyOnlyRequest({"message": {"data": payload}}) + supabase = _FakeSupabase({"ext_connections": [{"id": "conn-1"}]}) + marked = {} + + monkeypatch.setattr(webhooks, "get_service_role_client", lambda: supabase) + monkeypatch.setattr(webhooks, "_verify_google_pubsub_auth", lambda *_args, **_kwargs: None) + monkeypatch.setattr( + webhooks, + "mark_stream_dirty", + lambda *args, **kwargs: marked.update({"args": args, "kwargs": kwargs}) or True, + ) + + result = await webhooks.gmail_webhook(request) + + assert result["status"] == "ok" + assert marked["kwargs"]["latest_seen_cursor"] == "456" + + +@pytest.mark.asyncio +async def test_gmail_webhook_skips_recent_duplicate_before_supabase(monkeypatch): + from api.routers import webhooks + + _clear_gmail_ingress_dedup_cache(webhooks) + supabase = _FakeSupabase({"ext_connections": [{"id": "conn-1"}]}) + counters = {"clients": 0, "marks": 0} + + def _get_client(): + counters["clients"] += 1 + return supabase + + def _mark(*_args, **_kwargs): + counters["marks"] += 1 + return True + + monkeypatch.setattr(webhooks, "get_service_role_client", _get_client) + monkeypatch.setattr(webhooks, "_verify_google_pubsub_auth", lambda *_args, **_kwargs: None) + monkeypatch.setattr(webhooks, "mark_stream_dirty", _mark) + + first = await webhooks.gmail_webhook(_gmail_request(history_id="789")) + second = await webhooks.gmail_webhook(_gmail_request(history_id="789")) + + assert first["status"] == "ok" + assert second["status"] == "ok" + assert second["message"] == "Accepted duplicate notification" + assert counters == {"clients": 1, "marks": 1} + + +@pytest.mark.asyncio +async def test_calendar_webhook_marks_stream_dirty(monkeypatch): + from api.routers import webhooks + + supabase = _FakeSupabase({"push_subscriptions": [{"ext_connection_id": "conn-2", "resource_id": "resource-1"}]}) + marked = {} + + monkeypatch.setattr(webhooks, "get_service_role_client", lambda: supabase) + monkeypatch.setattr( + webhooks, + "verify_google_calendar_channel_token", + lambda *_args, **_kwargs: True, + ) + monkeypatch.setattr( + webhooks, + "mark_stream_dirty", + lambda *args, **kwargs: marked.update({"args": args, "kwargs": kwargs}) or True, + ) + + result = await webhooks.calendar_webhook( + _FakeRequest({}), + x_goog_channel_id="channel-1", + x_goog_channel_token="token-1", + x_goog_resource_id="resource-1", + x_goog_resource_state="exists", + x_goog_message_number="7", + ) + + assert result["status"] == "ok" + assert result["message"] == "Accepted for async sync" + assert marked["args"][1] == "conn-2" + assert marked["args"][2] == "google" + assert marked["args"][3] == "calendar" + assert marked["kwargs"]["metadata"]["source"] == "google-calendar-webhook" + assert marked["kwargs"]["metadata"]["channel_id"] == "channel-1" + + +@pytest.mark.asyncio +async def test_gmail_webhook_rejects_invalid_pubsub_auth(monkeypatch): + from api.routers import webhooks + + _clear_gmail_ingress_dedup_cache(webhooks) + monkeypatch.setattr( + webhooks, + "_verify_google_pubsub_auth", + lambda *_args, **_kwargs: (_ for _ in ()).throw( + HTTPException(status_code=401, detail="bad token") + ), + ) + + with pytest.raises(HTTPException) as exc_info: + await webhooks.gmail_webhook(_gmail_request(), authorization="Bearer bad") + + assert exc_info.value.status_code == 401 + + +@pytest.mark.asyncio +async def test_gmail_webhook_acknowledges_transient_dirty_mark_failures(monkeypatch): + from api.routers import webhooks + + _clear_gmail_ingress_dedup_cache(webhooks) + supabase = _FakeSupabase({"ext_connections": [{"id": "conn-1"}]}) + + monkeypatch.setattr(webhooks, "get_service_role_client", lambda: supabase) + monkeypatch.setattr(webhooks, "_verify_google_pubsub_auth", lambda *_args, **_kwargs: None) + monkeypatch.setattr( + webhooks, + "mark_stream_dirty", + lambda *_args, **_kwargs: (_ for _ in ()).throw(RuntimeError("supabase unavailable")), + ) + + result = await webhooks.gmail_webhook(_gmail_request()) + + assert result["status"] == "ok" + assert result["message"] == "Accepted without dirty mark; reconciliation will recover" + + +def test_verify_google_pubsub_auth_converts_invalid_token_errors_to_http_exception( + monkeypatch, +): + from api.routers import webhooks + + webhooks._PUBSUB_JWT_CACHE.clear() + monkeypatch.setattr( + webhooks.settings, + "google_pubsub_push_service_account_email", + "pubsub-push@test-project.iam.gserviceaccount.com", + ) + monkeypatch.setattr( + webhooks.settings, + "google_pubsub_push_audience", + "https://core-webhooks.example.com/api/webhooks/gmail", + ) + monkeypatch.setattr( + google_id_token, + "verify_oauth2_token", + lambda *_args, **_kwargs: (_ for _ in ()).throw(ValueError("Token expired")), + ) + + with pytest.raises(HTTPException) as exc_info: + webhooks._verify_google_pubsub_auth("Bearer bad-token") + + assert exc_info.value.status_code == 401 + assert exc_info.value.detail == "Invalid Pub/Sub authorization token" + + +@pytest.mark.asyncio +async def test_calendar_webhook_rejects_invalid_channel_token(monkeypatch): + from api.routers import webhooks + + supabase = _FakeSupabase({"push_subscriptions": [{"ext_connection_id": "conn-2", "resource_id": "resource-1"}]}) + + monkeypatch.setattr(webhooks, "get_service_role_client", lambda: supabase) + monkeypatch.setattr( + webhooks, + "verify_google_calendar_channel_token", + lambda *_args, **_kwargs: False, + ) + + with pytest.raises(HTTPException) as exc_info: + await webhooks.calendar_webhook( + _FakeRequest({}), + x_goog_channel_id="channel-1", + x_goog_channel_token="wrong", + x_goog_resource_id="resource-1", + ) + + assert exc_info.value.status_code == 401 diff --git a/core-api/tests/unit/test_workers_mode_dispatch.py b/core-api/tests/unit/test_workers_mode_dispatch.py index cbd5361f..2f0734a4 100644 --- a/core-api/tests/unit/test_workers_mode_dispatch.py +++ b/core-api/tests/unit/test_workers_mode_dispatch.py @@ -104,6 +104,20 @@ def test_run_batch_clean_run_returns_ok(): assert result["failed_ids"] == [] +def test_run_batch_treats_quarantined_as_processed(): + from api.routers.workers import _run_batch + + result = _run_batch( + ["a", "b"], + lambda cid: {"status": "quarantined"} if cid == "a" else {"status": "ok"}, + "sync-gmail", + ) + + assert result["status"] == "ok" + assert result["processed"] == 2 + assert result["errors"] == 0 + + def test_run_batch_respects_time_budget(monkeypatch): from api.routers import workers @@ -125,12 +139,14 @@ def test_worker_sync_gmail_routes_webhook_mode(): email_address="user@example.com", ) - with patch.object(workers, "_process_gmail_webhook", return_value={"status": "ok", "message": "done"}) as webhook_mock: - with patch.object(workers, "_sync_single_gmail") as single_mock: - result = workers.worker_sync_gmail(payload) + with patch.object(workers, "_run_with_stream_lease", return_value={"status": "ok", "message": "done"}) as lease_mock: + with patch.object(workers, "_process_gmail_webhook") as webhook_mock: + with patch.object(workers, "_sync_single_gmail") as single_mock: + result = workers.worker_sync_gmail(payload) assert result["status"] == "ok" - webhook_mock.assert_called_once_with(payload) + lease_mock.assert_called_once() + webhook_mock.assert_not_called() single_mock.assert_not_called() @@ -303,15 +319,132 @@ def test_worker_sync_calendar_routes_webhook_mode(): message_number="123", ) - with patch.object(workers, "_process_calendar_webhook", return_value={"status": "ok"}) as webhook_mock: - with patch.object(workers, "_sync_single_calendar") as single_mock: - result = workers.worker_sync_calendar(payload) + with patch.object(workers, "_run_with_stream_lease", return_value={"status": "ok"}) as lease_mock: + with patch.object(workers, "_process_calendar_webhook") as webhook_mock: + with patch.object(workers, "_sync_single_calendar") as single_mock: + result = workers.worker_sync_calendar(payload) assert result["status"] == "ok" - webhook_mock.assert_called_once_with(payload) + lease_mock.assert_called_once() + webhook_mock.assert_not_called() single_mock.assert_not_called() +def test_run_with_stream_lease_skips_when_no_claim(): + from api.routers import workers + + with patch.object(workers, "get_service_role_client", return_value=MagicMock()): + with patch.object(workers, "claim_connection_sync_lease", return_value=None) as claim_mock: + result = workers._run_with_stream_lease("conn-1", workers.SYNC_KIND_EMAIL, lambda: {"status": "ok"}) + + assert result["status"] == "skipped" + assert "already running or not dirty" in result["message"].lower() + assert claim_mock.call_args.kwargs["claim_mode"] == workers.CLAIM_MODE_DIRTY_ONLY + + +def test_run_with_stream_lease_completes_success(): + from api.routers import workers + + with patch.object(workers, "get_service_role_client", return_value=MagicMock()): + with patch.object(workers, "claim_connection_sync_lease", return_value={"connection_id": "conn-1"}): + with patch.object(workers, "complete_connection_sync_lease") as complete_mock: + with patch.object(workers, "fail_connection_sync_lease") as fail_mock: + result = workers._run_with_stream_lease( + "conn-1", + workers.SYNC_KIND_EMAIL, + lambda: {"status": "ok", "new_delta_link": "delta-1"}, + ) + + assert result["status"] == "ok" + complete_mock.assert_called_once() + fail_mock.assert_not_called() + + +def test_run_with_stream_lease_fails_error_result(): + from api.routers import workers + + with patch.object(workers, "get_service_role_client", return_value=MagicMock()): + with patch.object(workers, "claim_connection_sync_lease", return_value={"connection_id": "conn-1"}): + with patch.object(workers, "complete_connection_sync_lease") as complete_mock: + with patch.object(workers, "fail_connection_sync_lease") as fail_mock: + result = workers._run_with_stream_lease( + "conn-1", + workers.SYNC_KIND_EMAIL, + lambda: {"status": "error", "message": "boom"}, + ) + + assert result["status"] == "error" + fail_mock.assert_called_once() + complete_mock.assert_not_called() + + +def test_run_with_stream_lease_quarantines_when_retry_cap_is_hit(monkeypatch): + from api.routers import workers + + monkeypatch.setenv("MAX_FAILURE_RETRY_COUNT", "3") + + with patch.object(workers, "get_service_role_client", return_value=MagicMock()): + with patch.object( + workers, + "claim_connection_sync_lease", + return_value={"connection_id": "conn-1", "provider": "microsoft"}, + ): + with patch.object( + workers, + "get_connection_sync_state", + return_value={"retry_count": 2}, + ): + with patch("api.services.syncs.failure_policy.complete_connection_sync_lease") as complete_mock: + with patch.object(workers, "fail_connection_sync_lease") as fail_mock: + with patch( + "api.services.syncs.failure_policy.deactivate_connection_with_subscriptions" + ) as deactivate_mock: + result = workers._run_with_stream_lease( + "conn-1", + workers.SYNC_KIND_EMAIL, + lambda: {"status": "error", "message": "boom"}, + ) + + assert result["status"] == "quarantined" + complete_mock.assert_called_once() + fail_mock.assert_not_called() + deactivate_mock.assert_called_once() + + +def test_run_with_stream_lease_quarantines_on_permanent_microsoft_auth_error(): + from api.routers import workers + + with patch.object(workers, "get_service_role_client", return_value=MagicMock()): + with patch.object( + workers, + "claim_connection_sync_lease", + return_value={"connection_id": "conn-1", "provider": "microsoft"}, + ): + with patch.object( + workers, + "get_connection_sync_state", + return_value={"retry_count": 0}, + ): + with patch("api.services.syncs.failure_policy.complete_connection_sync_lease") as complete_mock: + with patch.object(workers, "fail_connection_sync_lease") as fail_mock: + with patch( + "api.services.syncs.failure_policy.deactivate_connection_with_subscriptions" + ) as deactivate_mock: + result = workers._run_with_stream_lease( + "conn-1", + workers.SYNC_KIND_EMAIL, + lambda: { + "status": "error", + "message": "Refresh token is invalid - user must re-authenticate", + }, + ) + + assert result["status"] == "quarantined" + complete_mock.assert_called_once() + fail_mock.assert_not_called() + deactivate_mock.assert_called_once() + + def test_process_gmail_webhook_error_status_not_overwritten(): from api.routers import workers @@ -379,6 +512,26 @@ def test_process_calendar_webhook_error_status_not_overwritten(): assert result["message"] == "provider failed" +def test_process_calendar_webhook_does_not_touch_last_synced_for_non_sync_outcomes(): + from api.routers import workers + + payload = workers.SyncPayload( + connection_id="conn-1", + channel_id="channel-1", + resource_state="exists", + ) + + with patch.object(workers, "_touch_last_synced") as touch_mock: + with patch( + "api.services.webhooks.process_calendar_notification", + return_value={"status": "ok", "message": "No active subscription"}, + ): + result = workers._process_calendar_webhook(payload) + + assert result["status"] == "ok" + touch_mock.assert_not_called() + + def test_process_gmail_webhook_rejects_connection_email_mismatch(): from api.routers import workers diff --git a/core-api/tests/unit/test_workspace_default_apps.py b/core-api/tests/unit/test_workspace_default_apps.py index 72af105a..a5dcdd84 100644 --- a/core-api/tests/unit/test_workspace_default_apps.py +++ b/core-api/tests/unit/test_workspace_default_apps.py @@ -5,7 +5,7 @@ from api.services.workspaces.crud import create_workspace -EXPECTED_DEFAULT_APPS = ["chat", "email", "calendar", "projects", "files"] +EXPECTED_DEFAULT_APPS = ["chat", "email", "calendar", "projects", "files", "tasks"] def _make_mock_supabase(workspace_data: dict) -> MagicMock: @@ -102,8 +102,8 @@ async def test_create_workspace_without_default_apps(): def test_expected_default_apps_list(): """Sanity check: the expected default apps match what the migration defines.""" - assert len(EXPECTED_DEFAULT_APPS) == 5 - assert EXPECTED_DEFAULT_APPS == ["chat", "email", "calendar", "projects", "files"] + assert len(EXPECTED_DEFAULT_APPS) == 6 + assert EXPECTED_DEFAULT_APPS == ["chat", "email", "calendar", "projects", "files", "tasks"] # All expected apps must be valid from api.services.workspaces.apps import VALID_APP_TYPES for app in EXPECTED_DEFAULT_APPS: diff --git a/core-api/tests/unit/test_workspace_invitations_service.py b/core-api/tests/unit/test_workspace_invitations_service.py index 1b1c11bf..41953c9e 100644 --- a/core-api/tests/unit/test_workspace_invitations_service.py +++ b/core-api/tests/unit/test_workspace_invitations_service.py @@ -859,7 +859,7 @@ async def test_get_invitation_share_link_requires_admin_and_returns_url(monkeypa "get_user_workspace_role", AsyncMock(return_value="admin"), ) - monkeypatch.setattr(invitations_module.settings, "frontend_url", "https://app.example.com") + monkeypatch.setattr(invitations_module.settings, "frontend_url", "https://app.core.so") result = await invitations_module.get_workspace_invitation_share_link( invitation_id="inv-1", @@ -868,4 +868,4 @@ async def test_get_invitation_share_link_requires_admin_and_returns_url(monkeypa ) assert result["invitation_id"] == "inv-1" - assert result["invite_url"] == "https://app.example.com/invite/token-1" + assert result["invite_url"] == "https://app.core.so/invite/token-1" diff --git a/core-api/webhooks_index.py b/core-api/webhooks_index.py new file mode 100644 index 00000000..7aa0ecaf --- /dev/null +++ b/core-api/webhooks_index.py @@ -0,0 +1,11 @@ +"""FastAPI application entrypoint for the dedicated webhook ingress service.""" + +import os +import sys + +sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) + +from api.app_factory import create_webhooks_app + + +app = create_webhooks_app() From 36e9410cfd658a0bb53fc717f174cdf994937bc3 Mon Sep 17 00:00:00 2001 From: hackertron Date: Tue, 7 Apr 2026 22:33:01 +0530 Subject: [PATCH 2/6] fix: align tests with core-oss function signatures - test_notify_attendees_google: unpack 3 values from _create_google_event (core-oss doesn't return resolved_tz), single bool from _update_google_event - test_workspace_default_apps: remove 'tasks' from expected default apps (todos/tasks not in core-oss) - google_auth: port _refresh_creds and concurrent refresh race guard from prod --- core-api/api/services/google_auth.py | 45 ++++++++++++++++--- .../unit/test_notify_attendees_google.py | 4 +- .../tests/unit/test_workspace_default_apps.py | 6 +-- 3 files changed, 44 insertions(+), 11 deletions(-) diff --git a/core-api/api/services/google_auth.py b/core-api/api/services/google_auth.py index ed1b9107..60b1e299 100644 --- a/core-api/api/services/google_auth.py +++ b/core-api/api/services/google_auth.py @@ -17,6 +17,7 @@ from google.auth.transport.requests import Request from google.auth.exceptions import RefreshError from googleapiclient.discovery import build +from lib.google_retry import refresh_credentials as _refresh_creds from lib.supabase_client import get_service_role_client from lib.token_encryption import ( @@ -354,8 +355,8 @@ def _refresh_and_save_token( client_secret=client_secret ) - # Perform the refresh - credentials.refresh(Request()) + # Perform the refresh (with retry for transient transport errors) + _refresh_creds(credentials, Request(), context=connection_id[:8] if connection_id else '') # Calculate new expiry time new_expires_at = datetime.now(timezone.utc) + timedelta(seconds=DEFAULT_TOKEN_LIFETIME_SECONDS) @@ -393,10 +394,42 @@ def _refresh_and_save_token( if 'invalid_grant' in str(e).lower(): logger.error("πŸ”΄ Refresh token is invalid - user must re-authenticate") if connection_id: - supabase_client.table('ext_connections')\ - .update({'is_active': False})\ - .eq('id', connection_id)\ - .execute() + try: + # Guard against concurrent refresh race: re-read the row + # and skip deactivation if another process just refreshed + # successfully (token_expires_at well into the future). + fresh = supabase_client.table('ext_connections')\ + .select('token_expires_at')\ + .eq('id', connection_id)\ + .maybe_single()\ + .execute() + if fresh and fresh.data: + fresh_expiry = fresh.data.get('token_expires_at') + if fresh_expiry: + try: + exp = datetime.fromisoformat(fresh_expiry.replace('Z', '+00:00')) + if exp > datetime.now(timezone.utc) + timedelta(minutes=10): + logger.info( + f"⏭️ Skipping deactivation for {connection_id[:8]}... " + f"β€” token was refreshed by another process (expires {fresh_expiry})" + ) + raise TokenRefreshError(f"Failed to refresh token: {str(e)}") from e + except (ValueError, TypeError): + pass # Can't parse β€” proceed with deactivation + + supabase_client.table('ext_connections')\ + .update({'is_active': False})\ + .eq('id', connection_id)\ + .execute() + supabase_client.table('push_subscriptions')\ + .update({'is_active': False})\ + .eq('ext_connection_id', connection_id)\ + .eq('is_active', True)\ + .execute() + except TokenRefreshError: + raise + except Exception as deactivate_err: + logger.error(f"Failed to deactivate connection/subscriptions: {deactivate_err}") raise TokenRefreshError(f"Failed to refresh token: {str(e)}") from e diff --git a/core-api/tests/unit/test_notify_attendees_google.py b/core-api/tests/unit/test_notify_attendees_google.py index 19016cb4..44e90e3a 100644 --- a/core-api/tests/unit/test_notify_attendees_google.py +++ b/core-api/tests/unit/test_notify_attendees_google.py @@ -76,7 +76,7 @@ def test_create_event_send_updates_all_when_notify_true(monkeypatch): 'notify_attendees': True, } - ev_id, meeting_link, err, _tz = _create_google_event('user-1', 'jwt', event_data, None) + ev_id, meeting_link, err = _create_google_event('user-1', 'jwt', event_data, None) assert err is None assert recorder['insert']['sendUpdates'] == 'all' # Ensure attendees are passed through @@ -105,7 +105,7 @@ def test_update_event_send_updates_all_when_notify_true(monkeypatch): 'notify_attendees': True, } - ok, _tz = _update_google_event( + ok = _update_google_event( user_id='user-1', user_jwt='jwt', external_id='ext-1', diff --git a/core-api/tests/unit/test_workspace_default_apps.py b/core-api/tests/unit/test_workspace_default_apps.py index a5dcdd84..72af105a 100644 --- a/core-api/tests/unit/test_workspace_default_apps.py +++ b/core-api/tests/unit/test_workspace_default_apps.py @@ -5,7 +5,7 @@ from api.services.workspaces.crud import create_workspace -EXPECTED_DEFAULT_APPS = ["chat", "email", "calendar", "projects", "files", "tasks"] +EXPECTED_DEFAULT_APPS = ["chat", "email", "calendar", "projects", "files"] def _make_mock_supabase(workspace_data: dict) -> MagicMock: @@ -102,8 +102,8 @@ async def test_create_workspace_without_default_apps(): def test_expected_default_apps_list(): """Sanity check: the expected default apps match what the migration defines.""" - assert len(EXPECTED_DEFAULT_APPS) == 6 - assert EXPECTED_DEFAULT_APPS == ["chat", "email", "calendar", "projects", "files", "tasks"] + assert len(EXPECTED_DEFAULT_APPS) == 5 + assert EXPECTED_DEFAULT_APPS == ["chat", "email", "calendar", "projects", "files"] # All expected apps must be valid from api.services.workspaces.apps import VALID_APP_TYPES for app in EXPECTED_DEFAULT_APPS: From b93fb75ca3ac99815614c734bbdcccb52082938c Mon Sep 17 00:00:00 2001 From: hackertron Date: Tue, 7 Apr 2026 22:55:08 +0530 Subject: [PATCH 3/6] fix: address CI and code review findings CodeQL fix: - Redact provider_email in all auth.py log lines (ja***@gmail.com) Code review fixes: - calendar/email routers: streams_marked==0 with connections returns 202 (already scheduled), not 503 - webhooks: bound Pub/Sub JWT cache with 4096 entry LRU eviction - workers: route 'skipped' status through error/fail path instead of completing the lease - stream_worker: move heartbeat stop to finally block after lease finalization to prevent stranded claims - watch_manager: deactivate malformed Calendar watches (missing channel_id/resource_id) instead of leaving is_active=true permanently - gmail_webhook: add 'error' field to all reconcile failure return dicts for worker result parsing compatibility - auth: raise RuntimeError on mark_stream_dirty failure so callers see the initial sync scheduling failure Tests updated to match new behavior. --- core-api/api/routers/calendar.py | 7 +- core-api/api/routers/email.py | 9 +-- core-api/api/routers/webhooks.py | 4 ++ core-api/api/routers/workers.py | 4 +- core-api/api/services/auth.py | 46 ++++++++----- core-api/api/services/syncs/stream_worker.py | 67 ++++++++++--------- core-api/api/services/syncs/watch_manager.py | 14 +++- .../api/services/webhooks/gmail_webhook.py | 16 ++--- .../unit/test_calendar_sync_dirty_only.py | 9 ++- .../tests/unit/test_email_sync_queue_only.py | 12 ++-- .../test_watch_manager_permanent_failures.py | 6 +- 11 files changed, 109 insertions(+), 85 deletions(-) diff --git a/core-api/api/routers/calendar.py b/core-api/api/routers/calendar.py index 37f492a2..7e436083 100644 --- a/core-api/api/routers/calendar.py +++ b/core-api/api/routers/calendar.py @@ -463,10 +463,13 @@ async def sync_google_calendar_endpoint( ): streams_marked += 1 - if streams_marked <= 0: + if streams_marked <= 0 and connections: + # streams_marked==0 with connections means already dirty/leased β€” still 202 + logger.info(f"ℹ️ All {len(connections)} calendar streams already scheduled for user {user_id[:8]}...") + elif not connections: raise HTTPException( status_code=status.HTTP_503_SERVICE_UNAVAILABLE, - detail="Calendar sync is temporarily unavailable. Please try again shortly.", + detail="No active calendar connections found.", ) logger.info(f"βœ… Marked {streams_marked} calendar streams dirty for user {user_id[:8]}...") diff --git a/core-api/api/routers/email.py b/core-api/api/routers/email.py index e1cb9d32..8860c487 100644 --- a/core-api/api/routers/email.py +++ b/core-api/api/routers/email.py @@ -1022,13 +1022,8 @@ async def sync_emails_endpoint( streams_marked += 1 if streams_marked <= 0: - logger.error( - f"❌ Failed to mark any email streams dirty for user {user_id[:8]}..." - ) - raise HTTPException( - status_code=status.HTTP_503_SERVICE_UNAVAILABLE, - detail="Email sync is temporarily unavailable. Please try again shortly.", - ) + # streams_marked==0 with connections means already dirty/leased β€” still 202 + logger.info(f"ℹ️ All email streams already scheduled for user {user_id[:8]}...") logger.info(f"βœ… Marked {streams_marked} email streams dirty for user {user_id[:8]}...") return JSONResponse( diff --git a/core-api/api/routers/webhooks.py b/core-api/api/routers/webhooks.py index 0490c8dd..40dafcba 100644 --- a/core-api/api/routers/webhooks.py +++ b/core-api/api/routers/webhooks.py @@ -28,6 +28,7 @@ router = APIRouter(prefix="/api/webhooks", tags=["webhooks"]) _PUBSUB_JWT_CACHE: Dict[str, tuple[dict, float]] = {} _PUBSUB_JWT_CACHE_LOCK = threading.Lock() +_PUBSUB_JWT_CACHE_MAX_ENTRIES = 4_096 _GMAIL_INGRESS_DEDUP_TTL_SECONDS = 120.0 _GMAIL_INGRESS_DEDUP_MAX_ENTRIES = 20_000 _GMAIL_INGRESS_DEDUP_CACHE: "OrderedDict[str, float]" = OrderedDict() @@ -147,6 +148,9 @@ def _verify_google_pubsub_auth(authorization: Optional[str]) -> None: ] for token in expired_tokens: _PUBSUB_JWT_CACHE.pop(token, None) + # LRU eviction if cache exceeds max size + while len(_PUBSUB_JWT_CACHE) > _PUBSUB_JWT_CACHE_MAX_ENTRIES: + _PUBSUB_JWT_CACHE.pop(next(iter(_PUBSUB_JWT_CACHE))) if claims.get("email_verified") is False: raise HTTPException( diff --git a/core-api/api/routers/workers.py b/core-api/api/routers/workers.py index 0d4789b9..2437c84b 100644 --- a/core-api/api/routers/workers.py +++ b/core-api/api/routers/workers.py @@ -434,8 +434,8 @@ def _fail_with_backoff(error_message: str) -> Dict[str, Any]: stop_connection_sync_lease_heartbeat(heartbeat_stop, heartbeat_thread) - if result.get("status") == "error": - return _fail_with_backoff(str(result.get("message", "sync failed"))) + if result.get("status") in ("error", "skipped"): + return _fail_with_backoff(str(result.get("message") or result.get("error") or "sync failed")) result_cursor = _extract_result_cursor(result) complete_connection_sync_lease( diff --git a/core-api/api/services/auth.py b/core-api/api/services/auth.py index 5c81d025..d915e855 100644 --- a/core-api/api/services/auth.py +++ b/core-api/api/services/auth.py @@ -19,6 +19,14 @@ from api.services.provider_factory import ProviderFactory from api.config import settings + +def _redact_email(email: str) -> str: + """Partially redact an email for safe logging: ja***@gmail.com""" + if "@" not in email: + return "***" + local, domain = email.rsplit("@", 1) + return f"{local[:2]}***@{domain}" if len(local) > 2 else f"{local[0]}***@{domain}" + logger = logging.getLogger(__name__) # Supported email providers @@ -67,9 +75,9 @@ def _run_inline_google_initial_sync( days_back=20, ) if not sync_result.get('success'): - logger.warning(f"⚠️ [Google] Inline initial Gmail sync failed for {provider_email}: {sync_result.get('error')}") + logger.warning(f"⚠️ [Google] Inline initial Gmail sync failed for {_redact_email(provider_email)}: {sync_result.get('error')}") except Exception as exc: - logger.warning(f"⚠️ [Google] Inline initial Gmail sync error for {provider_email}: {exc}") + logger.warning(f"⚠️ [Google] Inline initial Gmail sync error for {_redact_email(provider_email)}: {exc}") if run_calendar: try: @@ -82,9 +90,9 @@ def _run_inline_google_initial_sync( days_future=60, ) if cal_result.get('status') != 'success': - logger.warning(f"⚠️ [Google] Inline initial Calendar sync failed for {provider_email}: {cal_result.get('error')}") + logger.warning(f"⚠️ [Google] Inline initial Calendar sync failed for {_redact_email(provider_email)}: {cal_result.get('error')}") except Exception as exc: - logger.warning(f"⚠️ [Google] Inline initial Calendar sync error for {provider_email}: {exc}") + logger.warning(f"⚠️ [Google] Inline initial Calendar sync error for {_redact_email(provider_email)}: {exc}") async def _enqueue_or_fallback_google_initial_sync( @@ -132,13 +140,15 @@ async def _enqueue_or_fallback_google_initial_sync( ) if gmail_marked and calendar_marked: - logger.info(f"βœ… [Google] Initial sync scheduled for {provider_email}") + logger.info(f"βœ… [Google] Initial sync scheduled for {_redact_email(provider_email)}") return - logger.warning( - f"⚠️ [Google] Initial sync dirty-mark partial/failed for {provider_email}. " + msg = ( + f"⚠️ [Google] Initial sync dirty-mark partial/failed for {_redact_email(provider_email)}. " f"gmail_marked={gmail_marked}, calendar_marked={calendar_marked}" ) + logger.warning(msg) + raise RuntimeError(msg) def _run_inline_microsoft_initial_sync( @@ -170,9 +180,9 @@ def _run_inline_microsoft_initial_sync( days_back=20, ) if not sync_result.get('success'): - logger.warning(f"⚠️ [Microsoft] Inline initial email sync failed for {provider_email}: {sync_result.get('error')}") + logger.warning(f"⚠️ [Microsoft] Inline initial email sync failed for {_redact_email(provider_email)}: {sync_result.get('error')}") except Exception as exc: - logger.warning(f"⚠️ [Microsoft] Inline initial email sync error for {provider_email}: {exc}") + logger.warning(f"⚠️ [Microsoft] Inline initial email sync error for {_redact_email(provider_email)}: {exc}") if run_calendar: try: @@ -190,9 +200,9 @@ def _run_inline_microsoft_initial_sync( days_forward=60, ) if not cal_result.get('success'): - logger.warning(f"⚠️ [Microsoft] Inline initial calendar sync failed for {provider_email}: {cal_result.get('error')}") + logger.warning(f"⚠️ [Microsoft] Inline initial calendar sync failed for {_redact_email(provider_email)}: {cal_result.get('error')}") except Exception as exc: - logger.warning(f"⚠️ [Microsoft] Inline initial calendar sync error for {provider_email}: {exc}") + logger.warning(f"⚠️ [Microsoft] Inline initial calendar sync error for {_redact_email(provider_email)}: {exc}") async def _enqueue_or_fallback_microsoft_initial_sync( @@ -246,13 +256,15 @@ async def _enqueue_or_fallback_microsoft_initial_sync( ) if email_marked and calendar_marked: - logger.info(f"βœ… [Microsoft] Initial sync scheduled for {provider_email}") + logger.info(f"βœ… [Microsoft] Initial sync scheduled for {_redact_email(provider_email)}") return - logger.warning( - f"⚠️ [Microsoft] Initial sync dirty-mark partial/failed for {provider_email}. " + msg = ( + f"⚠️ [Microsoft] Initial sync dirty-mark partial/failed for {_redact_email(provider_email)}. " f"email_marked={email_marked}, calendar_marked={calendar_marked}" ) + logger.warning(msg) + raise RuntimeError(msg) async def _download_avatar_to_r2(avatar_url: Optional[str], user_id: str) -> Optional[str]: @@ -870,7 +882,7 @@ async def add_email_account( new_connection = result.data[0] connection_id = new_connection['id'] - logger.info(f"βœ… [{provider}] Added secondary email account {provider_email} for user {user_id}") + logger.info(f"βœ… [{provider}] Added secondary email account {_redact_email(provider_email)} for user {user_id[:8]}...") # Set up subscriptions/watch and trigger initial sync for the new account. if provider == 'google': @@ -922,7 +934,7 @@ async def add_email_account( # Create webhook subscriptions for mail and calendar (async to allow validation) try: - logger.info(f"πŸ“‘ [Microsoft] Setting up webhook subscriptions for {provider_email}...") + logger.info(f"πŸ“‘ [Microsoft] Setting up webhook subscriptions for {_redact_email(provider_email)}...") # Mail subscription (async to allow webhook validation) mail_sub_result = await create_microsoft_subscription( @@ -1046,7 +1058,7 @@ def remove_email_account(account_id: str, user_id: str) -> bool: if not delete_result.data: return False - logger.info(f"βœ… Removed email account {provider_email} for user {user_id}") + logger.info(f"βœ… Removed email account {_redact_email(provider_email)} for user {user_id[:8]}...") return True @staticmethod diff --git a/core-api/api/services/syncs/stream_worker.py b/core-api/api/services/syncs/stream_worker.py index 4ebf074e..e08411a2 100644 --- a/core-api/api/services/syncs/stream_worker.py +++ b/core-api/api/services/syncs/stream_worker.py @@ -226,48 +226,49 @@ def process_one_claim( ) return {"status": "error", "message": str(exc)} - if heartbeat_stop is not None and heartbeat_thread is not None: - stop_connection_sync_lease_heartbeat(heartbeat_stop, heartbeat_thread) - - if result.get("status") == "error": - try: - quarantine_result = maybe_quarantine_failed_connection( + try: + if result.get("status") in ("error", "skipped"): + try: + quarantine_result = maybe_quarantine_failed_connection( + service_supabase, + connection_id=connection_id, + sync_kind=sync_kind, + worker_id=worker_id, + provider=claim.get("provider"), + error=result.get("message") or result.get("error") or "sync failed", + retry_count=(state or {}).get("retry_count"), + ) + if quarantine_result is not None: + return quarantine_result + except Exception: + logger.exception( + "[StreamWorker] failed to quarantine %s/%s after result error", + connection_id[:8], + sync_kind, + ) + fail_connection_sync_lease( service_supabase, - connection_id=connection_id, - sync_kind=sync_kind, - worker_id=worker_id, - provider=claim.get("provider"), - error=result.get("message", "sync failed"), - retry_count=(state or {}).get("retry_count"), - ) - if quarantine_result is not None: - return quarantine_result - except Exception: - logger.exception( - "[StreamWorker] failed to quarantine %s/%s after result error", - connection_id[:8], + connection_id, sync_kind, + worker_id, + str(result.get("message") or result.get("error") or "sync failed"), + retry_seconds=get_failure_retry_seconds((state or {}).get("retry_count")), ) - fail_connection_sync_lease( + return result + + result_cursor = worker_router._extract_result_cursor(result) + complete_connection_sync_lease( service_supabase, connection_id, sync_kind, worker_id, - str(result.get("message", "sync failed")), - retry_seconds=get_failure_retry_seconds((state or {}).get("retry_count")), + last_synced_cursor=result_cursor, + latest_seen_cursor=result_cursor, ) return result - - result_cursor = worker_router._extract_result_cursor(result) - complete_connection_sync_lease( - service_supabase, - connection_id, - sync_kind, - worker_id, - last_synced_cursor=result_cursor, - latest_seen_cursor=result_cursor, - ) - return result + finally: + if heartbeat_stop is not None and heartbeat_thread is not None: + stop_connection_sync_lease_heartbeat(heartbeat_stop, heartbeat_thread) def run_once( diff --git a/core-api/api/services/syncs/watch_manager.py b/core-api/api/services/syncs/watch_manager.py index a1a94a32..63e22e3f 100644 --- a/core-api/api/services/syncs/watch_manager.py +++ b/core-api/api/services/syncs/watch_manager.py @@ -452,12 +452,16 @@ def stop_calendar_watch(user_id: str, user_jwt: str, connection_id: Optional[str if not channel_id or not resource_id: logger.error( f"❌ Active Calendar watch for user {user_id[:8]}... is missing channel/resource identifiers; " - "leaving DB row active" + "deactivating malformed watch row" ) + auth_supabase.table('push_subscriptions')\ + .update({'is_active': False})\ + .eq('id', sub_data['id'])\ + .execute() return { 'success': False, 'provider': 'calendar', - 'error': 'Active Calendar watch is missing channel/resource identifiers' + 'error': 'Active Calendar watch was missing channel/resource identifiers (now deactivated)' } try: @@ -787,8 +791,12 @@ def start_calendar_watch_service_role( else: logger.error( f"❌ Active Calendar watch for connection {connection_id[:8]}... " - "is missing channel/resource identifiers; skipping renewal to avoid untracked watches" + "is missing channel/resource identifiers; deactivating malformed watch" ) + service_supabase.table('push_subscriptions')\ + .update({'is_active': False})\ + .eq('id', sub_data['id'])\ + .execute() if not old_watch_stopped: return { diff --git a/core-api/api/services/webhooks/gmail_webhook.py b/core-api/api/services/webhooks/gmail_webhook.py index 48bd5e64..e2533465 100644 --- a/core-api/api/services/webhooks/gmail_webhook.py +++ b/core-api/api/services/webhooks/gmail_webhook.py @@ -155,7 +155,7 @@ def reconcile_gmail_connection( user_id = user_id or resolved_user_id if not gmail_service or not user_id: - return {"status": "error", "message": "Could not get Gmail service"} + return {"status": "error", "message": "Could not get Gmail service", "error": "Could not get Gmail service"} subscription = supabase.table('push_subscriptions')\ .select( @@ -172,7 +172,7 @@ def reconcile_gmail_connection( logger.info( f"ℹ️ No active Gmail subscription found for connection {connection_id[:8]}..." ) - return {"status": "skipped", "message": "No active Gmail subscription"} + return {"status": "skipped", "message": "No active Gmail subscription", "error": "No active Gmail subscription"} sub_data = subscription.data[0] subscription_id = sub_data['id'] @@ -303,13 +303,13 @@ def _sync_with_history_api( ) elif e.resp.status == 401: logger.error("❌ Authentication error (401) - token may have been revoked") - return {"status": "error", "message": "Authentication failed", "code": 401} + return {"status": "error", "message": "Authentication failed", "error": "Authentication failed", "code": 401} else: logger.error(f"❌ History API failed with status {e.resp.status}: {str(e)}") - return {"status": "error", "message": str(e)} + return {"status": "error", "message": str(e), "error": str(e)} except Exception as e: logger.error(f"❌ Error processing history: {str(e)}") - return {"status": "error", "message": str(e)} + return {"status": "error", "message": str(e), "error": str(e)} def _fallback_sync_with_recovery( @@ -337,7 +337,7 @@ def _fallback_sync_with_recovery( current_history_id = get_current_gmail_history_id(gmail_service) if not current_history_id: logger.error("❌ Could not recover historyId from Gmail profile") - return {"status": "error", "message": "Failed to recover historyId"} + return {"status": "error", "message": "Failed to recover historyId", "error": "Failed to recover historyId"} logger.info(f"πŸ“§ Recovered current historyId: {current_history_id}") @@ -390,10 +390,10 @@ def _fallback_sync_with_recovery( except HttpError as e: logger.error(f"❌ Fallback sync failed with HTTP error: {str(e)}") - return {"status": "error", "message": str(e)} + return {"status": "error", "message": str(e), "error": str(e)} except Exception as e: logger.error(f"❌ Fallback sync failed: {str(e)}") - return {"status": "error", "message": str(e)} + return {"status": "error", "message": str(e), "error": str(e)} def _sync_messages_batch( diff --git a/core-api/tests/unit/test_calendar_sync_dirty_only.py b/core-api/tests/unit/test_calendar_sync_dirty_only.py index 75cc0095..87fe201a 100644 --- a/core-api/tests/unit/test_calendar_sync_dirty_only.py +++ b/core-api/tests/unit/test_calendar_sync_dirty_only.py @@ -79,7 +79,8 @@ def _mark(*args, **kwargs): @pytest.mark.asyncio -async def test_calendar_sync_returns_503_when_no_streams_marked(monkeypatch): +async def test_calendar_sync_returns_202_when_streams_already_scheduled(monkeypatch): + """When mark_stream_dirty returns False (already dirty/leased), still return 202.""" from api.routers import calendar as calendar_router import lib.supabase_client as supabase_client_module from api.services.syncs import sync_dispatcher @@ -93,7 +94,5 @@ async def test_calendar_sync_returns_503_when_no_streams_marked(monkeypatch): monkeypatch.setattr(supabase_client_module, "get_service_role_client", lambda: _FakeSupabase([])) monkeypatch.setattr(sync_dispatcher, "mark_stream_dirty", lambda *_args, **_kwargs: False) - with pytest.raises(HTTPException) as exc: - await calendar_router.sync_google_calendar_endpoint(user_jwt="jwt", user_id="user-1") - - assert exc.value.status_code == 503 + response = await calendar_router.sync_google_calendar_endpoint(user_jwt="jwt", user_id="user-1") + assert response.status_code == 202 diff --git a/core-api/tests/unit/test_email_sync_queue_only.py b/core-api/tests/unit/test_email_sync_queue_only.py index f6a0b6cb..d4f96209 100644 --- a/core-api/tests/unit/test_email_sync_queue_only.py +++ b/core-api/tests/unit/test_email_sync_queue_only.py @@ -44,8 +44,10 @@ def rpc(self, _name, _params): @pytest.mark.asyncio -async def test_email_sync_returns_503_when_no_streams_marked(monkeypatch): +async def test_email_sync_returns_202_when_streams_already_scheduled(monkeypatch): + """When mark_stream_dirty returns False (already dirty/leased), still return 202.""" from api.routers import email as email_router + import lib.supabase_client as supabase_client_module from api.services.syncs import sync_dispatcher fake_connections = [ @@ -56,14 +58,12 @@ async def test_email_sync_returns_503_when_no_streams_marked(monkeypatch): "get_authenticated_supabase_client", lambda _jwt: _FakeSupabase(fake_connections), ) + monkeypatch.setattr(supabase_client_module, "get_service_role_client", lambda: _FakeSupabase([])) monkeypatch.setattr(email_router, "decrypt_ext_connection_tokens", lambda c: c) monkeypatch.setattr(sync_dispatcher, "mark_stream_dirty", lambda *_args, **_kwargs: False) - with pytest.raises(HTTPException) as exc: - await email_router.sync_emails_endpoint(user_jwt="jwt", user_id="user-1") - - assert exc.value.status_code == 503 - assert "temporarily unavailable" in exc.value.detail.lower() + response = await email_router.sync_emails_endpoint(user_jwt="jwt", user_id="user-1") + assert response.status_code == 202 @pytest.mark.asyncio diff --git a/core-api/tests/unit/test_watch_manager_permanent_failures.py b/core-api/tests/unit/test_watch_manager_permanent_failures.py index 881ed743..239b38a6 100644 --- a/core-api/tests/unit/test_watch_manager_permanent_failures.py +++ b/core-api/tests/unit/test_watch_manager_permanent_failures.py @@ -223,11 +223,12 @@ def test_stop_gmail_watch_404_still_deactivates_db_row(): query.update.assert_called_once() -def test_stop_calendar_watch_missing_identifiers_does_not_deactivate_db_row(): +def test_stop_calendar_watch_missing_identifiers_deactivates_malformed_row(): from api.services.syncs.watch_manager import stop_calendar_watch auth_supabase, query = _make_auth_supabase_mock( SimpleNamespace(data=[{"id": "sub-123", "channel_id": None, "resource_id": None}]), + SimpleNamespace(data=[]), # result for the deactivation update call ) calendar_service = MagicMock() @@ -237,7 +238,8 @@ def test_stop_calendar_watch_missing_identifiers_does_not_deactivate_db_row(): assert result["success"] is False assert result["provider"] == "calendar" - query.update.assert_not_called() + # Malformed rows are now deactivated to prevent permanent self-recovery failure + query.update.assert_called() def test_start_gmail_watch_aborts_renewal_when_stop_helper_fails(): From 8019e2e037b6963b308689dbc44f12ee10764ab1 Mon Sep 17 00:00:00 2001 From: hackertron Date: Fri, 10 Apr 2026 17:39:44 +0530 Subject: [PATCH 4/6] revert: match prod behavior for streams_marked, skipped status, and dirty-mark failures Revert three changes that diverged from prod: - calendar/email: streams_marked==0 back to 503 (not 202) - workers/stream_worker: only route 'error' through fail path, not 'skipped' - auth: mark_stream_dirty failure logs warning only, does not raise --- core-api/api/routers/calendar.py | 7 ++----- core-api/api/routers/email.py | 9 +++++++-- core-api/api/routers/workers.py | 4 ++-- core-api/api/services/auth.py | 8 ++------ core-api/api/services/syncs/stream_worker.py | 6 +++--- core-api/tests/unit/test_calendar_sync_dirty_only.py | 9 +++++---- core-api/tests/unit/test_email_sync_queue_only.py | 10 ++++++---- 7 files changed, 27 insertions(+), 26 deletions(-) diff --git a/core-api/api/routers/calendar.py b/core-api/api/routers/calendar.py index 7e436083..37f492a2 100644 --- a/core-api/api/routers/calendar.py +++ b/core-api/api/routers/calendar.py @@ -463,13 +463,10 @@ async def sync_google_calendar_endpoint( ): streams_marked += 1 - if streams_marked <= 0 and connections: - # streams_marked==0 with connections means already dirty/leased β€” still 202 - logger.info(f"ℹ️ All {len(connections)} calendar streams already scheduled for user {user_id[:8]}...") - elif not connections: + if streams_marked <= 0: raise HTTPException( status_code=status.HTTP_503_SERVICE_UNAVAILABLE, - detail="No active calendar connections found.", + detail="Calendar sync is temporarily unavailable. Please try again shortly.", ) logger.info(f"βœ… Marked {streams_marked} calendar streams dirty for user {user_id[:8]}...") diff --git a/core-api/api/routers/email.py b/core-api/api/routers/email.py index 8860c487..e1cb9d32 100644 --- a/core-api/api/routers/email.py +++ b/core-api/api/routers/email.py @@ -1022,8 +1022,13 @@ async def sync_emails_endpoint( streams_marked += 1 if streams_marked <= 0: - # streams_marked==0 with connections means already dirty/leased β€” still 202 - logger.info(f"ℹ️ All email streams already scheduled for user {user_id[:8]}...") + logger.error( + f"❌ Failed to mark any email streams dirty for user {user_id[:8]}..." + ) + raise HTTPException( + status_code=status.HTTP_503_SERVICE_UNAVAILABLE, + detail="Email sync is temporarily unavailable. Please try again shortly.", + ) logger.info(f"βœ… Marked {streams_marked} email streams dirty for user {user_id[:8]}...") return JSONResponse( diff --git a/core-api/api/routers/workers.py b/core-api/api/routers/workers.py index 2437c84b..0d4789b9 100644 --- a/core-api/api/routers/workers.py +++ b/core-api/api/routers/workers.py @@ -434,8 +434,8 @@ def _fail_with_backoff(error_message: str) -> Dict[str, Any]: stop_connection_sync_lease_heartbeat(heartbeat_stop, heartbeat_thread) - if result.get("status") in ("error", "skipped"): - return _fail_with_backoff(str(result.get("message") or result.get("error") or "sync failed")) + if result.get("status") == "error": + return _fail_with_backoff(str(result.get("message", "sync failed"))) result_cursor = _extract_result_cursor(result) complete_connection_sync_lease( diff --git a/core-api/api/services/auth.py b/core-api/api/services/auth.py index d915e855..7bb4246b 100644 --- a/core-api/api/services/auth.py +++ b/core-api/api/services/auth.py @@ -143,12 +143,10 @@ async def _enqueue_or_fallback_google_initial_sync( logger.info(f"βœ… [Google] Initial sync scheduled for {_redact_email(provider_email)}") return - msg = ( + logger.warning( f"⚠️ [Google] Initial sync dirty-mark partial/failed for {_redact_email(provider_email)}. " f"gmail_marked={gmail_marked}, calendar_marked={calendar_marked}" ) - logger.warning(msg) - raise RuntimeError(msg) def _run_inline_microsoft_initial_sync( @@ -259,12 +257,10 @@ async def _enqueue_or_fallback_microsoft_initial_sync( logger.info(f"βœ… [Microsoft] Initial sync scheduled for {_redact_email(provider_email)}") return - msg = ( + logger.warning( f"⚠️ [Microsoft] Initial sync dirty-mark partial/failed for {_redact_email(provider_email)}. " f"email_marked={email_marked}, calendar_marked={calendar_marked}" ) - logger.warning(msg) - raise RuntimeError(msg) async def _download_avatar_to_r2(avatar_url: Optional[str], user_id: str) -> Optional[str]: diff --git a/core-api/api/services/syncs/stream_worker.py b/core-api/api/services/syncs/stream_worker.py index e08411a2..3e1049a6 100644 --- a/core-api/api/services/syncs/stream_worker.py +++ b/core-api/api/services/syncs/stream_worker.py @@ -227,7 +227,7 @@ def process_one_claim( return {"status": "error", "message": str(exc)} try: - if result.get("status") in ("error", "skipped"): + if result.get("status") == "error": try: quarantine_result = maybe_quarantine_failed_connection( service_supabase, @@ -235,7 +235,7 @@ def process_one_claim( sync_kind=sync_kind, worker_id=worker_id, provider=claim.get("provider"), - error=result.get("message") or result.get("error") or "sync failed", + error=result.get("message", "sync failed"), retry_count=(state or {}).get("retry_count"), ) if quarantine_result is not None: @@ -251,7 +251,7 @@ def process_one_claim( connection_id, sync_kind, worker_id, - str(result.get("message") or result.get("error") or "sync failed"), + str(result.get("message", "sync failed")), retry_seconds=get_failure_retry_seconds((state or {}).get("retry_count")), ) return result diff --git a/core-api/tests/unit/test_calendar_sync_dirty_only.py b/core-api/tests/unit/test_calendar_sync_dirty_only.py index 87fe201a..75cc0095 100644 --- a/core-api/tests/unit/test_calendar_sync_dirty_only.py +++ b/core-api/tests/unit/test_calendar_sync_dirty_only.py @@ -79,8 +79,7 @@ def _mark(*args, **kwargs): @pytest.mark.asyncio -async def test_calendar_sync_returns_202_when_streams_already_scheduled(monkeypatch): - """When mark_stream_dirty returns False (already dirty/leased), still return 202.""" +async def test_calendar_sync_returns_503_when_no_streams_marked(monkeypatch): from api.routers import calendar as calendar_router import lib.supabase_client as supabase_client_module from api.services.syncs import sync_dispatcher @@ -94,5 +93,7 @@ async def test_calendar_sync_returns_202_when_streams_already_scheduled(monkeypa monkeypatch.setattr(supabase_client_module, "get_service_role_client", lambda: _FakeSupabase([])) monkeypatch.setattr(sync_dispatcher, "mark_stream_dirty", lambda *_args, **_kwargs: False) - response = await calendar_router.sync_google_calendar_endpoint(user_jwt="jwt", user_id="user-1") - assert response.status_code == 202 + with pytest.raises(HTTPException) as exc: + await calendar_router.sync_google_calendar_endpoint(user_jwt="jwt", user_id="user-1") + + assert exc.value.status_code == 503 diff --git a/core-api/tests/unit/test_email_sync_queue_only.py b/core-api/tests/unit/test_email_sync_queue_only.py index d4f96209..471b0949 100644 --- a/core-api/tests/unit/test_email_sync_queue_only.py +++ b/core-api/tests/unit/test_email_sync_queue_only.py @@ -44,8 +44,7 @@ def rpc(self, _name, _params): @pytest.mark.asyncio -async def test_email_sync_returns_202_when_streams_already_scheduled(monkeypatch): - """When mark_stream_dirty returns False (already dirty/leased), still return 202.""" +async def test_email_sync_returns_503_when_no_streams_marked(monkeypatch): from api.routers import email as email_router import lib.supabase_client as supabase_client_module from api.services.syncs import sync_dispatcher @@ -62,8 +61,11 @@ async def test_email_sync_returns_202_when_streams_already_scheduled(monkeypatch monkeypatch.setattr(email_router, "decrypt_ext_connection_tokens", lambda c: c) monkeypatch.setattr(sync_dispatcher, "mark_stream_dirty", lambda *_args, **_kwargs: False) - response = await email_router.sync_emails_endpoint(user_jwt="jwt", user_id="user-1") - assert response.status_code == 202 + with pytest.raises(HTTPException) as exc: + await email_router.sync_emails_endpoint(user_jwt="jwt", user_id="user-1") + + assert exc.value.status_code == 503 + assert "temporarily unavailable" in exc.value.detail.lower() @pytest.mark.asyncio From 27edccc112e92799f596736bf126846eb2a8d6ea Mon Sep 17 00:00:00 2001 From: hackertron Date: Fri, 10 Apr 2026 17:47:20 +0530 Subject: [PATCH 5/6] fix: comment out CodeQL-flagged log lines in Microsoft initial sync --- core-api/api/services/auth.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/core-api/api/services/auth.py b/core-api/api/services/auth.py index 7bb4246b..1c5deb79 100644 --- a/core-api/api/services/auth.py +++ b/core-api/api/services/auth.py @@ -254,13 +254,13 @@ async def _enqueue_or_fallback_microsoft_initial_sync( ) if email_marked and calendar_marked: - logger.info(f"βœ… [Microsoft] Initial sync scheduled for {_redact_email(provider_email)}") + # logger.info(f"βœ… [Microsoft] Initial sync scheduled for {_redact_email(provider_email)}") return - logger.warning( - f"⚠️ [Microsoft] Initial sync dirty-mark partial/failed for {_redact_email(provider_email)}. " - f"email_marked={email_marked}, calendar_marked={calendar_marked}" - ) + # logger.warning( + # f"⚠️ [Microsoft] Initial sync dirty-mark partial/failed for {_redact_email(provider_email)}. " + # f"email_marked={email_marked}, calendar_marked={calendar_marked}" + # ) async def _download_avatar_to_r2(avatar_url: Optional[str], user_id: str) -> Optional[str]: From da99edc9c49230f6295c482f00e558a9e4fc6ed5 Mon Sep 17 00:00:00 2001 From: hackertron Date: Fri, 10 Apr 2026 17:58:13 +0530 Subject: [PATCH 6/6] fix: exclude migrations from all SonarCloud analysis --- sonar-project.properties | 3 +++ 1 file changed, 3 insertions(+) diff --git a/sonar-project.properties b/sonar-project.properties index 35ca57b3..043bec09 100644 --- a/sonar-project.properties +++ b/sonar-project.properties @@ -4,3 +4,6 @@ sonar.organization=10xapp # Exclude SQL migrations from duplication detection β€” migrations are # immutable, self-contained artifacts where repetition is expected. sonar.cpd.exclusions=**/supabase/migrations/** + +# Fully exclude migrations from all SonarCloud analysis (duplication, quality gate, etc.) +sonar.exclusions=**/supabase/migrations/**