Enforce complete Paperboy type checks

.github/workflows/ci.yml

@@ -39,6 +39,8 @@ jobs:
           kill %1
       - name: Lint with ruff
         run: ruff check .
+      - name: Type-check package
+        run: mypy paperboy
 
   dependency-audit:
     runs-on: ubuntu-latest

paperboy/api/main.py

@@ -25,6 +25,7 @@
 from email import policy
 from email.parser import BytesParser
 from pathlib import Path
+from typing import Any
 
 from fastapi import FastAPI, HTTPException, Request
 from fastapi.responses import FileResponse, JSONResponse, PlainTextResponse, RedirectResponse, Response
@@ -433,8 +434,10 @@ async def capture_resend_event(request: Request) -> JSONResponse:
     if not _claim_email_provider_event(event_id, event_type):
         return JSONResponse({"ok": True, "duplicate": True})
     try:
-        data = event.get("data") if isinstance(event.get("data"), dict) else {}
-        recipients = data.get("to") if isinstance(data.get("to"), list) else []
+        raw_data = event.get("data")
+        data: dict[str, Any] = raw_data if isinstance(raw_data, dict) else {}
+        raw_recipients = data.get("to")
+        recipients: list[Any] = raw_recipients if isinstance(raw_recipients, list) else []
         email = str(recipients[0]).strip().lower() if recipients else ""
         subscription = get_subscription_by_email(email) if email else None
         subscription_id = int(subscription["id"]) if subscription is not None else None

paperboy/billing.py

@@ -49,6 +49,9 @@ def create_checkout(subscription: dict[str, Any]) -> str:
     billing_status = str(subscription.get("billing_status") or "unpaid")
     if billing_status in {"trialing", "active", "past_due"}:
         raise BillingStateError("manage the existing billing account instead")
+    price_id = settings.stripe_price_id
+    if not price_id:
+        raise BillingUnavailableError("checkout is temporarily unavailable")
 
     metadata = {"paperboy_subscription_id": str(subscription["id"])}
     customer_args: dict[str, Any]
@@ -59,7 +62,7 @@ def create_checkout(subscription: dict[str, Any]) -> str:
     session = stripe.checkout.Session.create(
         **customer_args,
         mode="subscription",
-        line_items=[{"price": settings.stripe_price_id, "quantity": 1}],
+        line_items=[{"price": price_id, "quantity": 1}],
         payment_method_collection="always",
         client_reference_id=str(subscription["id"]),
         metadata=metadata,

paperboy/digest/prompt_digest.py

@@ -44,7 +44,7 @@
           "proj": "🛠️ projects", "research": "🔬 research"}
 
 
-def _stream_from_actor(actor):
+def _stream_from_actor(actor: str | None) -> str:
     if not actor or ":" not in actor:
         return "research"
     prefix = actor.split(":", 1)[0]

paperboy/digest/research_digest.py

@@ -97,7 +97,7 @@ def batch_into_messages(lines: list[str], header: str) -> list[str]:
     return messages
 
 
-def main():
+def main() -> None:
     ap = argparse.ArgumentParser()
     ap.add_argument("--top", type=int, default=0)
     ap.add_argument("--min-relevance", type=int, default=5)

paperboy/discord_post.py

@@ -45,9 +45,10 @@ def post(content: str, *, username: str = "paperboy", dry_run: bool = False) ->
     )
     try:
         with urllib.request.urlopen(req, timeout=15) as r:
-            ok = 200 <= r.status < 300
+            status = int(r.status)
+            ok = 200 <= status < 300
             if not ok:
-                print(f"[discord_post] HTTP {r.status}", file=sys.stderr)
+                print(f"[discord_post] HTTP {status}", file=sys.stderr)
             return ok
     except urllib.error.HTTPError as e:
         print(f"[discord_post] HTTPError {e.code}: {e.read()[:200]!r}", file=sys.stderr)

paperboy/health.py

@@ -12,6 +12,7 @@
 import time
 import urllib.error
 import urllib.request
+from collections.abc import Callable
 from datetime import datetime, timezone
 from pathlib import Path
 
@@ -122,7 +123,7 @@ def check_backup_recency(max_age_hours: int = 48) -> tuple[bool, str]:
 
 def run_all() -> dict[str, dict]:
     """Run every probe and return a structured report."""
-    probes = {
+    probes: dict[str, Callable[[], tuple[bool, str]]] = {
         "database": check_database,
         "ollama": check_ollama,
         "disk": check_disk,

paperboy/ingest/research_papers.py

@@ -27,6 +27,7 @@
 import urllib.request
 from datetime import datetime, timezone
 from pathlib import Path
+from typing import Any
 from xml.etree import ElementTree
 
 from paperboy.db import connect
@@ -171,7 +172,7 @@ def fetch_arxiv(cfg: dict, ua: str, timeout: int, verbose: bool) -> list[dict]:
     except ElementTree.ParseError as e:
         print(f"[arxiv] XML parse error: {e}", file=sys.stderr)
         return []
-    papers = []
+    papers: list[dict] = []
     for entry in root.findall(f"{ATOM_NS}entry"):
         id_text = (entry.findtext(f"{ATOM_NS}id") or "").strip()
         arxiv_id = canonicalize_arxiv_id(id_text)
@@ -222,7 +223,7 @@ def fetch_hf_papers(cfg: dict, ua: str, timeout: int, verbose: bool) -> list[dic
     except json.JSONDecodeError as e:
         print(f"[hf] JSON parse error: {e}", file=sys.stderr)
         return []
-    papers = []
+    papers: list[dict] = []
     max_n = int(cfg.get("max_per_run", 50))
     for item in (data if isinstance(data, list) else [])[:max_n]:
         paper = item.get("paper") if isinstance(item, dict) else None
@@ -261,7 +262,7 @@ def fetch_semantic_scholar(cfg: dict, ua: str, timeout: int, verbose: bool) -> l
     max_total = int(cfg.get("max_per_run", 120))
     queries = cfg.get("topic_queries", [])
     sleep_s = float(cfg.get("inter_request_sleep_sec", 1.5))
-    papers = []
+    papers: list[dict] = []
     for q in queries:
         if len(papers) >= max_total:
             break
@@ -342,7 +343,7 @@ def dedup_and_merge(raw_papers: list[dict]) -> dict[str, dict]:
 
 # ---------------------------------------------------------------------------
 
-def main():
+def main() -> None:
     ap = argparse.ArgumentParser()
     ap.add_argument("--source", default="all",
                     choices=["all", "arxiv", "hf", "s2"])
@@ -392,7 +393,7 @@ def main():
     existing = load_existing_actors()
     existing_payloads = load_existing_payloads(existing & set(merged.keys()))
 
-    stats = {
+    stats: dict[str, Any] = {
         "raw_total": len(raw_papers), "unique_total": len(merged),
         "new_ingested": 0, "updated_feed_list": 0, "skipped_existing": 0,
         "errors": 0, "would_ingest": 0, "per_feed_counts": per_feed_counts,

paperboy/logging_config.py

@@ -75,7 +75,8 @@ def configure_logging(level: str | None = None, force_json: bool | None = None)
         force_json: True to force JSON output.  Defaults to truthy
                     PAPERBOY_LOG_FORMAT=json env var.
     """
-    lvl = (level or os.environ.get("PAPERBOY_LOG_LEVEL", "INFO")).upper()
+    configured_level = level or os.environ.get("PAPERBOY_LOG_LEVEL") or "INFO"
+    lvl = configured_level.upper()
     json_mode = force_json if force_json is not None else os.environ.get("PAPERBOY_LOG_FORMAT", "").lower() == "json"
 
     root = logging.getLogger("paperboy")

paperboy/scanners/news_opinion.py

@@ -44,7 +44,7 @@ def _sources_path() -> Path:
 
 def _load_sources() -> dict[str, list[str]]:
     try:
-        import yaml
+        import yaml  # type: ignore[import-untyped]
     except ImportError:
         print("[news_opinion] PyYAML not installed; run: pip install pyyaml", file=sys.stderr)
         return {}
@@ -53,7 +53,14 @@ def _load_sources() -> dict[str, list[str]]:
         print(f"[news_opinion] no sources file at {path}", file=sys.stderr)
         return {}
     with path.open(encoding="utf-8") as f:
-        return yaml.safe_load(f) or {}
+        loaded = yaml.safe_load(f) or {}
+    if not isinstance(loaded, dict):
+        return {}
+    return {
+        str(vertical): [str(url) for url in urls if isinstance(url, str)]
+        for vertical, urls in loaded.items()
+        if isinstance(urls, list)
+    }
 
 
 def _fetch_feed(url: str) -> list[dict]:
@@ -118,10 +125,13 @@ def _ollama_json(prompt: str) -> dict:
     )
     with urllib.request.urlopen(req, timeout=60) as resp:
         out = json.loads(resp.read())
+    if not isinstance(out, dict):
+        return {}
     try:
-        return json.loads(out.get("response", "{}"))
+        result = json.loads(out.get("response", "{}"))
     except json.JSONDecodeError:
         return {}
+    return result if isinstance(result, dict) else {}
 
 
 def llm_draft_prompt(vertical: str, item: dict) -> dict:

paperboy/score/research_papers.py

@@ -160,9 +160,14 @@ def call_ollama(system_prompt: str, user_prompt: str) -> dict:
     )
     with urllib.request.urlopen(req, timeout=TIMEOUT) as r:
         resp = json.load(r)
+    if not isinstance(resp, dict):
+        raise json.JSONDecodeError("Ollama response is not an object", str(resp), 0)
     raw = resp.get("response", "").strip()
     raw = re.sub(r"^```(?:json)?\s*|\s*```$", "", raw, flags=re.MULTILINE).strip()
-    return json.loads(raw)
+    verdict = json.loads(raw)
+    if not isinstance(verdict, dict):
+        raise json.JSONDecodeError("Ollama verdict is not an object", raw, 0)
+    return verdict
 
 
 def normalize_verdict(v: dict, parent_actor: str, parent_payload: dict) -> dict:
@@ -194,7 +199,7 @@ def normalize_verdict(v: dict, parent_actor: str, parent_payload: dict) -> dict:
     }
 
 
-def main():
+def main() -> None:
     ap = argparse.ArgumentParser()
     ap.add_argument("--limit", type=int, default=None)
     ap.add_argument("--dry-run", action="store_true")

paperboy/stream_common.py

@@ -56,21 +56,23 @@ def write_prompt_event(
             (actor,),
         ).fetchone()
         if existing:
-            return existing[0]
+            return int(existing[0])
         now = _now_iso()
         cur = conn.execute(
             "INSERT INTO events (ts, source, type, actor, payload_json, ingested_at) "
             "VALUES (?, 'pattern-scan', ?, ?, ?, ?)",
             (now, kind, actor, payload, now),
         )
         eid = cur.lastrowid
+        if eid is None:
+            raise RuntimeError("event insert did not return an id")
         tags = {f"stream:{stream}", *extra_tags}
         conn.executemany(
             "INSERT OR IGNORE INTO event_tags (event_id, tag) VALUES (?, ?)",
             [(eid, t) for t in tags],
         )
         conn.commit()
-        return eid
+        return int(eid)
     finally:
         conn.close()
 
@@ -85,7 +87,7 @@ def recent_actors_by_stream(stream: str, days: int = 14) -> list[str]:
             "WHERE source='pattern-scan' AND actor LIKE ? AND ts >= ?",
             (f"{stream}:%", cutoff),
         ).fetchall()
-        return [r[0] for r in rows]
+        return [str(r[0]) for r in rows]
     finally:
         conn.close()
 
@@ -112,21 +114,23 @@ def write_event(
                 (source, event_type, actor),
             ).fetchone()
             if existing:
-                return existing[0]
+                return int(existing[0])
         now = _now_iso()
         cur = conn.execute(
             "INSERT INTO events (ts, source, type, actor, payload_json, ingested_at) "
             "VALUES (?, ?, ?, ?, ?, ?)",
             (ts, source, event_type, actor, json.dumps(payload, ensure_ascii=False), now),
         )
         eid = cur.lastrowid
+        if eid is None:
+            raise RuntimeError("event insert did not return an id")
         if tags:
             conn.executemany(
                 "INSERT OR IGNORE INTO event_tags (event_id, tag) VALUES (?, ?)",
                 [(eid, t) for t in tags],
             )
         conn.commit()
-        return eid
+        return int(eid)
     finally:
         conn.close()