diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index c8a98ff..9c56158 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -39,6 +39,8 @@ jobs: kill %1 - name: Lint with ruff run: ruff check . + - name: Type-check package + run: mypy paperboy dependency-audit: runs-on: ubuntu-latest diff --git a/paperboy/api/main.py b/paperboy/api/main.py index e8a1db6..cd37d66 100644 --- a/paperboy/api/main.py +++ b/paperboy/api/main.py @@ -25,6 +25,7 @@ from email import policy from email.parser import BytesParser from pathlib import Path +from typing import Any from fastapi import FastAPI, HTTPException, Request from fastapi.responses import FileResponse, JSONResponse, PlainTextResponse, RedirectResponse, Response @@ -433,8 +434,10 @@ async def capture_resend_event(request: Request) -> JSONResponse: if not _claim_email_provider_event(event_id, event_type): return JSONResponse({"ok": True, "duplicate": True}) try: - data = event.get("data") if isinstance(event.get("data"), dict) else {} - recipients = data.get("to") if isinstance(data.get("to"), list) else [] + raw_data = event.get("data") + data: dict[str, Any] = raw_data if isinstance(raw_data, dict) else {} + raw_recipients = data.get("to") + recipients: list[Any] = raw_recipients if isinstance(raw_recipients, list) else [] email = str(recipients[0]).strip().lower() if recipients else "" subscription = get_subscription_by_email(email) if email else None subscription_id = int(subscription["id"]) if subscription is not None else None diff --git a/paperboy/billing.py b/paperboy/billing.py index a13cdd8..76dad49 100644 --- a/paperboy/billing.py +++ b/paperboy/billing.py @@ -49,6 +49,9 @@ def create_checkout(subscription: dict[str, Any]) -> str: billing_status = str(subscription.get("billing_status") or "unpaid") if billing_status in {"trialing", "active", "past_due"}: raise BillingStateError("manage the existing billing account instead") + price_id = settings.stripe_price_id + if not price_id: + raise BillingUnavailableError("checkout is temporarily unavailable") metadata = {"paperboy_subscription_id": str(subscription["id"])} customer_args: dict[str, Any] @@ -59,7 +62,7 @@ def create_checkout(subscription: dict[str, Any]) -> str: session = stripe.checkout.Session.create( **customer_args, mode="subscription", - line_items=[{"price": settings.stripe_price_id, "quantity": 1}], + line_items=[{"price": price_id, "quantity": 1}], payment_method_collection="always", client_reference_id=str(subscription["id"]), metadata=metadata, diff --git a/paperboy/digest/prompt_digest.py b/paperboy/digest/prompt_digest.py index 0aac124..8dcc1ad 100644 --- a/paperboy/digest/prompt_digest.py +++ b/paperboy/digest/prompt_digest.py @@ -44,7 +44,7 @@ "proj": "🛠️ projects", "research": "🔬 research"} -def _stream_from_actor(actor): +def _stream_from_actor(actor: str | None) -> str: if not actor or ":" not in actor: return "research" prefix = actor.split(":", 1)[0] diff --git a/paperboy/digest/research_digest.py b/paperboy/digest/research_digest.py index 6c83016..a7243b0 100644 --- a/paperboy/digest/research_digest.py +++ b/paperboy/digest/research_digest.py @@ -97,7 +97,7 @@ def batch_into_messages(lines: list[str], header: str) -> list[str]: return messages -def main(): +def main() -> None: ap = argparse.ArgumentParser() ap.add_argument("--top", type=int, default=0) ap.add_argument("--min-relevance", type=int, default=5) diff --git a/paperboy/discord_post.py b/paperboy/discord_post.py index 431be54..6bba42c 100644 --- a/paperboy/discord_post.py +++ b/paperboy/discord_post.py @@ -45,9 +45,10 @@ def post(content: str, *, username: str = "paperboy", dry_run: bool = False) -> ) try: with urllib.request.urlopen(req, timeout=15) as r: - ok = 200 <= r.status < 300 + status = int(r.status) + ok = 200 <= status < 300 if not ok: - print(f"[discord_post] HTTP {r.status}", file=sys.stderr) + print(f"[discord_post] HTTP {status}", file=sys.stderr) return ok except urllib.error.HTTPError as e: print(f"[discord_post] HTTPError {e.code}: {e.read()[:200]!r}", file=sys.stderr) diff --git a/paperboy/health.py b/paperboy/health.py index 2279b52..9553863 100644 --- a/paperboy/health.py +++ b/paperboy/health.py @@ -12,6 +12,7 @@ import time import urllib.error import urllib.request +from collections.abc import Callable from datetime import datetime, timezone from pathlib import Path @@ -122,7 +123,7 @@ def check_backup_recency(max_age_hours: int = 48) -> tuple[bool, str]: def run_all() -> dict[str, dict]: """Run every probe and return a structured report.""" - probes = { + probes: dict[str, Callable[[], tuple[bool, str]]] = { "database": check_database, "ollama": check_ollama, "disk": check_disk, diff --git a/paperboy/ingest/research_papers.py b/paperboy/ingest/research_papers.py index 7b66956..c120d4b 100644 --- a/paperboy/ingest/research_papers.py +++ b/paperboy/ingest/research_papers.py @@ -27,6 +27,7 @@ import urllib.request from datetime import datetime, timezone from pathlib import Path +from typing import Any from xml.etree import ElementTree from paperboy.db import connect @@ -171,7 +172,7 @@ def fetch_arxiv(cfg: dict, ua: str, timeout: int, verbose: bool) -> list[dict]: except ElementTree.ParseError as e: print(f"[arxiv] XML parse error: {e}", file=sys.stderr) return [] - papers = [] + papers: list[dict] = [] for entry in root.findall(f"{ATOM_NS}entry"): id_text = (entry.findtext(f"{ATOM_NS}id") or "").strip() arxiv_id = canonicalize_arxiv_id(id_text) @@ -222,7 +223,7 @@ def fetch_hf_papers(cfg: dict, ua: str, timeout: int, verbose: bool) -> list[dic except json.JSONDecodeError as e: print(f"[hf] JSON parse error: {e}", file=sys.stderr) return [] - papers = [] + papers: list[dict] = [] max_n = int(cfg.get("max_per_run", 50)) for item in (data if isinstance(data, list) else [])[:max_n]: paper = item.get("paper") if isinstance(item, dict) else None @@ -261,7 +262,7 @@ def fetch_semantic_scholar(cfg: dict, ua: str, timeout: int, verbose: bool) -> l max_total = int(cfg.get("max_per_run", 120)) queries = cfg.get("topic_queries", []) sleep_s = float(cfg.get("inter_request_sleep_sec", 1.5)) - papers = [] + papers: list[dict] = [] for q in queries: if len(papers) >= max_total: break @@ -342,7 +343,7 @@ def dedup_and_merge(raw_papers: list[dict]) -> dict[str, dict]: # --------------------------------------------------------------------------- -def main(): +def main() -> None: ap = argparse.ArgumentParser() ap.add_argument("--source", default="all", choices=["all", "arxiv", "hf", "s2"]) @@ -392,7 +393,7 @@ def main(): existing = load_existing_actors() existing_payloads = load_existing_payloads(existing & set(merged.keys())) - stats = { + stats: dict[str, Any] = { "raw_total": len(raw_papers), "unique_total": len(merged), "new_ingested": 0, "updated_feed_list": 0, "skipped_existing": 0, "errors": 0, "would_ingest": 0, "per_feed_counts": per_feed_counts, diff --git a/paperboy/logging_config.py b/paperboy/logging_config.py index 957c5cd..73fdb63 100644 --- a/paperboy/logging_config.py +++ b/paperboy/logging_config.py @@ -75,7 +75,8 @@ def configure_logging(level: str | None = None, force_json: bool | None = None) force_json: True to force JSON output. Defaults to truthy PAPERBOY_LOG_FORMAT=json env var. """ - lvl = (level or os.environ.get("PAPERBOY_LOG_LEVEL", "INFO")).upper() + configured_level = level or os.environ.get("PAPERBOY_LOG_LEVEL") or "INFO" + lvl = configured_level.upper() json_mode = force_json if force_json is not None else os.environ.get("PAPERBOY_LOG_FORMAT", "").lower() == "json" root = logging.getLogger("paperboy") diff --git a/paperboy/scanners/news_opinion.py b/paperboy/scanners/news_opinion.py index 4b4246b..76ed9e2 100644 --- a/paperboy/scanners/news_opinion.py +++ b/paperboy/scanners/news_opinion.py @@ -44,7 +44,7 @@ def _sources_path() -> Path: def _load_sources() -> dict[str, list[str]]: try: - import yaml + import yaml # type: ignore[import-untyped] except ImportError: print("[news_opinion] PyYAML not installed; run: pip install pyyaml", file=sys.stderr) return {} @@ -53,7 +53,14 @@ def _load_sources() -> dict[str, list[str]]: print(f"[news_opinion] no sources file at {path}", file=sys.stderr) return {} with path.open(encoding="utf-8") as f: - return yaml.safe_load(f) or {} + loaded = yaml.safe_load(f) or {} + if not isinstance(loaded, dict): + return {} + return { + str(vertical): [str(url) for url in urls if isinstance(url, str)] + for vertical, urls in loaded.items() + if isinstance(urls, list) + } def _fetch_feed(url: str) -> list[dict]: @@ -118,10 +125,13 @@ def _ollama_json(prompt: str) -> dict: ) with urllib.request.urlopen(req, timeout=60) as resp: out = json.loads(resp.read()) + if not isinstance(out, dict): + return {} try: - return json.loads(out.get("response", "{}")) + result = json.loads(out.get("response", "{}")) except json.JSONDecodeError: return {} + return result if isinstance(result, dict) else {} def llm_draft_prompt(vertical: str, item: dict) -> dict: diff --git a/paperboy/score/research_papers.py b/paperboy/score/research_papers.py index 02fa98a..045c120 100644 --- a/paperboy/score/research_papers.py +++ b/paperboy/score/research_papers.py @@ -160,9 +160,14 @@ def call_ollama(system_prompt: str, user_prompt: str) -> dict: ) with urllib.request.urlopen(req, timeout=TIMEOUT) as r: resp = json.load(r) + if not isinstance(resp, dict): + raise json.JSONDecodeError("Ollama response is not an object", str(resp), 0) raw = resp.get("response", "").strip() raw = re.sub(r"^```(?:json)?\s*|\s*```$", "", raw, flags=re.MULTILINE).strip() - return json.loads(raw) + verdict = json.loads(raw) + if not isinstance(verdict, dict): + raise json.JSONDecodeError("Ollama verdict is not an object", raw, 0) + return verdict def normalize_verdict(v: dict, parent_actor: str, parent_payload: dict) -> dict: @@ -194,7 +199,7 @@ def normalize_verdict(v: dict, parent_actor: str, parent_payload: dict) -> dict: } -def main(): +def main() -> None: ap = argparse.ArgumentParser() ap.add_argument("--limit", type=int, default=None) ap.add_argument("--dry-run", action="store_true") diff --git a/paperboy/stream_common.py b/paperboy/stream_common.py index b147393..bef4834 100644 --- a/paperboy/stream_common.py +++ b/paperboy/stream_common.py @@ -56,7 +56,7 @@ def write_prompt_event( (actor,), ).fetchone() if existing: - return existing[0] + return int(existing[0]) now = _now_iso() cur = conn.execute( "INSERT INTO events (ts, source, type, actor, payload_json, ingested_at) " @@ -64,13 +64,15 @@ def write_prompt_event( (now, kind, actor, payload, now), ) eid = cur.lastrowid + if eid is None: + raise RuntimeError("event insert did not return an id") tags = {f"stream:{stream}", *extra_tags} conn.executemany( "INSERT OR IGNORE INTO event_tags (event_id, tag) VALUES (?, ?)", [(eid, t) for t in tags], ) conn.commit() - return eid + return int(eid) finally: conn.close() @@ -85,7 +87,7 @@ def recent_actors_by_stream(stream: str, days: int = 14) -> list[str]: "WHERE source='pattern-scan' AND actor LIKE ? AND ts >= ?", (f"{stream}:%", cutoff), ).fetchall() - return [r[0] for r in rows] + return [str(r[0]) for r in rows] finally: conn.close() @@ -112,7 +114,7 @@ def write_event( (source, event_type, actor), ).fetchone() if existing: - return existing[0] + return int(existing[0]) now = _now_iso() cur = conn.execute( "INSERT INTO events (ts, source, type, actor, payload_json, ingested_at) " @@ -120,13 +122,15 @@ def write_event( (ts, source, event_type, actor, json.dumps(payload, ensure_ascii=False), now), ) eid = cur.lastrowid + if eid is None: + raise RuntimeError("event insert did not return an id") if tags: conn.executemany( "INSERT OR IGNORE INTO event_tags (event_id, tag) VALUES (?, ?)", [(eid, t) for t in tags], ) conn.commit() - return eid + return int(eid) finally: conn.close()