Add fetcher hardening tests and dedupe

scraping/fetchers.py

@@ -17,6 +17,13 @@
         "(KHTML, like Gecko) Chrome/124.0.0.0 Safari/537.36"
     ),
     "Accept-Language": "en-US,en;q=0.9",
+    "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8",
+    "Accept-Encoding": "gzip, br, deflate",
+    "Sec-Fetch-Dest": "document",
+    "Sec-Fetch-Mode": "navigate",
+    "Sec-Fetch-Site": "none",
+    "Sec-Fetch-User": "?1",
+    "Upgrade-Insecure-Requests": "1",
 }
 REQUEST_TIMEOUT = 20
 
@@ -37,6 +44,18 @@ def _parse_feed_date(entry) -> str:
     return _now_iso()
 
 
+def _dedupe_articles_by_url(articles: list) -> list:
+    seen = set()
+    deduped = []
+    for a in articles:
+        url = a.get("url", "")
+        if not url or url in seen:
+            continue
+        seen.add(url)
+        deduped.append(a)
+    return deduped
+
+
 # ─── RSS / ATOM ───────────────────────────────────────────────────────────────
 
 def fetch_rss_source(source: dict) -> list:
@@ -198,7 +217,7 @@ def fetch_sources_by_tier(tier: int) -> list:
             all_articles.extend(articles)
         except Exception as e:
             logger.error(f"  [{source['name']}] error: {e}")
-    return all_articles
+    return _dedupe_articles_by_url(all_articles)
 
 
 def fetch_all_sources() -> list:
@@ -208,4 +227,4 @@ def fetch_all_sources() -> list:
             all_articles.extend(fetch_source(source))
         except Exception as e:
             logger.error(f"  [{source['name']}] error: {e}")
-    return all_articles
+    return _dedupe_articles_by_url(all_articles)

tests/test_briefing.py

@@ -0,0 +1,188 @@
+"""Unit tests for briefing prompt parsing and helper extraction."""
+import unittest
+
+from analysis import briefing
+
+
+class _DummyProvider:
+    def __init__(self, reply: str):
+        self.reply = reply
+        self.calls = 0
+
+    def chat(self, messages, stream: bool = False):
+        self.calls += 1
+        return self.reply
+
+    def stream_chat(self, messages):
+        raise NotImplementedError
+
+
+class TestNovelty(unittest.TestCase):
+    def test_check_novelty_returns_skip(self):
+        provider = _DummyProvider("SKIP, not new")
+        article = {"title": "X", "url": "https://example.com/x"}
+        out = briefing.check_novelty([article], [{"id": 7, "headline": "old"}], provider)
+        self.assertEqual(out, "skip")
+        self.assertEqual(provider.calls, 1)
+
+    def test_check_novelty_returns_update_tuple(self):
+        provider = _DummyProvider("UPDATE 12 requested")
+        article = {"title": "X", "url": "https://example.com/x"}
+        out = briefing.check_novelty([article], [{"id": 2, "headline": "old"}], provider)
+        self.assertEqual(out, ("update", 12))
+
+    def test_check_novelty_defaults_to_full(self):
+        provider = _DummyProvider("NEW: this is a full item")
+        article = {"title": "X", "url": "https://example.com/x"}
+        out = briefing.check_novelty([article], [{"id": 2, "headline": "old"}], provider)
+        self.assertEqual(out, "full")
+
+    def test_check_novelty_no_recent_briefings(self):
+        provider = _DummyProvider("SKIP")
+        article = {"title": "X", "url": "https://example.com/x"}
+        out = briefing.check_novelty([article], [], provider)
+        self.assertEqual(out, "full")
+        self.assertEqual(provider.calls, 0)
+
+    def test_check_novelty_non_string_response(self):
+        provider = _DummyProvider(None)
+        article = {"title": "X", "url": "https://example.com/x"}
+        out = briefing.check_novelty([article], [{"id": 2, "headline": "old"}], provider)
+        self.assertEqual(out, "full")
+
+
+class TestParsingHelpers(unittest.TestCase):
+    def test_parse_json_list_supports_json_array_and_bullets(self):
+        self.assertEqual(
+            briefing._parse_json_list('["a", "b", "c", "d", "e", "f", "g", "h", "i", "j", "k"]')[:10],
+            ["a", "b", "c", "d", "e", "f", "g", "h", "i", "j"],
+        )
+        self.assertEqual(
+            briefing._parse_json_list("- a\n* b\n1. c"),
+            ["a", "b", "c"],
+        )
+
+    def test_parse_json_list_supports_quoted_lines(self):
+        self.assertEqual(
+            briefing._parse_json_list("'a'\n'b'\n'c'"),
+            ["a", "b", "c"],
+        )
+
+    def test_parse_topics_line_deduplicates_and_limits(self):
+        parsed = briefing._parse_topics_line("World, Politics, World, Security")
+        self.assertEqual(parsed, ["World", "Politics", "Security"])
+
+    def test_extract_section_respects_markers(self):
+        text = """
+<<<SEVERITY>>>
+4
+<<<CONFIDENCE>>>
+medium
+<<<HEADLINE>>>
+Signal heads
+<<<SUMMARY>>>
+something
+<<<END>>>
+"""
+        self.assertEqual(briefing._extract_section(text, "HEADLINE"), "Signal heads")
+        self.assertEqual(briefing._extract_section(text, "SUMMARY"), "something")
+
+    def test_first_line_like_headline_skips_metadata_lines(self):
+        text = """
+<<<CONFIDENCE>>>
+""" + "x" * 201 + """
+Real headline line
+"""
+        self.assertEqual(briefing._first_line_like_headline(text), "Real headline line")
+
+
+class TestParseBriefingResponse(unittest.TestCase):
+    def test_parse_briefing_response_extracts_fields(self):
+        raw = """
+<<<SEVERITY>>>
+5
+<<<CONFIDENCE>>>
+high
+<<<HEADLINE>>>
+Clear headline
+<<<SUMMARY>>>
+Short summary
+<<<DEVELOPMENTS>>>
+Developments body
+<<<CONTEXT>>>
+Context body
+<<<ACTORS>>>
+Actors body
+<<<OUTLOOK>>>
+Outlook body
+<<<WATCH>>>
+["Monitor sanctions", "Oil prices", "A", "B", "C", "D", "E", "F"]
+<<<QUESTIONS>>>
+1) What changed?
+2) Who is accountable?
+<<<TOPICS>>>
+World, Energy, World
+<<<END>>>
+"""
+        parsed = briefing.parse_briefing_response(raw)
+        self.assertEqual(parsed["severity"], 5)
+        self.assertEqual(parsed["confidence"], "high")
+        self.assertEqual(parsed["headline"], "Clear headline")
+        self.assertEqual(parsed["watch_indicators"], ["Monitor sanctions", "Oil prices", "A", "B", "C"])
+        self.assertEqual(parsed["suggested_questions"], ["What changed?", "Who is accountable?"])
+        self.assertEqual(parsed["topics"], ["World", "Energy"])
+
+
+class TestFallbackParsing(unittest.TestCase):
+    def test_apply_parsing_fallbacks_fills_required_fields(self):
+        parsed = {
+            "severity": 3,
+            "confidence": "medium",
+            "headline": "",
+            "summary": "",
+            "developments": "",
+            "context": "",
+            "actors": "",
+            "outlook": "",
+            "watch_indicators": [],
+            "suggested_questions": [],
+            "topics": [],
+            "topics_raw": "World, Economy",
+        }
+        articles = [
+            {
+                "title": "Fallback article title",
+                "summary": "Fallback article summary line",
+                "full_text": "",
+            }
+        ]
+        briefing.apply_parsing_fallbacks(parsed, "", articles)
+        self.assertEqual(parsed["headline"], "Fallback article title")
+        self.assertEqual(parsed["summary"], "Fallback article summary line")
+        self.assertIn("Fallback article title", parsed["developments"])
+
+
+class TestFormatArticlesForPrompt(unittest.TestCase):
+    def test_format_articles_for_prompt_truncates_timestamps(self):
+        articles = [
+            {
+                "title": "A",
+                "url": "https://example.com/a",
+                "source_name": "Source",
+                "published_at": "2026-07-13T12:34:56+00:00",
+                "summary": "x" * 700,
+            },
+            {
+                "title": "B",
+                "url": "https://example.com/b",
+                "source_name": "Source",
+                "published_at": "",
+                "full_text": "full body",
+                "summary": "y",
+            },
+        ]
+        out = briefing.format_articles_for_prompt(articles)
+        self.assertIn("[1] Source | 2026-07-13T12:34", out)
+        self.assertIn("TITLE: A", out)
+        self.assertIn("TITLE: B", out)
+        self.assertIn("full body", out)

tests/test_fetchers.py

@@ -0,0 +1,138 @@
+"""Unit tests for source fetch/parsing helpers."""
+from types import SimpleNamespace
+from unittest.mock import patch
+import unittest
+
+from scraping import fetchers
+
+
+class _FeedEntry(dict):
+    def __getattr__(self, name):
+        try:
+            return self[name]
+        except KeyError as e:
+            raise AttributeError(name) from e
+
+
+class _FakeResponse:
+    def __init__(self, text, status_code=200):
+        self.text = text
+        self.status_code = status_code
+
+    def raise_for_status(self):
+        if self.status_code >= 400:
+            raise Exception(f"HTTP {self.status_code}")
+
+
+def _feed(entries, bozo=False):
+    return SimpleNamespace(
+        bozo=bozo,
+        bozo_exception=Exception("bad feed") if bozo else None,
+        entries=entries,
+    )
+
+
+class TestFetchers(unittest.TestCase):
+    def test_parse_feed_date_prefers_published(self):
+        entry = _FeedEntry(published_parsed=(2026, 7, 13, 12, 34, 56, 0, 0, 0))
+        self.assertEqual(fetchers._parse_feed_date(entry), "2026-07-13T12:34:56+00:00")
+
+    def test_fetch_rss_source_parses_html_summary_and_skips_bad_items(self):
+        source = {"name": "World Feed", "url": "https://example.com/world.rss"}
+        entries = [
+            _FeedEntry({
+                "title": "  Stable update in the region  ",
+                "link": "https://example.com/1",
+                "summary": "<p>Good <b>story</b>.</p>",
+            }),
+            _FeedEntry({"title": "  ", "link": "https://example.com/no-title", "summary": "bad"}),
+            _FeedEntry({"title": "No link", "link": "", "summary": "bad"}),
+        ]
+
+        with patch.object(fetchers.feedparser, "parse", return_value=_feed(entries)):
+            articles = fetchers.fetch_rss_source(source)
+
+        self.assertEqual(len(articles), 1)
+        self.assertEqual(articles[0]["title"], "Stable update in the region")
+        self.assertEqual(articles[0]["summary"], "Good story .")
+
+    def test_fetch_rss_source_returns_empty_for_bad_feed(self):
+        source = {"name": "Bad Feed", "url": "https://example.com/bad.rss"}
+        with patch.object(fetchers.feedparser, "parse", return_value=_feed([], bozo=True)):
+            articles = fetchers.fetch_rss_source(source)
+        self.assertEqual(articles, [])
+
+    def test_fetch_scrape_source_builds_absolute_urls(self):
+        source = {
+            "name": "Scrape Source",
+            "url": "https://example.com/news/",
+            "scrape_config": {
+                "article_selector": "a",
+                "base_url": "https://example.com",
+            },
+            "tier": 3,
+            "region": "global",
+        }
+        html = "<html><a href=\"/item/1\">A long enough article headline</a></html>"
+        with patch("scraping.fetchers.requests.get", return_value=_FakeResponse(html)):
+            articles = fetchers.fetch_scrape_source(source)
+
+        self.assertEqual(len(articles), 1)
+        self.assertEqual(articles[0]["url"], "https://example.com/item/1")
+        self.assertEqual(articles[0]["source_name"], "Scrape Source")
+
+    def test_search_google_news_splits_source_from_title(self):
+        query_feed = _feed([
+            _FeedEntry({
+                "title": "Headline from test site - Reuters",
+                "link": "https://example.com/1",
+                "summary": "<p>Test summary</p>",
+            }),
+        ])
+        with patch.object(fetchers.feedparser, "parse", return_value=query_feed):
+            articles = fetchers.search_google_news("whatever", limit=5)
+
+        self.assertEqual(len(articles), 1)
+        self.assertEqual(articles[0]["title"], "Headline from test site")
+        self.assertEqual(articles[0]["source_name"], "Reuters")
+
+    def test_fetch_rss_source_includes_browser_headers_when_fetching_scrape_pages(self):
+        source = {
+            "name": "Scrape Source",
+            "url": "https://example.com/news/",
+            "scrape_config": {"article_selector": "a"},
+        }
+        html = "<html><a href=\"/item/1\">A long enough article headline</a></html>"
+        with patch("scraping.fetchers.requests.get", return_value=_FakeResponse(html)) as req_get:
+            fetchers.fetch_scrape_source(source)
+        req_get.assert_called_once_with(
+            source["url"],
+            headers=fetchers.HEADERS,
+            timeout=fetchers.REQUEST_TIMEOUT,
+        )
+
+    def test_fetch_sources_by_tier_deduplicates_duplicate_urls(self):
+        first = {"name": "Source A"}
+        second = {"name": "Source B"}
+
+        with patch.object(fetchers, "load_sources", return_value=[first, second]):
+            with patch.object(
+                fetchers,
+                "fetch_source",
+                side_effect=[
+                    [{"url": "https://example.com/article", "title": "A1"}],
+                    [{"url": "https://example.com/article", "title": "A2"},
+                     {"url": "https://example.com/other", "title": "B1"}],
+                ],
+            ):
+                articles = fetchers.fetch_sources_by_tier(1)
+
+        self.assertEqual(len(articles), 2)
+        self.assertEqual([a["url"] for a in articles], [
+            "https://example.com/article",
+            "https://example.com/other",
+        ])
+
+
+if __name__ == "__main__":
+    unittest.main()