diff --git a/scraping/fetchers.py b/scraping/fetchers.py index bee4fa4..d7d6e2c 100644 --- a/scraping/fetchers.py +++ b/scraping/fetchers.py @@ -17,6 +17,13 @@ "(KHTML, like Gecko) Chrome/124.0.0.0 Safari/537.36" ), "Accept-Language": "en-US,en;q=0.9", + "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8", + "Accept-Encoding": "gzip, br, deflate", + "Sec-Fetch-Dest": "document", + "Sec-Fetch-Mode": "navigate", + "Sec-Fetch-Site": "none", + "Sec-Fetch-User": "?1", + "Upgrade-Insecure-Requests": "1", } REQUEST_TIMEOUT = 20 @@ -37,6 +44,18 @@ def _parse_feed_date(entry) -> str: return _now_iso() +def _dedupe_articles_by_url(articles: list) -> list: + seen = set() + deduped = [] + for a in articles: + url = a.get("url", "") + if not url or url in seen: + continue + seen.add(url) + deduped.append(a) + return deduped + + # ─── RSS / ATOM ─────────────────────────────────────────────────────────────── def fetch_rss_source(source: dict) -> list: @@ -198,7 +217,7 @@ def fetch_sources_by_tier(tier: int) -> list: all_articles.extend(articles) except Exception as e: logger.error(f" [{source['name']}] error: {e}") - return all_articles + return _dedupe_articles_by_url(all_articles) def fetch_all_sources() -> list: @@ -208,4 +227,4 @@ def fetch_all_sources() -> list: all_articles.extend(fetch_source(source)) except Exception as e: logger.error(f" [{source['name']}] error: {e}") - return all_articles + return _dedupe_articles_by_url(all_articles) diff --git a/tests/test_briefing.py b/tests/test_briefing.py new file mode 100644 index 0000000..f078d0f --- /dev/null +++ b/tests/test_briefing.py @@ -0,0 +1,188 @@ +"""Unit tests for briefing prompt parsing and helper extraction.""" +import unittest + +from analysis import briefing + + +class _DummyProvider: + def __init__(self, reply: str): + self.reply = reply + self.calls = 0 + + def chat(self, messages, stream: bool = False): + self.calls += 1 + return self.reply + + def stream_chat(self, messages): + raise NotImplementedError + + +class TestNovelty(unittest.TestCase): + def test_check_novelty_returns_skip(self): + provider = _DummyProvider("SKIP, not new") + article = {"title": "X", "url": "https://example.com/x"} + out = briefing.check_novelty([article], [{"id": 7, "headline": "old"}], provider) + self.assertEqual(out, "skip") + self.assertEqual(provider.calls, 1) + + def test_check_novelty_returns_update_tuple(self): + provider = _DummyProvider("UPDATE 12 requested") + article = {"title": "X", "url": "https://example.com/x"} + out = briefing.check_novelty([article], [{"id": 2, "headline": "old"}], provider) + self.assertEqual(out, ("update", 12)) + + def test_check_novelty_defaults_to_full(self): + provider = _DummyProvider("NEW: this is a full item") + article = {"title": "X", "url": "https://example.com/x"} + out = briefing.check_novelty([article], [{"id": 2, "headline": "old"}], provider) + self.assertEqual(out, "full") + + def test_check_novelty_no_recent_briefings(self): + provider = _DummyProvider("SKIP") + article = {"title": "X", "url": "https://example.com/x"} + out = briefing.check_novelty([article], [], provider) + self.assertEqual(out, "full") + self.assertEqual(provider.calls, 0) + + def test_check_novelty_non_string_response(self): + provider = _DummyProvider(None) + article = {"title": "X", "url": "https://example.com/x"} + out = briefing.check_novelty([article], [{"id": 2, "headline": "old"}], provider) + self.assertEqual(out, "full") + + +class TestParsingHelpers(unittest.TestCase): + def test_parse_json_list_supports_json_array_and_bullets(self): + self.assertEqual( + briefing._parse_json_list('["a", "b", "c", "d", "e", "f", "g", "h", "i", "j", "k"]')[:10], + ["a", "b", "c", "d", "e", "f", "g", "h", "i", "j"], + ) + self.assertEqual( + briefing._parse_json_list("- a\n* b\n1. c"), + ["a", "b", "c"], + ) + + def test_parse_json_list_supports_quoted_lines(self): + self.assertEqual( + briefing._parse_json_list("'a'\n'b'\n'c'"), + ["a", "b", "c"], + ) + + def test_parse_topics_line_deduplicates_and_limits(self): + parsed = briefing._parse_topics_line("World, Politics, World, Security") + self.assertEqual(parsed, ["World", "Politics", "Security"]) + + def test_extract_section_respects_markers(self): + text = """ +<<>> +4 +<<>> +medium +<<>> +Signal heads +<<>> +something +<<>> +""" + self.assertEqual(briefing._extract_section(text, "HEADLINE"), "Signal heads") + self.assertEqual(briefing._extract_section(text, "SUMMARY"), "something") + + def test_first_line_like_headline_skips_metadata_lines(self): + text = """ +<<>> +""" + "x" * 201 + """ +Real headline line +""" + self.assertEqual(briefing._first_line_like_headline(text), "Real headline line") + + +class TestParseBriefingResponse(unittest.TestCase): + def test_parse_briefing_response_extracts_fields(self): + raw = """ +<<>> +5 +<<>> +high +<<>> +Clear headline +<<>> +Short summary +<<>> +Developments body +<<>> +Context body +<<>> +Actors body +<<>> +Outlook body +<<>> +["Monitor sanctions", "Oil prices", "A", "B", "C", "D", "E", "F"] +<<>> +1) What changed? +2) Who is accountable? +<<>> +World, Energy, World +<<>> +""" + parsed = briefing.parse_briefing_response(raw) + self.assertEqual(parsed["severity"], 5) + self.assertEqual(parsed["confidence"], "high") + self.assertEqual(parsed["headline"], "Clear headline") + self.assertEqual(parsed["watch_indicators"], ["Monitor sanctions", "Oil prices", "A", "B", "C"]) + self.assertEqual(parsed["suggested_questions"], ["What changed?", "Who is accountable?"]) + self.assertEqual(parsed["topics"], ["World", "Energy"]) + + +class TestFallbackParsing(unittest.TestCase): + def test_apply_parsing_fallbacks_fills_required_fields(self): + parsed = { + "severity": 3, + "confidence": "medium", + "headline": "", + "summary": "", + "developments": "", + "context": "", + "actors": "", + "outlook": "", + "watch_indicators": [], + "suggested_questions": [], + "topics": [], + "topics_raw": "World, Economy", + } + articles = [ + { + "title": "Fallback article title", + "summary": "Fallback article summary line", + "full_text": "", + } + ] + briefing.apply_parsing_fallbacks(parsed, "", articles) + self.assertEqual(parsed["headline"], "Fallback article title") + self.assertEqual(parsed["summary"], "Fallback article summary line") + self.assertIn("Fallback article title", parsed["developments"]) + + +class TestFormatArticlesForPrompt(unittest.TestCase): + def test_format_articles_for_prompt_truncates_timestamps(self): + articles = [ + { + "title": "A", + "url": "https://example.com/a", + "source_name": "Source", + "published_at": "2026-07-13T12:34:56+00:00", + "summary": "x" * 700, + }, + { + "title": "B", + "url": "https://example.com/b", + "source_name": "Source", + "published_at": "", + "full_text": "full body", + "summary": "y", + }, + ] + out = briefing.format_articles_for_prompt(articles) + self.assertIn("[1] Source | 2026-07-13T12:34", out) + self.assertIn("TITLE: A", out) + self.assertIn("TITLE: B", out) + self.assertIn("full body", out) diff --git a/tests/test_fetchers.py b/tests/test_fetchers.py new file mode 100644 index 0000000..3564285 --- /dev/null +++ b/tests/test_fetchers.py @@ -0,0 +1,138 @@ +"""Unit tests for source fetch/parsing helpers.""" +from types import SimpleNamespace +from unittest.mock import patch +import unittest + +from scraping import fetchers + + +class _FeedEntry(dict): + def __getattr__(self, name): + try: + return self[name] + except KeyError as e: + raise AttributeError(name) from e + + +class _FakeResponse: + def __init__(self, text, status_code=200): + self.text = text + self.status_code = status_code + + def raise_for_status(self): + if self.status_code >= 400: + raise Exception(f"HTTP {self.status_code}") + + +def _feed(entries, bozo=False): + return SimpleNamespace( + bozo=bozo, + bozo_exception=Exception("bad feed") if bozo else None, + entries=entries, + ) + + +class TestFetchers(unittest.TestCase): + def test_parse_feed_date_prefers_published(self): + entry = _FeedEntry(published_parsed=(2026, 7, 13, 12, 34, 56, 0, 0, 0)) + self.assertEqual(fetchers._parse_feed_date(entry), "2026-07-13T12:34:56+00:00") + + def test_fetch_rss_source_parses_html_summary_and_skips_bad_items(self): + source = {"name": "World Feed", "url": "https://example.com/world.rss"} + entries = [ + _FeedEntry({ + "title": " Stable update in the region ", + "link": "https://example.com/1", + "summary": "

Good story.

", + }), + _FeedEntry({"title": " ", "link": "https://example.com/no-title", "summary": "bad"}), + _FeedEntry({"title": "No link", "link": "", "summary": "bad"}), + ] + + with patch.object(fetchers.feedparser, "parse", return_value=_feed(entries)): + articles = fetchers.fetch_rss_source(source) + + self.assertEqual(len(articles), 1) + self.assertEqual(articles[0]["title"], "Stable update in the region") + self.assertEqual(articles[0]["summary"], "Good story .") + + def test_fetch_rss_source_returns_empty_for_bad_feed(self): + source = {"name": "Bad Feed", "url": "https://example.com/bad.rss"} + with patch.object(fetchers.feedparser, "parse", return_value=_feed([], bozo=True)): + articles = fetchers.fetch_rss_source(source) + self.assertEqual(articles, []) + + def test_fetch_scrape_source_builds_absolute_urls(self): + source = { + "name": "Scrape Source", + "url": "https://example.com/news/", + "scrape_config": { + "article_selector": "a", + "base_url": "https://example.com", + }, + "tier": 3, + "region": "global", + } + html = "A long enough article headline" + with patch("scraping.fetchers.requests.get", return_value=_FakeResponse(html)): + articles = fetchers.fetch_scrape_source(source) + + self.assertEqual(len(articles), 1) + self.assertEqual(articles[0]["url"], "https://example.com/item/1") + self.assertEqual(articles[0]["source_name"], "Scrape Source") + + def test_search_google_news_splits_source_from_title(self): + query_feed = _feed([ + _FeedEntry({ + "title": "Headline from test site - Reuters", + "link": "https://example.com/1", + "summary": "

Test summary

", + }), + ]) + with patch.object(fetchers.feedparser, "parse", return_value=query_feed): + articles = fetchers.search_google_news("whatever", limit=5) + + self.assertEqual(len(articles), 1) + self.assertEqual(articles[0]["title"], "Headline from test site") + self.assertEqual(articles[0]["source_name"], "Reuters") + + def test_fetch_rss_source_includes_browser_headers_when_fetching_scrape_pages(self): + source = { + "name": "Scrape Source", + "url": "https://example.com/news/", + "scrape_config": {"article_selector": "a"}, + } + html = "A long enough article headline" + with patch("scraping.fetchers.requests.get", return_value=_FakeResponse(html)) as req_get: + fetchers.fetch_scrape_source(source) + req_get.assert_called_once_with( + source["url"], + headers=fetchers.HEADERS, + timeout=fetchers.REQUEST_TIMEOUT, + ) + + def test_fetch_sources_by_tier_deduplicates_duplicate_urls(self): + first = {"name": "Source A"} + second = {"name": "Source B"} + + with patch.object(fetchers, "load_sources", return_value=[first, second]): + with patch.object( + fetchers, + "fetch_source", + side_effect=[ + [{"url": "https://example.com/article", "title": "A1"}], + [{"url": "https://example.com/article", "title": "A2"}, + {"url": "https://example.com/other", "title": "B1"}], + ], + ): + articles = fetchers.fetch_sources_by_tier(1) + + self.assertEqual(len(articles), 2) + self.assertEqual([a["url"] for a in articles], [ + "https://example.com/article", + "https://example.com/other", + ]) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_scheduler.py b/tests/test_scheduler.py new file mode 100644 index 0000000..af9cf93 --- /dev/null +++ b/tests/test_scheduler.py @@ -0,0 +1,86 @@ +"""Unit tests for scheduler reschedule behavior.""" +from unittest.mock import patch +import unittest + +from scraping import scheduler as scheduler_module +from scraping.scheduler import SmartScheduler +from storage.config import Config + + +class FakeTimer: + def __init__(self, interval, callback): + self.interval = interval + self.callback = callback + self.cancelled = False + self.started = False + + def start(self): + self.started = True + + def cancel(self): + self.cancelled = True + + +class FakeThread: + def __init__(self, target, *args, **kwargs): + self.target = target + + def start(self): + self.target() + + +class TestSchedulerReschedule(unittest.TestCase): + def test_reschedule_sentinel_uses_delay_timer(self): + sched = SmartScheduler() + sched._running = True + previous = FakeTimer(10, None) + sched._sentinel_timer = previous + timers = [] + + def fake_timer(interval, callback): + t = FakeTimer(interval, callback) + timers.append(t) + return t + + with patch.object(scheduler_module.threading, "Timer", side_effect=fake_timer): + with patch.object(sched, "_seconds_until_tier_allowed", return_value=120): + sched.reschedule_sentinel() + + self.assertTrue(previous.cancelled) + self.assertIsNotNone(sched._sentinel_timer) + self.assertEqual(len(timers), 1) + self.assertEqual(timers[0].interval, 120) + + def test_reschedule_sentinel_runs_cycle_when_allowed(self): + sched = SmartScheduler() + sched._running = True + calls = {"cycle": 0} + + with patch.object(scheduler_module.threading, "Thread", FakeThread): + with patch.object(sched, "_seconds_until_tier_allowed", return_value=0): + with patch.object(sched, "_sentinel_cycle", side_effect=lambda: calls.__setitem__("cycle", calls["cycle"] + 1)): + sched.reschedule_sentinel() + + self.assertEqual(calls["cycle"], 1) + self.assertIsNone(sched._sentinel_timer) + + def test_reschedule_briefing_respects_enabled_flag(self): + sched = SmartScheduler() + sched._running = True + current = FakeTimer(12, None) + sched._briefing_timer = current + + with patch.object(Config, "scheduled_briefing", return_value={"enabled": False}): + with patch.object(sched, "_schedule_briefing") as schedule: + sched.reschedule_briefing() + self.assertTrue(current.cancelled) + schedule.assert_not_called() + + with patch.object(Config, "scheduled_briefing", return_value={"enabled": True}): + with patch.object(sched, "_schedule_briefing") as schedule2: + sched.reschedule_briefing() + schedule2.assert_called_once() + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_triage.py b/tests/test_triage.py new file mode 100644 index 0000000..c38d362 --- /dev/null +++ b/tests/test_triage.py @@ -0,0 +1,98 @@ +"""Unit tests for triage scoring, topic matching, and enrichment.""" +from unittest import TestCase +from unittest.mock import patch + +from analysis import triage + + +class TestScoreSeverity(TestCase): + def test_score_severity_follows_priority(self): + self.assertEqual(triage.score_severity("Routine diplomatic update"), 1) + self.assertEqual( + triage.score_severity("Negotiations continue after election protests"), + 3, + ) + self.assertEqual( + triage.score_severity("Military operation confirmed near border"), + 4, + ) + self.assertEqual( + triage.score_severity("Nuclear weapon launch suspected after missile attack"), + 5, + ) + + def test_score_severity_ignores_case(self): + self.assertEqual( + triage.score_severity("MILITARY OPERATION in region"), + 4, + ) + + +class TestMatchTopics(TestCase): + def test_match_topics_keyword_matching(self): + topic_keywords = { + "Tech": ["GPU", "kernel", "OpenAI"], + "Conflict": ["summit", "threat"], + } + result = triage.match_topics( + "EU summit discusses AI chips", + "OpenAI announced a new model", + topic_keywords, + ) + self.assertIn("Tech", result) + self.assertIn("Conflict", result) + self.assertEqual(len(result), 2) + + def test_match_topics_from_empty_keyword_list(self): + topic_keywords = { + "Open Source Movement": [], + "United States": [], + "AI": [], + } + result = triage.match_topics( + "Policy report from the United States cabinet cites open-source software spending", + "", + topic_keywords, + ) + self.assertIn("Open Source Movement", result) + self.assertIn("United States", result) + self.assertNotIn("AI", result) + + def test_match_topics_empty_input(self): + self.assertEqual(triage.match_topics("Any title", "Any summary", {}), []) + + +class TestEnrichArticle(TestCase): + @patch("analysis.triage.extract_article_text") + def test_enrich_article_reuses_long_full_text(self, extract_mock): + article = { + "url": "https://example.com/a", + "summary": "x" * 600, + "full_text": "y" * 250, + } + out = triage.enrich_article(article) + self.assertEqual(out["full_text"], "y" * 250) + extract_mock.assert_not_called() + + @patch("analysis.triage.extract_article_text") + def test_enrich_article_promotes_summary(self, extract_mock): + article = { + "url": "https://example.com/b", + "summary": "x" * 320, + "full_text": "", + } + out = triage.enrich_article(article) + self.assertEqual(out["full_text"], "x" * 320) + extract_mock.assert_not_called() + + @patch("analysis.triage.extract_article_text") + def test_enrich_article_fetches_when_needed(self, extract_mock): + extract_mock.return_value = "extracted text" + article = { + "url": "https://example.com/c", + "summary": "short", + "full_text": "", + } + out = triage.enrich_article(article) + self.assertEqual(out["full_text"], "extracted text") + extract_mock.assert_called_once_with("https://example.com/c")