feat: Phase 2 新闻打分去重与 Agent 输入池裁剪

标题相似度合并、信源/时效/昨日重复加权排序，Agent 模式扩大 LLM 新闻候选池并记录抓取失败源。 Co-authored-by: Cursor <cursoragent@cursor.com>
2026-07-03 14:53:30 +08:00
parent 728bd1b9d1
commit 391f887d73
6 changed files with 370 additions and 46 deletions
--- a/tests/test_news_rank.py
+++ b/tests/test_news_rank.py
@@ -0,0 +1,92 @@
+from __future__ import annotations
+
+from datetime import datetime, timezone
+
+from daily.news.rank import (
+    apply_news_ranking,
+    fuzzy_dedupe_by_title,
+    jaccard_similarity,
+    score_news_item,
+    _title_tokens,
+)
+
+
+def test_jaccard_similarity_detects_duplicate_headlines():
+    left = _title_tokens("OpenAI launches GPT-5 with new reasoning")
+    right = _title_tokens("OpenAI launches GPT 5 with reasoning upgrade")
+    assert jaccard_similarity(left, right) >= 0.55
+
+
+def test_fuzzy_dedupe_keeps_higher_score():
+    items = [
+        {"title": "OpenAI launches GPT-5", "link": "https://a.example/1", "score": 20},
+        {"title": "OpenAI launches GPT 5 today", "link": "https://b.example/2", "score": 45},
+    ]
+    deduped = fuzzy_dedupe_by_title(items, threshold=0.55)
+    assert len(deduped) == 1
+    assert deduped[0]["link"] == "https://b.example/2"
+
+
+def test_score_news_item_prefers_fresh_official_and_penalizes_yesterday():
+    now = datetime(2026, 7, 3, 12, 0, tzinfo=timezone.utc)
+    fresh = {
+        "title": "Claude update",
+        "link": "https://anthropic.com/news/claude",
+        "category_id": "official",
+        "published": "Thu, 03 Jul 2026 10:00:00 GMT",
+    }
+    stale_seen = {
+        "title": "Old story",
+        "link": "https://example.com/old",
+        "category_id": "media",
+        "published": "Mon, 30 Jun 2026 10:00:00 GMT",
+    }
+    fresh_score = score_news_item(fresh, yesterday_links=set(), now=now)
+    stale_score = score_news_item(
+        stale_seen,
+        yesterday_links={"https://example.com/old"},
+        now=now,
+    )
+    assert fresh_score > stale_score
+
+
+def test_apply_news_ranking_sets_flat_scores(monkeypatch, tmp_path):
+    import daily.config as config
+    import daily.news.rank as rank
+
+    out = tmp_path / "output"
+    out.mkdir()
+    monkeypatch.setattr(config, "OUTPUT_DIR", out)
+    monkeypatch.setattr(rank, "load_yesterday_news_links", lambda _date: set())
+
+    news = {
+        "enabled": True,
+        "categories": [
+            {
+                "id": "official",
+                "name": "厂商官方",
+                "icon": "🏢",
+                "items": [
+                    {
+                        "title": "OpenAI ships new model",
+                        "link": "https://openai.com/a",
+                        "category_id": "official",
+                        "published": "Thu, 03 Jul 2026 08:00:00 GMT",
+                    },
+                    {
+                        "title": "OpenAI ships a new model today",
+                        "link": "https://techcrunch.com/a",
+                        "category_id": "media",
+                        "published": "Thu, 03 Jul 2026 07:00:00 GMT",
+                    },
+                ],
+            }
+        ],
+        "flat": [],
+        "stats": {},
+    }
+
+    ranked = apply_news_ranking(news, date_str="2026-07-03")
+    assert ranked["flat"]
+    assert all("score" in item for item in ranked["flat"])
+    assert len(ranked["flat"]) == 1