feat: Phase 2 新闻打分去重与 Agent 输入池裁剪

标题相似度合并、信源/时效/昨日重复加权排序，Agent 模式扩大 LLM 新闻候选池并记录抓取失败源。

Co-authored-by: Cursor <cursoragent@cursor.com>

.env.example

@@ -76,6 +76,13 @@ DAILY_AI_NEWS_HOURS=72
 DAILY_AI_NEWS_PER_FEED=3
 DAILY_AI_NEWS_PER_CATEGORY=5
 
+# 新闻排序 / 去重（Phase 2）
+# 标题 Jaccard 相似度阈值（0-95，默认 55 表示 0.55）
+# DAILY_NEWS_TITLE_SIM=55
+# Agent 模式 LLM 输入池（classic 仍用 DAILY_WECOM_* 条数）
+# DAILY_AGENT_NEWS_POOL=40
+# DAILY_AGENT_CN_NEWS_POOL=30
+
 # Reddit RSS（403/429 时在 Reddit 偏好设置 → RSS feeds 复制 user / feed 参数）
 # REDDIT_RSS_USER=your_username
 # REDDIT_RSS_FEED=your_feed_token

daily/generate.py

@@ -49,6 +49,7 @@ from daily.news.fetch import (
     prepare_wecom_cn_news_items,
     prepare_wecom_news_items,
 )
+from daily.news.rank import apply_news_ranking
 from daily.report_data import (
     build_full_payload,
     build_llm_input,
@@ -439,6 +440,8 @@ def generate_report() -> tuple[str, str, Path, Path]:
     topic_name, github_topic = fetch_topic_hot_repos(topic_fetch_n, exclude=seen_repos)
     ai_news = fetch_ai_news()
     cn_ai_news = fetch_cn_ai_news()
+    ai_news = apply_news_ranking(ai_news, date_str=date_str)
+    cn_ai_news = apply_news_ranking(cn_ai_news, date_str=date_str)
 
     wecom_limits = {
         "trending": wecom_trending,
@@ -463,6 +466,7 @@ def generate_report() -> tuple[str, str, Path, Path]:
         ai_news=ai_news,
         cn_ai_news=cn_ai_news,
         wecom_limits=wecom_limits,
+        agent_mode=is_agent_mode(),
     )
     save_json(
         data_json_path(date_str),

daily/news/fetch.py

@@ -286,7 +286,7 @@ def _fetch_one(
     client: httpx.Client,
     category: NewsCategory,
     feed: NewsFeed,
-) -> list[dict[str, Any]]:
+) -> tuple[list[dict[str, Any]], bool]:
     last_exc: Exception | None = None
     for url in _reddit_fetch_urls(feed.url):
         try:
@@ -294,12 +294,12 @@ def _fetch_one(
             resp = client.get(url, headers=headers)
             resp.raise_for_status()
             entries = _parse_feed(resp.text, feed.name, category)
-            return _filter_ai_entries(entries, ai_filter=feed.ai_filter)
+            return _filter_ai_entries(entries, ai_filter=feed.ai_filter), True
         except Exception as exc:
             last_exc = exc
             continue
     logger.warning("RSS fetch failed [%s] %s: %s", feed.name, feed.url, last_exc)
-    return []
+    return [], False
 
 
 def _dedupe_items(items: list[dict[str, Any]]) -> list[dict[str, Any]]:
@@ -344,7 +344,12 @@ def _fetch_news(categories: tuple[NewsCategory, ...]) -> dict[str, Any]:
             tasks.append((category, feed))
 
     raw_by_category: dict[str, list[dict[str, Any]]] = {c.id: [] for c in categories}
-    stats = {"feeds_total": len(tasks), "feeds_ok": 0, "items_raw": 0}
+    stats: dict[str, Any] = {
+        "feeds_total": len(tasks),
+        "feeds_ok": 0,
+        "items_raw": 0,
+        "feeds_failed": [],
+    }
 
     with httpx.Client(timeout=15.0, verify=certifi.where(), follow_redirects=True, headers=headers) as client:
         fast_tasks = [t for t in tasks if not t[1].slow]
@@ -358,19 +363,24 @@ def _fetch_news(categories: tuple[NewsCategory, ...]) -> dict[str, Any]:
             for future in as_completed(futures):
                 cat_id, feed_name = futures[future]
                 try:
-                    entries = future.result()
+                    entries, ok = future.result()
                 except Exception as exc:
                     logger.warning("RSS 任务异常 [%s]: %s", feed_name, exc)
+                    stats["feeds_failed"].append(feed_name)
                     continue
-                if entries:
+                if ok:
                     stats["feeds_ok"] += 1
+                else:
+                    stats["feeds_failed"].append(feed_name)
                 stats["items_raw"] += len(entries)
                 raw_by_category[cat_id].extend(entries[:per_feed])
 
         for cat, feed in slow_tasks:
-            entries = _fetch_one(client, cat, feed)
+            entries, ok = _fetch_one(client, cat, feed)
-            if entries:
+            if ok:
                 stats["feeds_ok"] += 1
+            else:
+                stats["feeds_failed"].append(feed.name)
             stats["items_raw"] += len(entries)
             raw_by_category[cat.id].extend(entries[:per_feed])
             if _is_reddit_url(feed.url):
@@ -467,6 +477,12 @@ def _format_news_section(
         f"> 近 **{hours}h** · {news.get('stats', {}).get('feeds_ok', 0)}/{news.get('stats', {}).get('feeds_total', 0)} 源可用",
         "",
     ]
+    failed = news.get("stats", {}).get("feeds_failed") or []
+    if failed:
+        preview = "、".join(failed[:5])
+        suffix = "…" if len(failed) > 5 else ""
+        lines.append(f"> ⚠️ {len(failed)} 个源抓取失败：{preview}{suffix}")
+        lines.append("")
 
     if not categories:
         lines.append("*暂无可用条目（网络/RSS 源异常或时间窗口内无更新）。*")
@@ -497,12 +513,20 @@ def _format_news_section(
     return lines
 
 
-def prepare_wecom_news_items(news: dict[str, Any]) -> list[dict[str, Any]]:
+def _sorted_flat(news: dict[str, Any]) -> list[dict[str, Any]]:
+    flat = list(news.get("flat") or [])
+    flat.sort(
+        key=lambda item: (float(item.get("score") or 0), _sort_key(item)[1]),
+        reverse=True,
+    )
+    return flat
+
+
+def prepare_wecom_news_items(news: dict[str, Any], *, limit: int | None = None) -> list[dict[str, Any]]:
     if not news.get("enabled"):
         return []
-    limit = _wecom_limit()
+    pick_limit = limit or _wecom_limit()
-    flat = _dedupe_items(news.get("flat") or [])
+    flat = _sorted_flat(news)
-    flat.sort(key=_sort_key, reverse=True)
     preferred = ("media", "newsletter", "official", "community", "research", "developer")
     picked: list[dict[str, Any]] = []
     seen: set[str] = set()
@@ -513,12 +537,12 @@ def prepare_wecom_news_items(news: dict[str, Any]) -> list[dict[str, Any]]:
                 continue
             picked.append(item)
             seen.add(link)
-            if len(picked) >= limit:
+            if len(picked) >= pick_limit:
                 break
-        if len(picked) >= limit:
+        if len(picked) >= pick_limit:
             break
     items: list[dict[str, Any]] = []
-    for item in picked[:limit]:
+    for item in picked[:pick_limit]:
         items.append(
             {
                 "title": item.get("title", "?"),
@@ -526,17 +550,17 @@ def prepare_wecom_news_items(news: dict[str, Any]) -> list[dict[str, Any]]:
                 "source_name": item.get("source_name", "?"),
                 "published_fmt": item.get("published_fmt", ""),
                 "desc_short": _clean_text(item.get("summary", ""), 36),
+                "score": item.get("score"),
             }
         )
     return items
 
 
-def prepare_wecom_cn_news_items(news: dict[str, Any]) -> list[dict[str, Any]]:
+def prepare_wecom_cn_news_items(news: dict[str, Any], *, limit: int | None = None) -> list[dict[str, Any]]:
     if not news.get("enabled"):
         return []
-    limit = _wecom_cn_limit()
+    pick_limit = limit or _wecom_cn_limit()
-    flat = _dedupe_items(news.get("flat") or [])
+    flat = _sorted_flat(news)
-    flat.sort(key=_sort_key, reverse=True)
     preferred = ("media", "tech", "dev")
     picked: list[dict[str, Any]] = []
     seen_links: set[str] = set()
@@ -551,23 +575,23 @@ def prepare_wecom_cn_news_items(news: dict[str, Any]) -> list[dict[str, Any]]:
             picked.append(item)
             seen_links.add(link)
             seen_sources.add(source)
-            if len(picked) >= limit:
+            if len(picked) >= pick_limit:
                 break
-        if len(picked) >= limit:
+        if len(picked) >= pick_limit:
             break
 
-    if len(picked) < limit:
+    if len(picked) < pick_limit:
         for item in flat:
             link = _normalize_link(item.get("link", ""))
             if not link or link in seen_links:
                 continue
             picked.append(item)
             seen_links.add(link)
-            if len(picked) >= limit:
+            if len(picked) >= pick_limit:
                 break
 
     items: list[dict[str, Any]] = []
-    for item in picked[:limit]:
+    for item in picked[:pick_limit]:
         items.append(
             {
                 "title": item.get("title", "?"),
@@ -575,6 +599,7 @@ def prepare_wecom_cn_news_items(news: dict[str, Any]) -> list[dict[str, Any]]:
                 "source_name": item.get("source_name", "?"),
                 "published_fmt": item.get("published_fmt", ""),
                 "desc_short": _clean_text(item.get("summary", ""), 36),
+                "score": item.get("score"),
             }
         )
     return items

daily/news/rank.py

@@ -0,0 +1,171 @@
+"""RSS 新闻去重、打分与排序。"""
+
+from __future__ import annotations
+
+import logging
+import re
+from datetime import datetime, timezone
+from typing import Any
+
+from daily.config import env_int
+from daily.delta import find_previous_data
+from daily.news.fetch import _entry_datetime, _normalize_link, _normalize_title
+
+logger = logging.getLogger(__name__)
+
+_WS = re.compile(r"\s+")
+_TOKEN = re.compile(r"[\w]{2,}", re.UNICODE)
+_CJK = re.compile(r"[\u4e00-\u9fff]")
+
+CATEGORY_TIER: dict[str, int] = {
+    "official": 20,
+    "developer": 16,
+    "research": 14,
+    "media": 12,
+    "newsletter": 10,
+    "community": 8,
+    "tech": 12,
+    "dev": 10,
+}
+
+
+def title_similarity_threshold() -> float:
+    raw = env_int("DAILY_NEWS_TITLE_SIM", 55)
+    return max(0, min(raw, 95)) / 100.0
+
+
+def _title_tokens(title: str) -> set[str]:
+    normalized = _normalize_title(title)
+    tokens = set(_TOKEN.findall(normalized))
+    cjk = "".join(_CJK.findall(title))
+    for i in range(max(0, len(cjk) - 1)):
+        tokens.add(cjk[i : i + 2])
+    if not tokens and normalized:
+        tokens.add(normalized)
+    return tokens
+
+
+def jaccard_similarity(left: set[str], right: set[str]) -> float:
+    if not left or not right:
+        return 0.0
+    inter = len(left & right)
+    union = len(left | right)
+    return inter / union if union else 0.0
+
+
+def fuzzy_dedupe_by_title(
+    items: list[dict[str, Any]],
+    *,
+    threshold: float | None = None,
+) -> list[dict[str, Any]]:
+    """按标题相似度合并重复报道，保留 score 更高（或更靠前）的条目。"""
+    if not items:
+        return []
+    limit = threshold if threshold is not None else title_similarity_threshold()
+    kept: list[dict[str, Any]] = []
+    kept_tokens: list[set[str]] = []
+    for item in items:
+        tokens = _title_tokens(str(item.get("title") or ""))
+        duplicate_idx: int | None = None
+        for idx, existing_tokens in enumerate(kept_tokens):
+            if jaccard_similarity(tokens, existing_tokens) >= limit:
+                duplicate_idx = idx
+                break
+        if duplicate_idx is None:
+            kept.append(item)
+            kept_tokens.append(tokens)
+            continue
+        existing = kept[duplicate_idx]
+        if float(item.get("score") or 0) > float(existing.get("score") or 0):
+            kept[duplicate_idx] = item
+            kept_tokens[duplicate_idx] = tokens
+    return kept
+
+
+def _freshness_points(item: dict[str, Any], *, now: datetime | None = None) -> int:
+    now = now or datetime.now(timezone.utc)
+    dt = _entry_datetime(item)
+    if dt is None:
+        return 4
+    age_hours = max(0.0, (now - dt).total_seconds() / 3600.0)
+    if age_hours <= 6:
+        return 30
+    if age_hours <= 24:
+        return 22
+    if age_hours <= 72:
+        return 8
+    return 0
+
+
+def _source_tier(item: dict[str, Any]) -> int:
+    category_id = str(item.get("category_id") or "")
+    return CATEGORY_TIER.get(category_id, 8)
+
+
+def _novelty_points(item: dict[str, Any], yesterday_links: set[str]) -> int:
+    link = _normalize_link(str(item.get("link") or ""))
+    if link and link in yesterday_links:
+        return -20
+    return 5
+
+
+def score_news_item(
+    item: dict[str, Any],
+    *,
+    yesterday_links: set[str] | None = None,
+    now: datetime | None = None,
+) -> int:
+    links = yesterday_links or set()
+    total = _source_tier(item) + _freshness_points(item, now=now) + _novelty_points(item, links)
+    return max(0, total)
+
+
+def load_yesterday_news_links(date_str: str) -> set[str]:
+    baseline = find_previous_data(date_str)
+    if not baseline:
+        return set()
+    _, data = baseline
+    links: set[str] = set()
+    for key in ("ai_news", "cn_ai_news"):
+        for item in data.get(key) or []:
+            if not isinstance(item, dict):
+                continue
+            link = _normalize_link(str(item.get("link") or ""))
+            if link:
+                links.add(link)
+    return links
+
+
+def apply_news_ranking(news: dict[str, Any], *, date_str: str) -> dict[str, Any]:
+    """对 categories / flat 打分、模糊去重并写回 score 字段。"""
+    if not news.get("enabled"):
+        return news
+
+    yesterday_links = load_yesterday_news_links(date_str)
+    threshold = title_similarity_threshold()
+    now = datetime.now(timezone.utc)
+
+    categories = news.get("categories") or []
+    for category in categories:
+        items = list(category.get("items") or [])
+        for item in items:
+            item["score"] = score_news_item(item, yesterday_links=yesterday_links, now=now)
+        items.sort(key=lambda x: float(x.get("score") or 0), reverse=True)
+        category["items"] = fuzzy_dedupe_by_title(items, threshold=threshold)
+
+    flat: list[dict[str, Any]] = []
+    for category in categories:
+        flat.extend(category.get("items") or [])
+    flat.sort(key=lambda x: float(x.get("score") or 0), reverse=True)
+    news["flat"] = fuzzy_dedupe_by_title(flat, threshold=threshold)
+
+    stats = dict(news.get("stats") or {})
+    stats["ranked_items"] = len(news["flat"])
+    stats["yesterday_links"] = len(yesterday_links)
+    news["stats"] = stats
+    logger.info(
+        "新闻排序完成：%d 条 flat，昨日链接基准 %d",
+        len(news["flat"]),
+        len(yesterday_links),
+    )
+    return news

daily/report_data.py

@@ -61,33 +61,47 @@ def _slim_news_items(
     prepare=prepare_wecom_news_items,
 ) -> list[dict[str, Any]]:
     items: list[dict[str, Any]] = []
-    for item in prepare(ai_news):
+    for item in prepare(ai_news, limit=limit):
-        items.append(
+        payload = {
-            {
+            "link": item.get("link", ""),
-                "link": item.get("link", ""),
+            "title": item.get("title", ""),
-                "title": item.get("title", ""),
+            "source_name": item.get("source_name", ""),
-                "source_name": item.get("source_name", ""),
+            "published_fmt": item.get("published_fmt", ""),
-                "published_fmt": item.get("published_fmt", ""),
+            "summary": item.get("desc_short") or "",
-                "summary": item.get("desc_short") or "",
+        }
-            }
+        if item.get("score") is not None:
-        )
+            payload["score"] = item.get("score")
+        items.append(payload)
         if len(items) >= limit:
             break
     if items:
         return items
-    for item in (ai_news.get("flat") or [])[:limit]:
+    flat = sorted(
-        items.append(
+        ai_news.get("flat") or [],
-            {
+        key=lambda row: float(row.get("score") or 0),
-                "link": item.get("link", ""),
+        reverse=True,
-                "title": item.get("title", ""),
+    )
-                "source_name": item.get("source_name", ""),
+    for item in flat[:limit]:
-                "published_fmt": item.get("published_fmt", ""),
+        payload = {
-                "summary": item.get("summary", ""),
+            "link": item.get("link", ""),
-            }
+            "title": item.get("title", ""),
-        )
+            "source_name": item.get("source_name", ""),
+            "published_fmt": item.get("published_fmt", ""),
+            "summary": item.get("summary", ""),
+        }
+        if item.get("score") is not None:
+            payload["score"] = item.get("score")
+        items.append(payload)
     return items
 
 
+def _news_pool_limit(wecom_limit: int, *, env_key: str, default_pool: int, agent_mode: bool) -> int:
+    if not agent_mode:
+        return wecom_limit
+    pool = env_int(env_key, default_pool)
+    return max(wecom_limit, pool)
+
+
 def _wecom_skill_pool() -> int:
     return max(10, env_int("DAILY_WECOM_SKILL_POOL", 200))
 
@@ -105,10 +119,21 @@ def build_llm_input(
     ai_news: dict[str, Any],
     cn_ai_news: dict[str, Any],
     wecom_limits: dict[str, int],
+    agent_mode: bool = False,
 ) -> dict[str, Any]:
     """供 Cursor 编辑的精简 JSON（不含完整 markdown）。"""
-    news_limit = wecom_limits.get("ai_news", 10)
+    news_limit = _news_pool_limit(
-    cn_news_limit = wecom_limits.get("cn_ai_news", 8)
+        wecom_limits.get("ai_news", 10),
+        env_key="DAILY_AGENT_NEWS_POOL",
+        default_pool=40,
+        agent_mode=agent_mode,
+    )
+    cn_news_limit = _news_pool_limit(
+        wecom_limits.get("cn_ai_news", 8),
+        env_key="DAILY_AGENT_CN_NEWS_POOL",
+        default_pool=30,
+        agent_mode=agent_mode,
+    )
     depth = compare_depth()
     trend_cmp = trending[:depth]
     hot_cmp = hot[:depth]

tests/test_news_rank.py

@@ -0,0 +1,92 @@
+from __future__ import annotations
+
+from datetime import datetime, timezone
+
+from daily.news.rank import (
+    apply_news_ranking,
+    fuzzy_dedupe_by_title,
+    jaccard_similarity,
+    score_news_item,
+    _title_tokens,
+)
+
+
+def test_jaccard_similarity_detects_duplicate_headlines():
+    left = _title_tokens("OpenAI launches GPT-5 with new reasoning")
+    right = _title_tokens("OpenAI launches GPT 5 with reasoning upgrade")
+    assert jaccard_similarity(left, right) >= 0.55
+
+
+def test_fuzzy_dedupe_keeps_higher_score():
+    items = [
+        {"title": "OpenAI launches GPT-5", "link": "https://a.example/1", "score": 20},
+        {"title": "OpenAI launches GPT 5 today", "link": "https://b.example/2", "score": 45},
+    ]
+    deduped = fuzzy_dedupe_by_title(items, threshold=0.55)
+    assert len(deduped) == 1
+    assert deduped[0]["link"] == "https://b.example/2"
+
+
+def test_score_news_item_prefers_fresh_official_and_penalizes_yesterday():
+    now = datetime(2026, 7, 3, 12, 0, tzinfo=timezone.utc)
+    fresh = {
+        "title": "Claude update",
+        "link": "https://anthropic.com/news/claude",
+        "category_id": "official",
+        "published": "Thu, 03 Jul 2026 10:00:00 GMT",
+    }
+    stale_seen = {
+        "title": "Old story",
+        "link": "https://example.com/old",
+        "category_id": "media",
+        "published": "Mon, 30 Jun 2026 10:00:00 GMT",
+    }
+    fresh_score = score_news_item(fresh, yesterday_links=set(), now=now)
+    stale_score = score_news_item(
+        stale_seen,
+        yesterday_links={"https://example.com/old"},
+        now=now,
+    )
+    assert fresh_score > stale_score
+
+
+def test_apply_news_ranking_sets_flat_scores(monkeypatch, tmp_path):
+    import daily.config as config
+    import daily.news.rank as rank
+
+    out = tmp_path / "output"
+    out.mkdir()
+    monkeypatch.setattr(config, "OUTPUT_DIR", out)
+    monkeypatch.setattr(rank, "load_yesterday_news_links", lambda _date: set())
+
+    news = {
+        "enabled": True,
+        "categories": [
+            {
+                "id": "official",
+                "name": "厂商官方",
+                "icon": "🏢",
+                "items": [
+                    {
+                        "title": "OpenAI ships new model",
+                        "link": "https://openai.com/a",
+                        "category_id": "official",
+                        "published": "Thu, 03 Jul 2026 08:00:00 GMT",
+                    },
+                    {
+                        "title": "OpenAI ships a new model today",
+                        "link": "https://techcrunch.com/a",
+                        "category_id": "media",
+                        "published": "Thu, 03 Jul 2026 07:00:00 GMT",
+                    },
+                ],
+            }
+        ],
+        "flat": [],
+        "stats": {},
+    }
+
+    ranked = apply_news_ranking(news, date_str="2026-07-03")
+    assert ranked["flat"]
+    assert all("score" in item for item in ranked["flat"])
+    assert len(ranked["flat"]) == 1