feat(news): research 时讯加可信过滤、去重与国内配额

候选池按独立事件拉取，白名单过滤低质源，同事件与 tech 主题去重后按展示上限打包。

Co-authored-by: Cursor <cursoragent@cursor.com>

daily/news/research.py

@@ -13,6 +13,7 @@ from daily.config import OUTPUT_DIR, ROOT, env, env_int, wecom_ai_news_tech_limi
 from daily.llm_client import cursor_agent_prompt, extract_json_object, has_cursor_configured
 from daily.news.fetch import brief_news_summary, _normalize_link
 from daily.news.pushed_links import filter_unpushed_items
+from daily.news.research_quality import post_process_research_news, research_cn_min
 
 logger = logging.getLogger(__name__)
 
@@ -40,6 +41,25 @@ def research_limit() -> int:
     return max(1, env_int("DAILY_WECOM_AI_NEWS", 10))
 
 
+def research_pool_limit(display_limit: int | None = None) -> int:
+    """Agent 原始候选条数（展示上限之上多拉，供可信/去重筛）。"""
+    lim = display_limit if display_limit is not None else research_limit()
+    explicit = env_int("DAILY_AI_NEWS_RESEARCH_POOL", 0)
+    if explicit > 0:
+        return max(lim, explicit)
+    return max(lim * 2, lim + 8)
+
+
+def research_tech_pool_limit(display_limit: int | None = None) -> int:
+    tech = display_limit if display_limit is not None else research_tech_limit()
+    if tech <= 0:
+        return 0
+    explicit = env_int("DAILY_AI_NEWS_RESEARCH_TECH_POOL", 0)
+    if explicit > 0:
+        return max(tech, explicit)
+    return max(tech * 2, tech + 4)
+
+
 def research_json_path(date_str: str) -> Path:
     return OUTPUT_DIR / f"{date_str}.ai-news-research.json"
 
@@ -90,7 +110,7 @@ def _normalize_research_item(raw: dict[str, Any]) -> dict[str, Any] | None:
     if not title or not link or not link.startswith("http"):
         return None
     desc = brief_news_summary(str(raw.get("desc_short") or raw.get("summary") or ""))
-    return {
+    item: dict[str, Any] = {
         "title": title,
         "link": link,
         "source_name": _guess_source_name(link, str(raw.get("source_name") or "")),
@@ -98,6 +118,10 @@ def _normalize_research_item(raw: dict[str, Any]) -> dict[str, Any] | None:
         "desc_short": desc,
         "summary_plain": desc,
     }
+    region = str(raw.get("region") or "").strip().lower()
+    if region:
+        item["region"] = region
+    return item
 
 
 def research_tech_limit() -> int:
@@ -133,11 +157,17 @@ def parse_research_response(
     *,
     limit: int,
     tech_limit: int = 0,
+    pool_limit: int | None = None,
+    tech_pool_limit: int | None = None,
 ) -> tuple[list[dict[str, Any]], list[dict[str, Any]]]:
     parsed = extract_json_object(raw)
+    item_cap = pool_limit if pool_limit is not None else limit
+    tech_cap = tech_pool_limit if tech_pool_limit is not None else tech_limit
     seen: set[str] = set()
-    items = _parse_items_array(parsed.get("items"), limit=limit, seen=seen)
-    tech_items = _parse_items_array(parsed.get("tech_items"), limit=tech_limit, seen=seen) if tech_limit else []
+    items = _parse_items_array(parsed.get("items"), limit=item_cap, seen=seen)
+    tech_items = (
+        _parse_items_array(parsed.get("tech_items"), limit=tech_cap, seen=seen) if tech_cap else []
+    )
     return items, tech_items
 
 
@@ -189,25 +219,38 @@ def fetch_ai_news_research(
 
     skill = _load_skill()
     now_cst = datetime.now(timezone(timedelta(hours=8)))
+    cn_min = research_cn_min(lim)
+    pool = research_pool_limit(lim)
+    tech_pool = research_tech_pool_limit(tech_lim)
+    # 候选池内国内目标略高于展示配额，避免筛完国内不足
+    cn_pool_target = max(cn_min * 2, cn_min + 2)
+    # 解析多收一点原始行，输出前/后处理再压成「去重后候选池」
+    raw_cap = max(pool + 10, (pool * 3) // 2)
+    tech_raw_cap = max(tech_pool + 4, (tech_pool * 3) // 2) if tech_pool else 0
     tech_clause = ""
-    if tech_lim:
+    if tech_pool:
         tech_clause = (
-            f"\n另输出 **tech_items 恰好 {tech_lim} 条**，聚焦工程技术："
+            f"\n另输出 **去重后** 约 **{tech_pool} 条** tech_items 候选（最终展示约 {tech_lim} 条），聚焦工程技术："
             "模型/框架发布、开源项目、芯片算力、开发者工具、推理与工程实践。"
-            "与 items 不得重复 link。"
+            "不得与 items 重复 link/同事件；输出前自行去重，候选池内每条应为独立事件。"
         )
     system = (
         f"{skill}\n\n"
         "当前执行 **早报 AI 时讯调研**。\n"
         f"时间窗口：近 **{h}** 小时（截至 {now_cst.strftime('%Y-%m-%d %H:%M')} UTC+8）。\n"
-        f"输出 **恰好 {lim} 条** items，按重要性排序。{tech_clause}\n"
-        "使用 WebSearch 检索；不要读取本项目文档或 RSS 配置。"
+        f"输出 **同事件去重后** 约 **{pool} 条** items 候选（按重要性排序；最终展示约 {lim} 条）。\n"
+        "候选池条数 = 独立事件数：同一事件多源报道只留一条最权威源，禁止用换源重复充数。\n"
+        f"去重后的候选中国内可信源尽量不少于 **{cn_pool_target}** 条（展示侧至少 {cn_min} 条）。\n"
+        f"禁止用低质源凑数；可信独立事件不足才少返回。{tech_clause}\n"
+        "只采用官方博客/新闻稿、政府监管原文、一线权威媒体、学术官方；"
+        "禁止二手搬运、标题党、营销号。使用 WebSearch 检索；不要读取本项目文档或 RSS 配置。"
     )
     user = (
         f"/deep-research 获取近 {h} 小时的 AI 人工智能新闻资讯，"
-        "不区分国内国外，合并精选。"
-        f"只输出 JSON，items 长度={lim}"
-        + (f"，tech_items 长度={tech_lim}" if tech_lim else "")
+        f"国内与国际合并；items 去重后约 {pool} 条独立事件（国内可信尽量 ≥{cn_pool_target}）；"
+        "输出前完成同事件去重；可信度不足则不写。"
+        f"只输出 JSON，items 去重后目标约 {pool} 条"
+        + (f"，tech_items 去重后目标约 {tech_pool} 条" if tech_pool else "")
         + "。"
     )
 
@@ -234,7 +277,13 @@ def fetch_ai_news_research(
             "stats": {"error": "empty_response"},
         }
 
-    items, tech_items = parse_research_response(raw, limit=lim, tech_limit=tech_lim)
+    items, tech_items = parse_research_response(
+        raw,
+        limit=lim,
+        tech_limit=tech_lim,
+        pool_limit=raw_cap,
+        tech_pool_limit=tech_raw_cap,
+    )
     payload = extract_json_object(raw)
     if payload:
         _save_research_json(research_json_path(date_str), payload)
@@ -250,10 +299,23 @@ def fetch_ai_news_research(
             "stats": {"error": "invalid_json"},
         }
 
+    items, tech_items = post_process_research_news(
+        items,
+        tech_items,
+        limit=lim,
+        tech_limit=tech_lim,
+        min_cn=cn_min,
+    )
     items = _apply_pushed_dedup(items, date_str=date_str, limit=lim)
     if tech_items:
         tech_items = _apply_pushed_dedup(tech_items, date_str=date_str, limit=tech_lim)
-    logger.info("AI 时讯 research 完成：%d 条 + %d 技术", len(items), len(tech_items))
+    logger.info(
+        "AI 时讯 research 完成：%d 条 + %d 技术（候选池 %d/%d）",
+        len(items),
+        len(tech_items),
+        pool,
+        tech_pool,
+    )
 
     flat = [
         {

daily/news/research_quality.py

@@ -0,0 +1,442 @@
+"""Research 时讯后处理：可信源、同事件去重、tech 主题过滤、国内配额。"""
+
+from __future__ import annotations
+
+import logging
+import re
+from typing import Any
+from urllib.parse import urlparse
+
+from daily.config import env_int
+
+logger = logging.getLogger(__name__)
+
+# 官方域（同事件去重时优先保留）
+_OFFICIAL_HOST_SUFFIXES: tuple[str, ...] = (
+    "openai.com",
+    "anthropic.com",
+    "deepmind.google",
+    "blog.google",
+    "ai.googleblog.com",
+    "microsoft.com",
+    "meta.com",
+    "engineering.fb.com",
+    "nvidia.com",
+    "huggingface.co",
+    "arxiv.org",
+    "github.com",
+    "github.blog",
+    "modelcontextprotocol.io",
+    "cursor.com",
+    "vercel.com",
+    "langchain.dev",
+    "cohere.com",
+    "moonshot.cn",
+    "moonshot.ai",
+    "sktelecom.com",
+    "tether.io",
+)
+
+# 权威媒体 / 可信站（国际 + 国内）
+_TRUSTED_HOST_SUFFIXES: tuple[str, ...] = _OFFICIAL_HOST_SUFFIXES + (
+    "techcrunch.com",
+    "theverge.com",
+    "wired.com",
+    "arstechnica.com",
+    "venturebeat.com",
+    "technologyreview.com",
+    "engadget.com",
+    "cnet.com",
+    "zdnet.com",
+    "axios.com",
+    "reuters.com",
+    "bloomberg.com",
+    "bloomberglaw.com",
+    "bbc.com",
+    "bbc.co.uk",
+    "nytimes.com",
+    "wsj.com",
+    "ft.com",
+    "nbcnews.com",
+    "time.com",
+    "theguardian.com",
+    "washingtonpost.com",
+    "theregister.com",
+    "nature.com",
+    "science.org",
+    "scmp.com",
+    "caixinglobal.com",
+    "caixin.com",
+    "qbitai.com",
+    "36kr.com",
+    "leiphone.com",
+    "jiqizhixin.com",
+    "ithome.com",
+    "tmtpost.com",
+    "huxiu.com",
+    "solidot.org",
+    "synched.cn",
+    "infoq.cn",
+    "yicai.com",
+    "news.cn",
+    "xinhuanet.com",
+    "people.com.cn",
+    "cls.cn",
+    "geekpark.net",
+    "standard.com",
+    "business-standard.com",
+    "siliconvalley.com",
+)
+
+_TRUSTED_SOURCE_NAMES: frozenset[str] = frozenset(
+    {
+        "techcrunch",
+        "the verge",
+        "wired",
+        "ars technica",
+        "engadget",
+        "reuters",
+        "bloomberg",
+        "bloomberg law",
+        "nbc news",
+        "time",
+        "the register",
+        "openai",
+        "anthropic",
+        "arxiv",
+        "hugging face",
+        "mcp blog",
+        "github",
+        "sk telecom",
+        "tether",
+        "量子位",
+        "36氪",
+        "36kr",
+        "雷锋网",
+        "机器之心",
+        "it之家",
+        "财新",
+        "caixin",
+        "钛媒体",
+        "虎嗅",
+        "第一财经",
+        "新华网",
+        "新华社",
+        "财联社",
+        "极客公园",
+        "人民日报",
+    }
+)
+
+_CN_HOST_SUFFIXES: tuple[str, ...] = (
+    "qbitai.com",
+    "36kr.com",
+    "leiphone.com",
+    "jiqizhixin.com",
+    "ithome.com",
+    "caixin.com",
+    "caixinglobal.com",
+    "tmtpost.com",
+    "huxiu.com",
+    "solidot.org",
+    "synched.cn",
+    "infoq.cn",
+    "moonshot.cn",
+    "yicai.com",
+    "news.cn",
+    "xinhuanet.com",
+    "people.com.cn",
+    "cls.cn",
+    "geekpark.net",
+    "zhihu.com",
+    "sina.com.cn",
+    "qq.com",
+    "163.com",
+)
+
+_CN_SOURCE_NAMES: frozenset[str] = frozenset(
+    {
+        "量子位",
+        "36氪",
+        "36kr",
+        "雷锋网",
+        "机器之心",
+        "it之家",
+        "财新",
+        "caixin",
+        "钛媒体",
+        "虎嗅",
+        "月之暗面",
+        "第一财经",
+        "新华网",
+        "新华社",
+        "财联社",
+        "极客公园",
+        "人民日报",
+    }
+)
+
+_ENTITIES: tuple[tuple[str, tuple[str, ...]], ...] = (
+    ("openai", ("openai", "altman", "chatgpt", "奥特曼")),
+    ("anthropic", ("anthropic", "amodei", "claude")),
+    ("kimi", ("kimi", "moonshot", "月之暗面")),
+    ("mcp", ("mcp", "model context protocol", "modelcontextprotocol")),
+    ("nvidia", ("nvidia", "英伟达")),
+    ("amd", ("amd",)),
+    ("hugging_face", ("hugging face", "huggingface")),
+    ("google", ("google", "deepmind", "gemini")),
+    ("meta", ("meta", "llama")),
+    ("microsoft", ("microsoft", "copilot")),
+    ("huawei", ("huawei", "华为", "昇腾", "ascend")),
+    ("moore", ("摩尔线程", "moore threads", "musa")),
+)
+
+_EVENT_CLUSTERS: tuple[tuple[str, tuple[str, ...]], ...] = (
+    ("petition", ("petition", "联名", "decelerat", "pace ai", "控制", "减速")),
+    ("hack", ("hack", "入侵", "siege", "breach", "攻击", "逃逸")),
+    ("open_source", ("open-source", "opensource", "open sources", "开源", "open-sources")),
+    ("adapt", ("适配", "adapt", "day-0", "day0", "day 0", "推理部署", "训练适配")),
+    ("release", ("release", "发布", "specification", "规范", "v2.0", "changelog")),
+    ("chip", ("chip", "芯片", "data center", "数据中心", "mi455")),
+    ("regulate", ("framework", "监管", "voluntary", "审核", "ban", "禁止")),
+)
+
+_RELEASE_FAMILY = frozenset({"open_source", "adapt", "release"})
+# 分发平台，不参与 tech↔items 主题冲突（避免 HF 上架与 HF 被黑误杀）
+_PLATFORM_ENTITIES = frozenset({"hugging_face"})
+
+
+def _host(link: str) -> str:
+    return urlparse(link).netloc.lower().removeprefix("www.")
+
+
+def _ends_with_any(host: str, suffixes: tuple[str, ...]) -> bool:
+    return any(host == s or host.endswith("." + s) for s in suffixes)
+
+
+def _norm_text(*parts: str) -> str:
+    text = " ".join(p for p in parts if p).lower()
+    text = re.sub(r"[\s\-_/|·，。、：:（）()【】\[\]]+", " ", text)
+    return text.strip()
+
+
+def _item_text(item: dict[str, Any]) -> str:
+    return _norm_text(
+        str(item.get("title") or ""),
+        str(item.get("desc_short") or item.get("summary_plain") or ""),
+    )
+
+
+def _match_labels(text: str, table: tuple[tuple[str, tuple[str, ...]], ...]) -> frozenset[str]:
+    hit: set[str] = set()
+    for label, kws in table:
+        if any(kw in text for kw in kws):
+            hit.add(label)
+    return frozenset(hit)
+
+
+def entities_of(item: dict[str, Any]) -> frozenset[str]:
+    return _match_labels(_item_text(item), _ENTITIES)
+
+
+def events_of(item: dict[str, Any]) -> frozenset[str]:
+    return _match_labels(_item_text(item), _EVENT_CLUSTERS)
+
+
+def event_key(item: dict[str, Any]) -> tuple[frozenset[str], frozenset[str]] | None:
+    ents = entities_of(item)
+    evs = events_of(item)
+    if not ents or not evs:
+        return None
+    return (ents, evs)
+
+
+def same_event(a: dict[str, Any], b: dict[str, Any]) -> bool:
+    """共享至少一实体且共享至少一事件簇 → 同事件（保守合并）。"""
+    ea, eva = entities_of(a), events_of(a)
+    eb, evb = entities_of(b), events_of(b)
+    if not ea or not eb or not eva or not evb:
+        return False
+    return bool(ea & eb) and bool(eva & evb)
+
+
+def is_official_item(item: dict[str, Any]) -> bool:
+    return _ends_with_any(_host(str(item.get("link") or "")), _OFFICIAL_HOST_SUFFIXES)
+
+
+def _is_institutional_cn_host(host: str) -> bool:
+    """新华社 / 政府站等机构域，默认可信。"""
+    if host.endswith(".gov.cn") or host.endswith(".gov.cn."):
+        return True
+    if host == "news.cn" or host.endswith(".news.cn"):
+        return True
+    if host.endswith("xinhuanet.com") or host.endswith("people.com.cn"):
+        return True
+    return False
+
+
+def is_trusted_item(item: dict[str, Any]) -> bool:
+    host = _host(str(item.get("link") or ""))
+    if _is_institutional_cn_host(host):
+        return True
+    if _ends_with_any(host, _TRUSTED_HOST_SUFFIXES):
+        return True
+    name = str(item.get("source_name") or "").strip().lower()
+    return name in _TRUSTED_SOURCE_NAMES
+
+
+def is_cn_item(item: dict[str, Any]) -> bool:
+    region = str(item.get("region") or "").strip().lower()
+    if region in {"cn", "china", "zh", "zh-cn"}:
+        return True
+    if region in {"intl", "international", "global", "en"}:
+        return False
+    host = _host(str(item.get("link") or ""))
+    if host.endswith(".cn") or _ends_with_any(host, _CN_HOST_SUFFIXES):
+        return True
+    name = str(item.get("source_name") or "").strip().lower()
+    return name in {n.lower() for n in _CN_SOURCE_NAMES}
+
+
+def research_cn_min(limit: int) -> int:
+    explicit = env_int("DAILY_WECOM_AI_NEWS_CN_MIN", 0)
+    if explicit > 0:
+        return min(explicit, max(1, limit))
+    return max(1, limit * 3 // 10)
+
+
+def _trust_rank(item: dict[str, Any]) -> int:
+    if is_official_item(item):
+        return 0
+    if is_trusted_item(item):
+        return 1
+    return 2
+
+
+def filter_trusted(items: list[dict[str, Any]]) -> list[dict[str, Any]]:
+    kept = [i for i in items if is_trusted_item(i)]
+    dropped = len(items) - len(kept)
+    if dropped:
+        logger.info("news_dedup_drop:trusted=%s", dropped)
+    return kept
+
+
+def dedupe_same_event(items: list[dict[str, Any]]) -> list[dict[str, Any]]:
+    """同事件只留一条；官方源优先，否则保留更靠前的。"""
+    kept: list[dict[str, Any]] = []
+    for item in items:
+        replaced = False
+        for idx, prev in enumerate(kept):
+            if not same_event(item, prev):
+                continue
+            if _trust_rank(item) < _trust_rank(prev):
+                kept[idx] = item
+            replaced = True
+            break
+        if not replaced:
+            kept.append(item)
+
+    dropped = len(items) - len(kept)
+    if dropped:
+        logger.info("news_dedup_drop:same_event=%s", dropped)
+    return kept
+
+
+def filter_tech_against_items(
+    tech_items: list[dict[str, Any]],
+    items: list[dict[str, Any]],
+) -> list[dict[str, Any]]:
+    kept: list[dict[str, Any]] = []
+    for tech in tech_items:
+        t_ents = entities_of(tech)
+        t_evs = events_of(tech)
+        conflict = False
+        for item in items:
+            if same_event(tech, item):
+                conflict = True
+                break
+            shared = (t_ents & entities_of(item)) - _PLATFORM_ENTITIES
+            if shared and ((t_evs | events_of(item)) & _RELEASE_FAMILY):
+                conflict = True
+                break
+        if not conflict:
+            kept.append(tech)
+    dropped = len(tech_items) - len(kept)
+    if dropped:
+        logger.info("news_dedup_drop:tech_topic=%s", dropped)
+    return kept
+
+
+def pack_with_cn_quota(
+    items: list[dict[str, Any]],
+    *,
+    limit: int,
+    min_cn: int,
+) -> list[dict[str, Any]]:
+    if limit <= 0:
+        return []
+    min_cn = max(0, min(min_cn, limit))
+    out: list[dict[str, Any]] = []
+    used: set[str] = set()
+    cn_got = 0
+
+    def _take(item: dict[str, Any]) -> None:
+        nonlocal cn_got
+        link = str(item.get("link") or "")
+        if not link or link in used:
+            return
+        out.append(item)
+        used.add(link)
+        if is_cn_item(item):
+            cn_got += 1
+
+    for item in items:
+        if len(out) >= limit:
+            break
+        link = str(item.get("link") or "")
+        if not link or link in used:
+            continue
+        slots_left = limit - len(out)
+        need_cn = max(0, min_cn - cn_got)
+        if not is_cn_item(item) and slots_left <= need_cn:
+            continue
+        _take(item)
+
+    if len(out) < limit:
+        for item in items:
+            if len(out) >= limit:
+                break
+            _take(item)
+
+    final_cn = sum(1 for i in out if is_cn_item(i))
+    if final_cn < min_cn:
+        logger.info("news_cn_short:%s", final_cn)
+    return out[:limit]
+
+
+def build_deduped_candidate_pool(
+    items: list[dict[str, Any]],
+    tech_items: list[dict[str, Any]],
+) -> tuple[list[dict[str, Any]], list[dict[str, Any]]]:
+    """拉取后立即得到去重候选池：可信过滤 + 同事件去重 + tech 相对 items 主题过滤。"""
+    items = dedupe_same_event(filter_trusted(items))
+    tech_items = dedupe_same_event(filter_trusted(tech_items))
+    tech_items = filter_tech_against_items(tech_items, items)
+    logger.info("research_pool_deduped:items=%s tech=%s", len(items), len(tech_items))
+    return items, tech_items
+
+
+def post_process_research_news(
+    items: list[dict[str, Any]],
+    tech_items: list[dict[str, Any]],
+    *,
+    limit: int,
+    tech_limit: int,
+    min_cn: int | None = None,
+) -> tuple[list[dict[str, Any]], list[dict[str, Any]]]:
+    """去重候选池 → 国内配额打包 → 截到展示上限。"""
+    cn_min = research_cn_min(limit) if min_cn is None else max(0, min_cn)
+    items, tech_items = build_deduped_candidate_pool(items, tech_items)
+    items = pack_with_cn_quota(items, limit=limit, min_cn=cn_min)
+    tech_items = filter_tech_against_items(tech_items, items)
+    return items, tech_items[: max(0, tech_limit)]