daily-robots/daily/news/research_quality.py

"""Research 时讯后处理：可信源、同事件去重、tech 主题过滤、国内配额。"""

from __future__ import annotations

import logging
import re
from typing import Any
from urllib.parse import urlparse

from daily.config import env_int

logger = logging.getLogger(__name__)

# 官方域（同事件去重时优先保留）
_OFFICIAL_HOST_SUFFIXES: tuple[str, ...] = (
    "openai.com",
    "anthropic.com",
    "deepmind.google",
    "blog.google",
    "ai.googleblog.com",
    "microsoft.com",
    "meta.com",
    "engineering.fb.com",
    "nvidia.com",
    "huggingface.co",
    "arxiv.org",
    "github.com",
    "github.blog",
    "modelcontextprotocol.io",
    "cursor.com",
    "vercel.com",
    "langchain.dev",
    "cohere.com",
    "moonshot.cn",
    "moonshot.ai",
    "sktelecom.com",
    "tether.io",
)

# 权威媒体 / 可信站（国际 + 国内）
_TRUSTED_HOST_SUFFIXES: tuple[str, ...] = _OFFICIAL_HOST_SUFFIXES + (
    "techcrunch.com",
    "theverge.com",
    "wired.com",
    "arstechnica.com",
    "venturebeat.com",
    "technologyreview.com",
    "engadget.com",
    "cnet.com",
    "zdnet.com",
    "axios.com",
    "reuters.com",
    "bloomberg.com",
    "bloomberglaw.com",
    "bbc.com",
    "bbc.co.uk",
    "nytimes.com",
    "wsj.com",
    "ft.com",
    "nbcnews.com",
    "time.com",
    "theguardian.com",
    "washingtonpost.com",
    "theregister.com",
    "nature.com",
    "science.org",
    "scmp.com",
    "caixinglobal.com",
    "caixin.com",
    "qbitai.com",
    "36kr.com",
    "leiphone.com",
    "jiqizhixin.com",
    "ithome.com",
    "tmtpost.com",
    "huxiu.com",
    "solidot.org",
    "synched.cn",
    "infoq.cn",
    "yicai.com",
    "news.cn",
    "xinhuanet.com",
    "people.com.cn",
    "cls.cn",
    "geekpark.net",
    "standard.com",
    "business-standard.com",
    "siliconvalley.com",
)

_TRUSTED_SOURCE_NAMES: frozenset[str] = frozenset(
    {
        "techcrunch",
        "the verge",
        "wired",
        "ars technica",
        "engadget",
        "reuters",
        "bloomberg",
        "bloomberg law",
        "nbc news",
        "time",
        "the register",
        "openai",
        "anthropic",
        "arxiv",
        "hugging face",
        "mcp blog",
        "github",
        "sk telecom",
        "tether",
        "量子位",
        "36氪",
        "36kr",
        "雷锋网",
        "机器之心",
        "it之家",
        "财新",
        "caixin",
        "钛媒体",
        "虎嗅",
        "第一财经",
        "新华网",
        "新华社",
        "财联社",
        "极客公园",
        "人民日报",
    }
)

_CN_HOST_SUFFIXES: tuple[str, ...] = (
    "qbitai.com",
    "36kr.com",
    "leiphone.com",
    "jiqizhixin.com",
    "ithome.com",
    "caixin.com",
    "caixinglobal.com",
    "tmtpost.com",
    "huxiu.com",
    "solidot.org",
    "synched.cn",
    "infoq.cn",
    "moonshot.cn",
    "yicai.com",
    "news.cn",
    "xinhuanet.com",
    "people.com.cn",
    "cls.cn",
    "geekpark.net",
    "zhihu.com",
    "sina.com.cn",
    "qq.com",
    "163.com",
)

_CN_SOURCE_NAMES: frozenset[str] = frozenset(
    {
        "量子位",
        "36氪",
        "36kr",
        "雷锋网",
        "机器之心",
        "it之家",
        "财新",
        "caixin",
        "钛媒体",
        "虎嗅",
        "月之暗面",
        "第一财经",
        "新华网",
        "新华社",
        "财联社",
        "极客公园",
        "人民日报",
    }
)

_ENTITIES: tuple[tuple[str, tuple[str, ...]], ...] = (
    ("openai", ("openai", "altman", "chatgpt", "奥特曼")),
    ("anthropic", ("anthropic", "amodei", "claude")),
    ("kimi", ("kimi", "moonshot", "月之暗面")),
    ("mcp", ("mcp", "model context protocol", "modelcontextprotocol")),
    ("nvidia", ("nvidia", "英伟达")),
    ("amd", ("amd",)),
    ("hugging_face", ("hugging face", "huggingface")),
    ("google", ("google", "deepmind", "gemini")),
    ("meta", ("meta", "llama")),
    ("microsoft", ("microsoft", "copilot")),
    ("huawei", ("huawei", "华为", "昇腾", "ascend")),
    ("moore", ("摩尔线程", "moore threads", "musa")),
)

_EVENT_CLUSTERS: tuple[tuple[str, tuple[str, ...]], ...] = (
    ("petition", ("petition", "联名", "decelerat", "pace ai", "控制", "减速")),
    ("hack", ("hack", "入侵", "siege", "breach", "攻击", "逃逸")),
    ("open_source", ("open-source", "opensource", "open sources", "开源", "open-sources")),
    ("adapt", ("适配", "adapt", "day-0", "day0", "day 0", "推理部署", "训练适配")),
    ("release", ("release", "发布", "specification", "规范", "v2.0", "changelog")),
    ("chip", ("chip", "芯片", "data center", "数据中心", "mi455")),
    ("regulate", ("framework", "监管", "voluntary", "审核", "ban", "禁止")),
)

_RELEASE_FAMILY = frozenset({"open_source", "adapt", "release"})
# 分发平台，不参与 tech↔items 主题冲突（避免 HF 上架与 HF 被黑误杀）
_PLATFORM_ENTITIES = frozenset({"hugging_face"})


def _host(link: str) -> str:
    return urlparse(link).netloc.lower().removeprefix("www.")


def _ends_with_any(host: str, suffixes: tuple[str, ...]) -> bool:
    return any(host == s or host.endswith("." + s) for s in suffixes)


def _norm_text(*parts: str) -> str:
    text = " ".join(p for p in parts if p).lower()
    text = re.sub(r"[\s\-_/|·，。、：:（）()【】\[\]]+", " ", text)
    return text.strip()


def _item_text(item: dict[str, Any]) -> str:
    return _norm_text(
        str(item.get("title") or ""),
        str(item.get("desc_short") or item.get("summary_plain") or ""),
    )


def _match_labels(text: str, table: tuple[tuple[str, tuple[str, ...]], ...]) -> frozenset[str]:
    hit: set[str] = set()
    for label, kws in table:
        if any(kw in text for kw in kws):
            hit.add(label)
    return frozenset(hit)


def entities_of(item: dict[str, Any]) -> frozenset[str]:
    return _match_labels(_item_text(item), _ENTITIES)


def events_of(item: dict[str, Any]) -> frozenset[str]:
    return _match_labels(_item_text(item), _EVENT_CLUSTERS)


def event_key(item: dict[str, Any]) -> tuple[frozenset[str], frozenset[str]] | None:
    ents = entities_of(item)
    evs = events_of(item)
    if not ents or not evs:
        return None
    return (ents, evs)


def same_event(a: dict[str, Any], b: dict[str, Any]) -> bool:
    """共享至少一实体且共享至少一事件簇 → 同事件（保守合并）。"""
    ea, eva = entities_of(a), events_of(a)
    eb, evb = entities_of(b), events_of(b)
    if not ea or not eb or not eva or not evb:
        return False
    return bool(ea & eb) and bool(eva & evb)


def is_official_item(item: dict[str, Any]) -> bool:
    return _ends_with_any(_host(str(item.get("link") or "")), _OFFICIAL_HOST_SUFFIXES)


def _is_institutional_cn_host(host: str) -> bool:
    """新华社 / 政府站等机构域，默认可信。"""
    if host.endswith(".gov.cn") or host.endswith(".gov.cn."):
        return True
    if host == "news.cn" or host.endswith(".news.cn"):
        return True
    if host.endswith("xinhuanet.com") or host.endswith("people.com.cn"):
        return True
    return False


def is_trusted_item(item: dict[str, Any]) -> bool:
    host = _host(str(item.get("link") or ""))
    if _is_institutional_cn_host(host):
        return True
    if _ends_with_any(host, _TRUSTED_HOST_SUFFIXES):
        return True
    name = str(item.get("source_name") or "").strip().lower()
    return name in _TRUSTED_SOURCE_NAMES


def is_cn_item(item: dict[str, Any]) -> bool:
    region = str(item.get("region") or "").strip().lower()
    if region in {"cn", "china", "zh", "zh-cn"}:
        return True
    if region in {"intl", "international", "global", "en"}:
        return False
    host = _host(str(item.get("link") or ""))
    if host.endswith(".cn") or _ends_with_any(host, _CN_HOST_SUFFIXES):
        return True
    name = str(item.get("source_name") or "").strip().lower()
    return name in {n.lower() for n in _CN_SOURCE_NAMES}


def research_cn_min(limit: int) -> int:
    explicit = env_int("DAILY_WECOM_AI_NEWS_CN_MIN", 0)
    if explicit > 0:
        return min(explicit, max(1, limit))
    return max(1, limit * 3 // 10)


def _trust_rank(item: dict[str, Any]) -> int:
    if is_official_item(item):
        return 0
    if is_trusted_item(item):
        return 1
    return 2


def filter_trusted(items: list[dict[str, Any]]) -> list[dict[str, Any]]:
    kept = [i for i in items if is_trusted_item(i)]
    dropped = len(items) - len(kept)
    if dropped:
        logger.info("news_dedup_drop:trusted=%s", dropped)
    return kept


def dedupe_same_event(items: list[dict[str, Any]]) -> list[dict[str, Any]]:
    """同事件只留一条；官方源优先，否则保留更靠前的。"""
    kept: list[dict[str, Any]] = []
    for item in items:
        replaced = False
        for idx, prev in enumerate(kept):
            if not same_event(item, prev):
                continue
            if _trust_rank(item) < _trust_rank(prev):
                kept[idx] = item
            replaced = True
            break
        if not replaced:
            kept.append(item)

    dropped = len(items) - len(kept)
    if dropped:
        logger.info("news_dedup_drop:same_event=%s", dropped)
    return kept


def filter_tech_against_items(
    tech_items: list[dict[str, Any]],
    items: list[dict[str, Any]],
) -> list[dict[str, Any]]:
    kept: list[dict[str, Any]] = []
    for tech in tech_items:
        t_ents = entities_of(tech)
        t_evs = events_of(tech)
        conflict = False
        for item in items:
            if same_event(tech, item):
                conflict = True
                break
            shared = (t_ents & entities_of(item)) - _PLATFORM_ENTITIES
            if shared and ((t_evs | events_of(item)) & _RELEASE_FAMILY):
                conflict = True
                break
        if not conflict:
            kept.append(tech)
    dropped = len(tech_items) - len(kept)
    if dropped:
        logger.info("news_dedup_drop:tech_topic=%s", dropped)
    return kept


def pack_with_cn_quota(
    items: list[dict[str, Any]],
    *,
    limit: int,
    min_cn: int,
) -> list[dict[str, Any]]:
    if limit <= 0:
        return []
    min_cn = max(0, min(min_cn, limit))
    out: list[dict[str, Any]] = []
    used: set[str] = set()
    cn_got = 0

    def _take(item: dict[str, Any]) -> None:
        nonlocal cn_got
        link = str(item.get("link") or "")
        if not link or link in used:
            return
        out.append(item)
        used.add(link)
        if is_cn_item(item):
            cn_got += 1

    for item in items:
        if len(out) >= limit:
            break
        link = str(item.get("link") or "")
        if not link or link in used:
            continue
        slots_left = limit - len(out)
        need_cn = max(0, min_cn - cn_got)
        if not is_cn_item(item) and slots_left <= need_cn:
            continue
        _take(item)

    if len(out) < limit:
        for item in items:
            if len(out) >= limit:
                break
            _take(item)

    final_cn = sum(1 for i in out if is_cn_item(i))
    if final_cn < min_cn:
        logger.info("news_cn_short:%s", final_cn)
    return out[:limit]


def build_deduped_candidate_pool(
    items: list[dict[str, Any]],
    tech_items: list[dict[str, Any]],
) -> tuple[list[dict[str, Any]], list[dict[str, Any]]]:
    """拉取后立即得到去重候选池：可信过滤 + 同事件去重 + tech 相对 items 主题过滤。"""
    items = dedupe_same_event(filter_trusted(items))
    tech_items = dedupe_same_event(filter_trusted(tech_items))
    tech_items = filter_tech_against_items(tech_items, items)
    logger.info("research_pool_deduped:items=%s tech=%s", len(items), len(tech_items))
    return items, tech_items


def post_process_research_news(
    items: list[dict[str, Any]],
    tech_items: list[dict[str, Any]],
    *,
    limit: int,
    tech_limit: int,
    min_cn: int | None = None,
) -> tuple[list[dict[str, Any]], list[dict[str, Any]]]:
    """去重候选池 → 国内配额打包 → 截到展示上限。"""
    cn_min = research_cn_min(limit) if min_cn is None else max(0, min_cn)
    items, tech_items = build_deduped_candidate_pool(items, tech_items)
    items = pack_with_cn_quota(items, limit=limit, min_cn=cn_min)
    tech_items = filter_tech_against_items(tech_items, items)
    return items, tech_items[: max(0, tech_limit)]