From 391f887d738454b5c551d879ce5a40dc85a9dc18 Mon Sep 17 00:00:00 2001 From: yumao Date: Fri, 3 Jul 2026 14:53:30 +0800 Subject: [PATCH] =?UTF-8?q?feat:=20Phase=202=20=E6=96=B0=E9=97=BB=E6=89=93?= =?UTF-8?q?=E5=88=86=E5=8E=BB=E9=87=8D=E4=B8=8E=20Agent=20=E8=BE=93?= =?UTF-8?q?=E5=85=A5=E6=B1=A0=E8=A3=81=E5=89=AA?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 标题相似度合并、信源/时效/昨日重复加权排序，Agent 模式扩大 LLM 新闻候选池并记录抓取失败源。 Co-authored-by: Cursor --- .env.example | 7 ++ daily/generate.py | 4 + daily/news/fetch.py | 73 +++++++++++------ daily/news/rank.py | 171 ++++++++++++++++++++++++++++++++++++++++ daily/report_data.py | 69 ++++++++++------ tests/test_news_rank.py | 92 +++++++++++++++++++++ 6 files changed, 370 insertions(+), 46 deletions(-) create mode 100644 daily/news/rank.py create mode 100644 tests/test_news_rank.py diff --git a/.env.example b/.env.example index 6e65f8e..8d343a7 100644 --- a/.env.example +++ b/.env.example @@ -76,6 +76,13 @@ DAILY_AI_NEWS_HOURS=72 DAILY_AI_NEWS_PER_FEED=3 DAILY_AI_NEWS_PER_CATEGORY=5 +# 新闻排序 / 去重（Phase 2） +# 标题 Jaccard 相似度阈值（0-95，默认 55 表示 0.55） +# DAILY_NEWS_TITLE_SIM=55 +# Agent 模式 LLM 输入池（classic 仍用 DAILY_WECOM_* 条数） +# DAILY_AGENT_NEWS_POOL=40 +# DAILY_AGENT_CN_NEWS_POOL=30 + # Reddit RSS（403/429 时在 Reddit 偏好设置 → RSS feeds 复制 user / feed 参数） # REDDIT_RSS_USER=your_username # REDDIT_RSS_FEED=your_feed_token diff --git a/daily/generate.py b/daily/generate.py index df0a8c9..b07586f 100644 --- a/daily/generate.py +++ b/daily/generate.py @@ -49,6 +49,7 @@ from daily.news.fetch import ( prepare_wecom_cn_news_items, prepare_wecom_news_items, ) +from daily.news.rank import apply_news_ranking from daily.report_data import ( build_full_payload, build_llm_input, @@ -439,6 +440,8 @@ def generate_report() -> tuple[str, str, Path, Path]: topic_name, github_topic = fetch_topic_hot_repos(topic_fetch_n, exclude=seen_repos) ai_news = fetch_ai_news() cn_ai_news = fetch_cn_ai_news() + ai_news = apply_news_ranking(ai_news, date_str=date_str) + cn_ai_news = apply_news_ranking(cn_ai_news, date_str=date_str) wecom_limits = { "trending": wecom_trending, @@ -463,6 +466,7 @@ def generate_report() -> tuple[str, str, Path, Path]: ai_news=ai_news, cn_ai_news=cn_ai_news, wecom_limits=wecom_limits, + agent_mode=is_agent_mode(), ) save_json( data_json_path(date_str), diff --git a/daily/news/fetch.py b/daily/news/fetch.py index 45230b1..afd264a 100644 --- a/daily/news/fetch.py +++ b/daily/news/fetch.py @@ -286,7 +286,7 @@ def _fetch_one( client: httpx.Client, category: NewsCategory, feed: NewsFeed, -) -> list[dict[str, Any]]: +) -> tuple[list[dict[str, Any]], bool]: last_exc: Exception | None = None for url in _reddit_fetch_urls(feed.url): try: @@ -294,12 +294,12 @@ def _fetch_one( resp = client.get(url, headers=headers) resp.raise_for_status() entries = _parse_feed(resp.text, feed.name, category) - return _filter_ai_entries(entries, ai_filter=feed.ai_filter) + return _filter_ai_entries(entries, ai_filter=feed.ai_filter), True except Exception as exc: last_exc = exc continue logger.warning("RSS fetch failed [%s] %s: %s", feed.name, feed.url, last_exc) - return [] + return [], False def _dedupe_items(items: list[dict[str, Any]]) -> list[dict[str, Any]]: @@ -344,7 +344,12 @@ def _fetch_news(categories: tuple[NewsCategory, ...]) -> dict[str, Any]: tasks.append((category, feed)) raw_by_category: dict[str, list[dict[str, Any]]] = {c.id: [] for c in categories} - stats = {"feeds_total": len(tasks), "feeds_ok": 0, "items_raw": 0} + stats: dict[str, Any] = { + "feeds_total": len(tasks), + "feeds_ok": 0, + "items_raw": 0, + "feeds_failed": [], + } with httpx.Client(timeout=15.0, verify=certifi.where(), follow_redirects=True, headers=headers) as client: fast_tasks = [t for t in tasks if not t[1].slow] @@ -358,19 +363,24 @@ def _fetch_news(categories: tuple[NewsCategory, ...]) -> dict[str, Any]: for future in as_completed(futures): cat_id, feed_name = futures[future] try: - entries = future.result() + entries, ok = future.result() except Exception as exc: logger.warning("RSS 任务异常 [%s]: %s", feed_name, exc) + stats["feeds_failed"].append(feed_name) continue - if entries: + if ok: stats["feeds_ok"] += 1 + else: + stats["feeds_failed"].append(feed_name) stats["items_raw"] += len(entries) raw_by_category[cat_id].extend(entries[:per_feed]) for cat, feed in slow_tasks: - entries = _fetch_one(client, cat, feed) - if entries: + entries, ok = _fetch_one(client, cat, feed) + if ok: stats["feeds_ok"] += 1 + else: + stats["feeds_failed"].append(feed.name) stats["items_raw"] += len(entries) raw_by_category[cat.id].extend(entries[:per_feed]) if _is_reddit_url(feed.url): @@ -467,6 +477,12 @@ def _format_news_section( f"> 近 **{hours}h** · {news.get('stats', {}).get('feeds_ok', 0)}/{news.get('stats', {}).get('feeds_total', 0)} 源可用", "", ] + failed = news.get("stats", {}).get("feeds_failed") or [] + if failed: + preview = "、".join(failed[:5]) + suffix = "…" if len(failed) > 5 else "" + lines.append(f"> ⚠️ {len(failed)} 个源抓取失败：{preview}{suffix}") + lines.append("") if not categories: lines.append("*暂无可用条目（网络/RSS 源异常或时间窗口内无更新）。*") @@ -497,12 +513,20 @@ def _format_news_section( return lines -def prepare_wecom_news_items(news: dict[str, Any]) -> list[dict[str, Any]]: +def _sorted_flat(news: dict[str, Any]) -> list[dict[str, Any]]: + flat = list(news.get("flat") or []) + flat.sort( + key=lambda item: (float(item.get("score") or 0), _sort_key(item)[1]), + reverse=True, + ) + return flat + + +def prepare_wecom_news_items(news: dict[str, Any], *, limit: int | None = None) -> list[dict[str, Any]]: if not news.get("enabled"): return [] - limit = _wecom_limit() - flat = _dedupe_items(news.get("flat") or []) - flat.sort(key=_sort_key, reverse=True) + pick_limit = limit or _wecom_limit() + flat = _sorted_flat(news) preferred = ("media", "newsletter", "official", "community", "research", "developer") picked: list[dict[str, Any]] = [] seen: set[str] = set() @@ -513,12 +537,12 @@ def prepare_wecom_news_items(news: dict[str, Any]) -> list[dict[str, Any]]: continue picked.append(item) seen.add(link) - if len(picked) >= limit: + if len(picked) >= pick_limit: break - if len(picked) >= limit: + if len(picked) >= pick_limit: break items: list[dict[str, Any]] = [] - for item in picked[:limit]: + for item in picked[:pick_limit]: items.append( { "title": item.get("title", "?"), @@ -526,17 +550,17 @@ def prepare_wecom_news_items(news: dict[str, Any]) -> list[dict[str, Any]]: "source_name": item.get("source_name", "?"), "published_fmt": item.get("published_fmt", ""), "desc_short": _clean_text(item.get("summary", ""), 36), + "score": item.get("score"), } ) return items -def prepare_wecom_cn_news_items(news: dict[str, Any]) -> list[dict[str, Any]]: +def prepare_wecom_cn_news_items(news: dict[str, Any], *, limit: int | None = None) -> list[dict[str, Any]]: if not news.get("enabled"): return [] - limit = _wecom_cn_limit() - flat = _dedupe_items(news.get("flat") or []) - flat.sort(key=_sort_key, reverse=True) + pick_limit = limit or _wecom_cn_limit() + flat = _sorted_flat(news) preferred = ("media", "tech", "dev") picked: list[dict[str, Any]] = [] seen_links: set[str] = set() @@ -551,23 +575,23 @@ def prepare_wecom_cn_news_items(news: dict[str, Any]) -> list[dict[str, Any]]: picked.append(item) seen_links.add(link) seen_sources.add(source) - if len(picked) >= limit: + if len(picked) >= pick_limit: break - if len(picked) >= limit: + if len(picked) >= pick_limit: break - if len(picked) < limit: + if len(picked) < pick_limit: for item in flat: link = _normalize_link(item.get("link", "")) if not link or link in seen_links: continue picked.append(item) seen_links.add(link) - if len(picked) >= limit: + if len(picked) >= pick_limit: break items: list[dict[str, Any]] = [] - for item in picked[:limit]: + for item in picked[:pick_limit]: items.append( { "title": item.get("title", "?"), @@ -575,6 +599,7 @@ def prepare_wecom_cn_news_items(news: dict[str, Any]) -> list[dict[str, Any]]: "source_name": item.get("source_name", "?"), "published_fmt": item.get("published_fmt", ""), "desc_short": _clean_text(item.get("summary", ""), 36), + "score": item.get("score"), } ) return items diff --git a/daily/news/rank.py b/daily/news/rank.py new file mode 100644 index 0000000..58fe213 --- /dev/null +++ b/daily/news/rank.py @@ -0,0 +1,171 @@ +"""RSS 新闻去重、打分与排序。""" + +from __future__ import annotations + +import logging +import re +from datetime import datetime, timezone +from typing import Any + +from daily.config import env_int +from daily.delta import find_previous_data +from daily.news.fetch import _entry_datetime, _normalize_link, _normalize_title + +logger = logging.getLogger(__name__) + +_WS = re.compile(r"\s+") +_TOKEN = re.compile(r"[\w]{2,}", re.UNICODE) +_CJK = re.compile(r"[\u4e00-\u9fff]") + +CATEGORY_TIER: dict[str, int] = { + "official": 20, + "developer": 16, + "research": 14, + "media": 12, + "newsletter": 10, + "community": 8, + "tech": 12, + "dev": 10, +} + + +def title_similarity_threshold() -> float: + raw = env_int("DAILY_NEWS_TITLE_SIM", 55) + return max(0, min(raw, 95)) / 100.0 + + +def _title_tokens(title: str) -> set[str]: + normalized = _normalize_title(title) + tokens = set(_TOKEN.findall(normalized)) + cjk = "".join(_CJK.findall(title)) + for i in range(max(0, len(cjk) - 1)): + tokens.add(cjk[i : i + 2]) + if not tokens and normalized: + tokens.add(normalized) + return tokens + + +def jaccard_similarity(left: set[str], right: set[str]) -> float: + if not left or not right: + return 0.0 + inter = len(left & right) + union = len(left | right) + return inter / union if union else 0.0 + + +def fuzzy_dedupe_by_title( + items: list[dict[str, Any]], + *, + threshold: float | None = None, +) -> list[dict[str, Any]]: + """按标题相似度合并重复报道，保留 score 更高（或更靠前）的条目。""" + if not items: + return [] + limit = threshold if threshold is not None else title_similarity_threshold() + kept: list[dict[str, Any]] = [] + kept_tokens: list[set[str]] = [] + for item in items: + tokens = _title_tokens(str(item.get("title") or "")) + duplicate_idx: int | None = None + for idx, existing_tokens in enumerate(kept_tokens): + if jaccard_similarity(tokens, existing_tokens) >= limit: + duplicate_idx = idx + break + if duplicate_idx is None: + kept.append(item) + kept_tokens.append(tokens) + continue + existing = kept[duplicate_idx] + if float(item.get("score") or 0) > float(existing.get("score") or 0): + kept[duplicate_idx] = item + kept_tokens[duplicate_idx] = tokens + return kept + + +def _freshness_points(item: dict[str, Any], *, now: datetime | None = None) -> int: + now = now or datetime.now(timezone.utc) + dt = _entry_datetime(item) + if dt is None: + return 4 + age_hours = max(0.0, (now - dt).total_seconds() / 3600.0) + if age_hours <= 6: + return 30 + if age_hours <= 24: + return 22 + if age_hours <= 72: + return 8 + return 0 + + +def _source_tier(item: dict[str, Any]) -> int: + category_id = str(item.get("category_id") or "") + return CATEGORY_TIER.get(category_id, 8) + + +def _novelty_points(item: dict[str, Any], yesterday_links: set[str]) -> int: + link = _normalize_link(str(item.get("link") or "")) + if link and link in yesterday_links: + return -20 + return 5 + + +def score_news_item( + item: dict[str, Any], + *, + yesterday_links: set[str] | None = None, + now: datetime | None = None, +) -> int: + links = yesterday_links or set() + total = _source_tier(item) + _freshness_points(item, now=now) + _novelty_points(item, links) + return max(0, total) + + +def load_yesterday_news_links(date_str: str) -> set[str]: + baseline = find_previous_data(date_str) + if not baseline: + return set() + _, data = baseline + links: set[str] = set() + for key in ("ai_news", "cn_ai_news"): + for item in data.get(key) or []: + if not isinstance(item, dict): + continue + link = _normalize_link(str(item.get("link") or "")) + if link: + links.add(link) + return links + + +def apply_news_ranking(news: dict[str, Any], *, date_str: str) -> dict[str, Any]: + """对 categories / flat 打分、模糊去重并写回 score 字段。""" + if not news.get("enabled"): + return news + + yesterday_links = load_yesterday_news_links(date_str) + threshold = title_similarity_threshold() + now = datetime.now(timezone.utc) + + categories = news.get("categories") or [] + for category in categories: + items = list(category.get("items") or []) + for item in items: + item["score"] = score_news_item(item, yesterday_links=yesterday_links, now=now) + items.sort(key=lambda x: float(x.get("score") or 0), reverse=True) + category["items"] = fuzzy_dedupe_by_title(items, threshold=threshold) + + flat: list[dict[str, Any]] = [] + for category in categories: + flat.extend(category.get("items") or []) + flat.sort(key=lambda x: float(x.get("score") or 0), reverse=True) + news["flat"] = fuzzy_dedupe_by_title(flat, threshold=threshold) + + stats = dict(news.get("stats") or {}) + stats["ranked_items"] = len(news["flat"]) + stats["yesterday_links"] = len(yesterday_links) + news["stats"] = stats + logger.info( + "新闻排序完成：%d 条 flat，昨日链接基准 %d", + len(news["flat"]), + len(yesterday_links), + ) + return news diff --git a/daily/report_data.py b/daily/report_data.py index ea8f2ed..2794455 100644 --- a/daily/report_data.py +++ b/daily/report_data.py @@ -61,33 +61,47 @@ def _slim_news_items( prepare=prepare_wecom_news_items, ) -> list[dict[str, Any]]: items: list[dict[str, Any]] = [] - for item in prepare(ai_news): - items.append( - { - "link": item.get("link", ""), - "title": item.get("title", ""), - "source_name": item.get("source_name", ""), - "published_fmt": item.get("published_fmt", ""), - "summary": item.get("desc_short") or "", - } - ) + for item in prepare(ai_news, limit=limit): + payload = { + "link": item.get("link", ""), + "title": item.get("title", ""), + "source_name": item.get("source_name", ""), + "published_fmt": item.get("published_fmt", ""), + "summary": item.get("desc_short") or "", + } + if item.get("score") is not None: + payload["score"] = item.get("score") + items.append(payload) if len(items) >= limit: break if items: return items - for item in (ai_news.get("flat") or [])[:limit]: - items.append( - { - "link": item.get("link", ""), - "title": item.get("title", ""), - "source_name": item.get("source_name", ""), - "published_fmt": item.get("published_fmt", ""), - "summary": item.get("summary", ""), - } - ) + flat = sorted( + ai_news.get("flat") or [], + key=lambda row: float(row.get("score") or 0), + reverse=True, + ) + for item in flat[:limit]: + payload = { + "link": item.get("link", ""), + "title": item.get("title", ""), + "source_name": item.get("source_name", ""), + "published_fmt": item.get("published_fmt", ""), + "summary": item.get("summary", ""), + } + if item.get("score") is not None: + payload["score"] = item.get("score") + items.append(payload) return items +def _news_pool_limit(wecom_limit: int, *, env_key: str, default_pool: int, agent_mode: bool) -> int: + if not agent_mode: + return wecom_limit + pool = env_int(env_key, default_pool) + return max(wecom_limit, pool) + + def _wecom_skill_pool() -> int: return max(10, env_int("DAILY_WECOM_SKILL_POOL", 200)) @@ -105,10 +119,21 @@ def build_llm_input( ai_news: dict[str, Any], cn_ai_news: dict[str, Any], wecom_limits: dict[str, int], + agent_mode: bool = False, ) -> dict[str, Any]: """供 Cursor 编辑的精简 JSON（不含完整 markdown）。""" - news_limit = wecom_limits.get("ai_news", 10) - cn_news_limit = wecom_limits.get("cn_ai_news", 8) + news_limit = _news_pool_limit( + wecom_limits.get("ai_news", 10), + env_key="DAILY_AGENT_NEWS_POOL", + default_pool=40, + agent_mode=agent_mode, + ) + cn_news_limit = _news_pool_limit( + wecom_limits.get("cn_ai_news", 8), + env_key="DAILY_AGENT_CN_NEWS_POOL", + default_pool=30, + agent_mode=agent_mode, + ) depth = compare_depth() trend_cmp = trending[:depth] hot_cmp = hot[:depth] diff --git a/tests/test_news_rank.py b/tests/test_news_rank.py new file mode 100644 index 0000000..efc1cc4 --- /dev/null +++ b/tests/test_news_rank.py @@ -0,0 +1,92 @@ +from __future__ import annotations + +from datetime import datetime, timezone + +from daily.news.rank import ( + apply_news_ranking, + fuzzy_dedupe_by_title, + jaccard_similarity, + score_news_item, + _title_tokens, +) + + +def test_jaccard_similarity_detects_duplicate_headlines(): + left = _title_tokens("OpenAI launches GPT-5 with new reasoning") + right = _title_tokens("OpenAI launches GPT 5 with reasoning upgrade") + assert jaccard_similarity(left, right) >= 0.55 + + +def test_fuzzy_dedupe_keeps_higher_score(): + items = [ + {"title": "OpenAI launches GPT-5", "link": "https://a.example/1", "score": 20}, + {"title": "OpenAI launches GPT 5 today", "link": "https://b.example/2", "score": 45}, + ] + deduped = fuzzy_dedupe_by_title(items, threshold=0.55) + assert len(deduped) == 1 + assert deduped[0]["link"] == "https://b.example/2" + + +def test_score_news_item_prefers_fresh_official_and_penalizes_yesterday(): + now = datetime(2026, 7, 3, 12, 0, tzinfo=timezone.utc) + fresh = { + "title": "Claude update", + "link": "https://anthropic.com/news/claude", + "category_id": "official", + "published": "Thu, 03 Jul 2026 10:00:00 GMT", + } + stale_seen = { + "title": "Old story", + "link": "https://example.com/old", + "category_id": "media", + "published": "Mon, 30 Jun 2026 10:00:00 GMT", + } + fresh_score = score_news_item(fresh, yesterday_links=set(), now=now) + stale_score = score_news_item( + stale_seen, + yesterday_links={"https://example.com/old"}, + now=now, + ) + assert fresh_score > stale_score + + +def test_apply_news_ranking_sets_flat_scores(monkeypatch, tmp_path): + import daily.config as config + import daily.news.rank as rank + + out = tmp_path / "output" + out.mkdir() + monkeypatch.setattr(config, "OUTPUT_DIR", out) + monkeypatch.setattr(rank, "load_yesterday_news_links", lambda _date: set()) + + news = { + "enabled": True, + "categories": [ + { + "id": "official", + "name": "厂商官方", + "icon": "🏢", + "items": [ + { + "title": "OpenAI ships new model", + "link": "https://openai.com/a", + "category_id": "official", + "published": "Thu, 03 Jul 2026 08:00:00 GMT", + }, + { + "title": "OpenAI ships a new model today", + "link": "https://techcrunch.com/a", + "category_id": "media", + "published": "Thu, 03 Jul 2026 07:00:00 GMT", + }, + ], + } + ], + "flat": [], + "stats": {}, + } + + ranked = apply_news_ranking(news, date_str="2026-07-03") + assert ranked["flat"] + assert all("score" in item for item in ranked["flat"]) + assert len(ranked["flat"]) == 1