ftb-diy/daily-robots
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: Phase 2 新闻打分去重与 Agent 输入池裁剪

Browse Source 
标题相似度合并、信源/时效/昨日重复加权排序，Agent 模式扩大 LLM 新闻候选池并记录抓取失败源。

Co-authored-by: Cursor <cursoragent@cursor.com>
This commit is contained in:
王礼
2026-07-03 14:53:30 +08:00
parent 728bd1b9d1
commit 391f887d73
6 changed files with 370 additions and 46 deletions
Download Patch File Download Diff File Expand all files Collapse all files

73
daily/news/fetch.py
View File

@@ -286,7 +286,7 @@ def _fetch_one(
client: httpx.Client,
category: NewsCategory,
feed: NewsFeed,
) -> list[dict[str, Any]]:
) -> tuple[list[dict[str, Any]], bool]:
last_exc: Exception | None = None
for url in _reddit_fetch_urls(feed.url):
try:
@@ -294,12 +294,12 @@ def _fetch_one(
resp = client.get(url, headers=headers)
resp.raise_for_status()
entries = _parse_feed(resp.text, feed.name, category)
return _filter_ai_entries(entries, ai_filter=feed.ai_filter)
return _filter_ai_entries(entries, ai_filter=feed.ai_filter), True
except Exception as exc:
last_exc = exc
continue
logger.warning("RSS fetch failed [%s] %s: %s", feed.name, feed.url, last_exc)
return []
return [], False
def _dedupe_items(items: list[dict[str, Any]]) -> list[dict[str, Any]]:
@@ -344,7 +344,12 @@ def _fetch_news(categories: tuple[NewsCategory, ...]) -> dict[str, Any]:
tasks.append((category, feed))
raw_by_category: dict[str, list[dict[str, Any]]] = {c.id: [] for c in categories}
stats = {"feeds_total": len(tasks), "feeds_ok": 0, "items_raw": 0}
stats: dict[str, Any] = {
"feeds_total": len(tasks),
"feeds_ok": 0,
"items_raw": 0,
"feeds_failed": [],
}
with httpx.Client(timeout=15.0, verify=certifi.where(), follow_redirects=True, headers=headers) as client:
fast_tasks = [t for t in tasks if not t[1].slow]
@@ -358,19 +363,24 @@ def _fetch_news(categories: tuple[NewsCategory, ...]) -> dict[str, Any]:
for future in as_completed(futures):
cat_id, feed_name = futures[future]
try:
entries = future.result()
entries, ok = future.result()
except Exception as exc:
logger.warning("RSS 任务异常 [%s]: %s", feed_name, exc)
stats["feeds_failed"].append(feed_name)
continue
if entries:
if ok:
stats["feeds_ok"] += 1
else:
stats["feeds_failed"].append(feed_name)
stats["items_raw"] += len(entries)
raw_by_category[cat_id].extend(entries[:per_feed])
for cat, feed in slow_tasks:
entries = _fetch_one(client, cat, feed)
if entries:
entries, ok = _fetch_one(client, cat, feed)
if ok:
stats["feeds_ok"] += 1
else:
stats["feeds_failed"].append(feed.name)
stats["items_raw"] += len(entries)
raw_by_category[cat.id].extend(entries[:per_feed])
if _is_reddit_url(feed.url):
@@ -467,6 +477,12 @@ def _format_news_section(
f"> 近 **{hours}h** · {news.get('stats', {}).get('feeds_ok', 0)}/{news.get('stats', {}).get('feeds_total', 0)} 源可用",
"",
]
failed = news.get("stats", {}).get("feeds_failed") or []
if failed:
preview = "、".join(failed[:5])
suffix = "…" if len(failed) > 5 else ""
lines.append(f"> ⚠️ {len(failed)} 个源抓取失败：{preview}{suffix}")
lines.append("")
if not categories:
lines.append("*暂无可用条目（网络/RSS 源异常或时间窗口内无更新）。*")
@@ -497,12 +513,20 @@ def _format_news_section(
return lines
def prepare_wecom_news_items(news: dict[str, Any]) -> list[dict[str, Any]]:
def _sorted_flat(news: dict[str, Any]) -> list[dict[str, Any]]:
flat = list(news.get("flat") or [])
flat.sort(
key=lambda item: (float(item.get("score") or 0), _sort_key(item)[1]),
reverse=True,
)
return flat
def prepare_wecom_news_items(news: dict[str, Any], *, limit: int | None = None) -> list[dict[str, Any]]:
if not news.get("enabled"):
return []
limit = _wecom_limit()
flat = _dedupe_items(news.get("flat") or [])
flat.sort(key=_sort_key, reverse=True)
pick_limit = limit or _wecom_limit()
flat = _sorted_flat(news)
preferred = ("media", "newsletter", "official", "community", "research", "developer")
picked: list[dict[str, Any]] = []
seen: set[str] = set()
@@ -513,12 +537,12 @@ def prepare_wecom_news_items(news: dict[str, Any]) -> list[dict[str, Any]]:
continue
picked.append(item)
seen.add(link)
if len(picked) >= limit:
if len(picked) >= pick_limit:
break
if len(picked) >= limit:
if len(picked) >= pick_limit:
break
items: list[dict[str, Any]] = []
for item in picked[:limit]:
for item in picked[:pick_limit]:
items.append(
{
"title": item.get("title", "?"),
@@ -526,17 +550,17 @@ def prepare_wecom_news_items(news: dict[str, Any]) -> list[dict[str, Any]]:
"source_name": item.get("source_name", "?"),
"published_fmt": item.get("published_fmt", ""),
"desc_short": _clean_text(item.get("summary", ""), 36),
"score": item.get("score"),
}
)
return items
def prepare_wecom_cn_news_items(news: dict[str, Any]) -> list[dict[str, Any]]:
def prepare_wecom_cn_news_items(news: dict[str, Any], *, limit: int | None = None) -> list[dict[str, Any]]:
if not news.get("enabled"):
return []
limit = _wecom_cn_limit()
flat = _dedupe_items(news.get("flat") or [])
flat.sort(key=_sort_key, reverse=True)
pick_limit = limit or _wecom_cn_limit()
flat = _sorted_flat(news)
preferred = ("media", "tech", "dev")
picked: list[dict[str, Any]] = []
seen_links: set[str] = set()
@@ -551,23 +575,23 @@ def prepare_wecom_cn_news_items(news: dict[str, Any]) -> list[dict[str, Any]]:
picked.append(item)
seen_links.add(link)
seen_sources.add(source)
if len(picked) >= limit:
if len(picked) >= pick_limit:
break
if len(picked) >= limit:
if len(picked) >= pick_limit:
break
if len(picked) < limit:
if len(picked) < pick_limit:
for item in flat:
link = _normalize_link(item.get("link", ""))
if not link or link in seen_links:
continue
picked.append(item)
seen_links.add(link)
if len(picked) >= limit:
if len(picked) >= pick_limit:
break
items: list[dict[str, Any]] = []
for item in picked[:limit]:
for item in picked[:pick_limit]:
items.append(
{
"title": item.get("title", "?"),
@@ -575,6 +599,7 @@ def prepare_wecom_cn_news_items(news: dict[str, Any]) -> list[dict[str, Any]]:
"source_name": item.get("source_name", "?"),
"published_fmt": item.get("published_fmt", ""),
"desc_short": _clean_text(item.get("summary", ""), 36),
"score": item.get("score"),
}
)
return items

171
daily/news/rank.py Normal file
View File

@@ -0,0 +1,171 @@
"""RSS 新闻去重、打分与排序。"""
from __future__ import annotations
import logging
import re
from datetime import datetime, timezone
from typing import Any
from daily.config import env_int
from daily.delta import find_previous_data
from daily.news.fetch import _entry_datetime, _normalize_link, _normalize_title
logger = logging.getLogger(__name__)
_WS = re.compile(r"\s+")
_TOKEN = re.compile(r"[\w]{2,}", re.UNICODE)
_CJK = re.compile(r"[\u4e00-\u9fff]")
CATEGORY_TIER: dict[str, int] = {
"official": 20,
"developer": 16,
"research": 14,
"media": 12,
"newsletter": 10,
"community": 8,
"tech": 12,
"dev": 10,
}
def title_similarity_threshold() -> float:
raw = env_int("DAILY_NEWS_TITLE_SIM", 55)
return max(0, min(raw, 95)) / 100.0
def _title_tokens(title: str) -> set[str]:
normalized = _normalize_title(title)
tokens = set(_TOKEN.findall(normalized))
cjk = "".join(_CJK.findall(title))
for i in range(max(0, len(cjk) - 1)):
tokens.add(cjk[i : i + 2])
if not tokens and normalized:
tokens.add(normalized)
return tokens
def jaccard_similarity(left: set[str], right: set[str]) -> float:
if not left or not right:
return 0.0
inter = len(left & right)
union = len(left | right)
return inter / union if union else 0.0
def fuzzy_dedupe_by_title(
items: list[dict[str, Any]],
*,
threshold: float | None = None,
) -> list[dict[str, Any]]:
"""按标题相似度合并重复报道，保留 score 更高（或更靠前）的条目。"""
if not items:
return []
limit = threshold if threshold is not None else title_similarity_threshold()
kept: list[dict[str, Any]] = []
kept_tokens: list[set[str]] = []
for item in items:
tokens = _title_tokens(str(item.get("title") or ""))
duplicate_idx: int | None = None
for idx, existing_tokens in enumerate(kept_tokens):
if jaccard_similarity(tokens, existing_tokens) >= limit:
duplicate_idx = idx
break
if duplicate_idx is None:
kept.append(item)
kept_tokens.append(tokens)
continue
existing = kept[duplicate_idx]
if float(item.get("score") or 0) > float(existing.get("score") or 0):
kept[duplicate_idx] = item
kept_tokens[duplicate_idx] = tokens
return kept
def _freshness_points(item: dict[str, Any], *, now: datetime | None = None) -> int:
now = now or datetime.now(timezone.utc)
dt = _entry_datetime(item)
if dt is None:
return 4
age_hours = max(0.0, (now - dt).total_seconds() / 3600.0)
if age_hours <= 6:
return 30
if age_hours <= 24:
return 22
if age_hours <= 72:
return 8
return 0
def _source_tier(item: dict[str, Any]) -> int:
category_id = str(item.get("category_id") or "")
return CATEGORY_TIER.get(category_id, 8)
def _novelty_points(item: dict[str, Any], yesterday_links: set[str]) -> int:
link = _normalize_link(str(item.get("link") or ""))
if link and link in yesterday_links:
return -20
return 5
def score_news_item(
item: dict[str, Any],
*,
yesterday_links: set[str] | None = None,
now: datetime | None = None,
) -> int:
links = yesterday_links or set()
total = _source_tier(item) + _freshness_points(item, now=now) + _novelty_points(item, links)
return max(0, total)
def load_yesterday_news_links(date_str: str) -> set[str]:
baseline = find_previous_data(date_str)
if not baseline:
return set()
_, data = baseline
links: set[str] = set()
for key in ("ai_news", "cn_ai_news"):
for item in data.get(key) or []:
if not isinstance(item, dict):
continue
link = _normalize_link(str(item.get("link") or ""))
if link:
links.add(link)
return links
def apply_news_ranking(news: dict[str, Any], *, date_str: str) -> dict[str, Any]:
"""对 categories / flat 打分、模糊去重并写回 score 字段。"""
if not news.get("enabled"):
return news
yesterday_links = load_yesterday_news_links(date_str)
threshold = title_similarity_threshold()
now = datetime.now(timezone.utc)
categories = news.get("categories") or []
for category in categories:
items = list(category.get("items") or [])
for item in items:
item["score"] = score_news_item(item, yesterday_links=yesterday_links, now=now)
items.sort(key=lambda x: float(x.get("score") or 0), reverse=True)
category["items"] = fuzzy_dedupe_by_title(items, threshold=threshold)
flat: list[dict[str, Any]] = []
for category in categories:
flat.extend(category.get("items") or [])
flat.sort(key=lambda x: float(x.get("score") or 0), reverse=True)
news["flat"] = fuzzy_dedupe_by_title(flat, threshold=threshold)
stats = dict(news.get("stats") or {})
stats["ranked_items"] = len(news["flat"])
stats["yesterday_links"] = len(yesterday_links)
news["stats"] = stats
logger.info(
"新闻排序完成：%d 条 flat，昨日链接基准 %d",
len(news["flat"]),
len(yesterday_links),
)
return news