ftb-diy/daily-robots
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: Phase 2 新闻打分去重与 Agent 输入池裁剪

Browse Source 
标题相似度合并、信源/时效/昨日重复加权排序，Agent 模式扩大 LLM 新闻候选池并记录抓取失败源。

Co-authored-by: Cursor <cursoragent@cursor.com>
This commit is contained in:
王礼
2026-07-03 14:53:30 +08:00
parent 728bd1b9d1
commit 391f887d73
6 changed files with 370 additions and 46 deletions
Download Patch File Download Diff File Expand all files Collapse all files

73
daily/news/fetch.py
View File

@@ -286,7 +286,7 @@ def _fetch_one(
client: httpx.Client,
category: NewsCategory,
feed: NewsFeed,
) -> list[dict[str, Any]]:
) -> tuple[list[dict[str, Any]], bool]:
last_exc: Exception | None = None
for url in _reddit_fetch_urls(feed.url):
try:
@@ -294,12 +294,12 @@ def _fetch_one(
resp = client.get(url, headers=headers)
resp.raise_for_status()
entries = _parse_feed(resp.text, feed.name, category)
return _filter_ai_entries(entries, ai_filter=feed.ai_filter)
return _filter_ai_entries(entries, ai_filter=feed.ai_filter), True
except Exception as exc:
last_exc = exc
continue
logger.warning("RSS fetch failed [%s] %s: %s", feed.name, feed.url, last_exc)
return []
return [], False
def _dedupe_items(items: list[dict[str, Any]]) -> list[dict[str, Any]]:
@@ -344,7 +344,12 @@ def _fetch_news(categories: tuple[NewsCategory, ...]) -> dict[str, Any]:
tasks.append((category, feed))
raw_by_category: dict[str, list[dict[str, Any]]] = {c.id: [] for c in categories}
stats = {"feeds_total": len(tasks), "feeds_ok": 0, "items_raw": 0}
stats: dict[str, Any] = {
"feeds_total": len(tasks),
"feeds_ok": 0,
"items_raw": 0,
"feeds_failed": [],
}
with httpx.Client(timeout=15.0, verify=certifi.where(), follow_redirects=True, headers=headers) as client:
fast_tasks = [t for t in tasks if not t[1].slow]
@@ -358,19 +363,24 @@ def _fetch_news(categories: tuple[NewsCategory, ...]) -> dict[str, Any]:
for future in as_completed(futures):
cat_id, feed_name = futures[future]
try:
entries = future.result()
entries, ok = future.result()
except Exception as exc:
logger.warning("RSS 任务异常 [%s]: %s", feed_name, exc)
stats["feeds_failed"].append(feed_name)
continue
if entries:
if ok:
stats["feeds_ok"] += 1
else:
stats["feeds_failed"].append(feed_name)
stats["items_raw"] += len(entries)
raw_by_category[cat_id].extend(entries[:per_feed])
for cat, feed in slow_tasks:
entries = _fetch_one(client, cat, feed)
if entries:
entries, ok = _fetch_one(client, cat, feed)
if ok:
stats["feeds_ok"] += 1
else:
stats["feeds_failed"].append(feed.name)
stats["items_raw"] += len(entries)
raw_by_category[cat.id].extend(entries[:per_feed])
if _is_reddit_url(feed.url):
@@ -467,6 +477,12 @@ def _format_news_section(
f"> 近 **{hours}h** · {news.get('stats', {}).get('feeds_ok', 0)}/{news.get('stats', {}).get('feeds_total', 0)} 源可用",
"",
]
failed = news.get("stats", {}).get("feeds_failed") or []
if failed:
preview = "、".join(failed[:5])
suffix = "…" if len(failed) > 5 else ""
lines.append(f"> ⚠️ {len(failed)} 个源抓取失败：{preview}{suffix}")
lines.append("")
if not categories:
lines.append("*暂无可用条目（网络/RSS 源异常或时间窗口内无更新）。*")
@@ -497,12 +513,20 @@ def _format_news_section(
return lines
def prepare_wecom_news_items(news: dict[str, Any]) -> list[dict[str, Any]]:
def _sorted_flat(news: dict[str, Any]) -> list[dict[str, Any]]:
flat = list(news.get("flat") or [])
flat.sort(
key=lambda item: (float(item.get("score") or 0), _sort_key(item)[1]),
reverse=True,
)
return flat
def prepare_wecom_news_items(news: dict[str, Any], *, limit: int | None = None) -> list[dict[str, Any]]:
if not news.get("enabled"):
return []
limit = _wecom_limit()
flat = _dedupe_items(news.get("flat") or [])
flat.sort(key=_sort_key, reverse=True)
pick_limit = limit or _wecom_limit()
flat = _sorted_flat(news)
preferred = ("media", "newsletter", "official", "community", "research", "developer")
picked: list[dict[str, Any]] = []
seen: set[str] = set()
@@ -513,12 +537,12 @@ def prepare_wecom_news_items(news: dict[str, Any]) -> list[dict[str, Any]]:
continue
picked.append(item)
seen.add(link)
if len(picked) >= limit:
if len(picked) >= pick_limit:
break
if len(picked) >= limit:
if len(picked) >= pick_limit:
break
items: list[dict[str, Any]] = []
for item in picked[:limit]:
for item in picked[:pick_limit]:
items.append(
{
"title": item.get("title", "?"),
@@ -526,17 +550,17 @@ def prepare_wecom_news_items(news: dict[str, Any]) -> list[dict[str, Any]]:
"source_name": item.get("source_name", "?"),
"published_fmt": item.get("published_fmt", ""),
"desc_short": _clean_text(item.get("summary", ""), 36),
"score": item.get("score"),
}
)
return items
def prepare_wecom_cn_news_items(news: dict[str, Any]) -> list[dict[str, Any]]:
def prepare_wecom_cn_news_items(news: dict[str, Any], *, limit: int | None = None) -> list[dict[str, Any]]:
if not news.get("enabled"):
return []
limit = _wecom_cn_limit()
flat = _dedupe_items(news.get("flat") or [])
flat.sort(key=_sort_key, reverse=True)
pick_limit = limit or _wecom_cn_limit()
flat = _sorted_flat(news)
preferred = ("media", "tech", "dev")
picked: list[dict[str, Any]] = []
seen_links: set[str] = set()
@@ -551,23 +575,23 @@ def prepare_wecom_cn_news_items(news: dict[str, Any]) -> list[dict[str, Any]]:
picked.append(item)
seen_links.add(link)
seen_sources.add(source)
if len(picked) >= limit:
if len(picked) >= pick_limit:
break
if len(picked) >= limit:
if len(picked) >= pick_limit:
break
if len(picked) < limit:
if len(picked) < pick_limit:
for item in flat:
link = _normalize_link(item.get("link", ""))
if not link or link in seen_links:
continue
picked.append(item)
seen_links.add(link)
if len(picked) >= limit:
if len(picked) >= pick_limit:
break
items: list[dict[str, Any]] = []
for item in picked[:limit]:
for item in picked[:pick_limit]:
items.append(
{
"title": item.get("title", "?"),
@@ -575,6 +599,7 @@ def prepare_wecom_cn_news_items(news: dict[str, Any]) -> list[dict[str, Any]]:
"source_name": item.get("source_name", "?"),
"published_fmt": item.get("published_fmt", ""),
"desc_short": _clean_text(item.get("summary", ""), 36),
"score": item.get("score"),
}
)
return items