ftb-diy/daily-robots
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

国内AI时讯

Browse Source
This commit is contained in:
王礼
2026-07-02 15:10:37 +08:00
parent eef4c76e3f
commit 1dc2fed5fa
10 changed files with 305 additions and 37 deletions
Download Patch File Download Diff File Expand all files Collapse all files

168
daily/news/fetch.py
View File

@@ -1,4 +1,4 @@
"""抓取并整理国际 AI 时讯 RSS。"""
"""抓取并整理国际 / 国内 AI 时讯 RSS。"""
from __future__ import annotations
@@ -17,7 +17,8 @@ import certifi
import httpx
from daily.config import env, env_int, news_summary_limit
from daily.news.feeds import NEWS_CATEGORIES, NewsCategory
from daily.news.feeds import NEWS_CATEGORIES, NewsCategory, NewsFeed
from daily.news.feeds_cn import CN_AI_TITLE_KEYWORDS, CN_NEWS_CATEGORIES
logger = logging.getLogger(__name__)
@@ -39,6 +40,11 @@ def _enabled() -> bool:
return raw not in {"0", "false", "no", "off"}
def _cn_enabled() -> bool:
raw = (env("DAILY_CN_AI_NEWS") or "1").strip().lower()
return raw not in {"0", "false", "no", "off"}
def _hours_window() -> int:
return max(1, env_int("DAILY_AI_NEWS_HOURS", 72))
@@ -55,6 +61,27 @@ def _wecom_limit() -> int:
return max(1, env_int("DAILY_WECOM_AI_NEWS", 10))
def _wecom_cn_limit() -> int:
return max(1, env_int("DAILY_WECOM_CN_AI_NEWS", 8))
def _matches_cn_ai_title(title: str) -> bool:
text = title.strip()
if not text:
return False
lower = text.lower()
for keyword in CN_AI_TITLE_KEYWORDS:
if keyword.lower() in lower:
return True
return False
def _filter_ai_entries(entries: list[dict[str, Any]], *, ai_filter: bool) -> list[dict[str, Any]]:
if not ai_filter:
return entries
return [item for item in entries if _matches_cn_ai_title(item.get("title", ""))]
def _now_utc() -> datetime:
return datetime.now(timezone.utc)
@@ -255,18 +282,23 @@ def _reddit_fetch_urls(feed_url: str) -> list[str]:
return [primary, fallback]
def _fetch_one(client: httpx.Client, category: NewsCategory, feed_url: str, feed_name: str) -> list[dict[str, Any]]:
def _fetch_one(
client: httpx.Client,
category: NewsCategory,
feed: NewsFeed,
) -> list[dict[str, Any]]:
last_exc: Exception | None = None
for url in _reddit_fetch_urls(feed_url):
for url in _reddit_fetch_urls(feed.url):
try:
headers = _request_headers(dict(client.headers), url)
resp = client.get(url, headers=headers)
resp.raise_for_status()
return _parse_feed(resp.text, feed_name, category)
entries = _parse_feed(resp.text, feed.name, category)
return _filter_ai_entries(entries, ai_filter=feed.ai_filter)
except Exception as exc:
last_exc = exc
continue
logger.warning("RSS fetch failed [%s] %s: %s", feed_name, feed_url, last_exc)
logger.warning("RSS fetch failed [%s] %s: %s", feed.name, feed.url, last_exc)
return []
@@ -299,33 +331,29 @@ def _sort_key(item: dict[str, Any]) -> tuple[int, datetime]:
return (0, dt)
def fetch_ai_news() -> dict[str, Any]:
"""按类别抓取 AI 时讯，返回 {enabled, hours, categories, flat, stats}。"""
if not _enabled():
return {"enabled": False, "categories": [], "flat": [], "stats": {}}
def _fetch_news(categories: tuple[NewsCategory, ...]) -> dict[str, Any]:
hours = _hours_window()
per_feed = _per_feed_limit()
per_category = _per_category_limit()
cutoff = _now_utc() - timedelta(hours=hours)
headers = {"User-Agent": USER_AGENT, "Accept": "application/rss+xml, application/atom+xml, application/xml, text/xml, */*"}
tasks: list[tuple[NewsCategory, str, str, bool]] = []
for category in NEWS_CATEGORIES:
tasks: list[tuple[NewsCategory, NewsFeed]] = []
for category in categories:
for feed in category.feeds:
tasks.append((category, feed.url, feed.name, feed.slow))
tasks.append((category, feed))
raw_by_category: dict[str, list[dict[str, Any]]] = {c.id: [] for c in NEWS_CATEGORIES}
raw_by_category: dict[str, list[dict[str, Any]]] = {c.id: [] for c in categories}
stats = {"feeds_total": len(tasks), "feeds_ok": 0, "items_raw": 0}
with httpx.Client(timeout=15.0, verify=certifi.where(), follow_redirects=True, headers=headers) as client:
fast_tasks = [t for t in tasks if not t[3]]
slow_tasks = [t for t in tasks if t[3]]
fast_tasks = [t for t in tasks if not t[1].slow]
slow_tasks = [t for t in tasks if t[1].slow]
with ThreadPoolExecutor(max_workers=8) as pool:
futures = {
pool.submit(_fetch_one, client, cat, url, name): (cat.id, name)
for cat, url, name, _slow in fast_tasks
pool.submit(_fetch_one, client, cat, feed): (cat.id, feed.name)
for cat, feed in fast_tasks
}
for future in as_completed(futures):
cat_id, feed_name = futures[future]
@@ -339,13 +367,13 @@ def fetch_ai_news() -> dict[str, Any]:
stats["items_raw"] += len(entries)
raw_by_category[cat_id].extend(entries[:per_feed])
for cat, url, name, _slow in slow_tasks:
entries = _fetch_one(client, cat, url, name)
for cat, feed in slow_tasks:
entries = _fetch_one(client, cat, feed)
if entries:
stats["feeds_ok"] += 1
stats["items_raw"] += len(entries)
raw_by_category[cat.id].extend(entries[:per_feed])
if _is_reddit_url(url):
if _is_reddit_url(feed.url):
time.sleep(2.0)
else:
time.sleep(1.0)
@@ -353,7 +381,7 @@ def fetch_ai_news() -> dict[str, Any]:
categories_out: list[dict[str, Any]] = []
flat: list[dict[str, Any]] = []
for category in NEWS_CATEGORIES:
for category in categories:
items = raw_by_category[category.id]
items = [i for i in items if _within_window(i, cutoff)]
items.sort(key=_sort_key, reverse=True)
@@ -384,16 +412,57 @@ def fetch_ai_news() -> dict[str, Any]:
}
def fetch_ai_news() -> dict[str, Any]:
"""按类别抓取国际 AI 时讯，返回 {enabled, hours, categories, flat, stats}。"""
if not _enabled():
return {"enabled": False, "categories": [], "flat": [], "stats": {}}
return _fetch_news(NEWS_CATEGORIES)
def fetch_cn_ai_news() -> dict[str, Any]:
"""按类别抓取国内 AI 时讯，返回 {enabled, hours, categories, flat, stats}。"""
if not _cn_enabled():
return {"enabled": False, "categories": [], "flat": [], "stats": {}}
return _fetch_news(CN_NEWS_CATEGORIES)
def format_news_section(news: dict[str, Any], *, section_no: int, wecom_limit: int | None = None) -> list[str]:
return _format_news_section(
news,
section_no=section_no,
title="国际 AI 时讯",
disabled_hint="AI 时讯已关闭（`DAILY_AI_NEWS=0`）。",
wecom_limit=wecom_limit,
)
def format_cn_news_section(news: dict[str, Any], *, section_no: int, wecom_limit: int | None = None) -> list[str]:
return _format_news_section(
news,
section_no=section_no,
title="国内 AI 时讯",
disabled_hint="国内 AI 时讯已关闭（`DAILY_CN_AI_NEWS=0`）。",
wecom_limit=wecom_limit,
)
def _format_news_section(
news: dict[str, Any],
*,
section_no: int,
title: str,
disabled_hint: str,
wecom_limit: int | None = None,
) -> list[str]:
if not news.get("enabled"):
return ["---", "", f"## {section_no}、国际 AI 时讯", "", "*AI 时讯已关闭（`DAILY_AI_NEWS=0`）。*", ""]
return ["---", "", f"## {section_no}、{title}", "", f"*{disabled_hint}*", ""]
categories = news.get("categories") or []
hours = news.get("hours", 72)
lines = [
"---",
"",
f"## {section_no}、国际 AI 时讯",
f"## {section_no}、{title}",
"",
f"> 近 **{hours}h** · {news.get('stats', {}).get('feeds_ok', 0)}/{news.get('stats', {}).get('feeds_total', 0)} 源可用",
"",
@@ -460,3 +529,52 @@ def prepare_wecom_news_items(news: dict[str, Any]) -> list[dict[str, Any]]:
}
)
return items
def prepare_wecom_cn_news_items(news: dict[str, Any]) -> list[dict[str, Any]]:
if not news.get("enabled"):
return []
limit = _wecom_cn_limit()
flat = _dedupe_items(news.get("flat") or [])
flat.sort(key=_sort_key, reverse=True)
preferred = ("media", "tech", "dev")
picked: list[dict[str, Any]] = []
seen_links: set[str] = set()
seen_sources: set[str] = set()
for cat in preferred:
for item in flat:
link = _normalize_link(item.get("link", ""))
source = item.get("source_name", "?")
if item.get("category_id") != cat or not link or link in seen_links or source in seen_sources:
continue
picked.append(item)
seen_links.add(link)
seen_sources.add(source)
if len(picked) >= limit:
break
if len(picked) >= limit:
break
if len(picked) < limit:
for item in flat:
link = _normalize_link(item.get("link", ""))
if not link or link in seen_links:
continue
picked.append(item)
seen_links.add(link)
if len(picked) >= limit:
break
items: list[dict[str, Any]] = []
for item in picked[:limit]:
items.append(
{
"title": item.get("title", "?"),
"link": item.get("link", ""),
"source_name": item.get("source_name", "?"),
"published_fmt": item.get("published_fmt", ""),
"desc_short": _clean_text(item.get("summary", ""), 36),
}
)
return items