ftb-diy/daily-robots
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: LLM 解耦 bot、RSS 外置与内容过滤

Browse Source 
daily 自建 Cursor bridge；DAILY_LLM_PROVIDER 控制后端；config/feeds.yaml 与 sensitive_words 可配置。

Co-authored-by: Cursor <cursoragent@cursor.com>
This commit is contained in:
王礼
2026-07-03 15:24:58 +08:00
parent ce04d5a342
commit 5742dc47ed
19 changed files with 771 additions and 239 deletions
Download Patch File Download Diff File Expand all files Collapse all files

77
daily/news/content_filter.py Normal file
View File

@@ -0,0 +1,77 @@
"""敏感词内容过滤。"""
from __future__ import annotations
import logging
from functools import lru_cache
from pathlib import Path
from typing import Any
from daily.config import ROOT, env
logger = logging.getLogger(__name__)
_WORDS_FILE = ROOT / "config" / "sensitive_words.yaml"
def content_filter_enabled() -> bool:
raw = (env("DAILY_CONTENT_FILTER") or "0").strip().lower()
return raw in {"1", "true", "yes", "on"}
@lru_cache(maxsize=1)
def load_sensitive_words() -> tuple[str, ...]:
if not _WORDS_FILE.exists():
return ()
try:
import yaml
except ImportError:
logger.warning("未安装 PyYAML，无法读取 %s", _WORDS_FILE)
return ()
try:
data = yaml.safe_load(_WORDS_FILE.read_text(encoding="utf-8"))
except OSError as exc:
logger.warning("读取敏感词配置失败: %s", exc)
return ()
except Exception as exc:
logger.warning("解析 sensitive_words.yaml 失败: %s", exc)
return ()
if not isinstance(data, dict):
return ()
words = data.get("words") or data.get("sensitive_words") or []
if not isinstance(words, list):
return ()
cleaned = tuple(str(word).strip() for word in words if str(word).strip())
return cleaned
def matches_sensitive_text(text: str, words: tuple[str, ...]) -> str | None:
haystack = (text or "").lower()
if not haystack:
return None
for word in words:
needle = word.lower()
if needle and needle in haystack:
return word
return None
def filter_news_items(items: list[dict[str, Any]]) -> tuple[list[dict[str, Any]], int]:
if not content_filter_enabled():
return items, 0
words = load_sensitive_words()
if not words:
return items, 0
kept: list[dict[str, Any]] = []
removed = 0
for item in items:
text = f"{item.get('title', '')} {item.get('summary', '')}"
hit = matches_sensitive_text(text, words)
if hit:
removed += 1
continue
kept.append(item)
if removed:
logger.info("内容过滤移除 %d 条（敏感词）", removed)
return kept, removed

125
daily/news/feeds.py
View File

@@ -1,125 +1,10 @@
"""国际 AI 时讯 RSS 源定义（按类别分组）。"""
"""国际 AI 时讯 RSS 源（优先 config/feeds.yaml）。"""
from __future__ import annotations
from dataclasses import dataclass
from daily.news.feeds_loader import load_categories
from daily.news.feeds_types import NewsCategory, NewsFeed
NEWS_CATEGORIES: tuple[NewsCategory, ...] = load_categories("intl")
@dataclass(frozen=True)
class NewsFeed:
name: str
url: str
slow: bool = False # 限速源（如 Reddit）串行抓取
ai_filter: bool = False # 综合源仅保留标题命中 AI 关键词的条目
@dataclass(frozen=True)
class NewsCategory:
id: str
name: str
icon: str
feeds: tuple[NewsFeed, ...]
NEWS_CATEGORIES: tuple[NewsCategory, ...] = (
NewsCategory(
id="official",
name="厂商官方",
icon="🏢",
feeds=(
NewsFeed("Anthropic Claude 更新", "https://docs.anthropic.com/en/release-notes/feed"),
NewsFeed("OpenAI", "https://openai.com/news/rss.xml"),
NewsFeed("Google AI", "https://blog.google/technology/ai/rss/"),
NewsFeed("DeepMind", "https://deepmind.google/blog/rss.xml"),
NewsFeed("Meta Engineering", "https://engineering.fb.com/feed/"),
NewsFeed("Microsoft Research", "https://www.microsoft.com/en-us/research/feed/"),
NewsFeed("Microsoft Blog", "https://blogs.microsoft.com/feed/"),
NewsFeed("Cohere", "https://cohere.com/blog/rss.xml"),
NewsFeed("Cursor Changelog", "https://cursor.com/changelog/rss.xml"),
),
),
NewsCategory(
id="developer",
name="Agent / LLM 开发者",
icon="🛠",
feeds=(
NewsFeed("LangChain", "https://blog.langchain.dev/rss/"),
NewsFeed("Hugging Face", "https://huggingface.co/blog/feed.xml"),
NewsFeed("Vercel Changelog", "https://vercel.com/changelog/rss.xml"),
NewsFeed("GitHub Copilot", "https://github.blog/changelog/label/copilot/feed/"),
),
),
NewsCategory(
id="media",
name="综合科技媒体",
icon="📰",
feeds=(
NewsFeed("The Verge AI", "https://www.theverge.com/rss/ai-artificial-intelligence/index.xml"),
NewsFeed("TechCrunch AI", "https://techcrunch.com/category/artificial-intelligence/feed/"),
NewsFeed("Ars Technica AI", "https://arstechnica.com/ai/feed/"),
NewsFeed("Wired AI", "https://www.wired.com/feed/tag/ai/latest/rss"),
NewsFeed("MIT Tech Review", "https://www.technologyreview.com/feed/"),
NewsFeed("VentureBeat AI", "https://venturebeat.com/category/ai/feed/"),
),
),
NewsCategory(
id="newsletter",
name="Newsletter 日报",
icon="✉️",
feeds=(
NewsFeed("Ben's Bites", "https://bensbites.substack.com/feed"),
NewsFeed("The Rundown AI", "https://therundown.substack.com/feed"),
NewsFeed("Latent Space", "https://www.latent.space/feed"),
NewsFeed("Simon Willison", "https://simonwillison.net/atom/everything/"),
NewsFeed("Import AI", "https://importai.substack.com/feed"),
NewsFeed("Last Week in AI", "https://lastweekin.ai/feed"),
NewsFeed("The Neuron", "https://www.theneuron.ai/feed"),
),
),
NewsCategory(
id="research",
name="研究 / 论文",
icon="📚",
feeds=(
NewsFeed("arXiv cs.CL", "https://arxiv.org/rss/cs.CL"),
NewsFeed("arXiv cs.AI", "https://arxiv.org/rss/cs.AI"),
NewsFeed("arXiv cs.LG", "https://arxiv.org/rss/cs.LG"),
),
),
NewsCategory(
id="trending",
name="热点 / 趋势",
icon="🔥",
feeds=(
NewsFeed(
"Google News · AI",
"https://news.google.com/rss/search?q=artificial+intelligence+OR+LLM+OR+Claude+OR+GPT&hl=en-US&gl=US&ceid=US:en",
),
NewsFeed(
"Google News · Technology",
"https://news.google.com/rss/headlines/section/topic/TECHNOLOGY?hl=en-US&gl=US&ceid=US:en",
),
NewsFeed("Techmeme", "https://www.techmeme.com/feed.xml"),
NewsFeed("HN · Front Page", "https://hnrss.org/frontpage"),
NewsFeed("HN · 100+ Points", "https://hnrss.org/newest?points=100"),
NewsFeed("Dev.to · AI", "https://dev.to/feed/tag/ai"),
NewsFeed("Lobsters", "https://lobste.rs/rss"),
),
),
NewsCategory(
id="community",
name="社区讨论",
icon="💬",
feeds=(
NewsFeed(
"HN · AI/LLM/Agent",
"https://hnrss.org/newest?q=AI+OR+LLM+OR+Claude+OR+agent+OR+GPT+OR+Gemini",
),
NewsFeed(
"Reddit · LLM/Claude/ML",
"https://old.reddit.com/r/LocalLLaMA+ClaudeAI+MachineLearning+OpenAI/.rss?limit=25",
slow=True,
),
),
),
)
__all__ = ["NewsCategory", "NewsFeed", "NEWS_CATEGORIES"]

72
daily/news/feeds_cn.py
View File

@@ -1,71 +1,11 @@
"""国内 AI 时讯 RSS 源定义（按类别分组）。"""
"""国内 AI 时讯 RSS 源（优先 config/feeds.yaml）。"""
from __future__ import annotations
from daily.news.feeds import NewsCategory, NewsFeed
from daily.news.feeds_loader import load_categories, load_cn_title_keywords
from daily.news.feeds_types import NewsCategory, NewsFeed
# 综合源 ai_filter=True 时，仅保留标题命中以下词之一的条目
CN_AI_TITLE_KEYWORDS: tuple[str, ...] = (
"人工智能",
"大模型",
"智能体",
"多模态",
"AIGC",
"LLM",
"GPT",
"Claude",
"Gemini",
"ChatGPT",
"OpenAI",
"Anthropic",
"Copilot",
"Agent",
"AI ",
" AI",
"AI·",
"AI业务",
"AI模型",
"AI助手",
"AI工具",
"AI编程",
"AI 编程",
"AI版",
"AI Agent",
"推理模型",
"深度学习",
"机器学习",
"Function Calling",
)
CN_AI_TITLE_KEYWORDS: tuple[str, ...] = load_cn_title_keywords()
CN_NEWS_CATEGORIES: tuple[NewsCategory, ...] = load_categories("cn")
CN_NEWS_CATEGORIES: tuple[NewsCategory, ...] = (
NewsCategory(
id="media",
name="AI 专业媒体",
icon="📰",
feeds=(
NewsFeed("量子位", "https://www.qbitai.com/feed"),
NewsFeed("InfoQ 中文", "https://www.infoq.cn/feed/AI"),
),
),
NewsCategory(
id="tech",
name="综合科技",
icon="📱",
feeds=(
NewsFeed("36氪", "https://36kr.com/feed", ai_filter=True),
NewsFeed("雷锋网", "https://www.leiphone.com/feed"),
NewsFeed(
"Google News · AI",
"https://news.google.com/rss/search?q=人工智能+OR+大模型+OR+Agent+OR+LLM&hl=zh-CN&gl=CN&ceid=CN:zh-Hans",
),
),
),
NewsCategory(
id="dev",
name="开发者社区",
icon="💻",
feeds=(
NewsFeed("掘金", "https://juejin.cn/rss", ai_filter=True),
),
),
)
__all__ = ["CN_AI_TITLE_KEYWORDS", "CN_NEWS_CATEGORIES", "NewsCategory", "NewsFeed"]

11
daily/news/feeds_defaults.py Normal file
View File

@@ -0,0 +1,11 @@
"""Built-in RSS defaults when config/feeds.yaml is missing or invalid."""
from __future__ import annotations
from daily.news.feeds_types import NewsCategory, NewsFeed
CN_AI_TITLE_KEYWORDS: tuple[str, ...] = ('人工智能', '大模型', '智能体', '多模态', 'AIGC', 'LLM', 'GPT', 'Claude', 'Gemini', 'ChatGPT', 'OpenAI', 'Anthropic', 'Copilot', 'Agent', 'AI ', ' AI', 'AI·', 'AI业务', 'AI模型', 'AI助手', 'AI工具', 'AI编程', 'AI 编程', 'AI版', 'AI Agent', '推理模型', '深度学习', '机器学习', 'Function Calling')
CN_NEWS_CATEGORIES: tuple[NewsCategory, ...] = (NewsCategory(id='media', name='AI 专业媒体', icon='📰', feeds=(NewsFeed(name='量子位', url='https://www.qbitai.com/feed', slow=False, ai_filter=False), NewsFeed(name='InfoQ 中文', url='https://www.infoq.cn/feed/AI', slow=False, ai_filter=False))), NewsCategory(id='tech', name='综合科技', icon='📱', feeds=(NewsFeed(name='36氪', url='https://36kr.com/feed', slow=False, ai_filter=True), NewsFeed(name='雷锋网', url='https://www.leiphone.com/feed', slow=False, ai_filter=False), NewsFeed(name='Google News · AI', url='https://news.google.com/rss/search?q=人工智能+OR+大模型+OR+Agent+OR+LLM&hl=zh-CN&gl=CN&ceid=CN:zh-Hans', slow=False, ai_filter=False))), NewsCategory(id='dev', name='开发者社区', icon='💻', feeds=(NewsFeed(name='掘金', url='https://juejin.cn/rss', slow=False, ai_filter=True),)))
NEWS_CATEGORIES: tuple[NewsCategory, ...] = (NewsCategory(id='official', name='厂商官方', icon='🏢', feeds=(NewsFeed(name='Anthropic Claude 更新', url='https://docs.anthropic.com/en/release-notes/feed', slow=False, ai_filter=False), NewsFeed(name='OpenAI', url='https://openai.com/news/rss.xml', slow=False, ai_filter=False), NewsFeed(name='Google AI', url='https://blog.google/technology/ai/rss/', slow=False, ai_filter=False), NewsFeed(name='DeepMind', url='https://deepmind.google/blog/rss.xml', slow=False, ai_filter=False), NewsFeed(name='Meta Engineering', url='https://engineering.fb.com/feed/', slow=False, ai_filter=False), NewsFeed(name='Microsoft Research', url='https://www.microsoft.com/en-us/research/feed/', slow=False, ai_filter=False), NewsFeed(name='Microsoft Blog', url='https://blogs.microsoft.com/feed/', slow=False, ai_filter=False), NewsFeed(name='Cohere', url='https://cohere.com/blog/rss.xml', slow=False, ai_filter=False), NewsFeed(name='Cursor Changelog', url='https://cursor.com/changelog/rss.xml', slow=False, ai_filter=False))), NewsCategory(id='developer', name='Agent / LLM 开发者', icon='🛠', feeds=(NewsFeed(name='LangChain', url='https://blog.langchain.dev/rss/', slow=False, ai_filter=False), NewsFeed(name='Hugging Face', url='https://huggingface.co/blog/feed.xml', slow=False, ai_filter=False), NewsFeed(name='Vercel Changelog', url='https://vercel.com/changelog/rss.xml', slow=False, ai_filter=False), NewsFeed(name='GitHub Copilot', url='https://github.blog/changelog/label/copilot/feed/', slow=False, ai_filter=False))), NewsCategory(id='media', name='综合科技媒体', icon='📰', feeds=(NewsFeed(name='The Verge AI', url='https://www.theverge.com/rss/ai-artificial-intelligence/index.xml', slow=False, ai_filter=False), NewsFeed(name='TechCrunch AI', url='https://techcrunch.com/category/artificial-intelligence/feed/', slow=False, ai_filter=False), NewsFeed(name='Ars Technica AI', url='https://arstechnica.com/ai/feed/', slow=False, ai_filter=False), NewsFeed(name='Wired AI', url='https://www.wired.com/feed/tag/ai/latest/rss', slow=False, ai_filter=False), NewsFeed(name='MIT Tech Review', url='https://www.technologyreview.com/feed/', slow=False, ai_filter=False), NewsFeed(name='VentureBeat AI', url='https://venturebeat.com/category/ai/feed/', slow=False, ai_filter=False))), NewsCategory(id='newsletter', name='Newsletter 日报', icon='✉️', feeds=(NewsFeed(name="Ben's Bites", url='https://bensbites.substack.com/feed', slow=False, ai_filter=False), NewsFeed(name='The Rundown AI', url='https://therundown.substack.com/feed', slow=False, ai_filter=False), NewsFeed(name='Latent Space', url='https://www.latent.space/feed', slow=False, ai_filter=False), NewsFeed(name='Simon Willison', url='https://simonwillison.net/atom/everything/', slow=False, ai_filter=False), NewsFeed(name='Import AI', url='https://importai.substack.com/feed', slow=False, ai_filter=False), NewsFeed(name='Last Week in AI', url='https://lastweekin.ai/feed', slow=False, ai_filter=False), NewsFeed(name='The Neuron', url='https://www.theneuron.ai/feed', slow=False, ai_filter=False))), NewsCategory(id='research', name='研究 / 论文', icon='📚', feeds=(NewsFeed(name='arXiv cs.CL', url='https://arxiv.org/rss/cs.CL', slow=False, ai_filter=False), NewsFeed(name='arXiv cs.AI', url='https://arxiv.org/rss/cs.AI', slow=False, ai_filter=False), NewsFeed(name='arXiv cs.LG', url='https://arxiv.org/rss/cs.LG', slow=False, ai_filter=False))), NewsCategory(id='trending', name='热点 / 趋势', icon='🔥', feeds=(NewsFeed(name='Google News · AI', url='https://news.google.com/rss/search?q=artificial+intelligence+OR+LLM+OR+Claude+OR+GPT&hl=en-US&gl=US&ceid=US:en', slow=False, ai_filter=False), NewsFeed(name='Google News · Technology', url='https://news.google.com/rss/headlines/section/topic/TECHNOLOGY?hl=en-US&gl=US&ceid=US:en', slow=False, ai_filter=False), NewsFeed(name='Techmeme', url='https://www.techmeme.com/feed.xml', slow=False, ai_filter=False), NewsFeed(name='HN · Front Page', url='https://hnrss.org/frontpage', slow=False, ai_filter=False), NewsFeed(name='HN · 100+ Points', url='https://hnrss.org/newest?points=100', slow=False, ai_filter=False), NewsFeed(name='Dev.to · AI', url='https://dev.to/feed/tag/ai', slow=False, ai_filter=False), NewsFeed(name='Lobsters', url='https://lobste.rs/rss', slow=False, ai_filter=False))), NewsCategory(id='community', name='社区讨论', icon='💬', feeds=(NewsFeed(name='HN · AI/LLM/Agent', url='https://hnrss.org/newest?q=AI+OR+LLM+OR+Claude+OR+agent+OR+GPT+OR+Gemini', slow=False, ai_filter=False), NewsFeed(name='Reddit · LLM/Claude/ML', url='https://old.reddit.com/r/LocalLLaMA+ClaudeAI+MachineLearning+OpenAI/.rss?limit=25', slow=True, ai_filter=False))))

113
daily/news/feeds_loader.py Normal file
View File

@@ -0,0 +1,113 @@
"""从 config/feeds.yaml 加载 RSS 源（失败时回退内置默认）。"""
from __future__ import annotations
import logging
from functools import lru_cache
from pathlib import Path
from typing import Any
from daily.config import ROOT
from daily.news.feeds_types import NewsCategory, NewsFeed
logger = logging.getLogger(__name__)
_FEEDS_FILE = ROOT / "config" / "feeds.yaml"
def _parse_feed(raw: dict[str, Any]) -> NewsFeed:
return NewsFeed(
name=str(raw.get("name") or "").strip(),
url=str(raw.get("url") or "").strip(),
slow=bool(raw.get("slow")),
ai_filter=bool(raw.get("ai_filter")),
)
def _parse_category(raw: dict[str, Any]) -> NewsCategory | None:
cat_id = str(raw.get("id") or "").strip()
if not cat_id:
return None
feeds_raw = raw.get("feeds") or []
feeds: list[NewsFeed] = []
for item in feeds_raw:
if not isinstance(item, dict):
continue
feed = _parse_feed(item)
if feed.name and feed.url:
feeds.append(feed)
if not feeds:
return None
return NewsCategory(
id=cat_id,
name=str(raw.get("name") or cat_id),
icon=str(raw.get("icon") or "📰"),
feeds=tuple(feeds),
)
def _parse_categories(items: Any) -> tuple[NewsCategory, ...]:
if not isinstance(items, list):
return ()
categories: list[NewsCategory] = []
for raw in items:
if not isinstance(raw, dict):
continue
cat = _parse_category(raw)
if cat:
categories.append(cat)
return tuple(categories)
def _load_yaml() -> dict[str, Any] | None:
if not _FEEDS_FILE.exists():
return None
try:
import yaml
except ImportError:
logger.warning("未安装 PyYAML，无法读取 %s", _FEEDS_FILE)
return None
try:
data = yaml.safe_load(_FEEDS_FILE.read_text(encoding="utf-8"))
except OSError as exc:
logger.warning("读取 feeds 配置失败: %s", exc)
return None
except Exception as exc:
logger.warning("解析 feeds.yaml 失败: %s", exc)
return None
return data if isinstance(data, dict) else None
@lru_cache(maxsize=1)
def _yaml_payload() -> dict[str, Any] | None:
return _load_yaml()
def load_categories(region: str) -> tuple[NewsCategory, ...]:
data = _yaml_payload()
if data:
block = data.get(region) or {}
categories = _parse_categories(block.get("categories"))
if categories:
return categories
logger.warning("feeds.yaml 中 %s.categories 为空，使用内置默认", region)
from daily.news import feeds_defaults as defaults
if region == "cn":
return defaults.CN_NEWS_CATEGORIES
return defaults.NEWS_CATEGORIES
def load_cn_title_keywords() -> tuple[str, ...]:
data = _yaml_payload()
if data:
block = data.get("cn") or {}
keywords = block.get("title_keywords")
if isinstance(keywords, list):
cleaned = tuple(str(x).strip() for x in keywords if str(x).strip())
if cleaned:
return cleaned
from daily.news.feeds_defaults import CN_AI_TITLE_KEYWORDS
return CN_AI_TITLE_KEYWORDS

21
daily/news/feeds_types.py Normal file
View File

@@ -0,0 +1,21 @@
"""RSS 源数据结构。"""
from __future__ import annotations
from dataclasses import dataclass
@dataclass(frozen=True)
class NewsFeed:
name: str
url: str
slow: bool = False
ai_filter: bool = False
@dataclass(frozen=True)
class NewsCategory:
id: str
name: str
icon: str
feeds: tuple[NewsFeed, ...]

5
daily/news/fetch.py
View File

@@ -17,6 +17,7 @@ import certifi
import httpx
from daily.config import env, env_int, news_summary_limit
from daily.news.content_filter import filter_news_items
from daily.news.feeds import NEWS_CATEGORIES, NewsCategory, NewsFeed
from daily.news.feeds_cn import CN_AI_TITLE_KEYWORDS, CN_NEWS_CATEGORIES
@@ -349,6 +350,7 @@ def _fetch_news(categories: tuple[NewsCategory, ...]) -> dict[str, Any]:
"feeds_ok": 0,
"items_raw": 0,
"feeds_failed": [],
"content_filtered": 0,
}
with httpx.Client(timeout=15.0, verify=certifi.where(), follow_redirects=True, headers=headers) as client:
@@ -394,6 +396,9 @@ def _fetch_news(categories: tuple[NewsCategory, ...]) -> dict[str, Any]:
for category in categories:
items = raw_by_category[category.id]
items = [i for i in items if _within_window(i, cutoff)]
items, filtered_count = filter_news_items(items)
if filtered_count:
stats["content_filtered"] = stats.get("content_filtered", 0) + filtered_count
items.sort(key=_sort_key, reverse=True)
items = _dedupe_items(items)[:per_category]
for item in items: