diff --git a/.env.example b/.env.example index 8d343a7..d9c6169 100644 --- a/.env.example +++ b/.env.example @@ -62,6 +62,10 @@ DAILY_CN_AI_NEWS=1 # Agent 工作流（推荐：叙事化早报） # DAILY_REPORT_MODE=agent +# DAILY_LLM_PROVIDER=auto +# auto — agent 模式优先 Cursor，否则 OpenAI；classic 优先 OpenAI +# openai — 强制 OpenAI 兼容 API +# cursor — 强制 Cursor SDK # CURSOR_API_KEY=cursor_... # CURSOR_MODEL=composer-2.5 # DAILY_CURSOR_CWD=. @@ -83,6 +87,10 @@ DAILY_AI_NEWS_PER_CATEGORY=5 # DAILY_AGENT_NEWS_POOL=40 # DAILY_AGENT_CN_NEWS_POOL=30 +# RSS 源：编辑 config/feeds.yaml（intl / cn）；缺失时回退内置默认 +# 内容过滤：config/sensitive_words.yaml + DAILY_CONTENT_FILTER=1 +# DAILY_CONTENT_FILTER=0 + # Reddit RSS（403/429 时在 Reddit 偏好设置 → RSS feeds 复制 user / feed 参数） # REDDIT_RSS_USER=your_username # REDDIT_RSS_FEED=your_feed_token diff --git a/Dockerfile b/Dockerfile index 579dcdd..d9ca38a 100644 --- a/Dockerfile +++ b/Dockerfile @@ -16,7 +16,7 @@ RUN pip install --no-cache-dir -r requirements.txt COPY daily/ daily/ COPY shared/ shared/ COPY skills/ skills/ -COPY bot/ bot/ +COPY config/ config/ COPY run-daily.sh . RUN chmod +x run-daily.sh \ diff --git a/README.md b/README.md index d78f39c..f29d8f4 100644 --- a/README.md +++ b/README.md @@ -116,10 +116,11 @@ Python 抓取 → Step1 趋势分析 (.trends.json) → Step2 写企微稿 (.wec ## 自定义 RSS 源 -国际 RSS 列表：`daily/news/feeds.py` -国内 RSS 列表：`daily/news/feeds_cn.py` +RSS 源配置在 **`config/feeds.yaml`**（国际 `intl` / 国内 `cn`），改 URL 或增删 feed 后无需改 Python。文件缺失或解析失败时，会回退 `daily/news/feeds_defaults.py` 内置列表。 -在对应文件的 `FEEDS` 列表中增删 URL 即可。常用开关（见 `.env.example`）： +可选 **`config/sensitive_words.yaml`** + `DAILY_CONTENT_FILTER=1`：标题/摘要命中敏感词则丢弃该条。 + +旧版硬编码路径（仍可读作参考）：`daily/news/feeds_defaults.py` ```env DAILY_AI_NEWS=1 diff --git a/config/feeds.yaml b/config/feeds.yaml new file mode 100644 index 0000000..c690b39 --- /dev/null +++ b/config/feeds.yaml @@ -0,0 +1,166 @@ +version: 1 +intl: + categories: + - id: official + name: 厂商官方 + icon: 🏢 + feeds: + - name: Anthropic Claude 更新 + url: https://docs.anthropic.com/en/release-notes/feed + - name: OpenAI + url: https://openai.com/news/rss.xml + - name: Google AI + url: https://blog.google/technology/ai/rss/ + - name: DeepMind + url: https://deepmind.google/blog/rss.xml + - name: Meta Engineering + url: https://engineering.fb.com/feed/ + - name: Microsoft Research + url: https://www.microsoft.com/en-us/research/feed/ + - name: Microsoft Blog + url: https://blogs.microsoft.com/feed/ + - name: Cohere + url: https://cohere.com/blog/rss.xml + - name: Cursor Changelog + url: https://cursor.com/changelog/rss.xml + - id: developer + name: Agent / LLM 开发者 + icon: 🛠 + feeds: + - name: LangChain + url: https://blog.langchain.dev/rss/ + - name: Hugging Face + url: https://huggingface.co/blog/feed.xml + - name: Vercel Changelog + url: https://vercel.com/changelog/rss.xml + - name: GitHub Copilot + url: https://github.blog/changelog/label/copilot/feed/ + - id: media + name: 综合科技媒体 + icon: 📰 + feeds: + - name: The Verge AI + url: https://www.theverge.com/rss/ai-artificial-intelligence/index.xml + - name: TechCrunch AI + url: https://techcrunch.com/category/artificial-intelligence/feed/ + - name: Ars Technica AI + url: https://arstechnica.com/ai/feed/ + - name: Wired AI + url: https://www.wired.com/feed/tag/ai/latest/rss + - name: MIT Tech Review + url: https://www.technologyreview.com/feed/ + - name: VentureBeat AI + url: https://venturebeat.com/category/ai/feed/ + - id: newsletter + name: Newsletter 日报 + icon: ✉️ + feeds: + - name: Ben's Bites + url: https://bensbites.substack.com/feed + - name: The Rundown AI + url: https://therundown.substack.com/feed + - name: Latent Space + url: https://www.latent.space/feed + - name: Simon Willison + url: https://simonwillison.net/atom/everything/ + - name: Import AI + url: https://importai.substack.com/feed + - name: Last Week in AI + url: https://lastweekin.ai/feed + - name: The Neuron + url: https://www.theneuron.ai/feed + - id: research + name: 研究 / 论文 + icon: 📚 + feeds: + - name: arXiv cs.CL + url: https://arxiv.org/rss/cs.CL + - name: arXiv cs.AI + url: https://arxiv.org/rss/cs.AI + - name: arXiv cs.LG + url: https://arxiv.org/rss/cs.LG + - id: trending + name: 热点 / 趋势 + icon: 🔥 + feeds: + - name: Google News · AI + url: https://news.google.com/rss/search?q=artificial+intelligence+OR+LLM+OR+Claude+OR+GPT&hl=en-US&gl=US&ceid=US:en + - name: Google News · Technology + url: https://news.google.com/rss/headlines/section/topic/TECHNOLOGY?hl=en-US&gl=US&ceid=US:en + - name: Techmeme + url: https://www.techmeme.com/feed.xml + - name: HN · Front Page + url: https://hnrss.org/frontpage + - name: HN · 100+ Points + url: https://hnrss.org/newest?points=100 + - name: Dev.to · AI + url: https://dev.to/feed/tag/ai + - name: Lobsters + url: https://lobste.rs/rss + - id: community + name: 社区讨论 + icon: 💬 + feeds: + - name: HN · AI/LLM/Agent + url: https://hnrss.org/newest?q=AI+OR+LLM+OR+Claude+OR+agent+OR+GPT+OR+Gemini + - name: Reddit · LLM/Claude/ML + url: https://old.reddit.com/r/LocalLLaMA+ClaudeAI+MachineLearning+OpenAI/.rss?limit=25 + slow: true +cn: + title_keywords: + - 人工智能 + - 大模型 + - 智能体 + - 多模态 + - AIGC + - LLM + - GPT + - Claude + - Gemini + - ChatGPT + - OpenAI + - Anthropic + - Copilot + - Agent + - 'AI ' + - ' AI' + - AI· + - AI业务 + - AI模型 + - AI助手 + - AI工具 + - AI编程 + - AI 编程 + - AI版 + - AI Agent + - 推理模型 + - 深度学习 + - 机器学习 + - Function Calling + categories: + - id: media + name: AI 专业媒体 + icon: 📰 + feeds: + - name: 量子位 + url: https://www.qbitai.com/feed + - name: InfoQ 中文 + url: https://www.infoq.cn/feed/AI + - id: tech + name: 综合科技 + icon: 📱 + feeds: + - name: 36氪 + url: https://36kr.com/feed + ai_filter: true + - name: 雷锋网 + url: https://www.leiphone.com/feed + - name: Google News · AI + url: https://news.google.com/rss/search?q=人工智能+OR+大模型+OR+Agent+OR+LLM&hl=zh-CN&gl=CN&ceid=CN:zh-Hans + - id: dev + name: 开发者社区 + icon: 💻 + feeds: + - name: 掘金 + url: https://juejin.cn/rss + ai_filter: true diff --git a/config/sensitive_words.yaml b/config/sensitive_words.yaml new file mode 100644 index 0000000..3eed45e --- /dev/null +++ b/config/sensitive_words.yaml @@ -0,0 +1,10 @@ +# 标题或摘要命中任一词则丢弃（不区分大小写） +words: + - 赌博 + - 六合彩 + - 网赌 + - 色情 + - 裸聊 + - 代孕 + - 办证 + - 刷单兼职 diff --git a/daily/cursor_bridge.py b/daily/cursor_bridge.py new file mode 100644 index 0000000..0f5111d --- /dev/null +++ b/daily/cursor_bridge.py @@ -0,0 +1,142 @@ +"""Windows 兼容的 Cursor SDK bridge（daily 包自用，不依赖 bot/）。""" + +from __future__ import annotations + +import codecs +import json +import logging +import os +import subprocess +import threading +import time +from pathlib import Path +from typing import Any, Mapping + +from daily.config import ROOT, env + +logger = logging.getLogger(__name__) + +READY_LINE_PREFIX = "cursor-sdk-bridge ready " +_bridge_lock = threading.Lock() +_bridge_process: subprocess.Popen[bytes] | None = None + + +def cursor_cwd() -> str: + return env("DAILY_CURSOR_CWD") or env("CURSOR_CWD") or str(ROOT) + + +def _parse_discovery_line(line: str) -> Mapping[str, Any] | None: + if not line.startswith(READY_LINE_PREFIX): + return None + payload = line[len(READY_LINE_PREFIX) :].strip() + loaded = json.loads(payload) + if not isinstance(loaded, dict): + raise RuntimeError("Bridge discovery payload must be an object") + return loaded + + +def _read_discovery_polling(process: subprocess.Popen[bytes], timeout: float = 60) -> Mapping[str, Any]: + if process.stderr is None: + raise RuntimeError("Bridge stderr unavailable") + + fd = process.stderr.fileno() + was_blocking = os.get_blocking(fd) + os.set_blocking(fd, False) + decoder = codecs.getincrementaldecoder("utf-8")(errors="replace") + pending = "" + stderr_lines: list[str] = [] + deadline = time.monotonic() + timeout + + try: + while time.monotonic() < deadline: + try: + chunk = os.read(fd, 8192) + except BlockingIOError: + chunk = b"" + + if chunk: + pending += decoder.decode(chunk) + while "\n" in pending: + line, pending = pending.split("\n", 1) + stderr_lines.append(line) + discovery = _parse_discovery_line(line) + if discovery is not None: + return discovery + else: + code = process.poll() + if code is not None: + pending += decoder.decode(b"", final=True) + if pending.strip(): + stderr_lines.append(pending.strip()) + joined = "\n".join(stderr_lines)[-2000:] + raise RuntimeError( + f"Bridge 启动失败 exit={code}: {joined or '无 stderr 输出'}" + ) + time.sleep(0.05) + finally: + os.set_blocking(fd, was_blocking) + + raise RuntimeError("等待 Cursor bridge 就绪超时") + + +def _auth_token_from_discovery(discovery: Mapping[str, Any]) -> str: + token = str(discovery.get("authToken") or "").strip() + if token: + return token + token_file = discovery.get("authTokenFile") + if token_file: + return Path(str(token_file)).read_text(encoding="utf-8").strip() + raise RuntimeError("Bridge discovery 缺少 auth token") + + +def warm_cursor_bridge(force: bool = False) -> None: + """启动 cursor-sdk-bridge 并写入 CURSOR_SDK_BRIDGE_* 环境变量。""" + global _bridge_process + + with _bridge_lock: + if ( + not force + and _bridge_process is not None + and _bridge_process.poll() is None + and os.environ.get("CURSOR_SDK_BRIDGE_URL") + and os.environ.get("CURSOR_SDK_BRIDGE_TOKEN") + ): + return + + if _bridge_process is not None and _bridge_process.poll() is None: + _bridge_process.terminate() + try: + _bridge_process.wait(timeout=5) + except subprocess.TimeoutExpired: + _bridge_process.kill() + + from cursor_sdk._vendor import resolve_bridge_path + + cwd = cursor_cwd() + os.environ["CURSOR_CWD"] = cwd + argv = [resolve_bridge_path(), "--workspace", cwd] + logger.info("启动 Cursor bridge workspace=%s", cwd) + + process = subprocess.Popen( + argv, + stdout=subprocess.DEVNULL, + stderr=subprocess.PIPE, + ) + try: + discovery = _read_discovery_polling(process) + except Exception: + process.kill() + process.wait(timeout=5) + raise + + url = str(discovery.get("url") or "").strip() + if not url: + host = str(discovery.get("host") or "127.0.0.1") + port = discovery.get("port") + url = f"http://{host}:{port}" + + token = _auth_token_from_discovery(discovery) + os.environ["CURSOR_SDK_BRIDGE_URL"] = url + os.environ["CURSOR_SDK_BRIDGE_TOKEN"] = token + _bridge_process = process + logger.info("Cursor bridge 就绪: %s", url) diff --git a/daily/cursor_client.py b/daily/cursor_client.py new file mode 100644 index 0000000..a6a7079 --- /dev/null +++ b/daily/cursor_client.py @@ -0,0 +1,33 @@ +"""Cursor SDK 调用（daily 包专用）。""" + +from __future__ import annotations + +from cursor_sdk import Agent, AgentOptions, CursorAgentError, LocalAgentOptions + +from daily.config import env +from daily.cursor_bridge import cursor_cwd, warm_cursor_bridge + + +def cursor_chat(system: str, user: str) -> str: + api_key = (env("CURSOR_API_KEY") or "").strip() + if not api_key: + return "" + + warm_cursor_bridge() + cwd = cursor_cwd() + model = env("CURSOR_MODEL") or "composer-2.5" + prompt = f"{system}\n\n{user}" + try: + result = Agent.prompt( + prompt, + AgentOptions( + api_key=api_key, + model=model, + local=LocalAgentOptions(cwd=cwd), + ), + ) + except CursorAgentError as exc: + raise RuntimeError(f"LLM 调用失败：{exc.message}") from exc + if result.status == "error": + raise RuntimeError(f"LLM 调用失败：{result.result or '未知错误'}") + return (result.result or "").strip() diff --git a/daily/llm_client.py b/daily/llm_client.py index 05caef0..d4dfaa5 100644 --- a/daily/llm_client.py +++ b/daily/llm_client.py @@ -3,17 +3,14 @@ from __future__ import annotations import json -import logging -import os import re from typing import Any import certifi import httpx -from daily.config import ROOT, env, env_int - -logger = logging.getLogger(__name__) +from daily.config import env, env_int +from daily.cursor_client import cursor_chat _JSON_BLOCK = re.compile(r"```(?:json)?\s*([\s\S]*?)```", re.IGNORECASE) @@ -44,6 +41,47 @@ def extract_json_object(text: str) -> dict[str, Any]: return {} +def _has_openai_configured() -> bool: + return bool(env("DAILY_LLM_API_KEY") or env("OPENAI_API_KEY")) + + +def _has_cursor_configured() -> bool: + return bool(env("CURSOR_API_KEY")) + + +def llm_provider() -> str: + raw = (env("DAILY_LLM_PROVIDER") or "auto").strip().lower() + if raw in {"openai", "cursor"}: + return raw + return "auto" + + +def resolve_llm_backend() -> str: + """返回 openai | cursor | 空字符串。""" + provider = llm_provider() + has_openai = _has_openai_configured() + has_cursor = _has_cursor_configured() + + if provider == "openai": + if has_openai: + return "openai" + return "cursor" if has_cursor else "" + + if provider == "cursor": + if has_cursor: + return "cursor" + return "openai" if has_openai else "" + + agent_mode = (env("DAILY_REPORT_MODE") or "").strip().lower() == "agent" + if agent_mode and has_cursor: + return "cursor" + if has_openai: + return "openai" + if has_cursor: + return "cursor" + return "" + + def _openai_chat(system: str, user: str) -> str: api_key = (env("DAILY_LLM_API_KEY") or env("OPENAI_API_KEY") or "").strip() if not api_key: @@ -70,53 +108,14 @@ def _openai_chat(system: str, user: str) -> str: return str(data["choices"][0]["message"]["content"] or "").strip() -def _cursor_chat(system: str, user: str) -> str: - api_key = (env("CURSOR_API_KEY") or "").strip() - if not api_key: - return "" - import sys - - from daily.config import ROOT - from cursor_sdk import Agent, AgentOptions, CursorAgentError, LocalAgentOptions - - _bot = str(ROOT / "bot") - if _bot not in sys.path: - sys.path.insert(0, _bot) - try: - from bridge_manager import warm_cursor_bridge - except ImportError: - warm_cursor_bridge = lambda: None # noqa: E731 - - cwd = env("DAILY_CURSOR_CWD") or str(ROOT) - # bridge_manager 读 bot env_config 的 CURSOR_CWD，早报侧须先对齐工作目录 - os.environ["CURSOR_CWD"] = cwd - warm_cursor_bridge() - model = env("CURSOR_MODEL") or "composer-2.5" - prompt = f"{system}\n\n{user}" - try: - result = Agent.prompt( - prompt, - AgentOptions( - api_key=api_key, - model=model, - local=LocalAgentOptions(cwd=cwd), - ), - ) - except CursorAgentError as exc: - raise RuntimeError(f"LLM 调用失败：{exc.message}") from exc - if result.status == "error": - raise RuntimeError(f"LLM 调用失败：{result.result or '未知错误'}") - return (result.result or "").strip() - - def llm_chat(system: str, user: str) -> str: - """优先 OpenAI 兼容 API，否则 Cursor SDK。""" - if env("DAILY_LLM_API_KEY") or env("OPENAI_API_KEY"): + backend = resolve_llm_backend() + if backend == "openai": return _openai_chat(system, user) - if env("CURSOR_API_KEY"): - return _cursor_chat(system, user) + if backend == "cursor": + return cursor_chat(system, user) return "" def has_llm_configured() -> bool: - return bool(env("DAILY_LLM_API_KEY") or env("OPENAI_API_KEY") or env("CURSOR_API_KEY")) + return bool(resolve_llm_backend()) diff --git a/daily/news/content_filter.py b/daily/news/content_filter.py new file mode 100644 index 0000000..6921fc4 --- /dev/null +++ b/daily/news/content_filter.py @@ -0,0 +1,77 @@ +"""敏感词内容过滤。""" + +from __future__ import annotations + +import logging +from functools import lru_cache +from pathlib import Path +from typing import Any + +from daily.config import ROOT, env + +logger = logging.getLogger(__name__) + +_WORDS_FILE = ROOT / "config" / "sensitive_words.yaml" + + +def content_filter_enabled() -> bool: + raw = (env("DAILY_CONTENT_FILTER") or "0").strip().lower() + return raw in {"1", "true", "yes", "on"} + + +@lru_cache(maxsize=1) +def load_sensitive_words() -> tuple[str, ...]: + if not _WORDS_FILE.exists(): + return () + try: + import yaml + except ImportError: + logger.warning("未安装 PyYAML，无法读取 %s", _WORDS_FILE) + return () + try: + data = yaml.safe_load(_WORDS_FILE.read_text(encoding="utf-8")) + except OSError as exc: + logger.warning("读取敏感词配置失败: %s", exc) + return () + except Exception as exc: + logger.warning("解析 sensitive_words.yaml 失败: %s", exc) + return () + if not isinstance(data, dict): + return () + words = data.get("words") or data.get("sensitive_words") or [] + if not isinstance(words, list): + return () + cleaned = tuple(str(word).strip() for word in words if str(word).strip()) + return cleaned + + +def matches_sensitive_text(text: str, words: tuple[str, ...]) -> str | None: + haystack = (text or "").lower() + if not haystack: + return None + for word in words: + needle = word.lower() + if needle and needle in haystack: + return word + return None + + +def filter_news_items(items: list[dict[str, Any]]) -> tuple[list[dict[str, Any]], int]: + if not content_filter_enabled(): + return items, 0 + words = load_sensitive_words() + if not words: + return items, 0 + + kept: list[dict[str, Any]] = [] + removed = 0 + for item in items: + text = f"{item.get('title', '')} {item.get('summary', '')}" + hit = matches_sensitive_text(text, words) + if hit: + removed += 1 + continue + kept.append(item) + if removed: + logger.info("内容过滤移除 %d 条（敏感词）", removed) + return kept, removed diff --git a/daily/news/feeds.py b/daily/news/feeds.py index 05676c2..cb4201d 100644 --- a/daily/news/feeds.py +++ b/daily/news/feeds.py @@ -1,125 +1,10 @@ -"""国际 AI 时讯 RSS 源定义（按类别分组）。""" +"""国际 AI 时讯 RSS 源（优先 config/feeds.yaml）。""" from __future__ import annotations -from dataclasses import dataclass +from daily.news.feeds_loader import load_categories +from daily.news.feeds_types import NewsCategory, NewsFeed +NEWS_CATEGORIES: tuple[NewsCategory, ...] = load_categories("intl") -@dataclass(frozen=True) -class NewsFeed: - name: str - url: str - slow: bool = False # 限速源（如 Reddit）串行抓取 - ai_filter: bool = False # 综合源仅保留标题命中 AI 关键词的条目 - - -@dataclass(frozen=True) -class NewsCategory: - id: str - name: str - icon: str - feeds: tuple[NewsFeed, ...] - - -NEWS_CATEGORIES: tuple[NewsCategory, ...] = ( - NewsCategory( - id="official", - name="厂商官方", - icon="🏢", - feeds=( - NewsFeed("Anthropic Claude 更新", "https://docs.anthropic.com/en/release-notes/feed"), - NewsFeed("OpenAI", "https://openai.com/news/rss.xml"), - NewsFeed("Google AI", "https://blog.google/technology/ai/rss/"), - NewsFeed("DeepMind", "https://deepmind.google/blog/rss.xml"), - NewsFeed("Meta Engineering", "https://engineering.fb.com/feed/"), - NewsFeed("Microsoft Research", "https://www.microsoft.com/en-us/research/feed/"), - NewsFeed("Microsoft Blog", "https://blogs.microsoft.com/feed/"), - NewsFeed("Cohere", "https://cohere.com/blog/rss.xml"), - NewsFeed("Cursor Changelog", "https://cursor.com/changelog/rss.xml"), - ), - ), - NewsCategory( - id="developer", - name="Agent / LLM 开发者", - icon="🛠", - feeds=( - NewsFeed("LangChain", "https://blog.langchain.dev/rss/"), - NewsFeed("Hugging Face", "https://huggingface.co/blog/feed.xml"), - NewsFeed("Vercel Changelog", "https://vercel.com/changelog/rss.xml"), - NewsFeed("GitHub Copilot", "https://github.blog/changelog/label/copilot/feed/"), - ), - ), - NewsCategory( - id="media", - name="综合科技媒体", - icon="📰", - feeds=( - NewsFeed("The Verge AI", "https://www.theverge.com/rss/ai-artificial-intelligence/index.xml"), - NewsFeed("TechCrunch AI", "https://techcrunch.com/category/artificial-intelligence/feed/"), - NewsFeed("Ars Technica AI", "https://arstechnica.com/ai/feed/"), - NewsFeed("Wired AI", "https://www.wired.com/feed/tag/ai/latest/rss"), - NewsFeed("MIT Tech Review", "https://www.technologyreview.com/feed/"), - NewsFeed("VentureBeat AI", "https://venturebeat.com/category/ai/feed/"), - ), - ), - NewsCategory( - id="newsletter", - name="Newsletter 日报", - icon="✉️", - feeds=( - NewsFeed("Ben's Bites", "https://bensbites.substack.com/feed"), - NewsFeed("The Rundown AI", "https://therundown.substack.com/feed"), - NewsFeed("Latent Space", "https://www.latent.space/feed"), - NewsFeed("Simon Willison", "https://simonwillison.net/atom/everything/"), - NewsFeed("Import AI", "https://importai.substack.com/feed"), - NewsFeed("Last Week in AI", "https://lastweekin.ai/feed"), - NewsFeed("The Neuron", "https://www.theneuron.ai/feed"), - ), - ), - NewsCategory( - id="research", - name="研究 / 论文", - icon="📚", - feeds=( - NewsFeed("arXiv cs.CL", "https://arxiv.org/rss/cs.CL"), - NewsFeed("arXiv cs.AI", "https://arxiv.org/rss/cs.AI"), - NewsFeed("arXiv cs.LG", "https://arxiv.org/rss/cs.LG"), - ), - ), - NewsCategory( - id="trending", - name="热点 / 趋势", - icon="🔥", - feeds=( - NewsFeed( - "Google News · AI", - "https://news.google.com/rss/search?q=artificial+intelligence+OR+LLM+OR+Claude+OR+GPT&hl=en-US&gl=US&ceid=US:en", - ), - NewsFeed( - "Google News · Technology", - "https://news.google.com/rss/headlines/section/topic/TECHNOLOGY?hl=en-US&gl=US&ceid=US:en", - ), - NewsFeed("Techmeme", "https://www.techmeme.com/feed.xml"), - NewsFeed("HN · Front Page", "https://hnrss.org/frontpage"), - NewsFeed("HN · 100+ Points", "https://hnrss.org/newest?points=100"), - NewsFeed("Dev.to · AI", "https://dev.to/feed/tag/ai"), - NewsFeed("Lobsters", "https://lobste.rs/rss"), - ), - ), - NewsCategory( - id="community", - name="社区讨论", - icon="💬", - feeds=( - NewsFeed( - "HN · AI/LLM/Agent", - "https://hnrss.org/newest?q=AI+OR+LLM+OR+Claude+OR+agent+OR+GPT+OR+Gemini", - ), - NewsFeed( - "Reddit · LLM/Claude/ML", - "https://old.reddit.com/r/LocalLLaMA+ClaudeAI+MachineLearning+OpenAI/.rss?limit=25", - slow=True, - ), - ), - ), -) +__all__ = ["NewsCategory", "NewsFeed", "NEWS_CATEGORIES"] diff --git a/daily/news/feeds_cn.py b/daily/news/feeds_cn.py index 1b37e21..1eff67e 100644 --- a/daily/news/feeds_cn.py +++ b/daily/news/feeds_cn.py @@ -1,71 +1,11 @@ -"""国内 AI 时讯 RSS 源定义（按类别分组）。""" +"""国内 AI 时讯 RSS 源（优先 config/feeds.yaml）。""" from __future__ import annotations -from daily.news.feeds import NewsCategory, NewsFeed +from daily.news.feeds_loader import load_categories, load_cn_title_keywords +from daily.news.feeds_types import NewsCategory, NewsFeed -# 综合源 ai_filter=True 时，仅保留标题命中以下词之一的条目 -CN_AI_TITLE_KEYWORDS: tuple[str, ...] = ( - "人工智能", - "大模型", - "智能体", - "多模态", - "AIGC", - "LLM", - "GPT", - "Claude", - "Gemini", - "ChatGPT", - "OpenAI", - "Anthropic", - "Copilot", - "Agent", - "AI ", - " AI", - "AI·", - "AI业务", - "AI模型", - "AI助手", - "AI工具", - "AI编程", - "AI 编程", - "AI版", - "AI Agent", - "推理模型", - "深度学习", - "机器学习", - "Function Calling", -) +CN_AI_TITLE_KEYWORDS: tuple[str, ...] = load_cn_title_keywords() +CN_NEWS_CATEGORIES: tuple[NewsCategory, ...] = load_categories("cn") -CN_NEWS_CATEGORIES: tuple[NewsCategory, ...] = ( - NewsCategory( - id="media", - name="AI 专业媒体", - icon="📰", - feeds=( - NewsFeed("量子位", "https://www.qbitai.com/feed"), - NewsFeed("InfoQ 中文", "https://www.infoq.cn/feed/AI"), - ), - ), - NewsCategory( - id="tech", - name="综合科技", - icon="📱", - feeds=( - NewsFeed("36氪", "https://36kr.com/feed", ai_filter=True), - NewsFeed("雷锋网", "https://www.leiphone.com/feed"), - NewsFeed( - "Google News · AI", - "https://news.google.com/rss/search?q=人工智能+OR+大模型+OR+Agent+OR+LLM&hl=zh-CN&gl=CN&ceid=CN:zh-Hans", - ), - ), - ), - NewsCategory( - id="dev", - name="开发者社区", - icon="💻", - feeds=( - NewsFeed("掘金", "https://juejin.cn/rss", ai_filter=True), - ), - ), -) +__all__ = ["CN_AI_TITLE_KEYWORDS", "CN_NEWS_CATEGORIES", "NewsCategory", "NewsFeed"] diff --git a/daily/news/feeds_defaults.py b/daily/news/feeds_defaults.py new file mode 100644 index 0000000..591a34e --- /dev/null +++ b/daily/news/feeds_defaults.py @@ -0,0 +1,11 @@ +"""Built-in RSS defaults when config/feeds.yaml is missing or invalid.""" + +from __future__ import annotations + +from daily.news.feeds_types import NewsCategory, NewsFeed + +CN_AI_TITLE_KEYWORDS: tuple[str, ...] = ('人工智能', '大模型', '智能体', '多模态', 'AIGC', 'LLM', 'GPT', 'Claude', 'Gemini', 'ChatGPT', 'OpenAI', 'Anthropic', 'Copilot', 'Agent', 'AI ', ' AI', 'AI·', 'AI业务', 'AI模型', 'AI助手', 'AI工具', 'AI编程', 'AI 编程', 'AI版', 'AI Agent', '推理模型', '深度学习', '机器学习', 'Function Calling') + +CN_NEWS_CATEGORIES: tuple[NewsCategory, ...] = (NewsCategory(id='media', name='AI 专业媒体', icon='📰', feeds=(NewsFeed(name='量子位', url='https://www.qbitai.com/feed', slow=False, ai_filter=False), NewsFeed(name='InfoQ 中文', url='https://www.infoq.cn/feed/AI', slow=False, ai_filter=False))), NewsCategory(id='tech', name='综合科技', icon='📱', feeds=(NewsFeed(name='36氪', url='https://36kr.com/feed', slow=False, ai_filter=True), NewsFeed(name='雷锋网', url='https://www.leiphone.com/feed', slow=False, ai_filter=False), NewsFeed(name='Google News · AI', url='https://news.google.com/rss/search?q=人工智能+OR+大模型+OR+Agent+OR+LLM&hl=zh-CN&gl=CN&ceid=CN:zh-Hans', slow=False, ai_filter=False))), NewsCategory(id='dev', name='开发者社区', icon='💻', feeds=(NewsFeed(name='掘金', url='https://juejin.cn/rss', slow=False, ai_filter=True),))) + +NEWS_CATEGORIES: tuple[NewsCategory, ...] = (NewsCategory(id='official', name='厂商官方', icon='🏢', feeds=(NewsFeed(name='Anthropic Claude 更新', url='https://docs.anthropic.com/en/release-notes/feed', slow=False, ai_filter=False), NewsFeed(name='OpenAI', url='https://openai.com/news/rss.xml', slow=False, ai_filter=False), NewsFeed(name='Google AI', url='https://blog.google/technology/ai/rss/', slow=False, ai_filter=False), NewsFeed(name='DeepMind', url='https://deepmind.google/blog/rss.xml', slow=False, ai_filter=False), NewsFeed(name='Meta Engineering', url='https://engineering.fb.com/feed/', slow=False, ai_filter=False), NewsFeed(name='Microsoft Research', url='https://www.microsoft.com/en-us/research/feed/', slow=False, ai_filter=False), NewsFeed(name='Microsoft Blog', url='https://blogs.microsoft.com/feed/', slow=False, ai_filter=False), NewsFeed(name='Cohere', url='https://cohere.com/blog/rss.xml', slow=False, ai_filter=False), NewsFeed(name='Cursor Changelog', url='https://cursor.com/changelog/rss.xml', slow=False, ai_filter=False))), NewsCategory(id='developer', name='Agent / LLM 开发者', icon='🛠', feeds=(NewsFeed(name='LangChain', url='https://blog.langchain.dev/rss/', slow=False, ai_filter=False), NewsFeed(name='Hugging Face', url='https://huggingface.co/blog/feed.xml', slow=False, ai_filter=False), NewsFeed(name='Vercel Changelog', url='https://vercel.com/changelog/rss.xml', slow=False, ai_filter=False), NewsFeed(name='GitHub Copilot', url='https://github.blog/changelog/label/copilot/feed/', slow=False, ai_filter=False))), NewsCategory(id='media', name='综合科技媒体', icon='📰', feeds=(NewsFeed(name='The Verge AI', url='https://www.theverge.com/rss/ai-artificial-intelligence/index.xml', slow=False, ai_filter=False), NewsFeed(name='TechCrunch AI', url='https://techcrunch.com/category/artificial-intelligence/feed/', slow=False, ai_filter=False), NewsFeed(name='Ars Technica AI', url='https://arstechnica.com/ai/feed/', slow=False, ai_filter=False), NewsFeed(name='Wired AI', url='https://www.wired.com/feed/tag/ai/latest/rss', slow=False, ai_filter=False), NewsFeed(name='MIT Tech Review', url='https://www.technologyreview.com/feed/', slow=False, ai_filter=False), NewsFeed(name='VentureBeat AI', url='https://venturebeat.com/category/ai/feed/', slow=False, ai_filter=False))), NewsCategory(id='newsletter', name='Newsletter 日报', icon='✉️', feeds=(NewsFeed(name="Ben's Bites", url='https://bensbites.substack.com/feed', slow=False, ai_filter=False), NewsFeed(name='The Rundown AI', url='https://therundown.substack.com/feed', slow=False, ai_filter=False), NewsFeed(name='Latent Space', url='https://www.latent.space/feed', slow=False, ai_filter=False), NewsFeed(name='Simon Willison', url='https://simonwillison.net/atom/everything/', slow=False, ai_filter=False), NewsFeed(name='Import AI', url='https://importai.substack.com/feed', slow=False, ai_filter=False), NewsFeed(name='Last Week in AI', url='https://lastweekin.ai/feed', slow=False, ai_filter=False), NewsFeed(name='The Neuron', url='https://www.theneuron.ai/feed', slow=False, ai_filter=False))), NewsCategory(id='research', name='研究 / 论文', icon='📚', feeds=(NewsFeed(name='arXiv cs.CL', url='https://arxiv.org/rss/cs.CL', slow=False, ai_filter=False), NewsFeed(name='arXiv cs.AI', url='https://arxiv.org/rss/cs.AI', slow=False, ai_filter=False), NewsFeed(name='arXiv cs.LG', url='https://arxiv.org/rss/cs.LG', slow=False, ai_filter=False))), NewsCategory(id='trending', name='热点 / 趋势', icon='🔥', feeds=(NewsFeed(name='Google News · AI', url='https://news.google.com/rss/search?q=artificial+intelligence+OR+LLM+OR+Claude+OR+GPT&hl=en-US&gl=US&ceid=US:en', slow=False, ai_filter=False), NewsFeed(name='Google News · Technology', url='https://news.google.com/rss/headlines/section/topic/TECHNOLOGY?hl=en-US&gl=US&ceid=US:en', slow=False, ai_filter=False), NewsFeed(name='Techmeme', url='https://www.techmeme.com/feed.xml', slow=False, ai_filter=False), NewsFeed(name='HN · Front Page', url='https://hnrss.org/frontpage', slow=False, ai_filter=False), NewsFeed(name='HN · 100+ Points', url='https://hnrss.org/newest?points=100', slow=False, ai_filter=False), NewsFeed(name='Dev.to · AI', url='https://dev.to/feed/tag/ai', slow=False, ai_filter=False), NewsFeed(name='Lobsters', url='https://lobste.rs/rss', slow=False, ai_filter=False))), NewsCategory(id='community', name='社区讨论', icon='💬', feeds=(NewsFeed(name='HN · AI/LLM/Agent', url='https://hnrss.org/newest?q=AI+OR+LLM+OR+Claude+OR+agent+OR+GPT+OR+Gemini', slow=False, ai_filter=False), NewsFeed(name='Reddit · LLM/Claude/ML', url='https://old.reddit.com/r/LocalLLaMA+ClaudeAI+MachineLearning+OpenAI/.rss?limit=25', slow=True, ai_filter=False)))) diff --git a/daily/news/feeds_loader.py b/daily/news/feeds_loader.py new file mode 100644 index 0000000..31d8e9c --- /dev/null +++ b/daily/news/feeds_loader.py @@ -0,0 +1,113 @@ +"""从 config/feeds.yaml 加载 RSS 源（失败时回退内置默认）。""" + +from __future__ import annotations + +import logging +from functools import lru_cache +from pathlib import Path +from typing import Any + +from daily.config import ROOT +from daily.news.feeds_types import NewsCategory, NewsFeed + +logger = logging.getLogger(__name__) + +_FEEDS_FILE = ROOT / "config" / "feeds.yaml" + + +def _parse_feed(raw: dict[str, Any]) -> NewsFeed: + return NewsFeed( + name=str(raw.get("name") or "").strip(), + url=str(raw.get("url") or "").strip(), + slow=bool(raw.get("slow")), + ai_filter=bool(raw.get("ai_filter")), + ) + + +def _parse_category(raw: dict[str, Any]) -> NewsCategory | None: + cat_id = str(raw.get("id") or "").strip() + if not cat_id: + return None + feeds_raw = raw.get("feeds") or [] + feeds: list[NewsFeed] = [] + for item in feeds_raw: + if not isinstance(item, dict): + continue + feed = _parse_feed(item) + if feed.name and feed.url: + feeds.append(feed) + if not feeds: + return None + return NewsCategory( + id=cat_id, + name=str(raw.get("name") or cat_id), + icon=str(raw.get("icon") or "📰"), + feeds=tuple(feeds), + ) + + +def _parse_categories(items: Any) -> tuple[NewsCategory, ...]: + if not isinstance(items, list): + return () + categories: list[NewsCategory] = [] + for raw in items: + if not isinstance(raw, dict): + continue + cat = _parse_category(raw) + if cat: + categories.append(cat) + return tuple(categories) + + +def _load_yaml() -> dict[str, Any] | None: + if not _FEEDS_FILE.exists(): + return None + try: + import yaml + except ImportError: + logger.warning("未安装 PyYAML，无法读取 %s", _FEEDS_FILE) + return None + try: + data = yaml.safe_load(_FEEDS_FILE.read_text(encoding="utf-8")) + except OSError as exc: + logger.warning("读取 feeds 配置失败: %s", exc) + return None + except Exception as exc: + logger.warning("解析 feeds.yaml 失败: %s", exc) + return None + return data if isinstance(data, dict) else None + + +@lru_cache(maxsize=1) +def _yaml_payload() -> dict[str, Any] | None: + return _load_yaml() + + +def load_categories(region: str) -> tuple[NewsCategory, ...]: + data = _yaml_payload() + if data: + block = data.get(region) or {} + categories = _parse_categories(block.get("categories")) + if categories: + return categories + logger.warning("feeds.yaml 中 %s.categories 为空，使用内置默认", region) + + from daily.news import feeds_defaults as defaults + + if region == "cn": + return defaults.CN_NEWS_CATEGORIES + return defaults.NEWS_CATEGORIES + + +def load_cn_title_keywords() -> tuple[str, ...]: + data = _yaml_payload() + if data: + block = data.get("cn") or {} + keywords = block.get("title_keywords") + if isinstance(keywords, list): + cleaned = tuple(str(x).strip() for x in keywords if str(x).strip()) + if cleaned: + return cleaned + from daily.news.feeds_defaults import CN_AI_TITLE_KEYWORDS + + return CN_AI_TITLE_KEYWORDS diff --git a/daily/news/feeds_types.py b/daily/news/feeds_types.py new file mode 100644 index 0000000..8d39225 --- /dev/null +++ b/daily/news/feeds_types.py @@ -0,0 +1,21 @@ +"""RSS 源数据结构。""" + +from __future__ import annotations + +from dataclasses import dataclass + + +@dataclass(frozen=True) +class NewsFeed: + name: str + url: str + slow: bool = False + ai_filter: bool = False + + +@dataclass(frozen=True) +class NewsCategory: + id: str + name: str + icon: str + feeds: tuple[NewsFeed, ...] diff --git a/daily/news/fetch.py b/daily/news/fetch.py index afd264a..4c4a4f6 100644 --- a/daily/news/fetch.py +++ b/daily/news/fetch.py @@ -17,6 +17,7 @@ import certifi import httpx from daily.config import env, env_int, news_summary_limit +from daily.news.content_filter import filter_news_items from daily.news.feeds import NEWS_CATEGORIES, NewsCategory, NewsFeed from daily.news.feeds_cn import CN_AI_TITLE_KEYWORDS, CN_NEWS_CATEGORIES @@ -349,6 +350,7 @@ def _fetch_news(categories: tuple[NewsCategory, ...]) -> dict[str, Any]: "feeds_ok": 0, "items_raw": 0, "feeds_failed": [], + "content_filtered": 0, } with httpx.Client(timeout=15.0, verify=certifi.where(), follow_redirects=True, headers=headers) as client: @@ -394,6 +396,9 @@ def _fetch_news(categories: tuple[NewsCategory, ...]) -> dict[str, Any]: for category in categories: items = raw_by_category[category.id] items = [i for i in items if _within_window(i, cutoff)] + items, filtered_count = filter_news_items(items) + if filtered_count: + stats["content_filtered"] = stats.get("content_filtered", 0) + filtered_count items.sort(key=_sort_key, reverse=True) items = _dedupe_items(items)[:per_category] for item in items: diff --git a/requirements.txt b/requirements.txt index fb640e1..5255c36 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,5 +1,6 @@ python-dotenv>=1.0.0 httpx>=0.27.0 certifi>=2024.0.0 +PyYAML>=6.0.0 # 英文描述转中文（使用 CURSOR_API_KEY 时需安装） cursor-sdk>=0.1.0 diff --git a/scripts/gen_feeds_defaults.py b/scripts/gen_feeds_defaults.py new file mode 100644 index 0000000..3c07096 --- /dev/null +++ b/scripts/gen_feeds_defaults.py @@ -0,0 +1,47 @@ +from pathlib import Path + +import yaml + +from daily.news.feeds_types import NewsCategory, NewsFeed + +data = yaml.safe_load(Path("config/feeds.yaml").read_text(encoding="utf-8")) + + +def parse_feed(raw: dict) -> NewsFeed: + return NewsFeed( + name=raw["name"], + url=raw["url"], + slow=bool(raw.get("slow")), + ai_filter=bool(raw.get("ai_filter")), + ) + + +def parse_cat(raw: dict) -> NewsCategory: + return NewsCategory( + id=raw["id"], + name=raw["name"], + icon=raw["icon"], + feeds=tuple(parse_feed(item) for item in raw["feeds"]), + ) + + +intl = tuple(parse_cat(item) for item in data["intl"]["categories"]) +cn = tuple(parse_cat(item) for item in data["cn"]["categories"]) +keywords = tuple(data["cn"]["title_keywords"]) + +lines = [ + '"""Built-in RSS defaults when config/feeds.yaml is missing or invalid."""', + "", + "from __future__ import annotations", + "", + "from daily.news.feeds_types import NewsCategory, NewsFeed", + "", + f"CN_AI_TITLE_KEYWORDS: tuple[str, ...] = {keywords!r}", + "", + f"CN_NEWS_CATEGORIES: tuple[NewsCategory, ...] = {cn!r}", + "", + f"NEWS_CATEGORIES: tuple[NewsCategory, ...] = {intl!r}", + "", +] +Path("daily/news/feeds_defaults.py").write_text("\n".join(lines), encoding="utf-8") +print("written") diff --git a/tests/test_feeds_and_filter.py b/tests/test_feeds_and_filter.py new file mode 100644 index 0000000..3b587cd --- /dev/null +++ b/tests/test_feeds_and_filter.py @@ -0,0 +1,40 @@ +from __future__ import annotations + +import pytest + + +def test_load_categories_from_yaml(): + from daily.news.feeds_loader import load_categories + + categories = load_categories("intl") + assert categories + assert any(cat.id == "official" for cat in categories) + assert categories[0].feeds + + +def test_load_categories_fallback_when_yaml_missing(monkeypatch, tmp_path): + import daily.config as config + import daily.news.feeds_loader as loader + + missing = tmp_path / "missing" / "feeds.yaml" + monkeypatch.setattr(loader, "_FEEDS_FILE", missing) + loader._yaml_payload.cache_clear() + + categories = loader.load_categories("intl") + assert categories + assert any(cat.id == "official" for cat in categories) + + +def test_content_filter_removes_matching_items(monkeypatch): + import daily.news.content_filter as cf + + monkeypatch.setenv("DAILY_CONTENT_FILTER", "1") + cf.load_sensitive_words.cache_clear() + items = [ + {"title": "正常 AI 新闻", "summary": "OpenAI 发布新模型"}, + {"title": "违规推广", "summary": "六合彩内幕消息"}, + ] + kept, removed = cf.filter_news_items(items) + assert removed == 1 + assert len(kept) == 1 + assert kept[0]["title"].startswith("正常") diff --git a/tests/test_llm_client.py b/tests/test_llm_client.py new file mode 100644 index 0000000..a7547b1 --- /dev/null +++ b/tests/test_llm_client.py @@ -0,0 +1,33 @@ +from __future__ import annotations + +import pytest + + +def test_resolve_llm_backend_prefers_cursor_in_agent_mode(monkeypatch): + import daily.llm_client as llm + + monkeypatch.setenv("DAILY_REPORT_MODE", "agent") + monkeypatch.setenv("CURSOR_API_KEY", "cursor_test") + monkeypatch.setenv("DAILY_LLM_API_KEY", "openai_test") + monkeypatch.delenv("DAILY_LLM_PROVIDER", raising=False) + assert llm.resolve_llm_backend() == "cursor" + + +def test_resolve_llm_backend_openai_override(monkeypatch): + import daily.llm_client as llm + + monkeypatch.setenv("DAILY_REPORT_MODE", "agent") + monkeypatch.setenv("DAILY_LLM_PROVIDER", "openai") + monkeypatch.setenv("CURSOR_API_KEY", "cursor_test") + monkeypatch.setenv("DAILY_LLM_API_KEY", "openai_test") + assert llm.resolve_llm_backend() == "openai" + + +def test_resolve_llm_backend_classic_defaults_openai(monkeypatch): + import daily.llm_client as llm + + monkeypatch.setenv("DAILY_REPORT_MODE", "classic") + monkeypatch.setenv("CURSOR_API_KEY", "cursor_test") + monkeypatch.setenv("DAILY_LLM_API_KEY", "openai_test") + monkeypatch.delenv("DAILY_LLM_PROVIDER", raising=False) + assert llm.resolve_llm_backend() == "openai"