feat: LLM 解耦 bot、RSS 外置与内容过滤
daily 自建 Cursor bridge；DAILY_LLM_PROVIDER 控制后端；config/feeds.yaml 与 sensitive_words 可配置。 Co-authored-by: Cursor <cursoragent@cursor.com>
This commit is contained in:
142
daily/cursor_bridge.py
Normal file
142
daily/cursor_bridge.py
Normal file
@@ -0,0 +1,142 @@
|
||||
"""Windows 兼容的 Cursor SDK bridge（daily 包自用，不依赖 bot/）。"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import codecs
|
||||
import json
|
||||
import logging
|
||||
import os
|
||||
import subprocess
|
||||
import threading
|
||||
import time
|
||||
from pathlib import Path
|
||||
from typing import Any, Mapping
|
||||
|
||||
from daily.config import ROOT, env
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
READY_LINE_PREFIX = "cursor-sdk-bridge ready "
|
||||
_bridge_lock = threading.Lock()
|
||||
_bridge_process: subprocess.Popen[bytes] | None = None
|
||||
|
||||
|
||||
def cursor_cwd() -> str:
|
||||
return env("DAILY_CURSOR_CWD") or env("CURSOR_CWD") or str(ROOT)
|
||||
|
||||
|
||||
def _parse_discovery_line(line: str) -> Mapping[str, Any] | None:
|
||||
if not line.startswith(READY_LINE_PREFIX):
|
||||
return None
|
||||
payload = line[len(READY_LINE_PREFIX) :].strip()
|
||||
loaded = json.loads(payload)
|
||||
if not isinstance(loaded, dict):
|
||||
raise RuntimeError("Bridge discovery payload must be an object")
|
||||
return loaded
|
||||
|
||||
|
||||
def _read_discovery_polling(process: subprocess.Popen[bytes], timeout: float = 60) -> Mapping[str, Any]:
|
||||
if process.stderr is None:
|
||||
raise RuntimeError("Bridge stderr unavailable")
|
||||
|
||||
fd = process.stderr.fileno()
|
||||
was_blocking = os.get_blocking(fd)
|
||||
os.set_blocking(fd, False)
|
||||
decoder = codecs.getincrementaldecoder("utf-8")(errors="replace")
|
||||
pending = ""
|
||||
stderr_lines: list[str] = []
|
||||
deadline = time.monotonic() + timeout
|
||||
|
||||
try:
|
||||
while time.monotonic() < deadline:
|
||||
try:
|
||||
chunk = os.read(fd, 8192)
|
||||
except BlockingIOError:
|
||||
chunk = b""
|
||||
|
||||
if chunk:
|
||||
pending += decoder.decode(chunk)
|
||||
while "\n" in pending:
|
||||
line, pending = pending.split("\n", 1)
|
||||
stderr_lines.append(line)
|
||||
discovery = _parse_discovery_line(line)
|
||||
if discovery is not None:
|
||||
return discovery
|
||||
else:
|
||||
code = process.poll()
|
||||
if code is not None:
|
||||
pending += decoder.decode(b"", final=True)
|
||||
if pending.strip():
|
||||
stderr_lines.append(pending.strip())
|
||||
joined = "\n".join(stderr_lines)[-2000:]
|
||||
raise RuntimeError(
|
||||
f"Bridge 启动失败 exit={code}: {joined or '无 stderr 输出'}"
|
||||
)
|
||||
time.sleep(0.05)
|
||||
finally:
|
||||
os.set_blocking(fd, was_blocking)
|
||||
|
||||
raise RuntimeError("等待 Cursor bridge 就绪超时")
|
||||
|
||||
|
||||
def _auth_token_from_discovery(discovery: Mapping[str, Any]) -> str:
|
||||
token = str(discovery.get("authToken") or "").strip()
|
||||
if token:
|
||||
return token
|
||||
token_file = discovery.get("authTokenFile")
|
||||
if token_file:
|
||||
return Path(str(token_file)).read_text(encoding="utf-8").strip()
|
||||
raise RuntimeError("Bridge discovery 缺少 auth token")
|
||||
|
||||
|
||||
def warm_cursor_bridge(force: bool = False) -> None:
|
||||
"""启动 cursor-sdk-bridge 并写入 CURSOR_SDK_BRIDGE_* 环境变量。"""
|
||||
global _bridge_process
|
||||
|
||||
with _bridge_lock:
|
||||
if (
|
||||
not force
|
||||
and _bridge_process is not None
|
||||
and _bridge_process.poll() is None
|
||||
and os.environ.get("CURSOR_SDK_BRIDGE_URL")
|
||||
and os.environ.get("CURSOR_SDK_BRIDGE_TOKEN")
|
||||
):
|
||||
return
|
||||
|
||||
if _bridge_process is not None and _bridge_process.poll() is None:
|
||||
_bridge_process.terminate()
|
||||
try:
|
||||
_bridge_process.wait(timeout=5)
|
||||
except subprocess.TimeoutExpired:
|
||||
_bridge_process.kill()
|
||||
|
||||
from cursor_sdk._vendor import resolve_bridge_path
|
||||
|
||||
cwd = cursor_cwd()
|
||||
os.environ["CURSOR_CWD"] = cwd
|
||||
argv = [resolve_bridge_path(), "--workspace", cwd]
|
||||
logger.info("启动 Cursor bridge workspace=%s", cwd)
|
||||
|
||||
process = subprocess.Popen(
|
||||
argv,
|
||||
stdout=subprocess.DEVNULL,
|
||||
stderr=subprocess.PIPE,
|
||||
)
|
||||
try:
|
||||
discovery = _read_discovery_polling(process)
|
||||
except Exception:
|
||||
process.kill()
|
||||
process.wait(timeout=5)
|
||||
raise
|
||||
|
||||
url = str(discovery.get("url") or "").strip()
|
||||
if not url:
|
||||
host = str(discovery.get("host") or "127.0.0.1")
|
||||
port = discovery.get("port")
|
||||
url = f"http://{host}:{port}"
|
||||
|
||||
token = _auth_token_from_discovery(discovery)
|
||||
os.environ["CURSOR_SDK_BRIDGE_URL"] = url
|
||||
os.environ["CURSOR_SDK_BRIDGE_TOKEN"] = token
|
||||
_bridge_process = process
|
||||
logger.info("Cursor bridge 就绪: %s", url)
|
||||
33
daily/cursor_client.py
Normal file
33
daily/cursor_client.py
Normal file
@@ -0,0 +1,33 @@
|
||||
"""Cursor SDK 调用（daily 包专用）。"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from cursor_sdk import Agent, AgentOptions, CursorAgentError, LocalAgentOptions
|
||||
|
||||
from daily.config import env
|
||||
from daily.cursor_bridge import cursor_cwd, warm_cursor_bridge
|
||||
|
||||
|
||||
def cursor_chat(system: str, user: str) -> str:
|
||||
api_key = (env("CURSOR_API_KEY") or "").strip()
|
||||
if not api_key:
|
||||
return ""
|
||||
|
||||
warm_cursor_bridge()
|
||||
cwd = cursor_cwd()
|
||||
model = env("CURSOR_MODEL") or "composer-2.5"
|
||||
prompt = f"{system}\n\n{user}"
|
||||
try:
|
||||
result = Agent.prompt(
|
||||
prompt,
|
||||
AgentOptions(
|
||||
api_key=api_key,
|
||||
model=model,
|
||||
local=LocalAgentOptions(cwd=cwd),
|
||||
),
|
||||
)
|
||||
except CursorAgentError as exc:
|
||||
raise RuntimeError(f"LLM 调用失败：{exc.message}") from exc
|
||||
if result.status == "error":
|
||||
raise RuntimeError(f"LLM 调用失败：{result.result or '未知错误'}")
|
||||
return (result.result or "").strip()
|
||||
@@ -3,17 +3,14 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import logging
|
||||
import os
|
||||
import re
|
||||
from typing import Any
|
||||
|
||||
import certifi
|
||||
import httpx
|
||||
|
||||
from daily.config import ROOT, env, env_int
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
from daily.config import env, env_int
|
||||
from daily.cursor_client import cursor_chat
|
||||
|
||||
_JSON_BLOCK = re.compile(r"```(?:json)?\s*([\s\S]*?)```", re.IGNORECASE)
|
||||
|
||||
@@ -44,6 +41,47 @@ def extract_json_object(text: str) -> dict[str, Any]:
|
||||
return {}
|
||||
|
||||
|
||||
def _has_openai_configured() -> bool:
|
||||
return bool(env("DAILY_LLM_API_KEY") or env("OPENAI_API_KEY"))
|
||||
|
||||
|
||||
def _has_cursor_configured() -> bool:
|
||||
return bool(env("CURSOR_API_KEY"))
|
||||
|
||||
|
||||
def llm_provider() -> str:
|
||||
raw = (env("DAILY_LLM_PROVIDER") or "auto").strip().lower()
|
||||
if raw in {"openai", "cursor"}:
|
||||
return raw
|
||||
return "auto"
|
||||
|
||||
|
||||
def resolve_llm_backend() -> str:
|
||||
"""返回 openai | cursor | 空字符串。"""
|
||||
provider = llm_provider()
|
||||
has_openai = _has_openai_configured()
|
||||
has_cursor = _has_cursor_configured()
|
||||
|
||||
if provider == "openai":
|
||||
if has_openai:
|
||||
return "openai"
|
||||
return "cursor" if has_cursor else ""
|
||||
|
||||
if provider == "cursor":
|
||||
if has_cursor:
|
||||
return "cursor"
|
||||
return "openai" if has_openai else ""
|
||||
|
||||
agent_mode = (env("DAILY_REPORT_MODE") or "").strip().lower() == "agent"
|
||||
if agent_mode and has_cursor:
|
||||
return "cursor"
|
||||
if has_openai:
|
||||
return "openai"
|
||||
if has_cursor:
|
||||
return "cursor"
|
||||
return ""
|
||||
|
||||
|
||||
def _openai_chat(system: str, user: str) -> str:
|
||||
api_key = (env("DAILY_LLM_API_KEY") or env("OPENAI_API_KEY") or "").strip()
|
||||
if not api_key:
|
||||
@@ -70,53 +108,14 @@ def _openai_chat(system: str, user: str) -> str:
|
||||
return str(data["choices"][0]["message"]["content"] or "").strip()
|
||||
|
||||
|
||||
def _cursor_chat(system: str, user: str) -> str:
|
||||
api_key = (env("CURSOR_API_KEY") or "").strip()
|
||||
if not api_key:
|
||||
return ""
|
||||
import sys
|
||||
|
||||
from daily.config import ROOT
|
||||
from cursor_sdk import Agent, AgentOptions, CursorAgentError, LocalAgentOptions
|
||||
|
||||
_bot = str(ROOT / "bot")
|
||||
if _bot not in sys.path:
|
||||
sys.path.insert(0, _bot)
|
||||
try:
|
||||
from bridge_manager import warm_cursor_bridge
|
||||
except ImportError:
|
||||
warm_cursor_bridge = lambda: None # noqa: E731
|
||||
|
||||
cwd = env("DAILY_CURSOR_CWD") or str(ROOT)
|
||||
# bridge_manager 读 bot env_config 的 CURSOR_CWD，早报侧须先对齐工作目录
|
||||
os.environ["CURSOR_CWD"] = cwd
|
||||
warm_cursor_bridge()
|
||||
model = env("CURSOR_MODEL") or "composer-2.5"
|
||||
prompt = f"{system}\n\n{user}"
|
||||
try:
|
||||
result = Agent.prompt(
|
||||
prompt,
|
||||
AgentOptions(
|
||||
api_key=api_key,
|
||||
model=model,
|
||||
local=LocalAgentOptions(cwd=cwd),
|
||||
),
|
||||
)
|
||||
except CursorAgentError as exc:
|
||||
raise RuntimeError(f"LLM 调用失败：{exc.message}") from exc
|
||||
if result.status == "error":
|
||||
raise RuntimeError(f"LLM 调用失败：{result.result or '未知错误'}")
|
||||
return (result.result or "").strip()
|
||||
|
||||
|
||||
def llm_chat(system: str, user: str) -> str:
|
||||
"""优先 OpenAI 兼容 API，否则 Cursor SDK。"""
|
||||
if env("DAILY_LLM_API_KEY") or env("OPENAI_API_KEY"):
|
||||
backend = resolve_llm_backend()
|
||||
if backend == "openai":
|
||||
return _openai_chat(system, user)
|
||||
if env("CURSOR_API_KEY"):
|
||||
return _cursor_chat(system, user)
|
||||
if backend == "cursor":
|
||||
return cursor_chat(system, user)
|
||||
return ""
|
||||
|
||||
|
||||
def has_llm_configured() -> bool:
|
||||
return bool(env("DAILY_LLM_API_KEY") or env("OPENAI_API_KEY") or env("CURSOR_API_KEY"))
|
||||
return bool(resolve_llm_backend())
|
||||
|
||||
77
daily/news/content_filter.py
Normal file
77
daily/news/content_filter.py
Normal file
@@ -0,0 +1,77 @@
|
||||
"""敏感词内容过滤。"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
from functools import lru_cache
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
from daily.config import ROOT, env
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
_WORDS_FILE = ROOT / "config" / "sensitive_words.yaml"
|
||||
|
||||
|
||||
def content_filter_enabled() -> bool:
|
||||
raw = (env("DAILY_CONTENT_FILTER") or "0").strip().lower()
|
||||
return raw in {"1", "true", "yes", "on"}
|
||||
|
||||
|
||||
@lru_cache(maxsize=1)
|
||||
def load_sensitive_words() -> tuple[str, ...]:
|
||||
if not _WORDS_FILE.exists():
|
||||
return ()
|
||||
try:
|
||||
import yaml
|
||||
except ImportError:
|
||||
logger.warning("未安装 PyYAML，无法读取 %s", _WORDS_FILE)
|
||||
return ()
|
||||
try:
|
||||
data = yaml.safe_load(_WORDS_FILE.read_text(encoding="utf-8"))
|
||||
except OSError as exc:
|
||||
logger.warning("读取敏感词配置失败: %s", exc)
|
||||
return ()
|
||||
except Exception as exc:
|
||||
logger.warning("解析 sensitive_words.yaml 失败: %s", exc)
|
||||
return ()
|
||||
if not isinstance(data, dict):
|
||||
return ()
|
||||
words = data.get("words") or data.get("sensitive_words") or []
|
||||
if not isinstance(words, list):
|
||||
return ()
|
||||
cleaned = tuple(str(word).strip() for word in words if str(word).strip())
|
||||
return cleaned
|
||||
|
||||
|
||||
def matches_sensitive_text(text: str, words: tuple[str, ...]) -> str | None:
|
||||
haystack = (text or "").lower()
|
||||
if not haystack:
|
||||
return None
|
||||
for word in words:
|
||||
needle = word.lower()
|
||||
if needle and needle in haystack:
|
||||
return word
|
||||
return None
|
||||
|
||||
|
||||
def filter_news_items(items: list[dict[str, Any]]) -> tuple[list[dict[str, Any]], int]:
|
||||
if not content_filter_enabled():
|
||||
return items, 0
|
||||
words = load_sensitive_words()
|
||||
if not words:
|
||||
return items, 0
|
||||
|
||||
kept: list[dict[str, Any]] = []
|
||||
removed = 0
|
||||
for item in items:
|
||||
text = f"{item.get('title', '')} {item.get('summary', '')}"
|
||||
hit = matches_sensitive_text(text, words)
|
||||
if hit:
|
||||
removed += 1
|
||||
continue
|
||||
kept.append(item)
|
||||
if removed:
|
||||
logger.info("内容过滤移除 %d 条（敏感词）", removed)
|
||||
return kept, removed
|
||||
@@ -1,125 +1,10 @@
|
||||
"""国际 AI 时讯 RSS 源定义（按类别分组）。"""
|
||||
"""国际 AI 时讯 RSS 源（优先 config/feeds.yaml）。"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from dataclasses import dataclass
|
||||
from daily.news.feeds_loader import load_categories
|
||||
from daily.news.feeds_types import NewsCategory, NewsFeed
|
||||
|
||||
NEWS_CATEGORIES: tuple[NewsCategory, ...] = load_categories("intl")
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class NewsFeed:
|
||||
name: str
|
||||
url: str
|
||||
slow: bool = False # 限速源（如 Reddit）串行抓取
|
||||
ai_filter: bool = False # 综合源仅保留标题命中 AI 关键词的条目
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class NewsCategory:
|
||||
id: str
|
||||
name: str
|
||||
icon: str
|
||||
feeds: tuple[NewsFeed, ...]
|
||||
|
||||
|
||||
NEWS_CATEGORIES: tuple[NewsCategory, ...] = (
|
||||
NewsCategory(
|
||||
id="official",
|
||||
name="厂商官方",
|
||||
icon="🏢",
|
||||
feeds=(
|
||||
NewsFeed("Anthropic Claude 更新", "https://docs.anthropic.com/en/release-notes/feed"),
|
||||
NewsFeed("OpenAI", "https://openai.com/news/rss.xml"),
|
||||
NewsFeed("Google AI", "https://blog.google/technology/ai/rss/"),
|
||||
NewsFeed("DeepMind", "https://deepmind.google/blog/rss.xml"),
|
||||
NewsFeed("Meta Engineering", "https://engineering.fb.com/feed/"),
|
||||
NewsFeed("Microsoft Research", "https://www.microsoft.com/en-us/research/feed/"),
|
||||
NewsFeed("Microsoft Blog", "https://blogs.microsoft.com/feed/"),
|
||||
NewsFeed("Cohere", "https://cohere.com/blog/rss.xml"),
|
||||
NewsFeed("Cursor Changelog", "https://cursor.com/changelog/rss.xml"),
|
||||
),
|
||||
),
|
||||
NewsCategory(
|
||||
id="developer",
|
||||
name="Agent / LLM 开发者",
|
||||
icon="🛠",
|
||||
feeds=(
|
||||
NewsFeed("LangChain", "https://blog.langchain.dev/rss/"),
|
||||
NewsFeed("Hugging Face", "https://huggingface.co/blog/feed.xml"),
|
||||
NewsFeed("Vercel Changelog", "https://vercel.com/changelog/rss.xml"),
|
||||
NewsFeed("GitHub Copilot", "https://github.blog/changelog/label/copilot/feed/"),
|
||||
),
|
||||
),
|
||||
NewsCategory(
|
||||
id="media",
|
||||
name="综合科技媒体",
|
||||
icon="📰",
|
||||
feeds=(
|
||||
NewsFeed("The Verge AI", "https://www.theverge.com/rss/ai-artificial-intelligence/index.xml"),
|
||||
NewsFeed("TechCrunch AI", "https://techcrunch.com/category/artificial-intelligence/feed/"),
|
||||
NewsFeed("Ars Technica AI", "https://arstechnica.com/ai/feed/"),
|
||||
NewsFeed("Wired AI", "https://www.wired.com/feed/tag/ai/latest/rss"),
|
||||
NewsFeed("MIT Tech Review", "https://www.technologyreview.com/feed/"),
|
||||
NewsFeed("VentureBeat AI", "https://venturebeat.com/category/ai/feed/"),
|
||||
),
|
||||
),
|
||||
NewsCategory(
|
||||
id="newsletter",
|
||||
name="Newsletter 日报",
|
||||
icon="✉️",
|
||||
feeds=(
|
||||
NewsFeed("Ben's Bites", "https://bensbites.substack.com/feed"),
|
||||
NewsFeed("The Rundown AI", "https://therundown.substack.com/feed"),
|
||||
NewsFeed("Latent Space", "https://www.latent.space/feed"),
|
||||
NewsFeed("Simon Willison", "https://simonwillison.net/atom/everything/"),
|
||||
NewsFeed("Import AI", "https://importai.substack.com/feed"),
|
||||
NewsFeed("Last Week in AI", "https://lastweekin.ai/feed"),
|
||||
NewsFeed("The Neuron", "https://www.theneuron.ai/feed"),
|
||||
),
|
||||
),
|
||||
NewsCategory(
|
||||
id="research",
|
||||
name="研究 / 论文",
|
||||
icon="📚",
|
||||
feeds=(
|
||||
NewsFeed("arXiv cs.CL", "https://arxiv.org/rss/cs.CL"),
|
||||
NewsFeed("arXiv cs.AI", "https://arxiv.org/rss/cs.AI"),
|
||||
NewsFeed("arXiv cs.LG", "https://arxiv.org/rss/cs.LG"),
|
||||
),
|
||||
),
|
||||
NewsCategory(
|
||||
id="trending",
|
||||
name="热点 / 趋势",
|
||||
icon="🔥",
|
||||
feeds=(
|
||||
NewsFeed(
|
||||
"Google News · AI",
|
||||
"https://news.google.com/rss/search?q=artificial+intelligence+OR+LLM+OR+Claude+OR+GPT&hl=en-US&gl=US&ceid=US:en",
|
||||
),
|
||||
NewsFeed(
|
||||
"Google News · Technology",
|
||||
"https://news.google.com/rss/headlines/section/topic/TECHNOLOGY?hl=en-US&gl=US&ceid=US:en",
|
||||
),
|
||||
NewsFeed("Techmeme", "https://www.techmeme.com/feed.xml"),
|
||||
NewsFeed("HN · Front Page", "https://hnrss.org/frontpage"),
|
||||
NewsFeed("HN · 100+ Points", "https://hnrss.org/newest?points=100"),
|
||||
NewsFeed("Dev.to · AI", "https://dev.to/feed/tag/ai"),
|
||||
NewsFeed("Lobsters", "https://lobste.rs/rss"),
|
||||
),
|
||||
),
|
||||
NewsCategory(
|
||||
id="community",
|
||||
name="社区讨论",
|
||||
icon="💬",
|
||||
feeds=(
|
||||
NewsFeed(
|
||||
"HN · AI/LLM/Agent",
|
||||
"https://hnrss.org/newest?q=AI+OR+LLM+OR+Claude+OR+agent+OR+GPT+OR+Gemini",
|
||||
),
|
||||
NewsFeed(
|
||||
"Reddit · LLM/Claude/ML",
|
||||
"https://old.reddit.com/r/LocalLLaMA+ClaudeAI+MachineLearning+OpenAI/.rss?limit=25",
|
||||
slow=True,
|
||||
),
|
||||
),
|
||||
),
|
||||
)
|
||||
__all__ = ["NewsCategory", "NewsFeed", "NEWS_CATEGORIES"]
|
||||
|
||||
@@ -1,71 +1,11 @@
|
||||
"""国内 AI 时讯 RSS 源定义（按类别分组）。"""
|
||||
"""国内 AI 时讯 RSS 源（优先 config/feeds.yaml）。"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from daily.news.feeds import NewsCategory, NewsFeed
|
||||
from daily.news.feeds_loader import load_categories, load_cn_title_keywords
|
||||
from daily.news.feeds_types import NewsCategory, NewsFeed
|
||||
|
||||
# 综合源 ai_filter=True 时，仅保留标题命中以下词之一的条目
|
||||
CN_AI_TITLE_KEYWORDS: tuple[str, ...] = (
|
||||
"人工智能",
|
||||
"大模型",
|
||||
"智能体",
|
||||
"多模态",
|
||||
"AIGC",
|
||||
"LLM",
|
||||
"GPT",
|
||||
"Claude",
|
||||
"Gemini",
|
||||
"ChatGPT",
|
||||
"OpenAI",
|
||||
"Anthropic",
|
||||
"Copilot",
|
||||
"Agent",
|
||||
"AI ",
|
||||
" AI",
|
||||
"AI·",
|
||||
"AI业务",
|
||||
"AI模型",
|
||||
"AI助手",
|
||||
"AI工具",
|
||||
"AI编程",
|
||||
"AI 编程",
|
||||
"AI版",
|
||||
"AI Agent",
|
||||
"推理模型",
|
||||
"深度学习",
|
||||
"机器学习",
|
||||
"Function Calling",
|
||||
)
|
||||
CN_AI_TITLE_KEYWORDS: tuple[str, ...] = load_cn_title_keywords()
|
||||
CN_NEWS_CATEGORIES: tuple[NewsCategory, ...] = load_categories("cn")
|
||||
|
||||
CN_NEWS_CATEGORIES: tuple[NewsCategory, ...] = (
|
||||
NewsCategory(
|
||||
id="media",
|
||||
name="AI 专业媒体",
|
||||
icon="📰",
|
||||
feeds=(
|
||||
NewsFeed("量子位", "https://www.qbitai.com/feed"),
|
||||
NewsFeed("InfoQ 中文", "https://www.infoq.cn/feed/AI"),
|
||||
),
|
||||
),
|
||||
NewsCategory(
|
||||
id="tech",
|
||||
name="综合科技",
|
||||
icon="📱",
|
||||
feeds=(
|
||||
NewsFeed("36氪", "https://36kr.com/feed", ai_filter=True),
|
||||
NewsFeed("雷锋网", "https://www.leiphone.com/feed"),
|
||||
NewsFeed(
|
||||
"Google News · AI",
|
||||
"https://news.google.com/rss/search?q=人工智能+OR+大模型+OR+Agent+OR+LLM&hl=zh-CN&gl=CN&ceid=CN:zh-Hans",
|
||||
),
|
||||
),
|
||||
),
|
||||
NewsCategory(
|
||||
id="dev",
|
||||
name="开发者社区",
|
||||
icon="💻",
|
||||
feeds=(
|
||||
NewsFeed("掘金", "https://juejin.cn/rss", ai_filter=True),
|
||||
),
|
||||
),
|
||||
)
|
||||
__all__ = ["CN_AI_TITLE_KEYWORDS", "CN_NEWS_CATEGORIES", "NewsCategory", "NewsFeed"]
|
||||
|
||||
11
daily/news/feeds_defaults.py
Normal file
11
daily/news/feeds_defaults.py
Normal file
@@ -0,0 +1,11 @@
|
||||
"""Built-in RSS defaults when config/feeds.yaml is missing or invalid."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from daily.news.feeds_types import NewsCategory, NewsFeed
|
||||
|
||||
CN_AI_TITLE_KEYWORDS: tuple[str, ...] = ('人工智能', '大模型', '智能体', '多模态', 'AIGC', 'LLM', 'GPT', 'Claude', 'Gemini', 'ChatGPT', 'OpenAI', 'Anthropic', 'Copilot', 'Agent', 'AI ', ' AI', 'AI·', 'AI业务', 'AI模型', 'AI助手', 'AI工具', 'AI编程', 'AI 编程', 'AI版', 'AI Agent', '推理模型', '深度学习', '机器学习', 'Function Calling')
|
||||
|
||||
CN_NEWS_CATEGORIES: tuple[NewsCategory, ...] = (NewsCategory(id='media', name='AI 专业媒体', icon='📰', feeds=(NewsFeed(name='量子位', url='https://www.qbitai.com/feed', slow=False, ai_filter=False), NewsFeed(name='InfoQ 中文', url='https://www.infoq.cn/feed/AI', slow=False, ai_filter=False))), NewsCategory(id='tech', name='综合科技', icon='📱', feeds=(NewsFeed(name='36氪', url='https://36kr.com/feed', slow=False, ai_filter=True), NewsFeed(name='雷锋网', url='https://www.leiphone.com/feed', slow=False, ai_filter=False), NewsFeed(name='Google News · AI', url='https://news.google.com/rss/search?q=人工智能+OR+大模型+OR+Agent+OR+LLM&hl=zh-CN&gl=CN&ceid=CN:zh-Hans', slow=False, ai_filter=False))), NewsCategory(id='dev', name='开发者社区', icon='💻', feeds=(NewsFeed(name='掘金', url='https://juejin.cn/rss', slow=False, ai_filter=True),)))
|
||||
|
||||
NEWS_CATEGORIES: tuple[NewsCategory, ...] = (NewsCategory(id='official', name='厂商官方', icon='🏢', feeds=(NewsFeed(name='Anthropic Claude 更新', url='https://docs.anthropic.com/en/release-notes/feed', slow=False, ai_filter=False), NewsFeed(name='OpenAI', url='https://openai.com/news/rss.xml', slow=False, ai_filter=False), NewsFeed(name='Google AI', url='https://blog.google/technology/ai/rss/', slow=False, ai_filter=False), NewsFeed(name='DeepMind', url='https://deepmind.google/blog/rss.xml', slow=False, ai_filter=False), NewsFeed(name='Meta Engineering', url='https://engineering.fb.com/feed/', slow=False, ai_filter=False), NewsFeed(name='Microsoft Research', url='https://www.microsoft.com/en-us/research/feed/', slow=False, ai_filter=False), NewsFeed(name='Microsoft Blog', url='https://blogs.microsoft.com/feed/', slow=False, ai_filter=False), NewsFeed(name='Cohere', url='https://cohere.com/blog/rss.xml', slow=False, ai_filter=False), NewsFeed(name='Cursor Changelog', url='https://cursor.com/changelog/rss.xml', slow=False, ai_filter=False))), NewsCategory(id='developer', name='Agent / LLM 开发者', icon='🛠', feeds=(NewsFeed(name='LangChain', url='https://blog.langchain.dev/rss/', slow=False, ai_filter=False), NewsFeed(name='Hugging Face', url='https://huggingface.co/blog/feed.xml', slow=False, ai_filter=False), NewsFeed(name='Vercel Changelog', url='https://vercel.com/changelog/rss.xml', slow=False, ai_filter=False), NewsFeed(name='GitHub Copilot', url='https://github.blog/changelog/label/copilot/feed/', slow=False, ai_filter=False))), NewsCategory(id='media', name='综合科技媒体', icon='📰', feeds=(NewsFeed(name='The Verge AI', url='https://www.theverge.com/rss/ai-artificial-intelligence/index.xml', slow=False, ai_filter=False), NewsFeed(name='TechCrunch AI', url='https://techcrunch.com/category/artificial-intelligence/feed/', slow=False, ai_filter=False), NewsFeed(name='Ars Technica AI', url='https://arstechnica.com/ai/feed/', slow=False, ai_filter=False), NewsFeed(name='Wired AI', url='https://www.wired.com/feed/tag/ai/latest/rss', slow=False, ai_filter=False), NewsFeed(name='MIT Tech Review', url='https://www.technologyreview.com/feed/', slow=False, ai_filter=False), NewsFeed(name='VentureBeat AI', url='https://venturebeat.com/category/ai/feed/', slow=False, ai_filter=False))), NewsCategory(id='newsletter', name='Newsletter 日报', icon='✉️', feeds=(NewsFeed(name="Ben's Bites", url='https://bensbites.substack.com/feed', slow=False, ai_filter=False), NewsFeed(name='The Rundown AI', url='https://therundown.substack.com/feed', slow=False, ai_filter=False), NewsFeed(name='Latent Space', url='https://www.latent.space/feed', slow=False, ai_filter=False), NewsFeed(name='Simon Willison', url='https://simonwillison.net/atom/everything/', slow=False, ai_filter=False), NewsFeed(name='Import AI', url='https://importai.substack.com/feed', slow=False, ai_filter=False), NewsFeed(name='Last Week in AI', url='https://lastweekin.ai/feed', slow=False, ai_filter=False), NewsFeed(name='The Neuron', url='https://www.theneuron.ai/feed', slow=False, ai_filter=False))), NewsCategory(id='research', name='研究 / 论文', icon='📚', feeds=(NewsFeed(name='arXiv cs.CL', url='https://arxiv.org/rss/cs.CL', slow=False, ai_filter=False), NewsFeed(name='arXiv cs.AI', url='https://arxiv.org/rss/cs.AI', slow=False, ai_filter=False), NewsFeed(name='arXiv cs.LG', url='https://arxiv.org/rss/cs.LG', slow=False, ai_filter=False))), NewsCategory(id='trending', name='热点 / 趋势', icon='🔥', feeds=(NewsFeed(name='Google News · AI', url='https://news.google.com/rss/search?q=artificial+intelligence+OR+LLM+OR+Claude+OR+GPT&hl=en-US&gl=US&ceid=US:en', slow=False, ai_filter=False), NewsFeed(name='Google News · Technology', url='https://news.google.com/rss/headlines/section/topic/TECHNOLOGY?hl=en-US&gl=US&ceid=US:en', slow=False, ai_filter=False), NewsFeed(name='Techmeme', url='https://www.techmeme.com/feed.xml', slow=False, ai_filter=False), NewsFeed(name='HN · Front Page', url='https://hnrss.org/frontpage', slow=False, ai_filter=False), NewsFeed(name='HN · 100+ Points', url='https://hnrss.org/newest?points=100', slow=False, ai_filter=False), NewsFeed(name='Dev.to · AI', url='https://dev.to/feed/tag/ai', slow=False, ai_filter=False), NewsFeed(name='Lobsters', url='https://lobste.rs/rss', slow=False, ai_filter=False))), NewsCategory(id='community', name='社区讨论', icon='💬', feeds=(NewsFeed(name='HN · AI/LLM/Agent', url='https://hnrss.org/newest?q=AI+OR+LLM+OR+Claude+OR+agent+OR+GPT+OR+Gemini', slow=False, ai_filter=False), NewsFeed(name='Reddit · LLM/Claude/ML', url='https://old.reddit.com/r/LocalLLaMA+ClaudeAI+MachineLearning+OpenAI/.rss?limit=25', slow=True, ai_filter=False))))
|
||||
113
daily/news/feeds_loader.py
Normal file
113
daily/news/feeds_loader.py
Normal file
@@ -0,0 +1,113 @@
|
||||
"""从 config/feeds.yaml 加载 RSS 源（失败时回退内置默认）。"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
from functools import lru_cache
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
from daily.config import ROOT
|
||||
from daily.news.feeds_types import NewsCategory, NewsFeed
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
_FEEDS_FILE = ROOT / "config" / "feeds.yaml"
|
||||
|
||||
|
||||
def _parse_feed(raw: dict[str, Any]) -> NewsFeed:
|
||||
return NewsFeed(
|
||||
name=str(raw.get("name") or "").strip(),
|
||||
url=str(raw.get("url") or "").strip(),
|
||||
slow=bool(raw.get("slow")),
|
||||
ai_filter=bool(raw.get("ai_filter")),
|
||||
)
|
||||
|
||||
|
||||
def _parse_category(raw: dict[str, Any]) -> NewsCategory | None:
|
||||
cat_id = str(raw.get("id") or "").strip()
|
||||
if not cat_id:
|
||||
return None
|
||||
feeds_raw = raw.get("feeds") or []
|
||||
feeds: list[NewsFeed] = []
|
||||
for item in feeds_raw:
|
||||
if not isinstance(item, dict):
|
||||
continue
|
||||
feed = _parse_feed(item)
|
||||
if feed.name and feed.url:
|
||||
feeds.append(feed)
|
||||
if not feeds:
|
||||
return None
|
||||
return NewsCategory(
|
||||
id=cat_id,
|
||||
name=str(raw.get("name") or cat_id),
|
||||
icon=str(raw.get("icon") or "📰"),
|
||||
feeds=tuple(feeds),
|
||||
)
|
||||
|
||||
|
||||
def _parse_categories(items: Any) -> tuple[NewsCategory, ...]:
|
||||
if not isinstance(items, list):
|
||||
return ()
|
||||
categories: list[NewsCategory] = []
|
||||
for raw in items:
|
||||
if not isinstance(raw, dict):
|
||||
continue
|
||||
cat = _parse_category(raw)
|
||||
if cat:
|
||||
categories.append(cat)
|
||||
return tuple(categories)
|
||||
|
||||
|
||||
def _load_yaml() -> dict[str, Any] | None:
|
||||
if not _FEEDS_FILE.exists():
|
||||
return None
|
||||
try:
|
||||
import yaml
|
||||
except ImportError:
|
||||
logger.warning("未安装 PyYAML，无法读取 %s", _FEEDS_FILE)
|
||||
return None
|
||||
try:
|
||||
data = yaml.safe_load(_FEEDS_FILE.read_text(encoding="utf-8"))
|
||||
except OSError as exc:
|
||||
logger.warning("读取 feeds 配置失败: %s", exc)
|
||||
return None
|
||||
except Exception as exc:
|
||||
logger.warning("解析 feeds.yaml 失败: %s", exc)
|
||||
return None
|
||||
return data if isinstance(data, dict) else None
|
||||
|
||||
|
||||
@lru_cache(maxsize=1)
|
||||
def _yaml_payload() -> dict[str, Any] | None:
|
||||
return _load_yaml()
|
||||
|
||||
|
||||
def load_categories(region: str) -> tuple[NewsCategory, ...]:
|
||||
data = _yaml_payload()
|
||||
if data:
|
||||
block = data.get(region) or {}
|
||||
categories = _parse_categories(block.get("categories"))
|
||||
if categories:
|
||||
return categories
|
||||
logger.warning("feeds.yaml 中 %s.categories 为空，使用内置默认", region)
|
||||
|
||||
from daily.news import feeds_defaults as defaults
|
||||
|
||||
if region == "cn":
|
||||
return defaults.CN_NEWS_CATEGORIES
|
||||
return defaults.NEWS_CATEGORIES
|
||||
|
||||
|
||||
def load_cn_title_keywords() -> tuple[str, ...]:
|
||||
data = _yaml_payload()
|
||||
if data:
|
||||
block = data.get("cn") or {}
|
||||
keywords = block.get("title_keywords")
|
||||
if isinstance(keywords, list):
|
||||
cleaned = tuple(str(x).strip() for x in keywords if str(x).strip())
|
||||
if cleaned:
|
||||
return cleaned
|
||||
from daily.news.feeds_defaults import CN_AI_TITLE_KEYWORDS
|
||||
|
||||
return CN_AI_TITLE_KEYWORDS
|
||||
21
daily/news/feeds_types.py
Normal file
21
daily/news/feeds_types.py
Normal file
@@ -0,0 +1,21 @@
|
||||
"""RSS 源数据结构。"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from dataclasses import dataclass
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class NewsFeed:
|
||||
name: str
|
||||
url: str
|
||||
slow: bool = False
|
||||
ai_filter: bool = False
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class NewsCategory:
|
||||
id: str
|
||||
name: str
|
||||
icon: str
|
||||
feeds: tuple[NewsFeed, ...]
|
||||
@@ -17,6 +17,7 @@ import certifi
|
||||
import httpx
|
||||
|
||||
from daily.config import env, env_int, news_summary_limit
|
||||
from daily.news.content_filter import filter_news_items
|
||||
from daily.news.feeds import NEWS_CATEGORIES, NewsCategory, NewsFeed
|
||||
from daily.news.feeds_cn import CN_AI_TITLE_KEYWORDS, CN_NEWS_CATEGORIES
|
||||
|
||||
@@ -349,6 +350,7 @@ def _fetch_news(categories: tuple[NewsCategory, ...]) -> dict[str, Any]:
|
||||
"feeds_ok": 0,
|
||||
"items_raw": 0,
|
||||
"feeds_failed": [],
|
||||
"content_filtered": 0,
|
||||
}
|
||||
|
||||
with httpx.Client(timeout=15.0, verify=certifi.where(), follow_redirects=True, headers=headers) as client:
|
||||
@@ -394,6 +396,9 @@ def _fetch_news(categories: tuple[NewsCategory, ...]) -> dict[str, Any]:
|
||||
for category in categories:
|
||||
items = raw_by_category[category.id]
|
||||
items = [i for i in items if _within_window(i, cutoff)]
|
||||
items, filtered_count = filter_news_items(items)
|
||||
if filtered_count:
|
||||
stats["content_filtered"] = stats.get("content_filtered", 0) + filtered_count
|
||||
items.sort(key=_sort_key, reverse=True)
|
||||
items = _dedupe_items(items)[:per_category]
|
||||
for item in items:
|
||||
|
||||
Reference in New Issue
Block a user