项目初始化

daily/news/__init__.py

@@ -0,0 +1,3 @@
+from daily.news.fetch import fetch_ai_news, format_news_section
+
+__all__ = ["fetch_ai_news", "format_news_section"]

daily/news/feeds.py

@@ -0,0 +1,124 @@
+"""国际 AI 时讯 RSS 源定义（按类别分组）。"""
+
+from __future__ import annotations
+
+from dataclasses import dataclass
+
+
+@dataclass(frozen=True)
+class NewsFeed:
+    name: str
+    url: str
+    slow: bool = False  # 限速源（如 Reddit）串行抓取
+
+
+@dataclass(frozen=True)
+class NewsCategory:
+    id: str
+    name: str
+    icon: str
+    feeds: tuple[NewsFeed, ...]
+
+
+NEWS_CATEGORIES: tuple[NewsCategory, ...] = (
+    NewsCategory(
+        id="official",
+        name="厂商官方",
+        icon="🏢",
+        feeds=(
+            NewsFeed("Anthropic Claude 更新", "https://docs.anthropic.com/en/release-notes/feed"),
+            NewsFeed("OpenAI", "https://openai.com/news/rss.xml"),
+            NewsFeed("Google AI", "https://blog.google/technology/ai/rss/"),
+            NewsFeed("DeepMind", "https://deepmind.google/blog/rss.xml"),
+            NewsFeed("Meta Engineering", "https://engineering.fb.com/feed/"),
+            NewsFeed("Microsoft Research", "https://www.microsoft.com/en-us/research/feed/"),
+            NewsFeed("Microsoft Blog", "https://blogs.microsoft.com/feed/"),
+            NewsFeed("Cohere", "https://cohere.com/blog/rss.xml"),
+            NewsFeed("Cursor Changelog", "https://cursor.com/changelog/rss.xml"),
+        ),
+    ),
+    NewsCategory(
+        id="developer",
+        name="Agent / LLM 开发者",
+        icon="🛠",
+        feeds=(
+            NewsFeed("LangChain", "https://blog.langchain.dev/rss/"),
+            NewsFeed("Hugging Face", "https://huggingface.co/blog/feed.xml"),
+            NewsFeed("Vercel Changelog", "https://vercel.com/changelog/rss.xml"),
+            NewsFeed("GitHub Copilot", "https://github.blog/changelog/label/copilot/feed/"),
+        ),
+    ),
+    NewsCategory(
+        id="media",
+        name="综合科技媒体",
+        icon="📰",
+        feeds=(
+            NewsFeed("The Verge AI", "https://www.theverge.com/rss/ai-artificial-intelligence/index.xml"),
+            NewsFeed("TechCrunch AI", "https://techcrunch.com/category/artificial-intelligence/feed/"),
+            NewsFeed("Ars Technica AI", "https://arstechnica.com/ai/feed/"),
+            NewsFeed("Wired AI", "https://www.wired.com/feed/tag/ai/latest/rss"),
+            NewsFeed("MIT Tech Review", "https://www.technologyreview.com/feed/"),
+            NewsFeed("VentureBeat AI", "https://venturebeat.com/category/ai/feed/"),
+        ),
+    ),
+    NewsCategory(
+        id="newsletter",
+        name="Newsletter 日报",
+        icon="✉️",
+        feeds=(
+            NewsFeed("Ben's Bites", "https://bensbites.substack.com/feed"),
+            NewsFeed("The Rundown AI", "https://therundown.substack.com/feed"),
+            NewsFeed("Latent Space", "https://www.latent.space/feed"),
+            NewsFeed("Simon Willison", "https://simonwillison.net/atom/everything/"),
+            NewsFeed("Import AI", "https://importai.substack.com/feed"),
+            NewsFeed("Last Week in AI", "https://lastweekin.ai/feed"),
+            NewsFeed("The Neuron", "https://www.theneuron.ai/feed"),
+        ),
+    ),
+    NewsCategory(
+        id="research",
+        name="研究 / 论文",
+        icon="📚",
+        feeds=(
+            NewsFeed("arXiv cs.CL", "https://arxiv.org/rss/cs.CL"),
+            NewsFeed("arXiv cs.AI", "https://arxiv.org/rss/cs.AI"),
+            NewsFeed("arXiv cs.LG", "https://arxiv.org/rss/cs.LG"),
+        ),
+    ),
+    NewsCategory(
+        id="trending",
+        name="热点 / 趋势",
+        icon="🔥",
+        feeds=(
+            NewsFeed(
+                "Google News · AI",
+                "https://news.google.com/rss/search?q=artificial+intelligence+OR+LLM+OR+Claude+OR+GPT&hl=en-US&gl=US&ceid=US:en",
+            ),
+            NewsFeed(
+                "Google News · Technology",
+                "https://news.google.com/rss/headlines/section/topic/TECHNOLOGY?hl=en-US&gl=US&ceid=US:en",
+            ),
+            NewsFeed("Techmeme", "https://www.techmeme.com/feed.xml"),
+            NewsFeed("HN · Front Page", "https://hnrss.org/frontpage"),
+            NewsFeed("HN · 100+ Points", "https://hnrss.org/newest?points=100"),
+            NewsFeed("Dev.to · AI", "https://dev.to/feed/tag/ai"),
+            NewsFeed("Lobsters", "https://lobste.rs/rss"),
+        ),
+    ),
+    NewsCategory(
+        id="community",
+        name="社区讨论",
+        icon="💬",
+        feeds=(
+            NewsFeed(
+                "HN · AI/LLM/Agent",
+                "https://hnrss.org/newest?q=AI+OR+LLM+OR+Claude+OR+agent+OR+GPT+OR+Gemini",
+            ),
+            NewsFeed(
+                "Reddit · LLM/Claude/ML",
+                "https://old.reddit.com/r/LocalLLaMA+ClaudeAI+MachineLearning+OpenAI/.rss?limit=25",
+                slow=True,
+            ),
+        ),
+    ),
+)

daily/news/fetch.py

@@ -0,0 +1,462 @@
+"""抓取并整理国际 AI 时讯 RSS。"""
+
+from __future__ import annotations
+
+import logging
+import re
+import time
+import html
+import xml.etree.ElementTree as ET
+from concurrent.futures import ThreadPoolExecutor, as_completed
+from datetime import datetime, timezone, timedelta
+from email.utils import parsedate_to_datetime
+from typing import Any
+from urllib.parse import parse_qs, urlencode, urlparse, urlunparse
+
+import certifi
+import httpx
+
+from daily.config import env, env_int, news_summary_limit
+from daily.news.feeds import NEWS_CATEGORIES, NewsCategory
+
+logger = logging.getLogger(__name__)
+
+USER_AGENT = "Mozilla/5.0 (compatible; skills-hot-daily/1.0; +https://skills.sh)"
+BROWSER_USER_AGENT = (
+    "Mozilla/5.0 (Windows NT 10.0; Win64; x64) "
+    "AppleWebKit/537.36 (KHTML, like Gecko) "
+    "Chrome/131.0.0.0 Safari/537.36"
+)
+STRIP_HTML = re.compile(r"<[^>]+>")
+WS = re.compile(r"\s+")
+
+ATOM_NS = {"a": "http://www.w3.org/2005/Atom"}
+RSS_NS = {"r": "http://purl.org/rss/1.0/modules/content/"}
+
+
+def _enabled() -> bool:
+    raw = (env("DAILY_AI_NEWS") or "1").strip().lower()
+    return raw not in {"0", "false", "no", "off"}
+
+
+def _hours_window() -> int:
+    return max(1, env_int("DAILY_AI_NEWS_HOURS", 72))
+
+
+def _per_feed_limit() -> int:
+    return max(1, env_int("DAILY_AI_NEWS_PER_FEED", 3))
+
+
+def _per_category_limit() -> int:
+    return max(1, env_int("DAILY_AI_NEWS_PER_CATEGORY", 5))
+
+
+def _wecom_limit() -> int:
+    return max(1, env_int("DAILY_WECOM_AI_NEWS", 10))
+
+
+def _now_utc() -> datetime:
+    return datetime.now(timezone.utc)
+
+
+def _parse_datetime(value: str | None) -> datetime | None:
+    if not value:
+        return None
+    text = value.strip()
+    if not text:
+        return None
+    try:
+        dt = parsedate_to_datetime(text)
+        if dt.tzinfo is None:
+            dt = dt.replace(tzinfo=timezone.utc)
+        return dt.astimezone(timezone.utc)
+    except (TypeError, ValueError, OverflowError):
+        pass
+    for fmt in (
+        "%Y-%m-%dT%H:%M:%SZ",
+        "%Y-%m-%dT%H:%M:%S%z",
+        "%Y-%m-%d",
+    ):
+        try:
+            dt = datetime.strptime(text[: len(fmt.replace("%z", "+0000"))], fmt.replace("%z", ""))
+            if dt.tzinfo is None:
+                dt = dt.replace(tzinfo=timezone.utc)
+            return dt.astimezone(timezone.utc)
+        except ValueError:
+            continue
+    return None
+
+
+def _clean_text(text: str | None, limit: int = 200) -> str:
+    if not text:
+        return ""
+    plain = STRIP_HTML.sub(" ", html.unescape(text))
+    plain = WS.sub(" ", plain).strip()
+    if limit <= 0 or len(plain) <= limit:
+        return plain
+    return plain[: limit - 3] + "..."
+
+
+def _normalize_link(link: str) -> str:
+    parsed = urlparse(link.strip())
+    query = parse_qs(parsed.query, keep_blank_values=False)
+    for key in list(query.keys()):
+        if key.lower().startswith("utm_") or key.lower() in {"ref", "source"}:
+            query.pop(key, None)
+    clean_query = urlencode({k: v[0] for k, v in query.items() if v}, doseq=False)
+    return urlunparse((parsed.scheme, parsed.netloc, parsed.path.rstrip("/"), "", clean_query, ""))
+
+
+def _normalize_title(title: str) -> str:
+    return WS.sub(" ", title.strip().lower())
+
+
+def _entry_datetime(entry: dict[str, Any]) -> datetime | None:
+    for key in ("published", "updated"):
+        dt = _parse_datetime(entry.get(key))
+        if dt:
+            return dt
+    return None
+
+
+def _parse_atom(content: str, feed_name: str, category: NewsCategory) -> list[dict[str, Any]]:
+    items: list[dict[str, Any]] = []
+    try:
+        root = ET.fromstring(content)
+    except ET.ParseError:
+        return items
+
+    for entry in root.findall("a:entry", ATOM_NS):
+        title_el = entry.find("a:title", ATOM_NS)
+        link_el = entry.find("a:link", ATOM_NS)
+        summary_el = entry.find("a:summary", ATOM_NS) or entry.find("a:content", ATOM_NS)
+        updated_el = entry.find("a:updated", ATOM_NS) or entry.find("a:published", ATOM_NS)
+        title = title_el.text.strip() if title_el is not None and title_el.text else ""
+        link = ""
+        if link_el is not None:
+            link = link_el.get("href") or (link_el.text or "").strip()
+        if not title or not link:
+            continue
+        items.append(
+            {
+                "title": title,
+                "link": link,
+                "summary": _clean_text(summary_el.text if summary_el is not None else ""),
+                "published": updated_el.text.strip() if updated_el is not None and updated_el.text else "",
+                "source_name": feed_name,
+                "category_id": category.id,
+                "category_name": category.name,
+                "category_icon": category.icon,
+            }
+        )
+    return items
+
+
+def _parse_rss(content: str, feed_name: str, category: NewsCategory) -> list[dict[str, Any]]:
+    items: list[dict[str, Any]] = []
+    try:
+        root = ET.fromstring(content)
+    except ET.ParseError:
+        return items
+
+    channel = root.find("channel")
+    if channel is None:
+        return items
+
+    for item in channel.findall("item"):
+        title_el = item.find("title")
+        link_el = item.find("link")
+        desc_el = item.find("description")
+        pub_el = item.find("pubDate")
+        title = title_el.text.strip() if title_el is not None and title_el.text else ""
+        link = link_el.text.strip() if link_el is not None and link_el.text else ""
+        if not title or not link:
+            continue
+        items.append(
+            {
+                "title": title,
+                "link": link,
+                "summary": _clean_text(desc_el.text if desc_el is not None else ""),
+                "published": pub_el.text.strip() if pub_el is not None and pub_el.text else "",
+                "source_name": feed_name,
+                "category_id": category.id,
+                "category_name": category.name,
+                "category_icon": category.icon,
+            }
+        )
+    return items
+
+
+def _parse_feed(content: str, feed_name: str, category: NewsCategory) -> list[dict[str, Any]]:
+    text = content.lstrip("\ufeff").strip()
+    if not text:
+        return []
+    if text.startswith("<rss") or "<channel>" in text[:500]:
+        return _parse_rss(text, feed_name, category)
+    if text.startswith("<feed") or "<entry>" in text[:500]:
+        return _parse_atom(text, feed_name, category)
+    if "<item>" in text[:2000]:
+        return _parse_rss(text, feed_name, category)
+    return _parse_atom(text, feed_name, category)
+
+
+def _is_reddit_url(url: str) -> bool:
+    host = urlparse(url).netloc.lower()
+    return host.endswith("reddit.com")
+
+
+def _reddit_auth_params() -> dict[str, str]:
+    user = (env("REDDIT_RSS_USER") or "").strip()
+    feed = (env("REDDIT_RSS_FEED") or "").strip()
+    if user and feed:
+        return {"user": user, "feed": feed}
+    return {}
+
+
+def _with_query_params(url: str, extra: dict[str, str]) -> str:
+    if not extra:
+        return url
+    parsed = urlparse(url)
+    query = parse_qs(parsed.query, keep_blank_values=True)
+    for key, value in extra.items():
+        if value and key not in query:
+            query[key] = [value]
+    clean_query = urlencode({k: v[0] for k, v in query.items() if v and v[0]}, doseq=False)
+    return urlunparse((parsed.scheme, parsed.netloc, parsed.path, "", clean_query, ""))
+
+
+def _reddit_old_url(url: str) -> str:
+    parsed = urlparse(url)
+    host = parsed.netloc.lower()
+    if host.startswith("old."):
+        return url
+    if host in {"www.reddit.com", "reddit.com"}:
+        return urlunparse((parsed.scheme, "old.reddit.com", parsed.path, "", parsed.query, ""))
+    return url
+
+
+def _request_headers(base: dict[str, str], url: str) -> dict[str, str]:
+    if not _is_reddit_url(url):
+        return base
+    return {
+        **base,
+        "User-Agent": BROWSER_USER_AGENT,
+        "Accept-Language": "en-US,en;q=0.9",
+    }
+
+
+def _reddit_fetch_urls(feed_url: str) -> list[str]:
+    primary = _with_query_params(feed_url, _reddit_auth_params())
+    if not _is_reddit_url(primary):
+        return [primary]
+    fallback = _reddit_old_url(primary)
+    if fallback == primary:
+        return [primary]
+    return [primary, fallback]
+
+
+def _fetch_one(client: httpx.Client, category: NewsCategory, feed_url: str, feed_name: str) -> list[dict[str, Any]]:
+    last_exc: Exception | None = None
+    for url in _reddit_fetch_urls(feed_url):
+        try:
+            headers = _request_headers(dict(client.headers), url)
+            resp = client.get(url, headers=headers)
+            resp.raise_for_status()
+            return _parse_feed(resp.text, feed_name, category)
+        except Exception as exc:
+            last_exc = exc
+            continue
+    logger.warning("RSS fetch failed [%s] %s: %s", feed_name, feed_url, last_exc)
+    return []
+
+
+def _dedupe_items(items: list[dict[str, Any]]) -> list[dict[str, Any]]:
+    seen_links: set[str] = set()
+    seen_titles: set[str] = set()
+    result: list[dict[str, Any]] = []
+    for item in items:
+        link_key = _normalize_link(item["link"])
+        title_key = _normalize_title(item["title"])
+        if link_key in seen_links or title_key in seen_titles:
+            continue
+        seen_links.add(link_key)
+        seen_titles.add(title_key)
+        result.append(item)
+    return result
+
+
+def _within_window(item: dict[str, Any], cutoff: datetime) -> bool:
+    dt = _entry_datetime(item)
+    if dt is None:
+        return True
+    return dt >= cutoff
+
+
+def _sort_key(item: dict[str, Any]) -> tuple[int, datetime]:
+    dt = _entry_datetime(item)
+    if dt is None:
+        return (1, datetime.min.replace(tzinfo=timezone.utc))
+    return (0, dt)
+
+
+def fetch_ai_news() -> dict[str, Any]:
+    """按类别抓取 AI 时讯，返回 {enabled, hours, categories, flat, stats}。"""
+    if not _enabled():
+        return {"enabled": False, "categories": [], "flat": [], "stats": {}}
+
+    hours = _hours_window()
+    per_feed = _per_feed_limit()
+    per_category = _per_category_limit()
+    cutoff = _now_utc() - timedelta(hours=hours)
+
+    headers = {"User-Agent": USER_AGENT, "Accept": "application/rss+xml, application/atom+xml, application/xml, text/xml, */*"}
+    tasks: list[tuple[NewsCategory, str, str, bool]] = []
+    for category in NEWS_CATEGORIES:
+        for feed in category.feeds:
+            tasks.append((category, feed.url, feed.name, feed.slow))
+
+    raw_by_category: dict[str, list[dict[str, Any]]] = {c.id: [] for c in NEWS_CATEGORIES}
+    stats = {"feeds_total": len(tasks), "feeds_ok": 0, "items_raw": 0}
+
+    with httpx.Client(timeout=15.0, verify=certifi.where(), follow_redirects=True, headers=headers) as client:
+        fast_tasks = [t for t in tasks if not t[3]]
+        slow_tasks = [t for t in tasks if t[3]]
+
+        with ThreadPoolExecutor(max_workers=8) as pool:
+            futures = {
+                pool.submit(_fetch_one, client, cat, url, name): (cat.id, name)
+                for cat, url, name, _slow in fast_tasks
+            }
+            for future in as_completed(futures):
+                cat_id, feed_name = futures[future]
+                try:
+                    entries = future.result()
+                except Exception as exc:
+                    logger.warning("RSS 任务异常 [%s]: %s", feed_name, exc)
+                    continue
+                if entries:
+                    stats["feeds_ok"] += 1
+                stats["items_raw"] += len(entries)
+                raw_by_category[cat_id].extend(entries[:per_feed])
+
+        for cat, url, name, _slow in slow_tasks:
+            entries = _fetch_one(client, cat, url, name)
+            if entries:
+                stats["feeds_ok"] += 1
+            stats["items_raw"] += len(entries)
+            raw_by_category[cat.id].extend(entries[:per_feed])
+            if _is_reddit_url(url):
+                time.sleep(2.0)
+            else:
+                time.sleep(1.0)
+
+    categories_out: list[dict[str, Any]] = []
+    flat: list[dict[str, Any]] = []
+
+    for category in NEWS_CATEGORIES:
+        items = raw_by_category[category.id]
+        items = [i for i in items if _within_window(i, cutoff)]
+        items.sort(key=_sort_key, reverse=True)
+        items = _dedupe_items(items)[:per_category]
+        for item in items:
+            dt = _entry_datetime(item)
+            item["published_fmt"] = dt.astimezone(timezone(timedelta(hours=8))).strftime("%m-%d %H:%M") if dt else ""
+        if items:
+            categories_out.append(
+                {
+                    "id": category.id,
+                    "name": category.name,
+                    "icon": category.icon,
+                    "items": items,
+                }
+            )
+            flat.extend(items)
+
+    flat.sort(key=_sort_key, reverse=True)
+    flat = _dedupe_items(flat)
+
+    return {
+        "enabled": True,
+        "hours": hours,
+        "categories": categories_out,
+        "flat": flat,
+        "stats": stats,
+    }
+
+
+def format_news_section(news: dict[str, Any], *, section_no: int, wecom_limit: int | None = None) -> list[str]:
+    if not news.get("enabled"):
+        return ["---", "", f"## {section_no}、国际 AI 时讯", "", "*AI 时讯已关闭（`DAILY_AI_NEWS=0`）。*", ""]
+
+    categories = news.get("categories") or []
+    hours = news.get("hours", 72)
+    lines = [
+        "---",
+        "",
+        f"## {section_no}、国际 AI 时讯",
+        "",
+        f"> 近 **{hours}h** · {news.get('stats', {}).get('feeds_ok', 0)}/{news.get('stats', {}).get('feeds_total', 0)} 源可用",
+        "",
+    ]
+
+    if not categories:
+        lines.append("*暂无可用条目（网络/RSS 源异常或时间窗口内无更新）。*")
+        lines.append("")
+        return lines
+
+    if wecom_limit is not None:
+        flat = (news.get("flat") or [])[:wecom_limit]
+        for i, item in enumerate(flat, 1):
+            pub = f" · {item['published_fmt']}" if item.get("published_fmt") else ""
+            lines.append(
+                f"{i}. [{item['title']}]({item['link']}) · `{item['source_name']}`{pub}"
+            )
+        lines.append("")
+        return lines
+
+    for cat in categories:
+        lines.append(f"### {cat['icon']} {cat['name']}")
+        lines.append("")
+        for i, item in enumerate(cat["items"], 1):
+            pub = f" · {item['published_fmt']}" if item.get("published_fmt") else ""
+            lines.append(f"{i}. **[{item['title']}]({item['link']})** · `{item['source_name']}`{pub}")
+            summary = item.get("summary", "")
+            if summary:
+                lines.append(f"   - {_clean_text(summary, news_summary_limit())}")
+        lines.append("")
+
+    return lines
+
+
+def prepare_wecom_news_items(news: dict[str, Any]) -> list[dict[str, Any]]:
+    if not news.get("enabled"):
+        return []
+    limit = _wecom_limit()
+    flat = _dedupe_items(news.get("flat") or [])
+    flat.sort(key=_sort_key, reverse=True)
+    preferred = ("media", "newsletter", "official", "community", "research", "developer")
+    picked: list[dict[str, Any]] = []
+    seen: set[str] = set()
+    for cat in preferred:
+        for item in flat:
+            link = _normalize_link(item.get("link", ""))
+            if item.get("category_id") != cat or link in seen:
+                continue
+            picked.append(item)
+            seen.add(link)
+            if len(picked) >= limit:
+                break
+        if len(picked) >= limit:
+            break
+    items: list[dict[str, Any]] = []
+    for item in picked[:limit]:
+        items.append(
+            {
+                "title": item.get("title", "?"),
+                "link": item.get("link", ""),
+                "source_name": item.get("source_name", "?"),
+                "published_fmt": item.get("published_fmt", ""),
+                "desc_short": _clean_text(item.get("summary", ""), 36),
+            }
+        )
+    return items