refactor: Phase 3 拆分 generate 流水线并补全测试

提取 collect/formatters/themes 等 pipeline 模块，新增 wecom 分条、RSS、delta、Agent 工作流测试。

Co-authored-by: Cursor <cursoragent@cursor.com>

daily/pipeline/__init__.py

@@ -0,0 +1 @@
+"""早报生成流水线子模块。"""

daily/pipeline/collect.py

@@ -0,0 +1,145 @@
+"""抓取与结构化输入组装。"""
+
+from __future__ import annotations
+
+from dataclasses import dataclass
+from datetime import datetime
+from typing import Any
+
+from daily.agent_workflow import is_agent_mode
+from daily.config import env_int
+from daily.delta import compare_depth
+from daily.github.search import fetch_emerging_repos, fetch_topic_hot_repos
+from daily.github.trending import fetch_github_trending
+from daily.news.fetch import fetch_ai_news, fetch_cn_ai_news
+from daily.news.rank import apply_news_ranking
+from daily.report_data import build_llm_input
+from daily.skills_board import load_boards
+from shared.skills_data import load_feed
+
+
+@dataclass
+class ReportLimits:
+    trending_n: int
+    hot_n: int
+    skill_pool: int
+    wecom_trending: int
+    wecom_hot: int
+    github_limit: int
+    wecom_github: int
+    emerging_limit: int
+    wecom_emerging: int
+    topic_limit: int
+    wecom_topic: int
+
+
+@dataclass
+class ReportContext:
+    feed: dict[str, Any]
+    date_str: str
+    time_str: str
+    updated: str
+    trending: list[dict[str, Any]]
+    hot: list[dict[str, Any]]
+    github_trending: list[dict[str, Any]]
+    github_emerging: list[dict[str, Any]]
+    github_topic: list[dict[str, Any]]
+    topic_name: str
+    ai_news: dict[str, Any]
+    cn_ai_news: dict[str, Any]
+    limits: ReportLimits
+    wecom_limits: dict[str, int]
+    llm_input: dict[str, Any]
+
+
+def resolve_limits() -> ReportLimits:
+    compare_n = compare_depth()
+    trending_n = env_int("DAILY_TRENDING_LIMIT", 150)
+    hot_n = max(env_int("DAILY_HOT_LIMIT", 150), compare_n)
+    skill_pool = max(10, env_int("DAILY_WECOM_SKILL_POOL", 200))
+    wecom_trending = env_int("DAILY_WECOM_TRENDING", 10)
+    wecom_hot = env_int("DAILY_WECOM_HOT", 10)
+    github_limit = env_int("DAILY_GITHUB_TRENDING_LIMIT", 10)
+    wecom_github = env_int("DAILY_WECOM_GITHUB_TRENDING", env_int("DAILY_WECOM_REPOS", 10))
+    emerging_limit = env_int("DAILY_GITHUB_EMERGING_LIMIT", 10)
+    wecom_emerging = env_int("DAILY_WECOM_GITHUB_EMERGING", 10)
+    topic_limit = env_int("DAILY_GITHUB_TOPIC_LIMIT", 10)
+    wecom_topic = env_int("DAILY_WECOM_GITHUB_TOPIC", 10)
+    return ReportLimits(
+        trending_n=trending_n,
+        hot_n=hot_n,
+        skill_pool=skill_pool,
+        wecom_trending=wecom_trending,
+        wecom_hot=wecom_hot,
+        github_limit=github_limit,
+        wecom_github=wecom_github,
+        emerging_limit=emerging_limit,
+        wecom_emerging=wecom_emerging,
+        topic_limit=topic_limit,
+        wecom_topic=wecom_topic,
+    )
+
+
+def collect_report_context(now: datetime) -> ReportContext:
+    limits = resolve_limits()
+    compare_n = compare_depth()
+    github_fetch_n = max(limits.github_limit, compare_n, limits.wecom_github)
+    emerging_fetch_n = max(limits.emerging_limit, compare_n, limits.wecom_emerging)
+    topic_fetch_n = max(limits.topic_limit, compare_n, limits.wecom_topic)
+
+    feed = load_feed(force=True)
+    date_str = now.strftime("%Y-%m-%d")
+    time_str = now.strftime("%H:%M") + " (UTC+8)"
+    updated = (feed.get("updatedAt") or "")[:10]
+
+    trending, hot = load_boards(feed, trending_limit=limits.trending_n, hot_limit=limits.hot_n)
+    github_trending = fetch_github_trending(github_fetch_n)
+    seen_repos = {r["repo"] for r in github_trending}
+    github_emerging = fetch_emerging_repos(emerging_fetch_n, exclude=seen_repos)
+    seen_repos.update(r["repo"] for r in github_emerging)
+    topic_name, github_topic = fetch_topic_hot_repos(topic_fetch_n, exclude=seen_repos)
+    ai_news = apply_news_ranking(fetch_ai_news(), date_str=date_str)
+    cn_ai_news = apply_news_ranking(fetch_cn_ai_news(), date_str=date_str)
+
+    wecom_limits = {
+        "trending": limits.wecom_trending,
+        "hot": limits.wecom_hot,
+        "trending_pool": limits.skill_pool,
+        "hot_pool": limits.skill_pool,
+        "github": limits.wecom_github,
+        "emerging": limits.wecom_emerging,
+        "topic": limits.wecom_topic,
+        "ai_news": env_int("DAILY_WECOM_AI_NEWS", 10),
+        "cn_ai_news": env_int("DAILY_WECOM_CN_AI_NEWS", 8),
+    }
+    llm_input = build_llm_input(
+        date_str=date_str,
+        updated=updated,
+        trending=trending,
+        hot=hot,
+        github_trending=github_trending,
+        github_emerging=github_emerging,
+        github_topic=github_topic,
+        topic_name=topic_name,
+        ai_news=ai_news,
+        cn_ai_news=cn_ai_news,
+        wecom_limits=wecom_limits,
+        agent_mode=is_agent_mode(),
+    )
+    return ReportContext(
+        feed=feed,
+        date_str=date_str,
+        time_str=time_str,
+        updated=updated,
+        trending=trending,
+        hot=hot,
+        github_trending=github_trending,
+        github_emerging=github_emerging,
+        github_topic=github_topic,
+        topic_name=topic_name,
+        ai_news=ai_news,
+        cn_ai_news=cn_ai_news,
+        limits=limits,
+        wecom_limits=wecom_limits,
+        llm_input=llm_input,
+    )

daily/pipeline/formatters.py

@@ -0,0 +1,180 @@
+"""归档 / 企微格式化与摘要构建。"""
+
+from __future__ import annotations
+
+import re
+import xml.etree.ElementTree as ET
+from typing import Any
+
+import certifi
+import httpx
+
+from daily.config import full_desc_limit, wecom_skill_desc_limit
+from daily.github.auth import github_html_headers
+from daily.news.fetch import prepare_wecom_cn_news_items, prepare_wecom_news_items
+from shared.skills_data import format_installs
+
+from daily.pipeline.snapshot import skill_id
+
+
+def short_desc(text: str, limit: int = 72) -> str:
+    text = re.sub(r"\s+", " ", text or "").strip()
+    if limit <= 0 or len(text) <= limit:
+        return text
+    return text[: limit - 3] + "..."
+
+
+def archive_desc(text: str) -> str:
+    return short_desc(text, full_desc_limit())
+
+
+def wecom_desc(text: str, limit: int = 36) -> str:
+    return short_desc(text, limit)
+
+
+def prepare_skill_item(item: dict[str, Any], prev_ids: set[str], rank: int) -> dict[str, Any]:
+    badge = ""
+    sid = skill_id(item)
+    if sid not in prev_ids and prev_ids:
+        badge = "🆕"
+    elif rank == 1:
+        badge = "👑"
+    installs_fmt = item.get("installs_fmt") or format_installs(item.get("installs", 0))
+    title = item.get("source", "?") if item.get("cluster") else item.get("title", "?")
+    desc = item.get("wecom_desc") or item.get("description") or item.get("cluster_titles") or ""
+    limit = wecom_skill_desc_limit()
+    desc_short = desc if item.get("wecom_desc") or limit <= 0 else wecom_desc(desc, limit)
+    return {
+        "title": title,
+        "source": item.get("source", "?"),
+        "installs_fmt": installs_fmt,
+        "link": item.get("link", ""),
+        "desc_short": desc_short,
+        "badge": badge,
+        "cluster": bool(item.get("cluster")),
+        "cluster_count": item.get("cluster_count"),
+        "cluster_titles": item.get("cluster_titles"),
+    }
+
+
+def prepare_github_item(item: dict[str, Any]) -> dict[str, Any]:
+    return {**item, "desc_short": wecom_desc(item.get("description", ""), 40)}
+
+
+def build_highlights(
+    trending: list[dict[str, Any]],
+    hot: list[dict[str, Any]],
+    github_trending: list[dict[str, Any]],
+    github_emerging: list[dict[str, Any]],
+    ai_news: dict[str, Any] | None = None,
+    cn_ai_news: dict[str, Any] | None = None,
+) -> list[str]:
+    points: list[str] = []
+    if ai_news and ai_news.get("enabled"):
+        top_news = prepare_wecom_news_items(ai_news)
+        if top_news:
+            n0 = top_news[0]
+            pub = f" · {n0['published_fmt']}" if n0.get("published_fmt") else ""
+            points.append(
+                f"🌍 AI 时讯 [{n0['title']}]({n0['link']})（`{n0.get('source_name', '?')}`{pub}）"
+            )
+        elif ai_news.get("flat"):
+            n0 = ai_news["flat"][0]
+            pub = f" · {n0['published_fmt']}" if n0.get("published_fmt") else ""
+            points.append(f"🌍 AI 时讯 [{n0['title']}]({n0['link']})（`{n0.get('source_name', '?')}`{pub}）")
+    if cn_ai_news and cn_ai_news.get("enabled"):
+        top_cn = prepare_wecom_cn_news_items(cn_ai_news)
+        if top_cn:
+            n0 = top_cn[0]
+            pub = f" · {n0['published_fmt']}" if n0.get("published_fmt") else ""
+            points.append(
+                f"🇨🇳 国内 AI [{n0['title']}]({n0['link']})（`{n0.get('source_name', '?')}`{pub}）"
+            )
+        elif cn_ai_news.get("flat"):
+            n0 = cn_ai_news["flat"][0]
+            pub = f" · {n0['published_fmt']}" if n0.get("published_fmt") else ""
+            points.append(
+                f"🇨🇳 国内 AI [{n0['title']}]({n0['link']})（`{n0.get('source_name', '?')}`{pub}）"
+            )
+    if trending:
+        t0 = trending[0]
+        points.append(f"📈 Skills 榜首 **{t0.get('title')}**（{format_installs(t0.get('installs', 0))}）")
+    if github_trending:
+        g0 = github_trending[0]
+        stars = g0.get("stars_today_fmt", "")
+        total = g0.get("total_stars_fmt", "")
+        star_hint = f"+{stars} today · " if stars else (f"⭐{total} · " if total else "")
+        points.append(f"🐙 GitHub Trending [{g0['repo']}]({g0['url']})（{star_hint}{g0.get('language', '')}）")
+    if github_emerging:
+        e0 = github_emerging[0]
+        points.append(f"🌱 新兴 [{e0['repo']}]({e0['url']})（⭐ {e0.get('total_stars_fmt', '?')}）")
+    elif hot:
+        h0 = hot[0]
+        points.append(f"🔥 Skills Hot 榜首 **{h0.get('title')}**（1H {format_installs(h0.get('installs', 0))}）")
+    while len(points) < 3 and len(trending) > len(points):
+        item = trending[len(points)]
+        points.append(f"✨ **{item.get('title')}** · `{item.get('source')}`")
+    return points[:3]
+
+
+def format_github_repo_section(repos: list[dict[str, Any]], *, show_created: bool = False) -> list[str]:
+    lines: list[str] = []
+    for i, repo in enumerate(repos, 1):
+        lang = repo.get("language") or "—"
+        stars_today = repo.get("stars_today_fmt") or ""
+        total = repo.get("total_stars_fmt") or ""
+        created = repo.get("created_at") or ""
+        meta_parts = [lang]
+        if stars_today:
+            meta_parts.append(f"+{stars_today} today")
+        if total:
+            meta_parts.append(f"总 ⭐ {total}")
+        if show_created and created:
+            meta_parts.append(f"创建于 {created}")
+        lines.append(f"{i}. **[{repo['repo']}]({repo['url']})** · {' · '.join(meta_parts)}")
+        desc = archive_desc(repo.get("description", ""))
+        if desc:
+            lines.append(f"   - {desc}")
+        lines.append("")
+    return lines
+
+
+def format_skill_section(items: list[dict[str, Any]], *, hot: bool = False) -> list[str]:
+    lines: list[str] = []
+    for i, item in enumerate(items, 1):
+        sid = item.get("id") or f"{item.get('source', '?')}/{item.get('title', '?')}"
+        link = item.get("link", "")
+        installs = format_installs(item.get("installs", 0))
+        meta = f"1H {installs}" if hot else f"总安装 {installs}"
+        if link:
+            lines.append(f"{i}. **[{sid}]({link})** · {meta}")
+        else:
+            lines.append(f"{i}. **{sid}** · {meta}")
+        desc = archive_desc(item.get("description", ""))
+        if desc:
+            lines.append(f"   - {desc}")
+        lines.append("")
+    return lines
+
+
+def fetch_latest_release_title(repo: str) -> str | None:
+    atom_url = f"https://github.com/{repo}/releases.atom"
+    try:
+        with httpx.Client(
+            timeout=12.0,
+            verify=certifi.where(),
+            follow_redirects=True,
+            headers=github_html_headers(),
+        ) as client:
+            resp = client.get(atom_url)
+            if resp.status_code != 200:
+                return None
+            root = ET.fromstring(resp.text)
+            ns = {"a": "http://www.w3.org/2005/Atom"}
+            entry = root.find("a:entry", ns)
+            if entry is None:
+                return None
+            title = entry.find("a:title", ns)
+            return title.text.strip() if title is not None and title.text else None
+    except Exception:
+        return None

daily/pipeline/localize.py

@@ -0,0 +1,116 @@
+"""归档内容中文化。"""
+
+from __future__ import annotations
+
+from typing import Any
+
+from daily.config import full_desc_limit, news_summary_limit
+from daily.localize import LocalizeJob, localize_descriptions, needs_chinese
+
+from daily.pipeline.snapshot import skill_id
+
+
+def localize_descriptions_in_place(
+    trending: list[dict[str, Any]],
+    hot: list[dict[str, Any]],
+    github_trending: list[dict[str, Any]],
+    github_emerging: list[dict[str, Any]],
+    github_topic: list[dict[str, Any]],
+    ai_news: dict[str, Any],
+    cn_ai_news: dict[str, Any] | None = None,
+) -> None:
+    full_limit = full_desc_limit()
+    news_limit = news_summary_limit()
+    jobs: list[LocalizeJob] = []
+    seen_skill: set[str] = set()
+    for item in trending + hot:
+        sid = skill_id(item)
+        if sid in seen_skill:
+            continue
+        seen_skill.add(sid)
+        desc = (item.get("description") or "").strip()
+        if desc:
+            jobs.append(LocalizeJob(f"skill:{sid}", desc, full_limit))
+
+    seen_repo: set[str] = set()
+    for repo_list in (github_trending, github_emerging, github_topic):
+        for item in repo_list:
+            repo = item.get("repo", "")
+            if not repo or repo in seen_repo:
+                continue
+            seen_repo.add(repo)
+            desc = (item.get("description") or "").strip()
+            if desc:
+                jobs.append(LocalizeJob(f"github:{repo}", desc, full_limit))
+
+    if ai_news.get("enabled"):
+        seen_news: set[str] = set()
+        for item in ai_news.get("flat") or []:
+            link = item.get("link", "")
+            if not link or link in seen_news:
+                continue
+            seen_news.add(link)
+            summary = (item.get("summary") or "").strip()
+            if summary:
+                jobs.append(LocalizeJob(f"news:{link}", summary, news_limit))
+
+    zh_map = localize_descriptions(jobs, archive=True)
+    if not zh_map and not jobs:
+        return
+
+    def _apply_zh(mapping: dict[str, str]) -> None:
+        for item in trending + hot:
+            key = f"skill:{skill_id(item)}"
+            if key in mapping:
+                item["description"] = mapping[key]
+        for repo_list in (github_trending, github_emerging, github_topic):
+            for item in repo_list:
+                key = f"github:{item.get('repo', '')}"
+                if key in mapping:
+                    item["description"] = mapping[key]
+        if ai_news.get("enabled"):
+            for cat in ai_news.get("categories") or []:
+                for item in cat.get("items") or []:
+                    key = f"news:{item.get('link', '')}"
+                    if key in mapping:
+                        item["summary"] = mapping[key]
+            for item in ai_news.get("flat") or []:
+                key = f"news:{item.get('link', '')}"
+                if key in mapping:
+                    item["summary"] = mapping[key]
+
+    _apply_zh(zh_map)
+
+    retry_jobs: list[LocalizeJob] = []
+    seen_skill.clear()
+    for item in trending + hot:
+        sid = skill_id(item)
+        if sid in seen_skill:
+            continue
+        seen_skill.add(sid)
+        desc = (item.get("description") or "").strip()
+        if needs_chinese(desc):
+            retry_jobs.append(LocalizeJob(f"skill:{sid}", desc, full_limit))
+    seen_repo.clear()
+    for repo_list in (github_trending, github_emerging, github_topic):
+        for item in repo_list:
+            repo = item.get("repo", "")
+            if not repo or repo in seen_repo:
+                continue
+            seen_repo.add(repo)
+            desc = (item.get("description") or "").strip()
+            if needs_chinese(desc):
+                retry_jobs.append(LocalizeJob(f"github:{repo}", desc, full_limit))
+    if ai_news.get("enabled"):
+        seen_news.clear()
+        for item in ai_news.get("flat") or []:
+            link = item.get("link", "")
+            if not link or link in seen_news:
+                continue
+            seen_news.add(link)
+            summary = (item.get("summary") or "").strip()
+            if needs_chinese(summary):
+                retry_jobs.append(LocalizeJob(f"news:{link}", summary, news_limit))
+
+    if retry_jobs:
+        _apply_zh(localize_descriptions(retry_jobs, archive=True))

daily/pipeline/snapshot.py

@@ -0,0 +1,36 @@
+"""Skills 快照读写。"""
+
+from __future__ import annotations
+
+import json
+from typing import Any
+
+from daily.config import CACHE_DIR, SNAPSHOT_FILE
+
+
+def skill_id(item: dict[str, Any]) -> str:
+    return str(item.get("id") or f"{item.get('source')}/{item.get('title')}")
+
+
+def load_snapshot() -> set[str]:
+    if not SNAPSHOT_FILE.exists():
+        return set()
+    try:
+        data = json.loads(SNAPSHOT_FILE.read_text(encoding="utf-8"))
+        return set(str(x) for x in (data.get("skill_ids") or []))
+    except (OSError, json.JSONDecodeError):
+        return set()
+
+
+def save_snapshot(feed: dict[str, Any], date_str: str) -> None:
+    CACHE_DIR.mkdir(parents=True, exist_ok=True)
+    ids: list[str] = []
+    for board in ("topTrending", "topHot"):
+        for item in feed.get(board, [])[:20]:
+            sid = skill_id(item)
+            if sid not in ids:
+                ids.append(sid)
+    SNAPSHOT_FILE.write_text(
+        json.dumps({"date": date_str, "skill_ids": ids}, ensure_ascii=False, indent=2),
+        encoding="utf-8",
+    )

daily/pipeline/themes.py

@@ -0,0 +1,53 @@
+"""主题检测与聚类。"""
+
+from __future__ import annotations
+
+from collections import defaultdict
+from typing import Any
+
+THEME_RULES: list[tuple[str, str, list[str]]] = [
+    ("🎬", "AI 多媒体 / 视频", ["runcomfy", "remotion", "video", "seedance", "inpaint", "lipsync"]),
+    ("🔧", "工程协作 / Skill 元能力", ["grill", "tdd", "architecture", "find-skills", "to-issues"]),
+    ("📱", "飞书 / Lark", ["lark", "feishu"]),
+    ("📣", "内容营销", ["viral", "tiktok", "instagram", "reels"]),
+    ("🎨", "设计 / 前端", ["frontend", "design", "ui-ux", "tailwind"]),
+]
+
+
+def detect_theme_line(feed: dict[str, Any]) -> str:
+    scores: dict[str, int] = defaultdict(int)
+    for board in ("topTrending", "topHot"):
+        for rank, item in enumerate(feed.get(board, [])[:10], 1):
+            haystack = " ".join(
+                [item.get("title", ""), item.get("source", ""), item.get("description", "")]
+            ).lower()
+            for _icon, label, keywords in THEME_RULES:
+                if any(k in haystack for k in keywords):
+                    scores[label] += max(1, 11 - rank)
+                    break
+    if not scores:
+        return "**今日主题**：Agent Skills 生态持续活跃"
+    return f"**今日主题**：{max(scores.items(), key=lambda x: x[1])[0]}"
+
+
+def theme_clusters(feed: dict[str, Any], limit: int = 5) -> list[tuple[str, list[str]]]:
+    from daily.pipeline.snapshot import skill_id
+
+    buckets: dict[str, list[str]] = defaultdict(list)
+    seen: set[str] = set()
+    for board in ("topTrending", "topHot"):
+        for item in feed.get(board, [])[:20]:
+            item_id = skill_id(item)
+            if item_id in seen:
+                continue
+            seen.add(item_id)
+            haystack = " ".join(
+                [item.get("title", ""), item.get("source", ""), item.get("description", "")]
+            ).lower()
+            for _icon, theme, keywords in THEME_RULES:
+                if any(k in haystack for k in keywords):
+                    label = f"**{item.get('title')}** (`{item.get('source')}`)"
+                    if label not in buckets[theme]:
+                        buckets[theme].append(label)
+                    break
+    return [(theme, examples[:limit]) for theme, examples in buckets.items() if examples]