feat: generate 以 board_select 为唯一列表主人并写回 shown keys

Co-authored-by: Cursor <cursoragent@cursor.com>
2026-07-14 10:36:53 +08:00
parent 0c324f9ace
commit d448002e7a
4 changed files with 1001 additions and 106 deletions
--- a/daily/generate.py
+++ b/daily/generate.py
@@ -22,17 +22,30 @@ from daily.config import (
    LOG_DIR,
    OUTPUT_DIR,
    SNAPSHOT_FILE,
+    board_pool_size,
    env,
    env_int,
    full_desc_limit,
    news_summary_limit,
+    wecom_delta_pad,
+    wecom_news_desc_limit,
+    wecom_pad_pool_size,
    wecom_skill_desc_limit,
 )
+from daily.board_history import (
+    BOARD_KEYS,
+    extract_shown_keys,
+    load_recent_shown_keys,
+    merge_wecom_shown_into_data,
+)
+from daily.board_select import board_select
 from daily.format_wecom import (
    build_wecom_report,
    finalize_wecom_skill_groups,
    replace_wecom_board_sections,
+    replace_wecom_news_sections,
    replace_wecom_skill_sections,
+    resolve_wecom_board_items,
 )
 from daily.agent_workflow import is_agent_mode, run_agent_workflow
 from daily.featured_pick import (
@@ -56,10 +69,17 @@ from daily.news.fetch import (
    fetch_cn_ai_news,
    format_cn_news_section,
    format_news_section,
+    finalize_wecom_news_items,
    prepare_wecom_cn_news_items,
    prepare_wecom_news_items,
+    sync_wecom_news_rows,
 )
 from daily.news.pushed_links import record_pushed_links
+from daily.news.research import (
+    fetch_ai_news_research,
+    format_research_news_section,
+    is_research_mode,
+)
 from daily.push_gate import evaluate_push_gate
 from daily.report_data import (
    build_full_payload,
@@ -68,7 +88,6 @@ from daily.report_data import (
    save_json,
 )
 from daily.skills_board import format_installs, load_boards, load_feed
-from daily.skills_group import group_skills_by_source

 THEME_RULES: list[tuple[str, str, list[str]]] = [
    ("🎬", "AI 多媒体 / 视频", ["runcomfy", "remotion", "video", "seedance", "inpaint", "lipsync"]),
@@ -140,7 +159,30 @@ def _localize_descriptions_in_place(
            seen_news.add(link)
            summary = (item.get("summary") or "").strip()
            if summary:
-                jobs.append(LocalizeJob(f"news:{link}", summary, news_limit))
+                jobs.append(
+                    LocalizeJob(
+                        f"news:{link}",
+                        summary,
+                        news_limit if news_limit > 0 else wecom_news_desc_limit(),
+                    )
+                )
+
+    if cn_ai_news and cn_ai_news.get("enabled"):
+        seen_cn: set[str] = set()
+        for item in cn_ai_news.get("flat") or []:
+            link = item.get("link", "")
+            if not link or link in seen_cn:
+                continue
+            seen_cn.add(link)
+            summary = (item.get("summary") or "").strip()
+            if summary and needs_chinese(summary):
+                jobs.append(
+                    LocalizeJob(
+                        f"news:{link}",
+                        summary,
+                        news_limit if news_limit > 0 else wecom_news_desc_limit(),
+                    )
+                )

    zh_map = localize_descriptions(jobs, archive=True)
    if not zh_map and not jobs:
@@ -166,6 +208,11 @@ def _localize_descriptions_in_place(
                key = f"news:{item.get('link', '')}"
                if key in mapping:
                    item["summary"] = mapping[key]
+        if cn_ai_news and cn_ai_news.get("enabled"):
+            for item in cn_ai_news.get("flat") or []:
+                key = f"news:{item.get('link', '')}"
+                if key in mapping:
+                    item["summary"] = mapping[key]

    _apply_zh(zh_map)

@@ -199,7 +246,29 @@ def _localize_descriptions_in_place(
            seen_news.add(link)
            summary = (item.get("summary") or "").strip()
            if needs_chinese(summary):
-                retry_jobs.append(LocalizeJob(f"news:{link}", summary, news_limit))
+                retry_jobs.append(
+                    LocalizeJob(
+                        f"news:{link}",
+                        summary,
+                        news_limit if news_limit > 0 else wecom_news_desc_limit(),
+                    )
+                )
+    if cn_ai_news and cn_ai_news.get("enabled"):
+        seen_cn: set[str] = set()
+        for item in cn_ai_news.get("flat") or []:
+            link = item.get("link", "")
+            if not link or link in seen_cn:
+                continue
+            seen_cn.add(link)
+            summary = (item.get("summary") or "").strip()
+            if needs_chinese(summary):
+                retry_jobs.append(
+                    LocalizeJob(
+                        f"news:{link}",
+                        summary,
+                        news_limit if news_limit > 0 else wecom_news_desc_limit(),
+                    )
+                )

    if retry_jobs:
        _apply_zh(localize_descriptions(retry_jobs, archive=True))
@@ -334,6 +403,27 @@ def _prepare_github_item(item: dict[str, Any]) -> dict[str, Any]:
    return {**item, "desc_short": _wecom_desc(item.get("description", ""), 40)}


+def _sync_movement_github_descriptions(
+    movement: dict[str, Any],
+    *,
+    github_trending: list[dict[str, Any]],
+    github_emerging: list[dict[str, Any]],
+    github_topic: list[dict[str, Any]],
+) -> None:
+    """将已中文化的 GitHub 描述同步到 movement 新入榜条目（供 Delta 企微列表使用）。"""
+    by_repo: dict[str, str] = {}
+    for item in github_trending + github_emerging + github_topic:
+        repo = str(item.get("repo") or "")
+        desc = (item.get("description") or "").strip()
+        if repo and desc:
+            by_repo[repo] = desc
+    for key in ("github_trending_moves", "github_emerging_moves", "github_topic_moves"):
+        for move in movement.get(key) or []:
+            repo = str(move.get("repo") or "")
+            if repo in by_repo:
+                move["description"] = by_repo[repo]
+
+
 def _fetch_latest_release_title(repo: str) -> str | None:
    atom_url = f"https://github.com/{repo}/releases.atom"
    try:
@@ -425,15 +515,17 @@ def generate_report() -> tuple[str, str, Path, Path]:
    skill_pool = max(10, env_int("DAILY_WECOM_SKILL_POOL", 200))
    wecom_trending = env_int("DAILY_WECOM_TRENDING", 10)
    wecom_hot = env_int("DAILY_WECOM_HOT", 10)
+    pad_pool = wecom_pad_pool_size(max(wecom_trending, wecom_hot, 10))
+    skill_pool = max(skill_pool, pad_pool)
    github_limit = env_int("DAILY_GITHUB_TRENDING_LIMIT", 10)
    wecom_github = env_int("DAILY_WECOM_GITHUB_TRENDING", env_int("DAILY_WECOM_REPOS", 10))
-    github_fetch_n = max(github_limit, compare_n, wecom_github)
+    github_fetch_n = max(github_limit, compare_n, wecom_github, pad_pool)
    emerging_limit = env_int("DAILY_GITHUB_EMERGING_LIMIT", 10)
    wecom_emerging = env_int("DAILY_WECOM_GITHUB_EMERGING", 10)
-    emerging_fetch_n = max(emerging_limit, compare_n, wecom_emerging)
+    emerging_fetch_n = max(emerging_limit, compare_n, wecom_emerging, pad_pool)
    topic_limit = env_int("DAILY_GITHUB_TOPIC_LIMIT", 10)
    wecom_topic = env_int("DAILY_WECOM_GITHUB_TOPIC", 10)
-    topic_fetch_n = max(topic_limit, compare_n, wecom_topic)
+    topic_fetch_n = max(topic_limit, compare_n, wecom_topic, pad_pool)

    feed = load_feed(force=True)
    prev_ids = _load_snapshot()
@@ -448,8 +540,27 @@ def generate_report() -> tuple[str, str, Path, Path]:
    github_emerging = fetch_emerging_repos(emerging_fetch_n, exclude=seen_repos)
    seen_repos.update(r["repo"] for r in github_emerging)
    topic_name, github_topic = fetch_topic_hot_repos(topic_fetch_n, exclude=seen_repos)
-    ai_news = fetch_ai_news()
-    cn_ai_news = fetch_cn_ai_news()
+
+    news_merged = is_research_mode()
+    ai_news_research: dict[str, Any] | None = None
+    wecom_news: list[dict[str, Any]] = []
+    wecom_tech_news: list[dict[str, Any]] = []
+    if news_merged:
+        ai_news_research = fetch_ai_news_research(date_str=date_str)
+        wecom_news = list(ai_news_research.get("items") or [])
+        wecom_tech_news = list(ai_news_research.get("tech_items") or [])
+        ai_news = {
+            "enabled": ai_news_research.get("enabled", False),
+            "mode": "research",
+            "hours": ai_news_research.get("hours", 24),
+            "flat": ai_news_research.get("flat") or [],
+            "categories": [],
+            "stats": ai_news_research.get("stats") or {},
+        }
+        cn_ai_news = {"enabled": False, "categories": [], "flat": [], "stats": {}}
+    else:
+        ai_news = fetch_ai_news()
+        cn_ai_news = fetch_cn_ai_news()

    wecom_limits = {
        "trending": wecom_trending,
@@ -460,7 +571,56 @@ def generate_report() -> tuple[str, str, Path, Path]:
        "emerging": wecom_emerging,
        "topic": wecom_topic,
        "ai_news": env_int("DAILY_WECOM_AI_NEWS", 10),
-        "cn_ai_news": env_int("DAILY_WECOM_CN_AI_NEWS", 8),
+        "cn_ai_news": env_int("DAILY_WECOM_CN_AI_NEWS", 10),
+    }
+    pool = max(board_pool_size(), skill_pool, pad_pool)
+    recent_shown = load_recent_shown_keys(date_str)
+    selected_trending = board_select(
+        board="skills_trending",
+        items=trending,
+        recent_keys=recent_shown["skills_trending"],
+        limit=wecom_trending,
+        pool_size=pool,
+        kind="skill",
+    )
+    selected_hot = board_select(
+        board="skills_hot",
+        items=hot,
+        recent_keys=recent_shown["skills_hot"],
+        limit=wecom_hot,
+        pool_size=pool,
+        kind="skill",
+    )
+    selected_github = board_select(
+        board="github_trending",
+        items=github_trending,
+        recent_keys=recent_shown["github_trending"],
+        limit=wecom_github,
+        pool_size=pool,
+        kind="github",
+    )
+    selected_emerging = board_select(
+        board="github_emerging",
+        items=github_emerging,
+        recent_keys=recent_shown["github_emerging"],
+        limit=wecom_emerging,
+        pool_size=pool,
+        kind="github",
+    )
+    selected_topic = board_select(
+        board="github_topic",
+        items=github_topic,
+        recent_keys=recent_shown["github_topic"],
+        limit=wecom_topic,
+        pool_size=pool,
+        kind="github",
+    )
+    boards_for_wecom = {
+        "skills_trending": selected_trending,
+        "skills_hot": selected_hot,
+        "github_trending": selected_github,
+        "github_emerging": selected_emerging,
+        "github_topic": selected_topic,
    }
    llm_input = build_llm_input(
        date_str=date_str,
@@ -474,15 +634,22 @@ def generate_report() -> tuple[str, str, Path, Path]:
        ai_news=ai_news,
        cn_ai_news=cn_ai_news,
        wecom_limits=wecom_limits,
+        research_items=wecom_news if news_merged else None,
+        research_tech_items=wecom_tech_news if news_merged else None,
+        boards_for_wecom=boards_for_wecom,
    )
    featured = apply_featured_pick(llm_input, date_str=date_str)
    movement = llm_input["movement"]
    eff_mode = llm_input["effective_wecom_mode"]
-    wecom_ai = prepare_wecom_news_items(ai_news, date_str=date_str)
-    wecom_cn = prepare_wecom_cn_news_items(cn_ai_news, date_str=date_str)
+    if news_merged:
+        wecom_ai: list[dict[str, Any]] = []
+        wecom_cn: list[dict[str, Any]] = []
+    else:
+        wecom_ai = prepare_wecom_news_items(ai_news, date_str=date_str)
+        wecom_cn = prepare_wecom_cn_news_items(cn_ai_news, date_str=date_str)
    push_gate = evaluate_push_gate(
        movement=movement,
-        ai_news_items=wecom_ai,
+        ai_news_items=wecom_news if news_merged else wecom_ai,
        cn_ai_news_items=wecom_cn,
        featured_pick=featured,
    )
@@ -526,6 +693,17 @@ def generate_report() -> tuple[str, str, Path, Path]:
    _localize_descriptions_in_place(
        trending, hot, github_trending, github_emerging, github_topic, ai_news, cn_ai_news
    )
+    if not news_merged:
+        sync_wecom_news_rows(wecom_ai, ai_news.get("flat") or [])
+        sync_wecom_news_rows(wecom_cn, cn_ai_news.get("flat") or [])
+        finalize_wecom_news_items(wecom_ai, force_chinese=True)
+        finalize_wecom_news_items(wecom_cn, force_chinese=False)
+    _sync_movement_github_descriptions(
+        movement,
+        github_trending=github_trending,
+        github_emerging=github_emerging,
+        github_topic=github_topic,
+    )

    themes = _theme_clusters(feed)

@@ -534,7 +712,8 @@ def generate_report() -> tuple[str, str, Path, Path]:
        "",
        f"> 生成时间：{now.strftime('%Y-%m-%d %H:%M')} (UTC+8)  ",
        f"> skills 数据更新：{updated}  ",
-        "> 数据来源：[skills.sh/trending](https://skills.sh/trending) · [skills.sh/hot](https://skills.sh/hot) · 国际/国内 AI RSS",
+        "> 数据来源：[skills.sh/trending](https://skills.sh/trending) · [skills.sh/hot](https://skills.sh/hot)"
+        + (" · AI 时讯 Deep Research" if news_merged else " · 国际/国内 AI RSS"),
        "",
        "---",
        "",
@@ -576,10 +755,14 @@ def generate_report() -> tuple[str, str, Path, Path]:
        lines.append("")

    section_no = 6
-    lines.extend(format_news_section(ai_news, section_no=section_no))
-    section_no += 1
-    lines.extend(format_cn_news_section(cn_ai_news, section_no=section_no))
-    section_no += 1
+    if news_merged and ai_news_research is not None:
+        lines.extend(format_research_news_section(ai_news_research, section_no=section_no))
+        section_no += 1
+    else:
+        lines.extend(format_news_section(ai_news, section_no=section_no))
+        section_no += 1
+        lines.extend(format_cn_news_section(cn_ai_news, section_no=section_no))
+        section_no += 1

    watch = (env("GITHUB_REPOS") or "").strip()
    if watch:
@@ -616,17 +799,85 @@ def generate_report() -> tuple[str, str, Path, Path]:
    lines.extend(["```", "", f"*企微短版见 `output/{date_str}.wecom.md`*"])

    markdown = "\n".join(lines)
-    gt = group_skills_by_source(trending, limit=wecom_trending, pool_size=skill_pool)
-    gh = group_skills_by_source(hot, limit=wecom_hot, pool_size=skill_pool)
-    if agent_wecom:
-        wecom_md = replace_wecom_skill_sections(
-            agent_wecom,
-            trending=gt,
-            hot=gh,
-            mode=eff_mode,
-            movement=movement,
-            topic_name=topic_name,
+    gt = selected_trending
+    gh = selected_hot
+    gt_pad = board_select(
+        board="skills_trending",
+        items=trending,
+        recent_keys=recent_shown["skills_trending"],
+        limit=pad_pool,
+        pool_size=pool,
+        kind="skill",
+    )
+    gh_pad = board_select(
+        board="skills_hot",
+        items=hot,
+        recent_keys=recent_shown["skills_hot"],
+        limit=pad_pool,
+        pool_size=pool,
+        kind="skill",
+    )
+    wecom_github_items = [_prepare_github_item(item) for item in selected_github]
+    wecom_emerging_items = [_prepare_github_item(item) for item in selected_emerging]
+    wecom_topic_items = [_prepare_github_item(item) for item in selected_topic]
+    wecom_github_pad = [
+        _prepare_github_item(item)
+        for item in board_select(
+            board="github_trending",
+            items=github_trending,
+            recent_keys=recent_shown["github_trending"],
+            limit=pad_pool,
+            pool_size=pool,
+            kind="github",
        )
+    ]
+    wecom_emerging_pad = [
+        _prepare_github_item(item)
+        for item in board_select(
+            board="github_emerging",
+            items=github_emerging,
+            recent_keys=recent_shown["github_emerging"],
+            limit=pad_pool,
+            pool_size=pool,
+            kind="github",
+        )
+    ]
+    wecom_topic_pad = [
+        _prepare_github_item(item)
+        for item in board_select(
+            board="github_topic",
+            items=github_topic,
+            recent_keys=recent_shown["github_topic"],
+            limit=pad_pool,
+            pool_size=pool,
+            kind="github",
+        )
+    ]
+    delta_pad = eff_mode == "delta" and wecom_delta_pad()
+    board_kwargs = {
+        "mode": eff_mode,
+        "movement": movement,
+        "trending": gt,
+        "hot": gh,
+        "topic_name": topic_name,
+        "github_trending": wecom_github_items,
+        "github_emerging": wecom_emerging_items,
+        "github_topic": wecom_topic_items,
+        "wecom_trending": wecom_trending,
+        "wecom_hot": wecom_hot,
+        "wecom_github": wecom_github,
+        "wecom_emerging": wecom_emerging,
+        "wecom_topic": wecom_topic,
+        "pad": delta_pad,
+        "date_str": date_str,
+        "trending_pad": gt_pad,
+        "hot_pad": gh_pad,
+        "github_trending_pad": wecom_github_pad,
+        "github_emerging_pad": wecom_emerging_pad,
+        "github_topic_pad": wecom_topic_pad,
+    }
+    if agent_wecom:
+        wecom_md = replace_wecom_skill_sections(agent_wecom, **board_kwargs)
    else:
        wecom_md = build_wecom_report(
            date_str=date_str,
@@ -635,8 +886,10 @@ def generate_report() -> tuple[str, str, Path, Path]:
            highlights=editorial_highlights
            or _build_highlights(trending, hot, github_trending, github_emerging, ai_news, cn_ai_news),
            theme_line=editorial_theme or _detect_theme_line(feed),
-            ai_news=wecom_ai,
-            cn_ai_news=wecom_cn,
+            ai_news=wecom_ai if not news_merged else None,
+            cn_ai_news=wecom_cn if not news_merged else None,
+            merged_ai_news=wecom_news if news_merged else None,
+            merged_tech_ai_news=wecom_tech_news if news_merged else None,
            trending=[
                _prepare_skill_item(item, prev_ids, r)
                for r, item in enumerate(finalize_wecom_skill_groups(gt), 1)
@@ -645,29 +898,40 @@ def generate_report() -> tuple[str, str, Path, Path]:
                _prepare_skill_item(item, prev_ids, r)
                for r, item in enumerate(finalize_wecom_skill_groups(gh), 1)
            ],
-            repos=[_prepare_github_item(item) for item in github_trending[:wecom_github]],
-            emerging=[_prepare_github_item(item) for item in github_emerging[:wecom_emerging]],
+            repos=wecom_github_items,
+            emerging=wecom_emerging_items,
            topic_name=topic_name,
-            topic_repos=[_prepare_github_item(item) for item in github_topic[:wecom_topic]],
+            topic_repos=wecom_topic_items,
            pick_command=pick_command,
            pick_why=pick_why,
            pick_title=pick_title,
            pick_url=pick_url,
            include_boards=(eff_mode == "full"),
        )
-        wecom_md = replace_wecom_board_sections(
-            wecom_md,
-            mode=eff_mode,
-            movement=movement,
-            trending=gt,
-            hot=gh,
-            topic_name=topic_name,
-        )
+        wecom_md = replace_wecom_board_sections(wecom_md, **board_kwargs)
+
+    wecom_md = replace_wecom_news_sections(
+        wecom_md,
+        ai_news=wecom_news if news_merged else wecom_ai,
+        cn_ai_news=None if news_merged else wecom_cn,
+        tech_ai_news=wecom_tech_news if news_merged else None,
+        merged=news_merged,
+    )

    if push_gate.should_push:
-        links = [x["link"] for x in wecom_ai + wecom_cn if x.get("link")]
+        if news_merged:
+            links = [x["link"] for x in wecom_news + wecom_tech_news if x.get("link")]
+        else:
+            links = [x["link"] for x in wecom_ai + wecom_cn if x.get("link")]
        record_pushed_links(date_str, links)

+    final_boards = resolve_wecom_board_items(**board_kwargs)
+    shown_keys = {
+        board: extract_shown_keys(board, final_boards.get(board) or [])
+        for board in BOARD_KEYS
+    }
+    llm_input = merge_wecom_shown_into_data(llm_input, shown_keys)
+
    save_json(
        data_json_path(date_str),
        build_full_payload(
@@ -675,6 +939,7 @@ def generate_report() -> tuple[str, str, Path, Path]:
            meta={
                "generated_at": now.isoformat(),
                "report_mode": "agent" if is_agent_mode() else "classic",
+                "ai_news_mode": "research" if news_merged else "rss",
                "cursor_editor": cursor_editor_enabled() and not is_agent_mode(),
                "featured_pick": featured.get("title") if featured else None,
                "effective_wecom_mode": eff_mode,