"""生成早报 Markdown(完整版 + 企微短版)。""" from __future__ import annotations import json import re import sys import xml.etree.ElementTree as ET from collections import defaultdict from datetime import datetime, timezone, timedelta from pathlib import Path from typing import Any import certifi import httpx import logging logger = logging.getLogger(__name__) from daily.config import ( CACHE_DIR, LOG_DIR, OUTPUT_DIR, SNAPSHOT_FILE, env, env_int, full_desc_limit, news_summary_limit, wecom_skill_desc_limit, ) from daily.format_wecom import build_wecom_report, finalize_wecom_skill_groups, replace_wecom_skill_sections from daily.agent_workflow import is_agent_mode, run_agent_workflow from daily.delta import compare_depth from daily.cursor_editor import ( apply_descriptions, is_enabled as cursor_editor_enabled, run_editorial, theme_line_from_editorial, ) from daily.github.auth import github_html_headers from daily.github.search import fetch_emerging_repos, fetch_topic_hot_repos from daily.github.trending import fetch_github_trending, trending_data_source_note from daily.localize import LocalizeJob, localize_descriptions, needs_chinese from daily.news.fetch import ( fetch_ai_news, fetch_cn_ai_news, format_cn_news_section, format_news_section, prepare_wecom_cn_news_items, prepare_wecom_news_items, ) from daily.news.rank import apply_news_ranking from daily.report_data import ( build_full_payload, build_llm_input, data_json_path, save_json, ) from daily.skills_board import load_boards from daily.skills_group import group_skills_by_source from shared.skills_data import format_installs, load_feed THEME_RULES: list[tuple[str, str, list[str]]] = [ ("🎬", "AI 多媒体 / 视频", ["runcomfy", "remotion", "video", "seedance", "inpaint", "lipsync"]), ("🔧", "工程协作 / Skill 元能力", ["grill", "tdd", "architecture", "find-skills", "to-issues"]), ("📱", "飞书 / Lark", ["lark", "feishu"]), ("📣", "内容营销", ["viral", "tiktok", "instagram", "reels"]), ("🎨", "设计 / 前端", ["frontend", "design", "ui-ux", "tailwind"]), ] def _now_cst() -> datetime: return datetime.now(timezone(timedelta(hours=8))) def _short_desc(text: str, limit: int = 72) -> str: text = re.sub(r"\s+", " ", text or "").strip() if limit <= 0 or len(text) <= limit: return text return text[: limit - 3] + "..." def _archive_desc(text: str) -> str: return _short_desc(text, full_desc_limit()) def _wecom_desc(text: str, limit: int = 36) -> str: return _short_desc(text, limit) def _localize_descriptions_in_place( trending: list[dict[str, Any]], hot: list[dict[str, Any]], github_trending: list[dict[str, Any]], github_emerging: list[dict[str, Any]], github_topic: list[dict[str, Any]], ai_news: dict[str, Any], cn_ai_news: dict[str, Any] | None = None, ) -> None: full_limit = full_desc_limit() news_limit = news_summary_limit() jobs: list[LocalizeJob] = [] seen_skill: set[str] = set() for item in trending + hot: sid = _skill_id(item) if sid in seen_skill: continue seen_skill.add(sid) desc = (item.get("description") or "").strip() if desc: jobs.append(LocalizeJob(f"skill:{sid}", desc, full_limit)) seen_repo: set[str] = set() for repo_list in (github_trending, github_emerging, github_topic): for item in repo_list: repo = item.get("repo", "") if not repo or repo in seen_repo: continue seen_repo.add(repo) desc = (item.get("description") or "").strip() if desc: jobs.append(LocalizeJob(f"github:{repo}", desc, full_limit)) if ai_news.get("enabled"): seen_news: set[str] = set() for item in ai_news.get("flat") or []: link = item.get("link", "") if not link or link in seen_news: continue seen_news.add(link) summary = (item.get("summary") or "").strip() if summary: jobs.append(LocalizeJob(f"news:{link}", summary, news_limit)) zh_map = localize_descriptions(jobs, archive=True) if not zh_map and not jobs: return def _apply_zh(mapping: dict[str, str]) -> None: for item in trending + hot: key = f"skill:{_skill_id(item)}" if key in mapping: item["description"] = mapping[key] for repo_list in (github_trending, github_emerging, github_topic): for item in repo_list: key = f"github:{item.get('repo', '')}" if key in mapping: item["description"] = mapping[key] if ai_news.get("enabled"): for cat in ai_news.get("categories") or []: for item in cat.get("items") or []: key = f"news:{item.get('link', '')}" if key in mapping: item["summary"] = mapping[key] for item in ai_news.get("flat") or []: key = f"news:{item.get('link', '')}" if key in mapping: item["summary"] = mapping[key] _apply_zh(zh_map) # 仍为英文的条目再译一轮(长描述或批次失败时) retry_jobs: list[LocalizeJob] = [] seen_skill.clear() for item in trending + hot: sid = _skill_id(item) if sid in seen_skill: continue seen_skill.add(sid) desc = (item.get("description") or "").strip() if needs_chinese(desc): retry_jobs.append(LocalizeJob(f"skill:{sid}", desc, full_limit)) seen_repo.clear() for repo_list in (github_trending, github_emerging, github_topic): for item in repo_list: repo = item.get("repo", "") if not repo or repo in seen_repo: continue seen_repo.add(repo) desc = (item.get("description") or "").strip() if needs_chinese(desc): retry_jobs.append(LocalizeJob(f"github:{repo}", desc, full_limit)) if ai_news.get("enabled"): seen_news.clear() for item in ai_news.get("flat") or []: link = item.get("link", "") if not link or link in seen_news: continue seen_news.add(link) summary = (item.get("summary") or "").strip() if needs_chinese(summary): retry_jobs.append(LocalizeJob(f"news:{link}", summary, news_limit)) if retry_jobs: _apply_zh(localize_descriptions(retry_jobs, archive=True)) def _skill_id(item: dict[str, Any]) -> str: return str(item.get("id") or f"{item.get('source')}/{item.get('title')}") def _load_snapshot() -> set[str]: if not SNAPSHOT_FILE.exists(): return set() try: data = json.loads(SNAPSHOT_FILE.read_text(encoding="utf-8")) return set(str(x) for x in (data.get("skill_ids") or [])) except (OSError, json.JSONDecodeError): return set() def _save_snapshot(feed: dict[str, Any], date_str: str) -> None: CACHE_DIR.mkdir(parents=True, exist_ok=True) ids: list[str] = [] for board in ("topTrending", "topHot"): for item in feed.get(board, [])[:20]: sid = _skill_id(item) if sid not in ids: ids.append(sid) SNAPSHOT_FILE.write_text( json.dumps({"date": date_str, "skill_ids": ids}, ensure_ascii=False, indent=2), encoding="utf-8", ) def _prepare_skill_item(item: dict[str, Any], prev_ids: set[str], rank: int) -> dict[str, Any]: badge = "" sid = _skill_id(item) if sid not in prev_ids and prev_ids: badge = "🆕" elif rank == 1: badge = "👑" installs_fmt = item.get("installs_fmt") or format_installs(item.get("installs", 0)) title = item.get("source", "?") if item.get("cluster") else item.get("title", "?") desc = item.get("wecom_desc") or item.get("description") or item.get("cluster_titles") or "" limit = wecom_skill_desc_limit() desc_short = desc if item.get("wecom_desc") or limit <= 0 else _wecom_desc(desc, limit) return { "title": title, "source": item.get("source", "?"), "installs_fmt": installs_fmt, "link": item.get("link", ""), "desc_short": desc_short, "badge": badge, "cluster": bool(item.get("cluster")), "cluster_count": item.get("cluster_count"), "cluster_titles": item.get("cluster_titles"), } def _detect_theme_line(feed: dict[str, Any]) -> str: scores: dict[str, int] = defaultdict(int) for board in ("topTrending", "topHot"): for rank, item in enumerate(feed.get(board, [])[:10], 1): haystack = " ".join( [item.get("title", ""), item.get("source", ""), item.get("description", "")] ).lower() for _icon, label, keywords in THEME_RULES: if any(k in haystack for k in keywords): scores[label] += max(1, 11 - rank) break if not scores: return "**今日主题**:Agent Skills 生态持续活跃" return f"**今日主题**:{max(scores.items(), key=lambda x: x[1])[0]}" def _build_highlights( trending: list[dict[str, Any]], hot: list[dict[str, Any]], github_trending: list[dict[str, Any]], github_emerging: list[dict[str, Any]], ai_news: dict[str, Any] | None = None, cn_ai_news: dict[str, Any] | None = None, ) -> list[str]: points: list[str] = [] if ai_news and ai_news.get("enabled"): top_news = prepare_wecom_news_items(ai_news) if top_news: n0 = top_news[0] pub = f" · {n0['published_fmt']}" if n0.get("published_fmt") else "" points.append( f"🌍 AI 时讯 [{n0['title']}]({n0['link']})(`{n0.get('source_name', '?')}`{pub})" ) elif ai_news.get("flat"): n0 = ai_news["flat"][0] pub = f" · {n0['published_fmt']}" if n0.get("published_fmt") else "" points.append(f"🌍 AI 时讯 [{n0['title']}]({n0['link']})(`{n0.get('source_name', '?')}`{pub})") if cn_ai_news and cn_ai_news.get("enabled"): top_cn = prepare_wecom_cn_news_items(cn_ai_news) if top_cn: n0 = top_cn[0] pub = f" · {n0['published_fmt']}" if n0.get("published_fmt") else "" points.append( f"🇨🇳 国内 AI [{n0['title']}]({n0['link']})(`{n0.get('source_name', '?')}`{pub})" ) elif cn_ai_news.get("flat"): n0 = cn_ai_news["flat"][0] pub = f" · {n0['published_fmt']}" if n0.get("published_fmt") else "" points.append( f"🇨🇳 国内 AI [{n0['title']}]({n0['link']})(`{n0.get('source_name', '?')}`{pub})" ) if trending: t0 = trending[0] points.append(f"📈 Skills 榜首 **{t0.get('title')}**({format_installs(t0.get('installs', 0))})") if github_trending: g0 = github_trending[0] stars = g0.get("stars_today_fmt", "") total = g0.get("total_stars_fmt", "") star_hint = f"+{stars} today · " if stars else (f"⭐{total} · " if total else "") points.append(f"🐙 GitHub Trending [{g0['repo']}]({g0['url']})({star_hint}{g0.get('language', '')})") if github_emerging: e0 = github_emerging[0] points.append(f"🌱 新兴 [{e0['repo']}]({e0['url']})(⭐ {e0.get('total_stars_fmt', '?')})") elif hot: h0 = hot[0] points.append(f"🔥 Skills Hot 榜首 **{h0.get('title')}**(1H {format_installs(h0.get('installs', 0))})") while len(points) < 3 and len(trending) > len(points): item = trending[len(points)] points.append(f"✨ **{item.get('title')}** · `{item.get('source')}`") return points[:3] def _prepare_github_item(item: dict[str, Any]) -> dict[str, Any]: return {**item, "desc_short": _wecom_desc(item.get("description", ""), 40)} def _fetch_latest_release_title(repo: str) -> str | None: atom_url = f"https://github.com/{repo}/releases.atom" try: with httpx.Client( timeout=12.0, verify=certifi.where(), follow_redirects=True, headers=github_html_headers(), ) as client: resp = client.get(atom_url) if resp.status_code != 200: return None root = ET.fromstring(resp.text) ns = {"a": "http://www.w3.org/2005/Atom"} entry = root.find("a:entry", ns) if entry is None: return None title = entry.find("a:title", ns) return title.text.strip() if title is not None and title.text else None except Exception: return None def _theme_clusters(feed: dict[str, Any], limit: int = 5) -> list[tuple[str, list[str]]]: buckets: dict[str, list[str]] = defaultdict(list) seen: set[str] = set() for board in ("topTrending", "topHot"): for item in feed.get(board, [])[:20]: item_id = _skill_id(item) if item_id in seen: continue seen.add(item_id) haystack = " ".join( [item.get("title", ""), item.get("source", ""), item.get("description", "")] ).lower() for _icon, theme, keywords in THEME_RULES: if any(k in haystack for k in keywords): label = f"**{item.get('title')}** (`{item.get('source')}`)" if label not in buckets[theme]: buckets[theme].append(label) break return [(theme, examples[:limit]) for theme, examples in buckets.items() if examples] def _format_github_repo_section(repos: list[dict[str, Any]], *, show_created: bool = False) -> list[str]: lines: list[str] = [] for i, repo in enumerate(repos, 1): lang = repo.get("language") or "—" stars_today = repo.get("stars_today_fmt") or "" total = repo.get("total_stars_fmt") or "" created = repo.get("created_at") or "" meta_parts = [lang] if stars_today: meta_parts.append(f"+{stars_today} today") if total: meta_parts.append(f"总 ⭐ {total}") if show_created and created: meta_parts.append(f"创建于 {created}") lines.append(f"{i}. **[{repo['repo']}]({repo['url']})** · {' · '.join(meta_parts)}") desc = _archive_desc(repo.get("description", "")) if desc: lines.append(f" - {desc}") lines.append("") return lines def _format_skill_section(items: list[dict[str, Any]], *, hot: bool = False) -> list[str]: lines: list[str] = [] for i, item in enumerate(items, 1): skill_id = item.get("id") or f"{item.get('source', '?')}/{item.get('title', '?')}" link = item.get("link", "") installs = format_installs(item.get("installs", 0)) meta = f"1H {installs}" if hot else f"总安装 {installs}" if link: lines.append(f"{i}. **[{skill_id}]({link})** · {meta}") else: lines.append(f"{i}. **{skill_id}** · {meta}") desc = _archive_desc(item.get("description", "")) if desc: lines.append(f" - {desc}") lines.append("") return lines def generate_report() -> tuple[str, str, Path, Path]: trending_n = env_int("DAILY_TRENDING_LIMIT", 150) hot_n = max(env_int("DAILY_HOT_LIMIT", 150), compare_depth()) compare_n = compare_depth() skill_pool = max(10, env_int("DAILY_WECOM_SKILL_POOL", 200)) wecom_trending = env_int("DAILY_WECOM_TRENDING", 10) wecom_hot = env_int("DAILY_WECOM_HOT", 10) github_limit = env_int("DAILY_GITHUB_TRENDING_LIMIT", 10) wecom_github = env_int("DAILY_WECOM_GITHUB_TRENDING", env_int("DAILY_WECOM_REPOS", 10)) github_fetch_n = max(github_limit, compare_n, wecom_github) emerging_limit = env_int("DAILY_GITHUB_EMERGING_LIMIT", 10) wecom_emerging = env_int("DAILY_WECOM_GITHUB_EMERGING", 10) emerging_fetch_n = max(emerging_limit, compare_n, wecom_emerging) topic_limit = env_int("DAILY_GITHUB_TOPIC_LIMIT", 10) wecom_topic = env_int("DAILY_WECOM_GITHUB_TOPIC", 10) topic_fetch_n = max(topic_limit, compare_n, wecom_topic) feed = load_feed(force=True) prev_ids = _load_snapshot() now = _now_cst() date_str = now.strftime("%Y-%m-%d") time_str = now.strftime("%H:%M") + " (UTC+8)" updated = (feed.get("updatedAt") or "")[:10] trending, hot = load_boards(feed, trending_limit=trending_n, hot_limit=hot_n) github_trending = fetch_github_trending(github_fetch_n) seen_repos = {r["repo"] for r in github_trending} github_emerging = fetch_emerging_repos(emerging_fetch_n, exclude=seen_repos) seen_repos.update(r["repo"] for r in github_emerging) topic_name, github_topic = fetch_topic_hot_repos(topic_fetch_n, exclude=seen_repos) ai_news = fetch_ai_news() cn_ai_news = fetch_cn_ai_news() ai_news = apply_news_ranking(ai_news, date_str=date_str) cn_ai_news = apply_news_ranking(cn_ai_news, date_str=date_str) wecom_limits = { "trending": wecom_trending, "hot": wecom_hot, "trending_pool": skill_pool, "hot_pool": skill_pool, "github": wecom_github, "emerging": wecom_emerging, "topic": wecom_topic, "ai_news": env_int("DAILY_WECOM_AI_NEWS", 10), "cn_ai_news": env_int("DAILY_WECOM_CN_AI_NEWS", 8), } llm_input = build_llm_input( date_str=date_str, updated=updated, trending=trending, hot=hot, github_trending=github_trending, github_emerging=github_emerging, github_topic=github_topic, topic_name=topic_name, ai_news=ai_news, cn_ai_news=cn_ai_news, wecom_limits=wecom_limits, agent_mode=is_agent_mode(), ) save_json( data_json_path(date_str), build_full_payload( llm_input, meta={ "generated_at": now.isoformat(), "report_mode": "agent" if is_agent_mode() else "classic", "cursor_editor": cursor_editor_enabled() and not is_agent_mode(), }, ), ) agent_wecom: str | None = None if is_agent_mode(): agent_wecom = run_agent_workflow( llm_input, date_str=date_str, time_str=time_str, updated=updated, ) if not agent_wecom: logger.warning("Agent 工作流失败,回退 classic 模式") editorial_theme: str | None = None editorial_highlights: list[str] | None = None if agent_wecom is None: editorial = run_editorial(llm_input, date_str=date_str) if editorial: apply_descriptions( trending=trending, hot=hot, github_trending=github_trending, github_emerging=github_emerging, github_topic=github_topic, ai_news=ai_news, cn_ai_news=cn_ai_news, descriptions=editorial.get("descriptions") or {}, ) editorial_theme = theme_line_from_editorial(editorial) or None hl = editorial.get("highlights") or [] editorial_highlights = hl if hl else None # 完整版归档:中文化 + 加长摘要(已是中文的条目会跳过翻译) _localize_descriptions_in_place( trending, hot, github_trending, github_emerging, github_topic, ai_news, cn_ai_news ) themes = _theme_clusters(feed) lines = [ f"# 早报 · {date_str}", "", f"> 生成时间:{now.strftime('%Y-%m-%d %H:%M')} (UTC+8) ", f"> skills 数据更新:{updated} ", "> 数据来源:[skills.sh/trending](https://skills.sh/trending) · [skills.sh/hot](https://skills.sh/hot) · 国际/国内 AI RSS", "", "---", "", f"## 一、Skills Trending Top {trending_n}", "", *_format_skill_section(trending), "---", "", f"## 二、Skills Hot Top {hot_n}", "", *_format_skill_section(hot, hot=True), "", "---", "", f"## 三、GitHub Trending Top {github_limit}", "", trending_data_source_note(), "", ] if github_trending: lines.extend(_format_github_repo_section(github_trending)) else: lines.append("*GitHub Trending 获取失败,请检查网络或配置 GITHUB_TOKEN。*") lines.append("") lines.extend(["---", "", f"## 四、新兴项目 Top {emerging_limit}", "", "> 数据来源:GitHub Search API(需 `GITHUB_TOKEN`)", ""]) if github_emerging: lines.extend(_format_github_repo_section(github_emerging, show_created=True)) else: lines.append("*新兴项目获取失败或未配置 GITHUB_TOKEN。*") lines.append("") lines.extend(["---", "", f"## 五、Topic `{topic_name}` Top {topic_limit}", "", "> 数据来源:GitHub Search API(需 `GITHUB_TOKEN`)", ""]) if github_topic: lines.extend(_format_github_repo_section(github_topic)) else: lines.append(f"*Topic `{topic_name}` 热点获取失败或未配置 GITHUB_TOKEN。*") lines.append("") section_no = 6 lines.extend(format_news_section(ai_news, section_no=section_no)) section_no += 1 lines.extend(format_cn_news_section(cn_ai_news, section_no=section_no)) section_no += 1 watch = (env("GITHUB_REPOS") or "").strip() if watch: lines.extend(["---", "", f"## {section_no}、关注仓库 Release", ""]) section_no += 1 for repo in [r.strip() for r in watch.split(",") if r.strip()]: release = _fetch_latest_release_title(repo) lines.append(f"- **{repo}**:{release or '暂无 release'}") lines.append("") lines.extend(["---", "", f"## {section_no}、主题聚类", ""]) for theme, examples in themes: lines.append(f"### {theme}") for ex in examples: lines.append(f"- {ex}") lines.append("") pick_src = trending[0].get("source", "") if trending else "" pick_name = trending[0].get("title", "") if trending else "" pick_command = ( f"npx skills add {pick_src}/{pick_name}" if pick_src and pick_name else "npx skills add vercel-labs/skills/find-skills" ) lines.extend(["---", "", "## 安装示例", "", "```bash"]) for item in trending[:4]: src, name = item.get("source", ""), item.get("title", "") if src and name: lines.append(f"npx skills add {src}/{name}") lines.extend(["```", "", f"*企微短版见 `output/{date_str}.wecom.md`*"]) markdown = "\n".join(lines) if agent_wecom: gt = group_skills_by_source(trending, limit=wecom_trending, pool_size=skill_pool) gh = group_skills_by_source(hot, limit=wecom_hot, pool_size=skill_pool) wecom_md = replace_wecom_skill_sections(agent_wecom, trending=gt, hot=gh) else: wecom_md = build_wecom_report( date_str=date_str, time_str=time_str, updated=updated, highlights=editorial_highlights or _build_highlights(trending, hot, github_trending, github_emerging, ai_news, cn_ai_news), theme_line=editorial_theme or _detect_theme_line(feed), ai_news=prepare_wecom_news_items(ai_news), cn_ai_news=prepare_wecom_cn_news_items(cn_ai_news), trending=[ _prepare_skill_item(item, prev_ids, r) for r, item in enumerate( finalize_wecom_skill_groups( group_skills_by_source(trending, limit=wecom_trending, pool_size=skill_pool) ), 1, ) ], hot=[ _prepare_skill_item(item, prev_ids, r) for r, item in enumerate( finalize_wecom_skill_groups( group_skills_by_source(hot, limit=wecom_hot, pool_size=skill_pool) ), 1, ) ], repos=[_prepare_github_item(item) for item in github_trending[:wecom_github]], emerging=[_prepare_github_item(item) for item in github_emerging[:wecom_emerging]], topic_name=topic_name, topic_repos=[_prepare_github_item(item) for item in github_topic[:wecom_topic]], pick_command=pick_command, ) _save_snapshot(feed, date_str) OUTPUT_DIR.mkdir(parents=True, exist_ok=True) out_md = OUTPUT_DIR / f"{date_str}.md" out_wecom = OUTPUT_DIR / f"{date_str}.wecom.md" out_md.write_text(markdown, encoding="utf-8") out_wecom.write_text(wecom_md, encoding="utf-8") return markdown, wecom_md, out_md, out_wecom def main() -> int: LOG_DIR.mkdir(parents=True, exist_ok=True) log_file = LOG_DIR / f"{_now_cst():%Y-%m-%d}.log" try: _, wecom_md, out_md, out_wecom = generate_report() nbytes = len(wecom_md.encode("utf-8")) msg = f"[{_now_cst():%H:%M:%S}] OK -> {out_md}, {out_wecom} ({nbytes} bytes)\n" log_file.write_text(msg, encoding="utf-8") print(msg.strip()) return 0 except Exception as exc: msg = f"[{_now_cst():%H:%M:%S}] FAIL: {exc}\n" log_file.write_text(msg, encoding="utf-8") print(msg.strip(), file=sys.stderr) return 1