项目初始化

This commit is contained in:
2026-07-02 11:31:16 +08:00
commit eef4c76e3f
64 changed files with 13369 additions and 0 deletions

6
daily/__init__.py Normal file
View File

@@ -0,0 +1,6 @@
"""早报生成与企微推送。"""
from daily.generate import generate_report, main as generate_main
from daily.webhook import send_report
__all__ = ["generate_report", "generate_main", "send_report"]

22
daily/__main__.py Normal file
View File

@@ -0,0 +1,22 @@
"""CLI: python -m daily [generate|push] [report_path]"""
from __future__ import annotations
import sys
from daily.generate import main as generate_main
from daily.webhook import main as push_main
def main() -> int:
cmd = (sys.argv[1] if len(sys.argv) > 1 else "generate").lower()
if cmd in {"generate", "gen", "g"}:
return generate_main()
if cmd in {"push", "send", "webhook"}:
return push_main(sys.argv[2:])
print(f"未知命令: {cmd}\n用法: python -m daily [generate|push] [report_path]", file=sys.stderr)
return 1
if __name__ == "__main__":
raise SystemExit(main())

134
daily/agent_workflow.py Normal file
View File

@@ -0,0 +1,134 @@
"""Agent 工作流:趋势分析 → 叙事化企微早报。"""
from __future__ import annotations
import json
import logging
import re
from pathlib import Path
from typing import Any
from daily.config import OUTPUT_DIR, ROOT, env
from daily.llm_client import extract_json_object, has_llm_configured, llm_chat
from daily.report_data import save_json
logger = logging.getLogger(__name__)
_SKILL_DIR = ROOT / "skills" / "daily-agent"
_MD_BLOCK = re.compile(r"```(?:markdown|md)?\s*([\s\S]*?)```", re.IGNORECASE)
_WECOM_NEW_ENTRY_NOTE = re.compile(r"(新入[^]*")
def _strip_new_entry_notes(md: str) -> str:
md = _WECOM_NEW_ENTRY_NOTE.sub("", md)
return re.sub(r"\*\*—", "** —", md)
def report_mode() -> str:
return (env("DAILY_REPORT_MODE") or "classic").strip().lower()
def is_agent_mode() -> bool:
if report_mode() != "agent":
return False
if not has_llm_configured():
logger.warning("DAILY_REPORT_MODE=agent 但未配置 LLM回退 classic")
return False
return True
def trends_json_path(date_str: str) -> Path:
return OUTPUT_DIR / f"{date_str}.trends.json"
def _load_skill() -> str:
path = _SKILL_DIR / "SKILL.md"
if path.exists():
return path.read_text(encoding="utf-8").strip()
return "你是早报主编 Agent。"
def _extract_markdown(text: str) -> str:
text = text.strip()
match = _MD_BLOCK.search(text)
if match:
return match.group(1).strip()
if text.startswith("📰"):
return text
return text
def analyze_trends(llm_input: dict[str, Any], *, date_str: str) -> dict[str, Any] | None:
skill = _load_skill()
system = (
f"{skill}\n\n"
"当前执行 **Step 1趋势分析**。\n"
"只输出 trends JSONheadline, opening, themes, top_picks, signals不要 Markdown。"
)
user = json.dumps(llm_input, ensure_ascii=False, indent=2)
try:
raw = llm_chat(system, user)
except Exception as exc:
logger.warning("Agent Step1 趋势分析失败:%s", exc)
return None
if not raw:
return None
parsed = extract_json_object(raw)
if not parsed.get("headline") and not parsed.get("opening"):
logger.warning("Agent Step1 JSON 无效")
return None
save_json(trends_json_path(date_str), parsed)
logger.info("Agent Step1 完成:%s", parsed.get("headline", "?"))
return parsed
def write_wecom_report(
llm_input: dict[str, Any],
trends: dict[str, Any],
*,
date_str: str,
time_str: str,
updated: str,
) -> str | None:
skill = _load_skill()
system = (
f"{skill}\n\n"
"当前执行 **Step 2撰写企微早报**。\n"
f"日期={date_str},时间={time_str},数据截至={updated}\n"
"只输出企微 Markdown 正文,不要代码块,不要 JSON。"
)
payload = {"data": llm_input, "trends": trends}
user = json.dumps(payload, ensure_ascii=False, indent=2)
try:
raw = llm_chat(system, user)
except Exception as exc:
logger.warning("Agent Step2 写稿失败:%s", exc)
return None
if not raw:
return None
md = _extract_markdown(raw)
if not md.startswith("📰"):
md = f"📰 **早报 · {date_str}**\n> ⏱ {time_str} · 数据截至 {updated}\n\n{md}"
md = _strip_new_entry_notes(md)
logger.info("Agent Step2 完成:%d bytes", len(md.encode("utf-8")))
return md
def run_agent_workflow(
llm_input: dict[str, Any],
*,
date_str: str,
time_str: str,
updated: str,
) -> str | None:
"""两步 Agent 工作流;成功返回企微 Markdown失败返回 None。"""
trends = analyze_trends(llm_input, date_str=date_str)
if not trends:
return None
return write_wecom_report(
llm_input,
trends,
date_str=date_str,
time_str=time_str,
updated=updated,
)

81
daily/config.py Normal file
View File

@@ -0,0 +1,81 @@
"""项目路径与环境变量加载。"""
from __future__ import annotations
import os
import sys
from pathlib import Path
from dotenv import load_dotenv
ROOT = Path(__file__).resolve().parent.parent
BOT_DIR = ROOT / "bot"
OUTPUT_DIR = ROOT / "output"
LOG_DIR = ROOT / "logs"
CACHE_DIR = ROOT / ".cache"
SNAPSHOT_FILE = CACHE_DIR / "last-report.json"
# 企微 webhook markdown.content 硬上限
WECOM_MARKDOWN_LIMIT = 4096
def wecom_chunk_bytes() -> int:
"""单条企微 markdown 消息字节上限。"""
chunk = env_int("DAILY_WECOM_CHUNK_BYTES", -1)
if chunk < 0:
chunk = env_int("DAILY_WECOM_MAX_BYTES", WECOM_MARKDOWN_LIMIT)
return min(chunk, WECOM_MARKDOWN_LIMIT)
def wecom_skill_desc_limit() -> int:
"""企微 Skills 榜单条简介建议字数。"""
return env_int("DAILY_WECOM_SKILL_DESC_LIMIT", 56)
def full_desc_limit() -> int:
"""完整版早报摘要长度0 表示不截断。"""
return env_int("DAILY_FULL_DESC_LIMIT", 0)
def news_summary_limit() -> int:
return env_int("DAILY_FULL_NEWS_SUMMARY_LIMIT", 0)
def wecom_max_bytes() -> int:
"""兼容旧配置名。"""
return wecom_chunk_bytes()
load_dotenv(ROOT / ".env")
load_dotenv(ROOT / ".env.local", override=True)
def ensure_bot_on_path() -> None:
bot = str(BOT_DIR)
if bot not in sys.path:
sys.path.insert(0, bot)
def _clean_env_value(raw: str | None) -> str | None:
if raw is None:
return None
value = raw.strip()
if not value:
return None
# .env 行内注释(未加引号时 python-dotenv 不会自动去掉)
if " #" in value:
value = value.split(" #", 1)[0].rstrip()
return value or None
def env(key: str, default: str | None = None) -> str | None:
return _clean_env_value(os.getenv(key, default))
def env_int(key: str, default: int) -> int:
raw = env(key)
if not raw:
return default
try:
return int(raw)
except ValueError:
return default

144
daily/cursor_editor.py Normal file
View File

@@ -0,0 +1,144 @@
"""Cursor 编辑层JSON 数据 → 主题 / 速览 / 中文描述。"""
from __future__ import annotations
import json
import logging
from pathlib import Path
from typing import Any
from daily.config import ROOT, env
from daily.llm_client import extract_json_object, has_llm_configured, llm_chat
from daily.text_utils import clip_text
from daily.report_data import editorial_json_path, save_json, skill_id
logger = logging.getLogger(__name__)
_SKILL_DIR = ROOT / "skills" / "daily-editor"
def is_enabled() -> bool:
raw = (env("DAILY_CURSOR_EDITOR") or "").strip().lower()
if raw in {"1", "true", "yes", "on"}:
return has_llm_configured()
if raw in {"0", "false", "no", "off"}:
return False
return False
def _load_skill_prompt() -> str:
skill_path = _SKILL_DIR / "SKILL.md"
if skill_path.exists():
return skill_path.read_text(encoding="utf-8").strip()
return "你是技术早报编辑。根据输入 JSON 输出编辑结果 JSON。"
def _build_system_prompt() -> str:
skill = _load_skill_prompt()
return (
f"{skill}\n\n"
"再次强调:只输出 JSON 对象,包含 theme_line、highlights3条、descriptions。"
)
def run_editorial(llm_input: dict[str, Any], *, date_str: str) -> dict[str, Any] | None:
"""调用 LLM 生成 editorial失败返回 None。"""
if not is_enabled():
return None
system = _build_system_prompt()
user = json.dumps(llm_input, ensure_ascii=False, indent=2)
try:
raw = llm_chat(system, user)
except Exception as exc:
logger.warning("Cursor 编辑失败,回退规则模式:%s", exc)
return None
if not raw:
logger.warning("Cursor 编辑无响应,回退规则模式")
return None
parsed = extract_json_object(raw)
if not parsed.get("theme_line") and not parsed.get("descriptions"):
logger.warning("Cursor 编辑 JSON 无效,回退规则模式")
return None
editorial = _normalize_editorial(parsed)
save_json(editorial_json_path(date_str), editorial)
return editorial
def _normalize_editorial(raw: dict[str, Any]) -> dict[str, Any]:
theme = str(raw.get("theme_line") or "").strip()
highlights_raw = raw.get("highlights") or []
highlights: list[str] = []
if isinstance(highlights_raw, list):
for item in highlights_raw:
if isinstance(item, str) and item.strip():
highlights.append(item.strip())
descriptions_raw = raw.get("descriptions") or {}
descriptions: dict[str, str] = {}
if isinstance(descriptions_raw, dict):
for key, value in descriptions_raw.items():
if isinstance(value, str) and value.strip():
limit = 40 if str(key).startswith("github:") else 36
descriptions[str(key)] = clip_text(value, limit)
return {
"theme_line": theme,
"highlights": highlights[:3],
"descriptions": descriptions,
}
def theme_line_from_editorial(editorial: dict[str, Any]) -> str:
theme = editorial.get("theme_line", "")
if not theme:
return ""
if "今日主题" in theme:
return theme if theme.startswith("**") else f"**{theme}**"
return f"**今日主题**{theme}"
def apply_descriptions(
*,
trending: list[dict[str, Any]],
hot: list[dict[str, Any]],
github_trending: list[dict[str, Any]],
github_emerging: list[dict[str, Any]],
github_topic: list[dict[str, Any]],
ai_news: dict[str, Any],
descriptions: dict[str, str],
) -> None:
if not descriptions:
return
for item in trending + hot:
key = f"skill:{skill_id(item)}"
if key in descriptions:
item["description"] = descriptions[key]
for repo_list in (github_trending, github_emerging, github_topic):
for item in repo_list:
key = f"github:{item.get('repo', '')}"
if key in descriptions:
item["description"] = descriptions[key]
if not ai_news.get("enabled"):
return
def _apply_news_item(item: dict[str, Any]) -> None:
key = f"news:{item.get('link', '')}"
if key in descriptions:
item["summary"] = descriptions[key]
for cat in ai_news.get("categories") or []:
for item in cat.get("items") or []:
_apply_news_item(item)
for item in ai_news.get("flat") or []:
_apply_news_item(item)
def load_cached_editorial(date_str: str) -> dict[str, Any] | None:
path = editorial_json_path(date_str)
if not path.exists():
return None
try:
return _normalize_editorial(json.loads(path.read_text(encoding="utf-8")))
except (OSError, json.JSONDecodeError):
return None

313
daily/delta.py Normal file
View File

@@ -0,0 +1,313 @@
"""榜单异动:对比昨日 Top N仅识别「新入榜」条目分榜、限条"""
from __future__ import annotations
import json
import logging
from datetime import datetime, timedelta
from pathlib import Path
from typing import Any, Callable
from daily.config import OUTPUT_DIR, env_int
logger = logging.getLogger(__name__)
KeyFn = Callable[[dict[str, Any]], str]
def compare_depth() -> int:
return env_int("DAILY_DELTA_COMPARE_DEPTH", 15)
def wecom_new_limit() -> int:
return env_int("DAILY_WECOM_NEW_MAX", 10)
def skill_id(item: dict[str, Any]) -> str:
return str(item.get("id") or f"{item.get('source')}/{item.get('title')}")
def _data_json_path(date_str: str) -> Path:
return OUTPUT_DIR / f"{date_str}.data.json"
def _load_data_json_file(path: Path) -> dict[str, Any]:
return json.loads(path.read_text(encoding="utf-8"))
def _key_set(items: list[dict[str, Any]], key_fn: KeyFn, *, depth: int) -> set[str]:
return {key_fn(item) for item in items[:depth] if key_fn(item)}
def find_previous_data(date_str: str) -> tuple[str, dict[str, Any]] | None:
"""查找最近一份早于 date_str 的 data.json。"""
try:
dt = datetime.strptime(date_str, "%Y-%m-%d")
except ValueError:
return None
lookback = env_int("DAILY_DELTA_LOOKBACK_DAYS", 7)
for days in range(1, lookback + 1):
prev_date = (dt - timedelta(days=days)).strftime("%Y-%m-%d")
path = _data_json_path(prev_date)
if not path.exists():
continue
try:
payload = _load_data_json_file(path)
except (OSError, ValueError) as exc:
logger.warning("读取异动基准 %s 失败:%s", path, exc)
continue
data = payload.get("data")
if isinstance(data, dict) and data.get("date"):
return prev_date, data
return None
def _prev_board_items(prev_data: dict[str, Any], board: str, depth: int) -> list[dict[str, Any]]:
baseline = prev_data.get("movement_baseline") or {}
if board in baseline and isinstance(baseline[board], list):
return baseline[board][:depth]
legacy = {
"skills_trending": "skills_trending",
"skills_hot": "skills_hot",
"github_trending": "github_trending",
"github_emerging": "github_emerging",
"github_topic": "github_topic",
}
if board == "github_topic":
topic = prev_data.get("github_topic") or {}
repos = topic.get("repos") if isinstance(topic, dict) else []
return (repos or [])[:depth]
field = legacy.get(board, board)
items = prev_data.get(field) or []
return items[:depth] if isinstance(items, list) else []
def _format_new_note(board: str, rank: int, *, topic_name: str = "llm") -> str:
labels = {
"trending": "Skills Trending",
"hot": "Skills Hot",
"github_trending": "GitHub Trending",
"github_emerging": "GitHub 新兴",
"github_topic": f"Topic `{topic_name}`",
}
return f"新入 {labels.get(board, board)} #{rank}"
def _build_skill_board_moves(
*,
board: str,
items: list[dict[str, Any]],
prev_data: dict[str, Any] | None,
depth: int,
) -> list[dict[str, Any]]:
board_key = f"skills_{board}"
prev_ids = (
_key_set(_prev_board_items(prev_data, board_key, depth), skill_id, depth=depth)
if prev_data
else set()
)
moves: list[dict[str, Any]] = []
for rank, item in enumerate(items[:depth], 1):
sid = skill_id(item)
if not sid or not prev_data or sid in prev_ids:
continue
moves.append(
{
"kind": "skill",
"board": board,
"id": sid,
"title": item.get("title", ""),
"source": item.get("source", ""),
"installs": int(item.get("installs") or 0),
"link": item.get("link", ""),
"description": item.get("description", ""),
"rank": rank,
"is_new": True,
"note": _format_new_note(board, rank),
}
)
return moves
def _repo_key(item: dict[str, Any]) -> str:
return str(item.get("repo") or "")
def _build_github_board_moves(
*,
board: str,
items: list[dict[str, Any]],
prev_data: dict[str, Any] | None,
depth: int,
topic_name: str = "llm",
) -> list[dict[str, Any]]:
prev_ids = (
_key_set(_prev_board_items(prev_data, board, depth), _repo_key, depth=depth)
if prev_data
else set()
)
moves: list[dict[str, Any]] = []
for rank, item in enumerate(items[:depth], 1):
repo = _repo_key(item)
if not repo or not prev_data or repo in prev_ids:
continue
moves.append(
{
"kind": "github",
"board": board,
"repo": repo,
"url": item.get("url", ""),
"language": item.get("language", ""),
"stars_today_fmt": item.get("stars_today_fmt", ""),
"total_stars_fmt": item.get("total_stars_fmt", ""),
"created_at": item.get("created_at", ""),
"description": item.get("description", ""),
"rank": rank,
"is_new": True,
"note": _format_new_note(board, rank, topic_name=topic_name),
}
)
return moves
def _board_summary(
*,
label: str,
baseline_date: str | None,
depth: int,
moves: list[dict[str, Any]],
capped: list[dict[str, Any]],
) -> str:
if not baseline_date:
return f"无历史基准,无法判断 {label} Top{depth} 新增"
if not moves:
return f"{baseline_date} Top{depth} 无新增 {label} 条目"
total = len(moves)
shown = len(capped)
if shown < total:
return f"{baseline_date} Top{depth} 新增 {total} 条,企微展示前 {shown}"
return f"{baseline_date} Top{depth} 新增 {total}"
def build_movement_baseline(
*,
trending: list[dict[str, Any]],
hot: list[dict[str, Any]],
github_trending: list[dict[str, Any]],
github_emerging: list[dict[str, Any]],
github_topic: list[dict[str, Any]],
depth: int | None = None,
) -> dict[str, Any]:
n = depth if depth is not None else compare_depth()
return {
"compare_depth": n,
"skills_trending": trending[:n],
"skills_hot": hot[:n],
"github_trending": github_trending[:n],
"github_emerging": github_emerging[:n],
"github_topic": github_topic[:n],
}
def build_movement_context(
*,
date_str: str,
trending: list[dict[str, Any]],
hot: list[dict[str, Any]],
github_trending: list[dict[str, Any]],
github_emerging: list[dict[str, Any]],
github_topic: list[dict[str, Any]],
topic_name: str = "llm",
) -> dict[str, Any]:
"""生成 Agent 可用的新增榜上下文:分 Trending/Hot/Topic企微每榜最多 wecom_new_limit 条。"""
depth = compare_depth()
cap = wecom_new_limit()
baseline = find_previous_data(date_str)
baseline_date = baseline[0] if baseline else None
prev_data = baseline[1] if baseline else None
topic = (topic_name or "llm").strip() or "llm"
skills_trending_all = _build_skill_board_moves(
board="trending", items=trending, prev_data=prev_data, depth=depth
)
skills_hot_all = _build_skill_board_moves(
board="hot", items=hot, prev_data=prev_data, depth=depth
)
github_trending_all = _build_github_board_moves(
board="github_trending", items=github_trending, prev_data=prev_data, depth=depth
)
github_emerging_all = _build_github_board_moves(
board="github_emerging", items=github_emerging, prev_data=prev_data, depth=depth
)
github_topic_all = _build_github_board_moves(
board="github_topic",
items=github_topic,
prev_data=prev_data,
depth=depth,
topic_name=topic,
)
skills_trending_moves = skills_trending_all[:cap]
skills_hot_moves = skills_hot_all[:cap]
github_trending_moves = github_trending_all[:cap]
github_emerging_moves = github_emerging_all[:cap]
github_topic_moves = github_topic_all[:cap]
topic_label = f"Topic `{topic}`"
return {
"baseline_date": baseline_date,
"compare_depth": depth,
"wecom_new_limit": cap,
"selection_mode": "top_n",
"topic_name": topic,
"skills_trending_moves": skills_trending_moves,
"skills_hot_moves": skills_hot_moves,
"skills_trending_stable": not skills_trending_all,
"skills_hot_stable": not skills_hot_all,
"skills_trending_summary": _board_summary(
label="Skills Trending",
baseline_date=baseline_date,
depth=depth,
moves=skills_trending_all,
capped=skills_trending_moves,
),
"skills_hot_summary": _board_summary(
label="Skills Hot",
baseline_date=baseline_date,
depth=depth,
moves=skills_hot_all,
capped=skills_hot_moves,
),
"github_trending_moves": github_trending_moves,
"github_emerging_moves": github_emerging_moves,
"github_topic_moves": github_topic_moves,
"github_trending_stable": not github_trending_all,
"github_emerging_stable": not github_emerging_all,
"github_topic_stable": not github_topic_all,
"github_trending_summary": _board_summary(
label="GitHub Trending",
baseline_date=baseline_date,
depth=depth,
moves=github_trending_all,
capped=github_trending_moves,
),
"github_emerging_summary": _board_summary(
label="GitHub 新兴",
baseline_date=baseline_date,
depth=depth,
moves=github_emerging_all,
capped=github_emerging_moves,
),
"github_topic_summary": _board_summary(
label=topic_label,
baseline_date=baseline_date,
depth=depth,
moves=github_topic_all,
capped=github_topic_moves,
),
# 兼容旧字段(合并,仅供调试)
"skills_moves": skills_trending_moves + skills_hot_moves,
"github_moves": github_trending_moves + github_emerging_moves + github_topic_moves,
"skills_stable": not skills_trending_all and not skills_hot_all,
"github_stable": not github_trending_all and not github_emerging_all and not github_topic_all,
}

281
daily/format_wecom.py Normal file
View File

@@ -0,0 +1,281 @@
"""企微早报排版。"""
from __future__ import annotations
import re
from typing import Any
from daily.config import wecom_skill_desc_limit
from daily.localize import LocalizeJob, localize_brief_descriptions, needs_chinese
from daily.text_utils import trim_brief
ICONS = {
"header": "📰",
"highlights": "💡",
"trending": "📈",
"hot": "🔥",
"github": "🐙",
"emerging": "🌱",
"topic": "🤖",
"ainews": "🌍",
"pick": "📦",
"theme": "🎯",
"file": "📄",
}
def _skill_line(rank: int, item: dict[str, Any], *, badge: str = "") -> list[str]:
source = item.get("source", "?")
installs = item.get("installs_fmt", "?")
link = item.get("link", "")
desc = item.get("desc_short", "")
badge_prefix = f"{badge} " if badge else ""
if item.get("cluster"):
count = int(item.get("cluster_count") or 1)
sample = item.get("cluster_titles") or item.get("title", "")
label = f"**{source}** · {count} skills · **{installs}**"
if link:
head = f"{rank}. {badge_prefix}[{label}]({link})"
else:
head = f"{rank}. {badge_prefix}{label}"
lines = [head]
if sample or desc:
hint = desc or sample
lines.append(f" > {hint}")
return lines
title = item.get("title", "?")
if link:
head = f"{rank}. {badge_prefix}[**{title}**]({link}) · `{source}` · **{installs}**"
else:
head = f"{rank}. {badge_prefix}**{title}** · `{source}` · **{installs}**"
lines = [head]
if desc:
lines.append(f" > {desc}")
return lines
def _ai_news_lines(items: list[dict[str, Any]]) -> list[str]:
lines: list[str] = []
for i, item in enumerate(items, 1):
title = item.get("title", "?")
link = item.get("link", "")
source = item.get("source_name", "?")
pub = item.get("published_fmt", "")
desc = item.get("desc_short", "")
pub_suffix = f" · {pub}" if pub else ""
if link:
head = f"{i}. [**{title}**]({link}) · `{source}`{pub_suffix}"
else:
head = f"{i}. **{title}** · `{source}`{pub_suffix}"
lines.append(head)
if desc:
lines.append(f" > {desc}")
return lines
def _github_repo_lines(repos: list[dict[str, Any]], *, show_created: bool = False) -> list[str]:
lines: list[str] = []
for i, repo in enumerate(repos, 1):
name = repo["repo"]
url = repo["url"]
lang = repo.get("language", "")
stars_today = repo.get("stars_today_fmt", "")
total = repo.get("total_stars_fmt", "")
created = repo.get("created_at", "")
meta_parts: list[str] = []
if lang:
meta_parts.append(lang)
if stars_today:
meta_parts.append(f"+{stars_today} today")
elif total:
meta_parts.append(f"{total}")
if show_created and created:
meta_parts.append(f"创建于 {created}")
meta = f" · {' · '.join(meta_parts)}" if meta_parts else ""
lines.append(f"{i}. [{name}]({url}){meta}")
desc = repo.get("desc_short") or repo.get("description", "")
if desc:
lines.append(f" > {desc}")
return lines
def _fallback_skill_desc(item: dict[str, Any]) -> str:
if item.get("cluster"):
count = int(item.get("cluster_count") or 1)
source = item.get("source") or "unknown"
sample = item.get("cluster_titles") or item.get("title") or ""
return f"{count} agent skills from {source}, including {sample}"
title = item.get("title") or "skill"
source = item.get("source") or "unknown"
return f"{title} skill from {source}"
def _skill_group_key(item: dict[str, Any]) -> str:
return str(item.get("id") or f"{item.get('source')}/{item.get('title')}")
def finalize_wecom_skill_groups(
items: list[dict[str, Any]],
*,
desc_limit: int | None = None,
) -> list[dict[str, Any]]:
"""为企微 Skills 榜生成一句简要中文简介(与完整版 .md 长描述分离)。"""
if desc_limit is None:
desc_limit = wecom_skill_desc_limit()
limit = desc_limit if desc_limit > 0 else 48
copies: list[tuple[str, dict[str, Any]]] = []
jobs: list[LocalizeJob] = []
for item in items:
copy = dict(item)
desc = (copy.get("description") or "").strip()
if not desc:
desc = _fallback_skill_desc(copy)
key = _skill_group_key(copy)
job_key = f"wecom:{key}"
if needs_chinese(desc) or len(desc) > limit:
jobs.append(LocalizeJob(job_key, desc, limit))
else:
copy["wecom_desc"] = desc
copies.append((job_key, copy))
zh_map = localize_brief_descriptions(jobs, archive=True)
out: list[dict[str, Any]] = []
for job_key, copy in copies:
if job_key in zh_map:
copy["wecom_desc"] = zh_map[job_key]
elif "wecom_desc" not in copy:
copy["wecom_desc"] = _brief_fallback_desc(
(copy.get("description") or "").strip() or _fallback_skill_desc(copy),
limit,
)
out.append(copy)
return out
def _brief_fallback_desc(text: str, limit: int) -> str:
return trim_brief(text, limit)
def _grouped_skill_to_wecom_item(
item: dict[str, Any],
*,
desc_limit: int | None = None,
) -> dict[str, Any]:
if desc_limit is None:
desc_limit = wecom_skill_desc_limit()
limit = desc_limit if desc_limit > 0 else 48
installs = int(item.get("installs") or 0)
installs_fmt = item.get("installs_fmt") or str(installs)
desc = (item.get("wecom_desc") or item.get("description") or "").strip()
if not desc and item.get("cluster"):
desc = item.get("cluster_titles") or ""
if not item.get("wecom_desc"):
desc = _brief_fallback_desc(desc, limit)
return {
"title": item.get("title", ""),
"source": item.get("source", "?"),
"installs_fmt": installs_fmt,
"link": item.get("link", ""),
"desc_short": desc,
"cluster": bool(item.get("cluster")),
"cluster_count": item.get("cluster_count"),
"cluster_titles": item.get("cluster_titles"),
}
def build_skills_board_section(icon_key: str, board_label: str, items: list[dict[str, Any]]) -> str:
prepared = finalize_wecom_skill_groups(items)
wecom_items = [_grouped_skill_to_wecom_item(x) for x in prepared]
lines = [f"{ICONS[icon_key]} **{board_label} Top {len(wecom_items)}**"]
for rank, item in enumerate(wecom_items, 1):
lines.extend(_skill_line(rank, item))
return "\n".join(lines)
_SKILL_SECTIONS = re.compile(
r"📈 \*\*Skills Trending.*?(?=🐙 \*\*GitHub Trending)",
re.DOTALL,
)
def replace_wecom_skill_sections(
md: str,
*,
trending: list[dict[str, Any]],
hot: list[dict[str, Any]],
) -> str:
"""用 Python 合并后的 Skills 榜替换或插入 Agent 早报中的对应区块。"""
trending_sec = build_skills_board_section("trending", "Skills Trending", trending)
hot_sec = build_skills_board_section("hot", "Skills Hot", hot)
replacement = f"{trending_sec}\n\n{hot_sec}\n\n"
if _SKILL_SECTIONS.search(md):
return _SKILL_SECTIONS.sub(replacement, md)
github_marker = "🐙 **GitHub Trending"
idx = md.find(github_marker)
if idx >= 0:
return md[:idx] + replacement + md[idx:]
return md.rstrip() + "\n\n" + replacement
def build_wecom_report(
*,
date_str: str,
time_str: str,
updated: str,
highlights: list[str],
theme_line: str,
trending: list[dict[str, Any]],
hot: list[dict[str, Any]],
repos: list[dict[str, Any]],
emerging: list[dict[str, Any]],
topic_name: str,
topic_repos: list[dict[str, Any]],
ai_news: list[dict[str, Any]] | None = None,
pick_command: str,
) -> str:
lines = [
f"{ICONS['header']} **早报 · {date_str}**",
f"> ⏱ {time_str} · 数据截至 {updated}",
"",
f"{ICONS['highlights']} **今日速览**",
]
for point in highlights[:3]:
lines.append(f"> {point}")
lines.append("")
lines.append(f"{ICONS['theme']} {theme_line}")
lines.append("")
if ai_news:
lines.append(f"{ICONS['ainews']} **国际 AI 时讯 Top {len(ai_news)}**")
lines.extend(_ai_news_lines(ai_news))
lines.append("")
lines.append(f"{ICONS['trending']} **Skills Trending Top {len(trending)}**")
for rank, item in enumerate(trending, 1):
lines.extend(_skill_line(rank, item, badge=item.get("badge", "")))
lines.append("")
lines.append(f"{ICONS['hot']} **Skills Hot Top {len(hot)}**")
for rank, item in enumerate(hot, 1):
lines.extend(_skill_line(rank, item, badge=item.get("badge", "")))
lines.append("")
if repos:
lines.append(f"{ICONS['github']} **GitHub Trending Top {len(repos)}**")
lines.extend(_github_repo_lines(repos))
lines.append("")
if emerging:
lines.append(f"{ICONS['emerging']} **新兴项目 Top {len(emerging)}**")
lines.extend(_github_repo_lines(emerging, show_created=True))
lines.append("")
if topic_repos:
lines.append(f"{ICONS['topic']} **Topic `{topic_name}` Top {len(topic_repos)}**")
lines.extend(_github_repo_lines(topic_repos))
lines.append("")
lines.append(f"{ICONS['pick']} **今日首推**")
lines.append(f"`{pick_command}`")
return "\n".join(lines)

635
daily/generate.py Normal file
View File

@@ -0,0 +1,635 @@
"""生成早报 Markdown完整版 + 企微短版)。"""
from __future__ import annotations
import json
import re
import sys
import xml.etree.ElementTree as ET
from collections import defaultdict
from datetime import datetime, timezone, timedelta
from pathlib import Path
from typing import Any
import certifi
import httpx
import logging
logger = logging.getLogger(__name__)
from daily.config import (
CACHE_DIR,
LOG_DIR,
OUTPUT_DIR,
SNAPSHOT_FILE,
ensure_bot_on_path,
env,
env_int,
full_desc_limit,
news_summary_limit,
wecom_skill_desc_limit,
)
from daily.format_wecom import build_wecom_report, finalize_wecom_skill_groups, replace_wecom_skill_sections
from daily.agent_workflow import is_agent_mode, run_agent_workflow
from daily.delta import compare_depth
from daily.cursor_editor import (
apply_descriptions,
is_enabled as cursor_editor_enabled,
run_editorial,
theme_line_from_editorial,
)
from daily.github.auth import github_html_headers
from daily.github.search import fetch_emerging_repos, fetch_topic_hot_repos
from daily.github.trending import fetch_github_trending, trending_data_source_note
from daily.localize import LocalizeJob, localize_descriptions, needs_chinese
from daily.news.fetch import fetch_ai_news, format_news_section, prepare_wecom_news_items
from daily.report_data import (
build_full_payload,
build_llm_input,
data_json_path,
save_json,
)
from daily.skills_board import load_boards
from daily.skills_group import group_skills_by_source
ensure_bot_on_path()
from skills_service import _format_installs, load_feed # noqa: E402
THEME_RULES: list[tuple[str, str, list[str]]] = [
("🎬", "AI 多媒体 / 视频", ["runcomfy", "remotion", "video", "seedance", "inpaint", "lipsync"]),
("🔧", "工程协作 / Skill 元能力", ["grill", "tdd", "architecture", "find-skills", "to-issues"]),
("📱", "飞书 / Lark", ["lark", "feishu"]),
("📣", "内容营销", ["viral", "tiktok", "instagram", "reels"]),
("🎨", "设计 / 前端", ["frontend", "design", "ui-ux", "tailwind"]),
]
def _now_cst() -> datetime:
return datetime.now(timezone(timedelta(hours=8)))
def _short_desc(text: str, limit: int = 72) -> str:
text = re.sub(r"\s+", " ", text or "").strip()
if limit <= 0 or len(text) <= limit:
return text
return text[: limit - 3] + "..."
def _archive_desc(text: str) -> str:
return _short_desc(text, full_desc_limit())
def _wecom_desc(text: str, limit: int = 36) -> str:
return _short_desc(text, limit)
def _localize_descriptions_in_place(
trending: list[dict[str, Any]],
hot: list[dict[str, Any]],
github_trending: list[dict[str, Any]],
github_emerging: list[dict[str, Any]],
github_topic: list[dict[str, Any]],
ai_news: dict[str, Any],
) -> None:
full_limit = full_desc_limit()
news_limit = news_summary_limit()
jobs: list[LocalizeJob] = []
seen_skill: set[str] = set()
for item in trending + hot:
sid = _skill_id(item)
if sid in seen_skill:
continue
seen_skill.add(sid)
desc = (item.get("description") or "").strip()
if desc:
jobs.append(LocalizeJob(f"skill:{sid}", desc, full_limit))
seen_repo: set[str] = set()
for repo_list in (github_trending, github_emerging, github_topic):
for item in repo_list:
repo = item.get("repo", "")
if not repo or repo in seen_repo:
continue
seen_repo.add(repo)
desc = (item.get("description") or "").strip()
if desc:
jobs.append(LocalizeJob(f"github:{repo}", desc, full_limit))
if ai_news.get("enabled"):
seen_news: set[str] = set()
for item in ai_news.get("flat") or []:
link = item.get("link", "")
if not link or link in seen_news:
continue
seen_news.add(link)
summary = (item.get("summary") or "").strip()
if summary:
jobs.append(LocalizeJob(f"news:{link}", summary, news_limit))
zh_map = localize_descriptions(jobs, archive=True)
if not zh_map and not jobs:
return
def _apply_zh(mapping: dict[str, str]) -> None:
for item in trending + hot:
key = f"skill:{_skill_id(item)}"
if key in mapping:
item["description"] = mapping[key]
for repo_list in (github_trending, github_emerging, github_topic):
for item in repo_list:
key = f"github:{item.get('repo', '')}"
if key in mapping:
item["description"] = mapping[key]
if ai_news.get("enabled"):
for cat in ai_news.get("categories") or []:
for item in cat.get("items") or []:
key = f"news:{item.get('link', '')}"
if key in mapping:
item["summary"] = mapping[key]
for item in ai_news.get("flat") or []:
key = f"news:{item.get('link', '')}"
if key in mapping:
item["summary"] = mapping[key]
_apply_zh(zh_map)
# 仍为英文的条目再译一轮(长描述或批次失败时)
retry_jobs: list[LocalizeJob] = []
seen_skill.clear()
for item in trending + hot:
sid = _skill_id(item)
if sid in seen_skill:
continue
seen_skill.add(sid)
desc = (item.get("description") or "").strip()
if needs_chinese(desc):
retry_jobs.append(LocalizeJob(f"skill:{sid}", desc, full_limit))
seen_repo.clear()
for repo_list in (github_trending, github_emerging, github_topic):
for item in repo_list:
repo = item.get("repo", "")
if not repo or repo in seen_repo:
continue
seen_repo.add(repo)
desc = (item.get("description") or "").strip()
if needs_chinese(desc):
retry_jobs.append(LocalizeJob(f"github:{repo}", desc, full_limit))
if ai_news.get("enabled"):
seen_news.clear()
for item in ai_news.get("flat") or []:
link = item.get("link", "")
if not link or link in seen_news:
continue
seen_news.add(link)
summary = (item.get("summary") or "").strip()
if needs_chinese(summary):
retry_jobs.append(LocalizeJob(f"news:{link}", summary, news_limit))
if retry_jobs:
_apply_zh(localize_descriptions(retry_jobs, archive=True))
def _skill_id(item: dict[str, Any]) -> str:
return str(item.get("id") or f"{item.get('source')}/{item.get('title')}")
def _load_snapshot() -> set[str]:
if not SNAPSHOT_FILE.exists():
return set()
try:
data = json.loads(SNAPSHOT_FILE.read_text(encoding="utf-8"))
return set(str(x) for x in (data.get("skill_ids") or []))
except (OSError, json.JSONDecodeError):
return set()
def _save_snapshot(feed: dict[str, Any], date_str: str) -> None:
CACHE_DIR.mkdir(parents=True, exist_ok=True)
ids: list[str] = []
for board in ("topTrending", "topHot"):
for item in feed.get(board, [])[:20]:
sid = _skill_id(item)
if sid not in ids:
ids.append(sid)
SNAPSHOT_FILE.write_text(
json.dumps({"date": date_str, "skill_ids": ids}, ensure_ascii=False, indent=2),
encoding="utf-8",
)
def _prepare_skill_item(item: dict[str, Any], prev_ids: set[str], rank: int) -> dict[str, Any]:
badge = ""
sid = _skill_id(item)
if sid not in prev_ids and prev_ids:
badge = "🆕"
elif rank == 1:
badge = "👑"
installs_fmt = item.get("installs_fmt") or _format_installs(item.get("installs", 0))
title = item.get("source", "?") if item.get("cluster") else item.get("title", "?")
desc = item.get("wecom_desc") or item.get("description") or item.get("cluster_titles") or ""
limit = wecom_skill_desc_limit()
desc_short = desc if item.get("wecom_desc") or limit <= 0 else _wecom_desc(desc, limit)
return {
"title": title,
"source": item.get("source", "?"),
"installs_fmt": installs_fmt,
"link": item.get("link", ""),
"desc_short": desc_short,
"badge": badge,
"cluster": bool(item.get("cluster")),
"cluster_count": item.get("cluster_count"),
"cluster_titles": item.get("cluster_titles"),
}
def _detect_theme_line(feed: dict[str, Any]) -> str:
scores: dict[str, int] = defaultdict(int)
for board in ("topTrending", "topHot"):
for rank, item in enumerate(feed.get(board, [])[:10], 1):
haystack = " ".join(
[item.get("title", ""), item.get("source", ""), item.get("description", "")]
).lower()
for _icon, label, keywords in THEME_RULES:
if any(k in haystack for k in keywords):
scores[label] += max(1, 11 - rank)
break
if not scores:
return "**今日主题**Agent Skills 生态持续活跃"
return f"**今日主题**{max(scores.items(), key=lambda x: x[1])[0]}"
def _build_highlights(
trending: list[dict[str, Any]],
hot: list[dict[str, Any]],
github_trending: list[dict[str, Any]],
github_emerging: list[dict[str, Any]],
ai_news: dict[str, Any] | None = None,
) -> list[str]:
points: list[str] = []
if ai_news and ai_news.get("enabled"):
top_news = prepare_wecom_news_items(ai_news)
if top_news:
n0 = top_news[0]
pub = f" · {n0['published_fmt']}" if n0.get("published_fmt") else ""
points.append(
f"🌍 AI 时讯 [{n0['title']}]({n0['link']})`{n0.get('source_name', '?')}`{pub}"
)
elif ai_news.get("flat"):
n0 = ai_news["flat"][0]
pub = f" · {n0['published_fmt']}" if n0.get("published_fmt") else ""
points.append(f"🌍 AI 时讯 [{n0['title']}]({n0['link']})`{n0.get('source_name', '?')}`{pub}")
if trending:
t0 = trending[0]
points.append(f"📈 Skills 榜首 **{t0.get('title')}**{_format_installs(t0.get('installs', 0))}")
if github_trending:
g0 = github_trending[0]
stars = g0.get("stars_today_fmt", "")
total = g0.get("total_stars_fmt", "")
star_hint = f"+{stars} today · " if stars else (f"{total} · " if total else "")
points.append(f"🐙 GitHub Trending [{g0['repo']}]({g0['url']}){star_hint}{g0.get('language', '')}")
if github_emerging:
e0 = github_emerging[0]
points.append(f"🌱 新兴 [{e0['repo']}]({e0['url']})(⭐ {e0.get('total_stars_fmt', '?')}")
elif hot:
h0 = hot[0]
points.append(f"🔥 Skills Hot 榜首 **{h0.get('title')}**1H {_format_installs(h0.get('installs', 0))}")
while len(points) < 3 and len(trending) > len(points):
item = trending[len(points)]
points.append(f"✨ **{item.get('title')}** · `{item.get('source')}`")
return points[:3]
def _prepare_github_item(item: dict[str, Any]) -> dict[str, Any]:
return {**item, "desc_short": _wecom_desc(item.get("description", ""), 40)}
def _fetch_latest_release_title(repo: str) -> str | None:
atom_url = f"https://github.com/{repo}/releases.atom"
try:
with httpx.Client(
timeout=12.0,
verify=certifi.where(),
follow_redirects=True,
headers=github_html_headers(),
) as client:
resp = client.get(atom_url)
if resp.status_code != 200:
return None
root = ET.fromstring(resp.text)
ns = {"a": "http://www.w3.org/2005/Atom"}
entry = root.find("a:entry", ns)
if entry is None:
return None
title = entry.find("a:title", ns)
return title.text.strip() if title is not None and title.text else None
except Exception:
return None
def _theme_clusters(feed: dict[str, Any], limit: int = 5) -> list[tuple[str, list[str]]]:
buckets: dict[str, list[str]] = defaultdict(list)
seen: set[str] = set()
for board in ("topTrending", "topHot"):
for item in feed.get(board, [])[:20]:
item_id = _skill_id(item)
if item_id in seen:
continue
seen.add(item_id)
haystack = " ".join(
[item.get("title", ""), item.get("source", ""), item.get("description", "")]
).lower()
for _icon, theme, keywords in THEME_RULES:
if any(k in haystack for k in keywords):
label = f"**{item.get('title')}** (`{item.get('source')}`)"
if label not in buckets[theme]:
buckets[theme].append(label)
break
return [(theme, examples[:limit]) for theme, examples in buckets.items() if examples]
def _format_github_repo_section(repos: list[dict[str, Any]], *, show_created: bool = False) -> list[str]:
lines: list[str] = []
for i, repo in enumerate(repos, 1):
lang = repo.get("language") or ""
stars_today = repo.get("stars_today_fmt") or ""
total = repo.get("total_stars_fmt") or ""
created = repo.get("created_at") or ""
meta_parts = [lang]
if stars_today:
meta_parts.append(f"+{stars_today} today")
if total:
meta_parts.append(f"总 ⭐ {total}")
if show_created and created:
meta_parts.append(f"创建于 {created}")
lines.append(f"{i}. **[{repo['repo']}]({repo['url']})** · {' · '.join(meta_parts)}")
desc = _archive_desc(repo.get("description", ""))
if desc:
lines.append(f" - {desc}")
lines.append("")
return lines
def _format_skill_section(items: list[dict[str, Any]], *, hot: bool = False) -> list[str]:
lines: list[str] = []
for i, item in enumerate(items, 1):
skill_id = item.get("id") or f"{item.get('source', '?')}/{item.get('title', '?')}"
link = item.get("link", "")
installs = _format_installs(item.get("installs", 0))
meta = f"1H {installs}" if hot else f"总安装 {installs}"
if link:
lines.append(f"{i}. **[{skill_id}]({link})** · {meta}")
else:
lines.append(f"{i}. **{skill_id}** · {meta}")
desc = _archive_desc(item.get("description", ""))
if desc:
lines.append(f" - {desc}")
lines.append("")
return lines
def generate_report() -> tuple[str, str, Path, Path]:
trending_n = env_int("DAILY_TRENDING_LIMIT", 150)
hot_n = max(env_int("DAILY_HOT_LIMIT", 150), compare_depth())
compare_n = compare_depth()
skill_pool = max(10, env_int("DAILY_WECOM_SKILL_POOL", 200))
wecom_trending = env_int("DAILY_WECOM_TRENDING", 10)
wecom_hot = env_int("DAILY_WECOM_HOT", 10)
github_limit = env_int("DAILY_GITHUB_TRENDING_LIMIT", 10)
wecom_github = env_int("DAILY_WECOM_GITHUB_TRENDING", env_int("DAILY_WECOM_REPOS", 10))
github_fetch_n = max(github_limit, compare_n, wecom_github)
emerging_limit = env_int("DAILY_GITHUB_EMERGING_LIMIT", 10)
wecom_emerging = env_int("DAILY_WECOM_GITHUB_EMERGING", 10)
emerging_fetch_n = max(emerging_limit, compare_n, wecom_emerging)
topic_limit = env_int("DAILY_GITHUB_TOPIC_LIMIT", 10)
wecom_topic = env_int("DAILY_WECOM_GITHUB_TOPIC", 10)
topic_fetch_n = max(topic_limit, compare_n, wecom_topic)
feed = load_feed(force=True)
prev_ids = _load_snapshot()
now = _now_cst()
date_str = now.strftime("%Y-%m-%d")
time_str = now.strftime("%H:%M") + " (UTC+8)"
updated = (feed.get("updatedAt") or "")[:10]
trending, hot = load_boards(feed, trending_limit=trending_n, hot_limit=hot_n)
github_trending = fetch_github_trending(github_fetch_n)
seen_repos = {r["repo"] for r in github_trending}
github_emerging = fetch_emerging_repos(emerging_fetch_n, exclude=seen_repos)
seen_repos.update(r["repo"] for r in github_emerging)
topic_name, github_topic = fetch_topic_hot_repos(topic_fetch_n, exclude=seen_repos)
ai_news = fetch_ai_news()
wecom_limits = {
"trending": wecom_trending,
"hot": wecom_hot,
"trending_pool": skill_pool,
"hot_pool": skill_pool,
"github": wecom_github,
"emerging": wecom_emerging,
"topic": wecom_topic,
"ai_news": env_int("DAILY_WECOM_AI_NEWS", 10),
}
llm_input = build_llm_input(
date_str=date_str,
updated=updated,
trending=trending,
hot=hot,
github_trending=github_trending,
github_emerging=github_emerging,
github_topic=github_topic,
topic_name=topic_name,
ai_news=ai_news,
wecom_limits=wecom_limits,
)
save_json(
data_json_path(date_str),
build_full_payload(
llm_input,
meta={
"generated_at": now.isoformat(),
"report_mode": "agent" if is_agent_mode() else "classic",
"cursor_editor": cursor_editor_enabled() and not is_agent_mode(),
},
),
)
agent_wecom: str | None = None
if is_agent_mode():
agent_wecom = run_agent_workflow(
llm_input,
date_str=date_str,
time_str=time_str,
updated=updated,
)
if not agent_wecom:
logger.warning("Agent 工作流失败,回退 classic 模式")
editorial_theme: str | None = None
editorial_highlights: list[str] | None = None
if agent_wecom is None:
editorial = run_editorial(llm_input, date_str=date_str)
if editorial:
apply_descriptions(
trending=trending,
hot=hot,
github_trending=github_trending,
github_emerging=github_emerging,
github_topic=github_topic,
ai_news=ai_news,
descriptions=editorial.get("descriptions") or {},
)
editorial_theme = theme_line_from_editorial(editorial) or None
hl = editorial.get("highlights") or []
editorial_highlights = hl if hl else None
# 完整版归档:中文化 + 加长摘要(已是中文的条目会跳过翻译)
_localize_descriptions_in_place(
trending, hot, github_trending, github_emerging, github_topic, ai_news
)
themes = _theme_clusters(feed)
lines = [
f"# 早报 · {date_str}",
"",
f"> 生成时间:{now.strftime('%Y-%m-%d %H:%M')} (UTC+8) ",
f"> skills 数据更新:{updated} ",
"> 数据来源:[skills.sh/trending](https://skills.sh/trending) · [skills.sh/hot](https://skills.sh/hot) · 国际 AI RSS",
"",
"---",
"",
f"## 一、Skills Trending Top {trending_n}",
"",
*_format_skill_section(trending),
"---",
"",
f"## 二、Skills Hot Top {hot_n}",
"",
*_format_skill_section(hot, hot=True),
"",
"---",
"",
f"## 三、GitHub Trending Top {github_limit}",
"",
trending_data_source_note(),
"",
]
if github_trending:
lines.extend(_format_github_repo_section(github_trending))
else:
lines.append("*GitHub Trending 获取失败,请检查网络或配置 GITHUB_TOKEN。*")
lines.append("")
lines.extend(["---", "", f"## 四、新兴项目 Top {emerging_limit}", "", "> 数据来源GitHub Search API需 `GITHUB_TOKEN`", ""])
if github_emerging:
lines.extend(_format_github_repo_section(github_emerging, show_created=True))
else:
lines.append("*新兴项目获取失败或未配置 GITHUB_TOKEN。*")
lines.append("")
lines.extend(["---", "", f"## 五、Topic `{topic_name}` Top {topic_limit}", "", "> 数据来源GitHub Search API需 `GITHUB_TOKEN`", ""])
if github_topic:
lines.extend(_format_github_repo_section(github_topic))
else:
lines.append(f"*Topic `{topic_name}` 热点获取失败或未配置 GITHUB_TOKEN。*")
lines.append("")
section_no = 6
lines.extend(format_news_section(ai_news, section_no=section_no))
section_no += 1
watch = (env("GITHUB_REPOS") or "").strip()
if watch:
lines.extend(["---", "", f"## {section_no}、关注仓库 Release", ""])
section_no += 1
for repo in [r.strip() for r in watch.split(",") if r.strip()]:
release = _fetch_latest_release_title(repo)
lines.append(f"- **{repo}**{release or '暂无 release'}")
lines.append("")
lines.extend(["---", "", f"## {section_no}、主题聚类", ""])
for theme, examples in themes:
lines.append(f"### {theme}")
for ex in examples:
lines.append(f"- {ex}")
lines.append("")
pick_src = trending[0].get("source", "") if trending else ""
pick_name = trending[0].get("title", "") if trending else ""
pick_command = (
f"npx skills add {pick_src}/{pick_name}"
if pick_src and pick_name
else "npx skills add vercel-labs/skills/find-skills"
)
lines.extend(["---", "", "## 安装示例", "", "```bash"])
for item in trending[:4]:
src, name = item.get("source", ""), item.get("title", "")
if src and name:
lines.append(f"npx skills add {src}/{name}")
lines.extend(["```", "", f"*企微短版见 `output/{date_str}.wecom.md`*"])
markdown = "\n".join(lines)
if agent_wecom:
gt = group_skills_by_source(trending, limit=wecom_trending, pool_size=skill_pool)
gh = group_skills_by_source(hot, limit=wecom_hot, pool_size=skill_pool)
wecom_md = replace_wecom_skill_sections(agent_wecom, trending=gt, hot=gh)
else:
wecom_md = build_wecom_report(
date_str=date_str,
time_str=time_str,
updated=updated,
highlights=editorial_highlights
or _build_highlights(trending, hot, github_trending, github_emerging, ai_news),
theme_line=editorial_theme or _detect_theme_line(feed),
ai_news=prepare_wecom_news_items(ai_news),
trending=[
_prepare_skill_item(item, prev_ids, r)
for r, item in enumerate(
finalize_wecom_skill_groups(
group_skills_by_source(trending, limit=wecom_trending, pool_size=skill_pool)
),
1,
)
],
hot=[
_prepare_skill_item(item, prev_ids, r)
for r, item in enumerate(
finalize_wecom_skill_groups(
group_skills_by_source(hot, limit=wecom_hot, pool_size=skill_pool)
),
1,
)
],
repos=[_prepare_github_item(item) for item in github_trending[:wecom_github]],
emerging=[_prepare_github_item(item) for item in github_emerging[:wecom_emerging]],
topic_name=topic_name,
topic_repos=[_prepare_github_item(item) for item in github_topic[:wecom_topic]],
pick_command=pick_command,
)
_save_snapshot(feed, date_str)
OUTPUT_DIR.mkdir(parents=True, exist_ok=True)
out_md = OUTPUT_DIR / f"{date_str}.md"
out_wecom = OUTPUT_DIR / f"{date_str}.wecom.md"
out_md.write_text(markdown, encoding="utf-8")
out_wecom.write_text(wecom_md, encoding="utf-8")
return markdown, wecom_md, out_md, out_wecom
def main() -> int:
LOG_DIR.mkdir(parents=True, exist_ok=True)
log_file = LOG_DIR / f"{_now_cst():%Y-%m-%d}.log"
try:
_, wecom_md, out_md, out_wecom = generate_report()
nbytes = len(wecom_md.encode("utf-8"))
msg = f"[{_now_cst():%H:%M:%S}] OK -> {out_md}, {out_wecom} ({nbytes} bytes)\n"
log_file.write_text(msg, encoding="utf-8")
print(msg.strip())
return 0
except Exception as exc:
msg = f"[{_now_cst():%H:%M:%S}] FAIL: {exc}\n"
log_file.write_text(msg, encoding="utf-8")
print(msg.strip(), file=sys.stderr)
return 1

9
daily/github/__init__.py Normal file
View File

@@ -0,0 +1,9 @@
from daily.github.search import fetch_emerging_repos, fetch_topic_hot_repos
from daily.github.trending import fetch_github_trending, trending_data_source_note
__all__ = [
"fetch_emerging_repos",
"fetch_topic_hot_repos",
"fetch_github_trending",
"trending_data_source_note",
]

81
daily/github/auth.py Normal file
View File

@@ -0,0 +1,81 @@
"""GitHub 请求共用GITHUB_TOKEN、请求头、仓库 API。"""
from __future__ import annotations
import logging
from typing import Any
import certifi
import httpx
from daily.config import env
logger = logging.getLogger(__name__)
DEFAULT_USER_AGENT = (
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 "
"(KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36"
)
def github_token() -> str | None:
raw = (env("GITHUB_TOKEN") or "").strip()
return raw or None
def github_api_headers() -> dict[str, str]:
headers = {
"User-Agent": env("GITHUB_TRENDING_USER_AGENT", DEFAULT_USER_AGENT),
"Accept": "application/vnd.github+json",
"X-GitHub-Api-Version": "2022-11-28",
}
token = github_token()
if token:
headers["Authorization"] = f"Bearer {token}"
return headers
def github_html_headers() -> dict[str, str]:
headers = {
"User-Agent": env("GITHUB_TRENDING_USER_AGENT", DEFAULT_USER_AGENT),
"Accept": "text/html,application/xhtml+xml",
"Accept-Language": "en-US,en;q=0.9",
}
token = github_token()
if token:
headers["Authorization"] = f"Bearer {token}"
return headers
def fetch_repo_api(repo: str) -> dict[str, Any] | None:
url = f"https://api.github.com/repos/{repo}"
try:
with httpx.Client(
timeout=12.0,
verify=certifi.where(),
headers=github_api_headers(),
) as client:
resp = client.get(url)
if resp.status_code != 200:
return None
data = resp.json()
return {
"description": data.get("description") or "",
"language": data.get("language") or "",
"stars": data.get("stargazers_count", 0),
}
except Exception as exc:
logger.debug("GitHub API repo %s 失败: %s", repo, exc)
return None
def format_star_count(value: int | float | str) -> str:
try:
n = int(value)
except (TypeError, ValueError):
return str(value)
if n >= 1_000_000:
return f"{n / 1_000_000:.1f}M".replace(".0M", "M")
if n >= 1_000:
return f"{n / 1_000:.1f}K".replace(".0K", "K")
return f"{n:,}"

121
daily/github/search.py Normal file
View File

@@ -0,0 +1,121 @@
"""GitHub Search API新兴项目、Topic 热点。"""
from __future__ import annotations
import logging
from datetime import datetime, timedelta, timezone
from typing import Any
import certifi
import httpx
from daily.config import env, env_int
from daily.github.auth import format_star_count, github_api_headers, github_token
logger = logging.getLogger(__name__)
def _repo_from_api_item(item: dict[str, Any], *, source: str) -> dict[str, Any]:
full_name = item.get("full_name") or ""
stars = item.get("stargazers_count", 0)
created = (item.get("created_at") or "")[:10]
return {
"repo": full_name,
"url": item.get("html_url") or f"https://github.com/{full_name}",
"description": item.get("description") or "",
"language": item.get("language") or "",
"stars_today": None,
"stars_today_fmt": "",
"total_stars_fmt": format_star_count(stars),
"created_at": created,
"source": source,
}
def search_github_repos(
query: str,
limit: int,
*,
sort: str = "stars",
require_token: bool = True,
) -> list[dict[str, Any]]:
if require_token and not github_token():
logger.warning("GitHub Search 需要 GITHUB_TOKEN: %s", query[:80])
return []
try:
with httpx.Client(
timeout=20.0,
verify=certifi.where(),
headers=github_api_headers(),
) as client:
resp = client.get(
"https://api.github.com/search/repositories",
params={
"q": query,
"sort": sort,
"order": "desc",
"per_page": min(max(limit, 1), 30),
},
)
if resp.status_code != 200:
logger.warning("GitHub Search 失败 (%s): %s", resp.status_code, query[:80])
return []
items = resp.json().get("items") or []
except Exception as exc:
logger.warning("GitHub Search 异常: %s", exc)
return []
repos: list[dict[str, Any]] = []
for item in items:
full_name = item.get("full_name") or ""
if not full_name:
continue
repos.append(_repo_from_api_item(item, source="api-search"))
if len(repos) >= limit:
break
return repos
def _date_days_ago(days: int) -> str:
return (datetime.now(timezone.utc) - timedelta(days=days)).strftime("%Y-%m-%d")
def fetch_emerging_repos(
limit: int = 3,
*,
days: int | None = None,
min_stars: int | None = None,
exclude: set[str] | None = None,
) -> list[dict[str, Any]]:
days = days if days is not None else env_int("GITHUB_EMERGING_DAYS", 14)
min_stars = min_stars if min_stars is not None else env_int("GITHUB_EMERGING_MIN_STARS", 200)
created_after = _date_days_ago(days)
query = f"created:>{created_after} stars:>{min_stars} fork:false"
repos = search_github_repos(query, limit + len(exclude or set()))
if exclude:
repos = [r for r in repos if r["repo"] not in exclude]
for item in repos:
item["source"] = "api-emerging"
return repos[:limit]
def fetch_topic_hot_repos(
limit: int = 3,
*,
topic: str | None = None,
pushed_days: int | None = None,
min_stars: int | None = None,
exclude: set[str] | None = None,
) -> tuple[str, list[dict[str, Any]]]:
topic = (topic or env("GITHUB_TOPIC") or "llm").strip()
pushed_days = pushed_days if pushed_days is not None else env_int("GITHUB_TOPIC_PUSHED_DAYS", 7)
min_stars = min_stars if min_stars is not None else env_int("GITHUB_TOPIC_MIN_STARS", 50)
pushed_after = _date_days_ago(pushed_days)
query = f"topic:{topic} pushed:>{pushed_after} stars:>{min_stars} fork:false"
repos = search_github_repos(query, limit + len(exclude or set()))
if exclude:
repos = [r for r in repos if r["repo"] not in exclude]
for item in repos:
item["source"] = "api-topic"
return topic, repos[:limit]

206
daily/github/trending.py Normal file
View File

@@ -0,0 +1,206 @@
"""GitHub Trending页面爬取或 Search API。"""
from __future__ import annotations
import logging
import re
from datetime import datetime, timedelta, timezone
from html import unescape
from typing import Any, Literal
from urllib.parse import urlencode
import certifi
import httpx
from daily.config import env
from daily.github.auth import (
fetch_repo_api,
format_star_count,
github_html_headers,
github_token,
)
from daily.github.search import search_github_repos
logger = logging.getLogger(__name__)
TrendingSince = Literal["daily", "weekly", "monthly"]
TrendingMode = Literal["scrape", "api"]
DEFAULT_LIMIT = 5
DEFAULT_SINCE: TrendingSince = "daily"
DEFAULT_MODE: TrendingMode = "scrape"
TRENDING_URL = "https://github.com/trending"
_ARTICLE_RE = re.compile(r'<article class="Box-row">.*?</article>', re.S)
_REPO_HREF_RE = re.compile(r'h2[^>]*>\s*<a[^>]+href="([^"]+)"')
_DESC_RE = re.compile(r'<p class="col-9[^"]*"[^>]*>([^<]*)</p>')
_STARS_TODAY_RE = re.compile(r"([\d,]+)\s+stars?\s+today", re.I)
_LANG_RE = re.compile(r'itemprop="programmingLanguage"[^>]*>([^<]+)<')
_TOTAL_STARS_RE = re.compile(
r'href="/[^/]+/[^/]+/stargazers"[^>]*>\s*<svg[^>]*octicon-star[^>]*>.*?</svg>\s*([\d.,kKmM]+)',
re.S,
)
def _strip_html(text: str) -> str:
return unescape(re.sub(r"\s+", " ", text or "")).strip()
def trending_mode() -> TrendingMode:
raw = (env("GITHUB_TRENDING_MODE") or DEFAULT_MODE).strip().lower()
if raw in {"api", "token", "search"}:
return "api"
return "scrape"
def trending_data_source_note() -> str:
if trending_mode() == "api":
return "> 数据来源GitHub Search API`GITHUB_TRENDING_MODE=api`,需 `GITHUB_TOKEN`"
return "> 数据来源:[github.com/trending](https://github.com/trending?since=daily)(页面爬取,失败时 API 降级)"
def _parse_article(article_html: str) -> dict[str, Any] | None:
href_match = _REPO_HREF_RE.search(article_html)
if not href_match:
return None
href = href_match.group(1).strip("/")
if href.count("/") != 1:
return None
owner, name = href.split("/", 1)
repo = f"{owner}/{name}"
desc_match = _DESC_RE.search(article_html)
stars_today_match = _STARS_TODAY_RE.search(article_html)
lang_match = _LANG_RE.search(article_html)
total_stars_match = _TOTAL_STARS_RE.search(article_html)
stars_today_raw = stars_today_match.group(1).replace(",", "") if stars_today_match else ""
stars_today = int(stars_today_raw) if stars_today_raw.isdigit() else None
return {
"repo": repo,
"url": f"https://github.com/{repo}",
"description": _strip_html(desc_match.group(1)) if desc_match else "",
"language": _strip_html(lang_match.group(1)) if lang_match else "",
"stars_today": stars_today,
"stars_today_fmt": stars_today_match.group(1) if stars_today_match else "",
"total_stars_fmt": _strip_html(total_stars_match.group(1)) if total_stars_match else "",
"source": "scrape",
}
def _build_trending_url(*, since: TrendingSince = DEFAULT_SINCE, language: str = "") -> str:
if language:
return f"{TRENDING_URL}/{language}?{urlencode({'since': since})}"
return f"{TRENDING_URL}?{urlencode({'since': since})}"
def _since_push_date(since: TrendingSince) -> str:
now = datetime.now(timezone.utc)
if since == "weekly":
delta = timedelta(days=7)
elif since == "monthly":
delta = timedelta(days=30)
else:
delta = timedelta(days=1)
return (now - delta).strftime("%Y-%m-%d")
def _enrich_repo_from_api(item: dict[str, Any]) -> dict[str, Any]:
if not github_token() and env("GITHUB_API_ENRICH", "1") != "1":
return item
meta = fetch_repo_api(item["repo"])
if not meta:
return item
enriched = dict(item)
if not enriched.get("description"):
enriched["description"] = meta["description"]
if not enriched.get("language"):
enriched["language"] = meta["language"]
if not enriched.get("total_stars_fmt") and meta["stars"]:
enriched["total_stars_fmt"] = format_star_count(meta["stars"])
enriched["source"] = enriched.get("source", "scrape") + "+api"
return enriched
def _fetch_trending_html(url: str) -> str | None:
try:
with httpx.Client(
timeout=20.0,
verify=certifi.where(),
follow_redirects=True,
headers=github_html_headers(),
) as client:
resp = client.get(url)
if resp.status_code in {403, 429} and not github_token():
logger.warning("GitHub Trending %s(匿名可能被限),可配置 GITHUB_TOKEN", resp.status_code)
resp.raise_for_status()
return resp.text
except Exception as exc:
logger.warning("GitHub Trending 页面抓取失败: %s", exc)
return None
def _parse_trending_html(html: str, limit: int) -> list[dict[str, Any]]:
repos: list[dict[str, Any]] = []
for article_html in _ARTICLE_RE.findall(html):
item = _parse_article(article_html)
if item:
repos.append(_enrich_repo_from_api(item))
if len(repos) >= limit:
break
return repos
def _fetch_trending_via_api(limit: int, since: TrendingSince, language: str) -> list[dict[str, Any]]:
pushed_after = _since_push_date(since)
parts = [f"pushed:>{pushed_after}", "stars:>50", "fork:false"]
if language:
parts.append(f"language:{language}")
query = " ".join(parts)
repos = search_github_repos(query, limit, require_token=True)
for item in repos:
item["source"] = "api-search"
return repos
def fetch_github_trending(
limit: int = DEFAULT_LIMIT,
*,
since: TrendingSince | None = None,
language: str = "",
) -> list[dict[str, Any]]:
since = since or env("GITHUB_TRENDING_SINCE", DEFAULT_SINCE) # type: ignore[assignment]
if since not in ("daily", "weekly", "monthly"):
since = DEFAULT_SINCE
lang = (language or env("GITHUB_TRENDING_LANGUAGE") or "").strip()
mode = trending_mode()
if mode == "api":
repos = _fetch_trending_via_api(limit, since, lang)
if not repos and not github_token():
logger.warning("GITHUB_TRENDING_MODE=api 需要配置 GITHUB_TOKEN")
elif repos:
logger.info("GitHub Trending 使用 API 模式,共 %d", len(repos))
return repos[:limit]
url = _build_trending_url(since=since, language=lang)
html = _fetch_trending_html(url)
repos: list[dict[str, Any]] = []
if html:
repos = _parse_trending_html(html, limit)
if not repos:
logger.warning("GitHub Trending 页面解析为空: %s", url)
if len(repos) < limit:
before = len(repos)
fallback = _fetch_trending_via_api(limit, since, lang)
seen = {r["repo"] for r in repos}
for item in fallback:
if item["repo"] in seen:
continue
repos.append(item)
seen.add(item["repo"])
if len(repos) >= limit:
break
if len(repos) > before:
logger.info("已用 GitHub API 补充 %d 条 Trending 数据", len(repos) - before)
return repos[:limit]

119
daily/llm_client.py Normal file
View File

@@ -0,0 +1,119 @@
"""LLM 调用共享工具OpenAI 兼容 API / Cursor SDK"""
from __future__ import annotations
import json
import logging
import os
import re
from typing import Any
import certifi
import httpx
from daily.config import ROOT, env, env_int
logger = logging.getLogger(__name__)
_JSON_BLOCK = re.compile(r"```(?:json)?\s*([\s\S]*?)```", re.IGNORECASE)
def extract_json_object(text: str) -> dict[str, Any]:
text = text.strip()
if not text:
return {}
try:
data = json.loads(text)
return data if isinstance(data, dict) else {}
except json.JSONDecodeError:
pass
match = _JSON_BLOCK.search(text)
if match:
try:
data = json.loads(match.group(1).strip())
return data if isinstance(data, dict) else {}
except json.JSONDecodeError:
pass
start, end = text.find("{"), text.rfind("}")
if start >= 0 and end > start:
try:
data = json.loads(text[start : end + 1])
return data if isinstance(data, dict) else {}
except json.JSONDecodeError:
pass
return {}
def _openai_chat(system: str, user: str) -> str:
api_key = (env("DAILY_LLM_API_KEY") or env("OPENAI_API_KEY") or "").strip()
if not api_key:
return ""
base = (env("DAILY_LLM_API_BASE") or env("OPENAI_API_BASE") or "https://api.openai.com/v1").rstrip("/")
model = env("DAILY_LLM_MODEL") or env("OPENAI_MODEL") or "gpt-4o-mini"
timeout = env_int("DAILY_LLM_TIMEOUT", 120)
payload = {
"model": model,
"temperature": 0.2,
"messages": [
{"role": "system", "content": system},
{"role": "user", "content": user},
],
}
with httpx.Client(timeout=timeout, verify=certifi.where()) as client:
resp = client.post(
f"{base}/chat/completions",
headers={"Authorization": f"Bearer {api_key}", "Content-Type": "application/json"},
json=payload,
)
resp.raise_for_status()
data = resp.json()
return str(data["choices"][0]["message"]["content"] or "").strip()
def _cursor_chat(system: str, user: str) -> str:
api_key = (env("CURSOR_API_KEY") or "").strip()
if not api_key:
return ""
from cursor_sdk import Agent, AgentOptions, CursorAgentError, LocalAgentOptions
from daily.config import ensure_bot_on_path
ensure_bot_on_path()
try:
from bridge_manager import warm_cursor_bridge
except ImportError:
warm_cursor_bridge = lambda: None # noqa: E731
cwd = env("DAILY_CURSOR_CWD") or str(ROOT)
# bridge_manager 读 bot env_config 的 CURSOR_CWD早报侧须先对齐工作目录
os.environ["CURSOR_CWD"] = cwd
warm_cursor_bridge()
model = env("CURSOR_MODEL") or "composer-2.5"
prompt = f"{system}\n\n{user}"
try:
result = Agent.prompt(
prompt,
AgentOptions(
api_key=api_key,
model=model,
local=LocalAgentOptions(cwd=cwd),
),
)
except CursorAgentError as exc:
raise RuntimeError(f"LLM 调用失败:{exc.message}") from exc
if result.status == "error":
raise RuntimeError(f"LLM 调用失败:{result.result or '未知错误'}")
return (result.result or "").strip()
def llm_chat(system: str, user: str) -> str:
"""优先 OpenAI 兼容 API否则 Cursor SDK。"""
if env("DAILY_LLM_API_KEY") or env("OPENAI_API_KEY"):
return _openai_chat(system, user)
if env("CURSOR_API_KEY"):
return _cursor_chat(system, user)
return ""
def has_llm_configured() -> bool:
return bool(env("DAILY_LLM_API_KEY") or env("OPENAI_API_KEY") or env("CURSOR_API_KEY"))

236
daily/localize.py Normal file
View File

@@ -0,0 +1,236 @@
"""将英文描述批量改写为简短中文(大模型 + 本地缓存)。"""
from __future__ import annotations
import hashlib
import json
import logging
from dataclasses import dataclass
from daily.config import CACHE_DIR, env, env_int
from daily.cursor_editor import is_enabled as cursor_editor_enabled
from daily.llm_client import extract_json_object, llm_chat
from daily.text_utils import clip_text, trim_brief
logger = logging.getLogger(__name__)
_CACHE_FILE = CACHE_DIR / "zh-desc-cache.json"
@dataclass(frozen=True)
class LocalizeJob:
key: str
text: str
limit: int
def _enabled(*, archive: bool = False) -> bool:
if not archive and cursor_editor_enabled():
return False
raw = (env("DAILY_ZH_DESC") or "1").strip().lower()
return raw not in {"0", "false", "no", "off"}
def _is_mostly_chinese(text: str) -> bool:
text = text.strip()
if not text:
return True
cjk = sum(1 for c in text if "\u4e00" <= c <= "\u9fff")
latin = sum(1 for c in text if c.isascii() and c.isalpha())
return cjk >= max(latin, 1)
def needs_chinese(text: str) -> bool:
"""文本非空且尚未以中文为主。"""
text = (text or "").strip()
if not text:
return False
return not _is_mostly_chinese(text)
def _cache_key(text: str) -> str:
return hashlib.sha1(text.encode("utf-8")).hexdigest()[:16]
def _legacy_cache_key(text: str, limit: int) -> str:
return hashlib.sha1(f"{limit}:{text}".encode("utf-8")).hexdigest()[:16]
def _cache_is_truncated(text: str) -> bool:
t = (text or "").rstrip()
return t.endswith("") or t.endswith("...")
def _lookup_cached(cache: dict[str, str], text: str, limit: int) -> str | None:
hit = cache.get(_cache_key(text))
if hit and not (limit <= 0 and _cache_is_truncated(hit)):
return hit
for legacy_limit in (72, 120, 200):
legacy = cache.get(_legacy_cache_key(text, legacy_limit))
if legacy and not (limit <= 0 and _cache_is_truncated(legacy)):
cache[_cache_key(text)] = legacy
return legacy
return None
def _load_cache() -> dict[str, str]:
if not _CACHE_FILE.exists():
return {}
try:
data = json.loads(_CACHE_FILE.read_text(encoding="utf-8"))
return {str(k): str(v) for k, v in (data.get("entries") or {}).items()}
except (OSError, json.JSONDecodeError, TypeError):
return {}
def _save_cache(entries: dict[str, str]) -> None:
CACHE_DIR.mkdir(parents=True, exist_ok=True)
_CACHE_FILE.write_text(
json.dumps({"entries": entries}, ensure_ascii=False, indent=2),
encoding="utf-8",
)
def _build_prompt(jobs: list[LocalizeJob], *, brief: bool = False) -> tuple[str, str]:
if brief:
system = (
"你是技术早报编辑。把输入 JSON 中每条描述改写为**一句**中文简要介绍。"
"要求:输出必须是中文;只保留核心能力与典型场景;不要逐字翻译;不要加引号或编号;"
"每条必须语义完整、可独立阅读limit 为建议最大字数,请控制在 limit 以内且不要用省略号截断;"
"已是中文且足够简短时可适度精简;"
"只输出 JSON 对象key 与输入一致value 为中文简介字符串。"
)
else:
system = (
"你是技术早报编辑。把输入 JSON 中每条英文描述改写为**完整**中文介绍。"
"要求:输出必须是中文;保留关键能力与使用场景;不要逐字翻译;不要加引号或编号;"
"不要以省略号截断;已是中文则原样或适度精简;"
"只输出 JSON 对象key 与输入一致value 为中文简介字符串。"
)
payload = {job.key: {"text": job.text, "limit": job.limit} for job in jobs}
user = json.dumps(payload, ensure_ascii=False, indent=2)
return system, user
def _brief_cache_key(text: str, limit: int) -> str:
return hashlib.sha1(f"brief:{limit}:{text}".encode("utf-8")).hexdigest()[:16]
def _lookup_brief_cached(cache: dict[str, str], text: str, limit: int) -> str | None:
hit = cache.get(_brief_cache_key(text, limit))
if hit and not _cache_is_truncated(hit):
return hit
return None
def _translate_batch(jobs: list[LocalizeJob], *, brief: bool = False) -> dict[str, str]:
if not jobs:
return {}
system, user = _build_prompt(jobs, brief=brief)
raw = llm_chat(system, user)
if not raw:
return {}
parsed = extract_json_object(raw)
out: dict[str, str] = {}
for job in jobs:
value = parsed.get(job.key)
if isinstance(value, str) and value.strip():
trimmed = value.strip()
if brief and job.limit > 0:
out[job.key] = trim_brief(trimmed, job.limit)
else:
out[job.key] = clip_text(trimmed, job.limit) if job.limit > 0 else trimmed
return out
def localize_brief_descriptions(jobs: list[LocalizeJob], *, archive: bool = False) -> dict[str, str]:
"""企微用:将描述改写为一句简要中文。"""
if not _enabled(archive=archive) or not jobs:
return {}
cache = _load_cache()
result: dict[str, str] = {}
pending: list[LocalizeJob] = []
for job in jobs:
if not job.text.strip():
continue
limit = job.limit if job.limit > 0 else 48
text = job.text.strip()
if _is_mostly_chinese(text) and (limit <= 0 or len(text) <= limit):
result[job.key] = text
continue
cached = _lookup_brief_cached(cache, text, limit)
if cached:
result[job.key] = trim_brief(cached, limit) if limit > 0 else cached
else:
pending.append(LocalizeJob(job.key, text, limit))
if not pending:
return result
batch_size = max(3, min(10, env_int("DAILY_ZH_DESC_BATCH", 20)))
for i in range(0, len(pending), batch_size):
chunk = pending[i : i + batch_size]
try:
translated = _translate_batch(chunk, brief=True)
except Exception as exc:
logger.warning("企微简要摘要批次失败,保留原文:%s", exc)
continue
for job in chunk:
zh = translated.get(job.key)
if not zh:
continue
ck = _brief_cache_key(job.text, job.limit if job.limit > 0 else 48)
cache[ck] = zh
result[job.key] = zh
if cache:
_save_cache(cache)
return result
def localize_descriptions(jobs: list[LocalizeJob], *, archive: bool = False) -> dict[str, str]:
"""返回 job.key -> 中文简介。archive=True 时用于完整版 .md不受 Cursor 编辑层开关影响。"""
if not _enabled(archive=archive) or not jobs:
return {}
cache = _load_cache()
result: dict[str, str] = {}
pending: list[LocalizeJob] = []
for job in jobs:
if not job.text.strip():
continue
if _is_mostly_chinese(job.text):
result[job.key] = clip_text(job.text, job.limit)
continue
ck = _cache_key(job.text)
cached = _lookup_cached(cache, job.text, job.limit)
if cached:
result[job.key] = clip_text(cached, job.limit)
else:
pending.append(job)
if not pending:
return result
batch_size = max(5, env_int("DAILY_ZH_DESC_BATCH", 20))
for i in range(0, len(pending), batch_size):
chunk = pending[i : i + batch_size]
try:
translated = _translate_batch(chunk, brief=False)
except Exception as exc:
logger.warning("中文摘要批次失败,保留英文:%s", exc)
continue
for job in chunk:
zh = translated.get(job.key)
if not zh:
continue
ck = _cache_key(job.text)
cache[ck] = zh
result[job.key] = clip_text(zh, job.limit)
if cache:
_save_cache(cache)
return result

3
daily/news/__init__.py Normal file
View File

@@ -0,0 +1,3 @@
from daily.news.fetch import fetch_ai_news, format_news_section
__all__ = ["fetch_ai_news", "format_news_section"]

124
daily/news/feeds.py Normal file
View File

@@ -0,0 +1,124 @@
"""国际 AI 时讯 RSS 源定义(按类别分组)。"""
from __future__ import annotations
from dataclasses import dataclass
@dataclass(frozen=True)
class NewsFeed:
name: str
url: str
slow: bool = False # 限速源(如 Reddit串行抓取
@dataclass(frozen=True)
class NewsCategory:
id: str
name: str
icon: str
feeds: tuple[NewsFeed, ...]
NEWS_CATEGORIES: tuple[NewsCategory, ...] = (
NewsCategory(
id="official",
name="厂商官方",
icon="🏢",
feeds=(
NewsFeed("Anthropic Claude 更新", "https://docs.anthropic.com/en/release-notes/feed"),
NewsFeed("OpenAI", "https://openai.com/news/rss.xml"),
NewsFeed("Google AI", "https://blog.google/technology/ai/rss/"),
NewsFeed("DeepMind", "https://deepmind.google/blog/rss.xml"),
NewsFeed("Meta Engineering", "https://engineering.fb.com/feed/"),
NewsFeed("Microsoft Research", "https://www.microsoft.com/en-us/research/feed/"),
NewsFeed("Microsoft Blog", "https://blogs.microsoft.com/feed/"),
NewsFeed("Cohere", "https://cohere.com/blog/rss.xml"),
NewsFeed("Cursor Changelog", "https://cursor.com/changelog/rss.xml"),
),
),
NewsCategory(
id="developer",
name="Agent / LLM 开发者",
icon="🛠",
feeds=(
NewsFeed("LangChain", "https://blog.langchain.dev/rss/"),
NewsFeed("Hugging Face", "https://huggingface.co/blog/feed.xml"),
NewsFeed("Vercel Changelog", "https://vercel.com/changelog/rss.xml"),
NewsFeed("GitHub Copilot", "https://github.blog/changelog/label/copilot/feed/"),
),
),
NewsCategory(
id="media",
name="综合科技媒体",
icon="📰",
feeds=(
NewsFeed("The Verge AI", "https://www.theverge.com/rss/ai-artificial-intelligence/index.xml"),
NewsFeed("TechCrunch AI", "https://techcrunch.com/category/artificial-intelligence/feed/"),
NewsFeed("Ars Technica AI", "https://arstechnica.com/ai/feed/"),
NewsFeed("Wired AI", "https://www.wired.com/feed/tag/ai/latest/rss"),
NewsFeed("MIT Tech Review", "https://www.technologyreview.com/feed/"),
NewsFeed("VentureBeat AI", "https://venturebeat.com/category/ai/feed/"),
),
),
NewsCategory(
id="newsletter",
name="Newsletter 日报",
icon="✉️",
feeds=(
NewsFeed("Ben's Bites", "https://bensbites.substack.com/feed"),
NewsFeed("The Rundown AI", "https://therundown.substack.com/feed"),
NewsFeed("Latent Space", "https://www.latent.space/feed"),
NewsFeed("Simon Willison", "https://simonwillison.net/atom/everything/"),
NewsFeed("Import AI", "https://importai.substack.com/feed"),
NewsFeed("Last Week in AI", "https://lastweekin.ai/feed"),
NewsFeed("The Neuron", "https://www.theneuron.ai/feed"),
),
),
NewsCategory(
id="research",
name="研究 / 论文",
icon="📚",
feeds=(
NewsFeed("arXiv cs.CL", "https://arxiv.org/rss/cs.CL"),
NewsFeed("arXiv cs.AI", "https://arxiv.org/rss/cs.AI"),
NewsFeed("arXiv cs.LG", "https://arxiv.org/rss/cs.LG"),
),
),
NewsCategory(
id="trending",
name="热点 / 趋势",
icon="🔥",
feeds=(
NewsFeed(
"Google News · AI",
"https://news.google.com/rss/search?q=artificial+intelligence+OR+LLM+OR+Claude+OR+GPT&hl=en-US&gl=US&ceid=US:en",
),
NewsFeed(
"Google News · Technology",
"https://news.google.com/rss/headlines/section/topic/TECHNOLOGY?hl=en-US&gl=US&ceid=US:en",
),
NewsFeed("Techmeme", "https://www.techmeme.com/feed.xml"),
NewsFeed("HN · Front Page", "https://hnrss.org/frontpage"),
NewsFeed("HN · 100+ Points", "https://hnrss.org/newest?points=100"),
NewsFeed("Dev.to · AI", "https://dev.to/feed/tag/ai"),
NewsFeed("Lobsters", "https://lobste.rs/rss"),
),
),
NewsCategory(
id="community",
name="社区讨论",
icon="💬",
feeds=(
NewsFeed(
"HN · AI/LLM/Agent",
"https://hnrss.org/newest?q=AI+OR+LLM+OR+Claude+OR+agent+OR+GPT+OR+Gemini",
),
NewsFeed(
"Reddit · LLM/Claude/ML",
"https://old.reddit.com/r/LocalLLaMA+ClaudeAI+MachineLearning+OpenAI/.rss?limit=25",
slow=True,
),
),
),
)

462
daily/news/fetch.py Normal file
View File

@@ -0,0 +1,462 @@
"""抓取并整理国际 AI 时讯 RSS。"""
from __future__ import annotations
import logging
import re
import time
import html
import xml.etree.ElementTree as ET
from concurrent.futures import ThreadPoolExecutor, as_completed
from datetime import datetime, timezone, timedelta
from email.utils import parsedate_to_datetime
from typing import Any
from urllib.parse import parse_qs, urlencode, urlparse, urlunparse
import certifi
import httpx
from daily.config import env, env_int, news_summary_limit
from daily.news.feeds import NEWS_CATEGORIES, NewsCategory
logger = logging.getLogger(__name__)
USER_AGENT = "Mozilla/5.0 (compatible; skills-hot-daily/1.0; +https://skills.sh)"
BROWSER_USER_AGENT = (
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) "
"AppleWebKit/537.36 (KHTML, like Gecko) "
"Chrome/131.0.0.0 Safari/537.36"
)
STRIP_HTML = re.compile(r"<[^>]+>")
WS = re.compile(r"\s+")
ATOM_NS = {"a": "http://www.w3.org/2005/Atom"}
RSS_NS = {"r": "http://purl.org/rss/1.0/modules/content/"}
def _enabled() -> bool:
raw = (env("DAILY_AI_NEWS") or "1").strip().lower()
return raw not in {"0", "false", "no", "off"}
def _hours_window() -> int:
return max(1, env_int("DAILY_AI_NEWS_HOURS", 72))
def _per_feed_limit() -> int:
return max(1, env_int("DAILY_AI_NEWS_PER_FEED", 3))
def _per_category_limit() -> int:
return max(1, env_int("DAILY_AI_NEWS_PER_CATEGORY", 5))
def _wecom_limit() -> int:
return max(1, env_int("DAILY_WECOM_AI_NEWS", 10))
def _now_utc() -> datetime:
return datetime.now(timezone.utc)
def _parse_datetime(value: str | None) -> datetime | None:
if not value:
return None
text = value.strip()
if not text:
return None
try:
dt = parsedate_to_datetime(text)
if dt.tzinfo is None:
dt = dt.replace(tzinfo=timezone.utc)
return dt.astimezone(timezone.utc)
except (TypeError, ValueError, OverflowError):
pass
for fmt in (
"%Y-%m-%dT%H:%M:%SZ",
"%Y-%m-%dT%H:%M:%S%z",
"%Y-%m-%d",
):
try:
dt = datetime.strptime(text[: len(fmt.replace("%z", "+0000"))], fmt.replace("%z", ""))
if dt.tzinfo is None:
dt = dt.replace(tzinfo=timezone.utc)
return dt.astimezone(timezone.utc)
except ValueError:
continue
return None
def _clean_text(text: str | None, limit: int = 200) -> str:
if not text:
return ""
plain = STRIP_HTML.sub(" ", html.unescape(text))
plain = WS.sub(" ", plain).strip()
if limit <= 0 or len(plain) <= limit:
return plain
return plain[: limit - 3] + "..."
def _normalize_link(link: str) -> str:
parsed = urlparse(link.strip())
query = parse_qs(parsed.query, keep_blank_values=False)
for key in list(query.keys()):
if key.lower().startswith("utm_") or key.lower() in {"ref", "source"}:
query.pop(key, None)
clean_query = urlencode({k: v[0] for k, v in query.items() if v}, doseq=False)
return urlunparse((parsed.scheme, parsed.netloc, parsed.path.rstrip("/"), "", clean_query, ""))
def _normalize_title(title: str) -> str:
return WS.sub(" ", title.strip().lower())
def _entry_datetime(entry: dict[str, Any]) -> datetime | None:
for key in ("published", "updated"):
dt = _parse_datetime(entry.get(key))
if dt:
return dt
return None
def _parse_atom(content: str, feed_name: str, category: NewsCategory) -> list[dict[str, Any]]:
items: list[dict[str, Any]] = []
try:
root = ET.fromstring(content)
except ET.ParseError:
return items
for entry in root.findall("a:entry", ATOM_NS):
title_el = entry.find("a:title", ATOM_NS)
link_el = entry.find("a:link", ATOM_NS)
summary_el = entry.find("a:summary", ATOM_NS) or entry.find("a:content", ATOM_NS)
updated_el = entry.find("a:updated", ATOM_NS) or entry.find("a:published", ATOM_NS)
title = title_el.text.strip() if title_el is not None and title_el.text else ""
link = ""
if link_el is not None:
link = link_el.get("href") or (link_el.text or "").strip()
if not title or not link:
continue
items.append(
{
"title": title,
"link": link,
"summary": _clean_text(summary_el.text if summary_el is not None else ""),
"published": updated_el.text.strip() if updated_el is not None and updated_el.text else "",
"source_name": feed_name,
"category_id": category.id,
"category_name": category.name,
"category_icon": category.icon,
}
)
return items
def _parse_rss(content: str, feed_name: str, category: NewsCategory) -> list[dict[str, Any]]:
items: list[dict[str, Any]] = []
try:
root = ET.fromstring(content)
except ET.ParseError:
return items
channel = root.find("channel")
if channel is None:
return items
for item in channel.findall("item"):
title_el = item.find("title")
link_el = item.find("link")
desc_el = item.find("description")
pub_el = item.find("pubDate")
title = title_el.text.strip() if title_el is not None and title_el.text else ""
link = link_el.text.strip() if link_el is not None and link_el.text else ""
if not title or not link:
continue
items.append(
{
"title": title,
"link": link,
"summary": _clean_text(desc_el.text if desc_el is not None else ""),
"published": pub_el.text.strip() if pub_el is not None and pub_el.text else "",
"source_name": feed_name,
"category_id": category.id,
"category_name": category.name,
"category_icon": category.icon,
}
)
return items
def _parse_feed(content: str, feed_name: str, category: NewsCategory) -> list[dict[str, Any]]:
text = content.lstrip("\ufeff").strip()
if not text:
return []
if text.startswith("<rss") or "<channel>" in text[:500]:
return _parse_rss(text, feed_name, category)
if text.startswith("<feed") or "<entry>" in text[:500]:
return _parse_atom(text, feed_name, category)
if "<item>" in text[:2000]:
return _parse_rss(text, feed_name, category)
return _parse_atom(text, feed_name, category)
def _is_reddit_url(url: str) -> bool:
host = urlparse(url).netloc.lower()
return host.endswith("reddit.com")
def _reddit_auth_params() -> dict[str, str]:
user = (env("REDDIT_RSS_USER") or "").strip()
feed = (env("REDDIT_RSS_FEED") or "").strip()
if user and feed:
return {"user": user, "feed": feed}
return {}
def _with_query_params(url: str, extra: dict[str, str]) -> str:
if not extra:
return url
parsed = urlparse(url)
query = parse_qs(parsed.query, keep_blank_values=True)
for key, value in extra.items():
if value and key not in query:
query[key] = [value]
clean_query = urlencode({k: v[0] for k, v in query.items() if v and v[0]}, doseq=False)
return urlunparse((parsed.scheme, parsed.netloc, parsed.path, "", clean_query, ""))
def _reddit_old_url(url: str) -> str:
parsed = urlparse(url)
host = parsed.netloc.lower()
if host.startswith("old."):
return url
if host in {"www.reddit.com", "reddit.com"}:
return urlunparse((parsed.scheme, "old.reddit.com", parsed.path, "", parsed.query, ""))
return url
def _request_headers(base: dict[str, str], url: str) -> dict[str, str]:
if not _is_reddit_url(url):
return base
return {
**base,
"User-Agent": BROWSER_USER_AGENT,
"Accept-Language": "en-US,en;q=0.9",
}
def _reddit_fetch_urls(feed_url: str) -> list[str]:
primary = _with_query_params(feed_url, _reddit_auth_params())
if not _is_reddit_url(primary):
return [primary]
fallback = _reddit_old_url(primary)
if fallback == primary:
return [primary]
return [primary, fallback]
def _fetch_one(client: httpx.Client, category: NewsCategory, feed_url: str, feed_name: str) -> list[dict[str, Any]]:
last_exc: Exception | None = None
for url in _reddit_fetch_urls(feed_url):
try:
headers = _request_headers(dict(client.headers), url)
resp = client.get(url, headers=headers)
resp.raise_for_status()
return _parse_feed(resp.text, feed_name, category)
except Exception as exc:
last_exc = exc
continue
logger.warning("RSS fetch failed [%s] %s: %s", feed_name, feed_url, last_exc)
return []
def _dedupe_items(items: list[dict[str, Any]]) -> list[dict[str, Any]]:
seen_links: set[str] = set()
seen_titles: set[str] = set()
result: list[dict[str, Any]] = []
for item in items:
link_key = _normalize_link(item["link"])
title_key = _normalize_title(item["title"])
if link_key in seen_links or title_key in seen_titles:
continue
seen_links.add(link_key)
seen_titles.add(title_key)
result.append(item)
return result
def _within_window(item: dict[str, Any], cutoff: datetime) -> bool:
dt = _entry_datetime(item)
if dt is None:
return True
return dt >= cutoff
def _sort_key(item: dict[str, Any]) -> tuple[int, datetime]:
dt = _entry_datetime(item)
if dt is None:
return (1, datetime.min.replace(tzinfo=timezone.utc))
return (0, dt)
def fetch_ai_news() -> dict[str, Any]:
"""按类别抓取 AI 时讯,返回 {enabled, hours, categories, flat, stats}。"""
if not _enabled():
return {"enabled": False, "categories": [], "flat": [], "stats": {}}
hours = _hours_window()
per_feed = _per_feed_limit()
per_category = _per_category_limit()
cutoff = _now_utc() - timedelta(hours=hours)
headers = {"User-Agent": USER_AGENT, "Accept": "application/rss+xml, application/atom+xml, application/xml, text/xml, */*"}
tasks: list[tuple[NewsCategory, str, str, bool]] = []
for category in NEWS_CATEGORIES:
for feed in category.feeds:
tasks.append((category, feed.url, feed.name, feed.slow))
raw_by_category: dict[str, list[dict[str, Any]]] = {c.id: [] for c in NEWS_CATEGORIES}
stats = {"feeds_total": len(tasks), "feeds_ok": 0, "items_raw": 0}
with httpx.Client(timeout=15.0, verify=certifi.where(), follow_redirects=True, headers=headers) as client:
fast_tasks = [t for t in tasks if not t[3]]
slow_tasks = [t for t in tasks if t[3]]
with ThreadPoolExecutor(max_workers=8) as pool:
futures = {
pool.submit(_fetch_one, client, cat, url, name): (cat.id, name)
for cat, url, name, _slow in fast_tasks
}
for future in as_completed(futures):
cat_id, feed_name = futures[future]
try:
entries = future.result()
except Exception as exc:
logger.warning("RSS 任务异常 [%s]: %s", feed_name, exc)
continue
if entries:
stats["feeds_ok"] += 1
stats["items_raw"] += len(entries)
raw_by_category[cat_id].extend(entries[:per_feed])
for cat, url, name, _slow in slow_tasks:
entries = _fetch_one(client, cat, url, name)
if entries:
stats["feeds_ok"] += 1
stats["items_raw"] += len(entries)
raw_by_category[cat.id].extend(entries[:per_feed])
if _is_reddit_url(url):
time.sleep(2.0)
else:
time.sleep(1.0)
categories_out: list[dict[str, Any]] = []
flat: list[dict[str, Any]] = []
for category in NEWS_CATEGORIES:
items = raw_by_category[category.id]
items = [i for i in items if _within_window(i, cutoff)]
items.sort(key=_sort_key, reverse=True)
items = _dedupe_items(items)[:per_category]
for item in items:
dt = _entry_datetime(item)
item["published_fmt"] = dt.astimezone(timezone(timedelta(hours=8))).strftime("%m-%d %H:%M") if dt else ""
if items:
categories_out.append(
{
"id": category.id,
"name": category.name,
"icon": category.icon,
"items": items,
}
)
flat.extend(items)
flat.sort(key=_sort_key, reverse=True)
flat = _dedupe_items(flat)
return {
"enabled": True,
"hours": hours,
"categories": categories_out,
"flat": flat,
"stats": stats,
}
def format_news_section(news: dict[str, Any], *, section_no: int, wecom_limit: int | None = None) -> list[str]:
if not news.get("enabled"):
return ["---", "", f"## {section_no}、国际 AI 时讯", "", "*AI 时讯已关闭(`DAILY_AI_NEWS=0`)。*", ""]
categories = news.get("categories") or []
hours = news.get("hours", 72)
lines = [
"---",
"",
f"## {section_no}、国际 AI 时讯",
"",
f"> 近 **{hours}h** · {news.get('stats', {}).get('feeds_ok', 0)}/{news.get('stats', {}).get('feeds_total', 0)} 源可用",
"",
]
if not categories:
lines.append("*暂无可用条目(网络/RSS 源异常或时间窗口内无更新)。*")
lines.append("")
return lines
if wecom_limit is not None:
flat = (news.get("flat") or [])[:wecom_limit]
for i, item in enumerate(flat, 1):
pub = f" · {item['published_fmt']}" if item.get("published_fmt") else ""
lines.append(
f"{i}. [{item['title']}]({item['link']}) · `{item['source_name']}`{pub}"
)
lines.append("")
return lines
for cat in categories:
lines.append(f"### {cat['icon']} {cat['name']}")
lines.append("")
for i, item in enumerate(cat["items"], 1):
pub = f" · {item['published_fmt']}" if item.get("published_fmt") else ""
lines.append(f"{i}. **[{item['title']}]({item['link']})** · `{item['source_name']}`{pub}")
summary = item.get("summary", "")
if summary:
lines.append(f" - {_clean_text(summary, news_summary_limit())}")
lines.append("")
return lines
def prepare_wecom_news_items(news: dict[str, Any]) -> list[dict[str, Any]]:
if not news.get("enabled"):
return []
limit = _wecom_limit()
flat = _dedupe_items(news.get("flat") or [])
flat.sort(key=_sort_key, reverse=True)
preferred = ("media", "newsletter", "official", "community", "research", "developer")
picked: list[dict[str, Any]] = []
seen: set[str] = set()
for cat in preferred:
for item in flat:
link = _normalize_link(item.get("link", ""))
if item.get("category_id") != cat or link in seen:
continue
picked.append(item)
seen.add(link)
if len(picked) >= limit:
break
if len(picked) >= limit:
break
items: list[dict[str, Any]] = []
for item in picked[:limit]:
items.append(
{
"title": item.get("title", "?"),
"link": item.get("link", ""),
"source_name": item.get("source_name", "?"),
"published_fmt": item.get("published_fmt", ""),
"desc_short": _clean_text(item.get("summary", ""), 36),
}
)
return items

183
daily/report_data.py Normal file
View File

@@ -0,0 +1,183 @@
"""早报结构化数据:抓取结果 → JSON 中间层。"""
from __future__ import annotations
import json
from pathlib import Path
from typing import Any
from daily.config import OUTPUT_DIR, env_int
from daily.delta import build_movement_baseline, build_movement_context, compare_depth
from daily.news.fetch import prepare_wecom_news_items
from daily.skills_group import group_skills_by_source
def skill_id(item: dict[str, Any]) -> str:
return str(item.get("id") or f"{item.get('source')}/{item.get('title')}")
def _slim_skill(item: dict[str, Any]) -> dict[str, Any]:
payload = {
"id": skill_id(item),
"title": item.get("title", ""),
"source": item.get("source", ""),
"installs": item.get("installs", 0),
"link": item.get("link", ""),
"description": item.get("description", ""),
}
if item.get("cluster"):
payload.update(
{
"cluster": True,
"cluster_count": item.get("cluster_count", 1),
"cluster_skills": item.get("cluster_skills", []),
"cluster_titles": item.get("cluster_titles", ""),
"installs_fmt": item.get("installs_fmt", ""),
"installs_min": item.get("installs_min"),
"installs_max": item.get("installs_max"),
}
)
elif item.get("installs_fmt"):
payload["installs_fmt"] = item.get("installs_fmt")
return payload
def _slim_github(item: dict[str, Any]) -> dict[str, Any]:
return {
"repo": item.get("repo", ""),
"url": item.get("url", ""),
"language": item.get("language", ""),
"stars_today_fmt": item.get("stars_today_fmt", ""),
"total_stars_fmt": item.get("total_stars_fmt", ""),
"created_at": item.get("created_at", ""),
"description": item.get("description", ""),
}
def _slim_news_items(ai_news: dict[str, Any], limit: int) -> list[dict[str, Any]]:
items: list[dict[str, Any]] = []
for item in prepare_wecom_news_items(ai_news):
items.append(
{
"link": item.get("link", ""),
"title": item.get("title", ""),
"source_name": item.get("source_name", ""),
"published_fmt": item.get("published_fmt", ""),
"summary": item.get("desc_short") or "",
}
)
if len(items) >= limit:
break
if items:
return items
for item in (ai_news.get("flat") or [])[:limit]:
items.append(
{
"link": item.get("link", ""),
"title": item.get("title", ""),
"source_name": item.get("source_name", ""),
"published_fmt": item.get("published_fmt", ""),
"summary": item.get("summary", ""),
}
)
return items
def _wecom_skill_pool() -> int:
return max(10, env_int("DAILY_WECOM_SKILL_POOL", 200))
def build_llm_input(
*,
date_str: str,
updated: str,
trending: list[dict[str, Any]],
hot: list[dict[str, Any]],
github_trending: list[dict[str, Any]],
github_emerging: list[dict[str, Any]],
github_topic: list[dict[str, Any]],
topic_name: str,
ai_news: dict[str, Any],
wecom_limits: dict[str, int],
) -> dict[str, Any]:
"""供 Cursor 编辑的精简 JSON不含完整 markdown"""
news_limit = wecom_limits.get("ai_news", 10)
depth = compare_depth()
trend_cmp = trending[:depth]
hot_cmp = hot[:depth]
github_cmp = github_trending[:depth]
emerging_cmp = github_emerging[:depth]
topic_cmp = github_topic[:depth]
trending_slice = group_skills_by_source(
trending,
limit=wecom_limits.get("trending", 10),
pool_size=wecom_limits.get("trending_pool", _wecom_skill_pool()),
)
hot_slice = group_skills_by_source(
hot,
limit=wecom_limits.get("hot", 10),
pool_size=wecom_limits.get("hot_pool", _wecom_skill_pool()),
)
github_slice = github_trending[: wecom_limits.get("github", 5)]
emerging_slice = github_emerging[: wecom_limits.get("emerging", 3)]
topic_slice = github_topic[: wecom_limits.get("topic", 3)]
movement = build_movement_context(
date_str=date_str,
trending=[_slim_skill(x) for x in trend_cmp],
hot=[_slim_skill(x) for x in hot_cmp],
github_trending=[_slim_github(x) for x in github_cmp],
github_emerging=[_slim_github(x) for x in emerging_cmp],
github_topic=[_slim_github(x) for x in topic_cmp],
topic_name=topic_name,
)
movement_baseline = build_movement_baseline(
trending=[_slim_skill(x) for x in trend_cmp],
hot=[_slim_skill(x) for x in hot_cmp],
github_trending=[_slim_github(x) for x in github_cmp],
github_emerging=[_slim_github(x) for x in emerging_cmp],
github_topic=[_slim_github(x) for x in topic_cmp],
depth=depth,
)
return {
"date": date_str,
"data_updated": updated,
"skills_trending": [_slim_skill(x) for x in trending_slice],
"skills_hot": [_slim_skill(x) for x in hot_slice],
"github_trending": [_slim_github(x) for x in github_slice],
"github_emerging": [_slim_github(x) for x in emerging_slice],
"github_topic": {
"topic": topic_name,
"repos": [_slim_github(x) for x in topic_slice],
},
"ai_news": _slim_news_items(ai_news, news_limit) if ai_news.get("enabled") else [],
"movement": movement,
"movement_baseline": movement_baseline,
}
def build_full_payload(
llm_input: dict[str, Any],
*,
meta: dict[str, Any],
) -> dict[str, Any]:
return {"meta": meta, "data": llm_input}
def data_json_path(date_str: str) -> Path:
return OUTPUT_DIR / f"{date_str}.data.json"
def editorial_json_path(date_str: str) -> Path:
return OUTPUT_DIR / f"{date_str}.editorial.json"
def save_json(path: Path, data: dict[str, Any]) -> None:
path.parent.mkdir(parents=True, exist_ok=True)
path.write_text(json.dumps(data, ensure_ascii=False, indent=2), encoding="utf-8")
def load_json(path: Path) -> dict[str, Any]:
return json.loads(path.read_text(encoding="utf-8"))

120
daily/skills_board.py Normal file
View File

@@ -0,0 +1,120 @@
"""从 skills.sh 官网抓取 Trending / Hot 完整榜单(突破 feed.json 50 条限制)。"""
from __future__ import annotations
import logging
import re
from typing import Any, Literal
import certifi
import httpx
from daily.config import env
logger = logging.getLogger(__name__)
Board = Literal["trending", "hot"]
SKILLS_SITE = "https://www.skills.sh"
USER_AGENT = "Mozilla/5.0 (compatible; skills-hot-daily/1.0; +https://skills.sh)"
_SKILL_RE = re.compile(
r'\{"source":"(?P<source>[^"]+)","skillId":"(?P<skill_id>[^"]+)",'
r'"name":"(?P<name>[^"]+)","installs":(?P<installs>\d+)'
)
_RSC_CHUNK_RE = re.compile(r"self\.__next_f\.push\(\[1,\"(.*?)\"\]\)", re.DOTALL)
def board_source() -> str:
return (env("SKILLS_BOARD_SOURCE") or "website").strip().lower()
def _fetch_html(path: str) -> str:
url = f"{SKILLS_SITE}{path}"
headers = {"User-Agent": USER_AGENT, "Accept": "text/html"}
with httpx.Client(timeout=30.0, verify=certifi.where(), follow_redirects=True) as client:
resp = client.get(url, headers=headers)
resp.raise_for_status()
return resp.text
def _rsc_blob(html: str) -> str:
chunks = _RSC_CHUNK_RE.findall(html)
blob = "\n".join(chunks)
return blob.encode("utf-8").decode("unicode_escape", errors="ignore")
def _parse_initial_skills(blob: str, *, limit: int) -> list[dict[str, Any]]:
items: list[dict[str, Any]] = []
seen: set[str] = set()
for match in _SKILL_RE.finditer(blob):
source = match.group("source")
skill_id = match.group("skill_id")
uid = f"{source}/{skill_id}"
if uid in seen:
continue
seen.add(uid)
items.append(
{
"id": uid,
"title": skill_id,
"source": source,
"installs": int(match.group("installs")),
"link": f"{SKILLS_SITE}/{source}/{skill_id}",
"description": "",
}
)
if len(items) >= limit:
break
return items
def fetch_board(board: Board, *, limit: int) -> list[dict[str, Any]]:
path = "/trending" if board == "trending" else "/hot"
try:
html = _fetch_html(path)
items = _parse_initial_skills(_rsc_blob(html), limit=limit)
if items:
logger.info("skills.sh %s: %d items (limit=%d)", board, len(items), limit)
return items
except Exception as exc:
logger.warning("skills.sh %s fetch failed, fallback to feed.json: %s", board, exc)
return []
def enrich_from_feed(items: list[dict[str, Any]], feed: dict[str, Any]) -> None:
desc_by_id: dict[str, str] = {}
for key in ("topTrending", "topHot", "topAllTime"):
for row in feed.get(key, []):
uid = str(row.get("id") or f"{row.get('source')}/{row.get('title')}")
desc = (row.get("description") or "").strip()
if desc:
desc_by_id[uid] = desc
for item in items:
if not item.get("description"):
item["description"] = desc_by_id.get(item["id"], "")
def load_boards(
feed: dict[str, Any],
*,
trending_limit: int,
hot_limit: int,
) -> tuple[list[dict[str, Any]], list[dict[str, Any]]]:
"""优先 skills.sh 官网;失败时回退 feed.json。"""
if board_source() == "feed":
return (
list(feed.get("topTrending", [])[:trending_limit]),
list(feed.get("topHot", [])[:hot_limit]),
)
trending = fetch_board("trending", limit=trending_limit)
hot = fetch_board("hot", limit=hot_limit)
if not trending:
trending = list(feed.get("topTrending", [])[:trending_limit])
else:
enrich_from_feed(trending, feed)
if not hot:
hot = list(feed.get("topHot", [])[:hot_limit])
else:
enrich_from_feed(hot, feed)
return trending, hot

103
daily/skills_group.py Normal file
View File

@@ -0,0 +1,103 @@
"""Skills 榜单:同 source 合并为一条(企微 Top N"""
from __future__ import annotations
from typing import Any
def skill_id(item: dict[str, Any]) -> str:
return str(item.get("id") or f"{item.get('source')}/{item.get('title')}")
def format_installs(n: int | float) -> str:
if n >= 1_000_000:
return f"{n / 1_000_000:.1f}M"
if n >= 1_000:
return f"{n / 1_000:.1f}K"
return str(int(n))
def _installs_range(items: list[dict[str, Any]]) -> tuple[int, int, str]:
values = [int(item.get("installs") or 0) for item in items]
lo, hi = min(values), max(values)
if lo == hi:
return lo, hi, format_installs(hi)
return lo, hi, f"{format_installs(lo)}{format_installs(hi)}"
def _best_description(items: list[dict[str, Any]]) -> str:
for item in items:
desc = (item.get("description") or "").strip()
if desc:
return desc
return ""
def _cluster_skill(items: list[dict[str, Any]]) -> dict[str, Any]:
ranked = sorted(items, key=lambda x: int(x.get("installs") or 0), reverse=True)
top = ranked[0]
_lo, _hi, installs_fmt = _installs_range(ranked)
titles = [str(x.get("title") or "") for x in ranked if x.get("title")]
sample = ", ".join(titles[:4])
if len(titles) > 4:
sample = f"{sample}"
return {
"id": skill_id(top),
"title": top.get("title", ""),
"source": top.get("source", ""),
"installs": int(top.get("installs") or 0),
"installs_min": _lo,
"installs_max": _hi,
"installs_fmt": installs_fmt,
"link": top.get("link", ""),
"description": _best_description(ranked),
"cluster": True,
"cluster_count": len(ranked),
"cluster_skills": titles,
"cluster_titles": sample,
}
def _single_skill(item: dict[str, Any]) -> dict[str, Any]:
installs = int(item.get("installs") or 0)
return {
"id": skill_id(item),
"title": item.get("title", ""),
"source": item.get("source", ""),
"installs": installs,
"installs_fmt": format_installs(installs),
"link": item.get("link", ""),
"description": item.get("description", ""),
"cluster": False,
}
def group_skills_by_source(
items: list[dict[str, Any]],
*,
limit: int = 10,
pool_size: int = 50,
) -> list[dict[str, Any]]:
"""按 source 去重合并;保留各 source 在榜内的最佳名次顺序。"""
if not items or limit <= 0:
return []
pool = items[: max(pool_size, limit)]
by_source: dict[str, list[dict[str, Any]]] = {}
first_rank: dict[str, int] = {}
for rank, item in enumerate(pool, 1):
source = (item.get("source") or "?").strip() or "?"
by_source.setdefault(source, []).append(item)
first_rank.setdefault(source, rank)
ordered_sources = sorted(by_source.keys(), key=lambda s: first_rank[s])
result: list[dict[str, Any]] = []
for source in ordered_sources:
group = by_source[source]
if len(group) == 1:
result.append(_single_skill(group[0]))
else:
result.append(_cluster_skill(group))
if len(result) >= limit:
break
return result

30
daily/text_utils.py Normal file
View File

@@ -0,0 +1,30 @@
"""文本裁剪等轻量工具。"""
from __future__ import annotations
import re
_WS = re.compile(r"\s+")
def clip_text(text: str, limit: int) -> str:
text = _WS.sub(" ", (text or "").strip())
if limit <= 0 or len(text) <= limit:
return text
return text[: max(1, limit - 1)] + ""
def trim_brief(text: str, limit: int) -> str:
"""企微简要:控制在 limit 内,优先在句读处截断,不加省略号。"""
text = _WS.sub(" ", (text or "").strip())
if not text or limit <= 0 or len(text) <= limit:
return text
for sep in ("", "", "", ""):
pos = text.find(sep)
if pos != -1 and pos + 1 <= limit + 8:
return text[: pos + 1]
for sep in ("", ""):
pos = text.find(sep)
if pos != -1 and pos + 1 <= limit:
return text[: pos + 1]
return text[:limit].rstrip(",、;: ")

96
daily/webhook.py Normal file
View File

@@ -0,0 +1,96 @@
"""推送早报至企业微信群 webhook超长自动分多条"""
from __future__ import annotations
import sys
import time
from pathlib import Path
import certifi
import httpx
from daily.config import OUTPUT_DIR, ROOT, env_int, wecom_chunk_bytes
from daily.wecom_split import split_wecom_messages
_PUSH_GAP_MS = 300
def _load_webhook_key() -> str:
from daily.config import env
key = (env("WECOM_WEBHOOK_KEY") or "").strip()
if not key:
raise RuntimeError("请设置 WECOM_WEBHOOK_KEY项目根 .env")
return key
def _resolve_report_path(arg: str | None) -> Path:
if arg:
path = Path(arg)
if not path.is_absolute():
path = ROOT / path
return path
candidates = sorted(
OUTPUT_DIR.glob("*.wecom.md"),
key=lambda p: p.stat().st_mtime,
reverse=True,
)
if candidates:
return candidates[0]
legacy = sorted(ROOT.glob("*.wecom.md"), key=lambda p: p.stat().st_mtime, reverse=True)
if legacy:
return legacy[0]
raise RuntimeError("未找到 .wecom.md 报告,请先运行 python -m daily")
def _post_markdown(client: httpx.Client, url: str, content: str) -> None:
payload = {"msgtype": "markdown", "markdown": {"content": content}}
resp = client.post(url, json=payload)
resp.raise_for_status()
data = resp.json()
if data.get("errcode", 0) != 0:
raise RuntimeError(f"推送失败: errcode={data.get('errcode')} errmsg={data.get('errmsg')}")
def send_report(report_path: Path | None = None) -> None:
path = _resolve_report_path(str(report_path) if report_path else None)
if not path.exists():
raise RuntimeError(f"报告文件不存在: {path}")
content = path.read_text(encoding="utf-8").strip()
if not content:
raise RuntimeError(f"报告内容为空: {path}")
chunk_limit = wecom_chunk_bytes()
max_parts = env_int("DAILY_WECOM_MAX_PARTS", 5)
parts = split_wecom_messages(content, chunk_limit)
if len(parts) > max_parts:
raise RuntimeError(
f"早报需 {len(parts)} 条消息,超过 DAILY_WECOM_MAX_PARTS={max_parts},请调小各区块条数"
)
key = _load_webhook_key()
url = f"https://qyapi.weixin.qq.com/cgi-bin/webhook/send?key={key}"
with httpx.Client(timeout=20.0, verify=certifi.where()) as client:
for i, part in enumerate(parts, 1):
if i > 1:
time.sleep(_PUSH_GAP_MS / 1000.0)
_post_markdown(client, url, part)
total_bytes = len(content.encode("utf-8"))
if len(parts) == 1:
print(f"已推送至企业微信: {path.name} ({total_bytes} bytes)")
else:
sizes = ", ".join(str(len(p.encode("utf-8"))) for p in parts)
print(f"已推送至企业微信: {path.name} ({total_bytes} bytes → {len(parts)} 条: {sizes})")
def main(argv: list[str] | None = None) -> int:
args = argv if argv is not None else sys.argv[1:]
try:
send_report(Path(args[0]) if args else None)
return 0
except Exception as exc:
print(f"ERROR: {exc}", file=sys.stderr)
return 1

94
daily/wecom_split.py Normal file
View File

@@ -0,0 +1,94 @@
"""企微 markdown 按字节上限拆分为多条消息(按区块,不截断正文)。"""
from __future__ import annotations
import re
_SECTION_START = re.compile(r"^[📰💡🎯🌍📈🔥🐙🌱🤖📦📄]")
def _utf8_len(text: str) -> int:
return len(text.encode("utf-8"))
def _split_lines_by_budget(text: str, limit: int) -> list[str]:
lines = text.splitlines()
chunks: list[str] = []
buf: list[str] = []
for line in lines:
candidate = "\n".join(buf + [line]) if buf else line
if _utf8_len(candidate) <= limit:
buf.append(line)
continue
if buf:
chunks.append("\n".join(buf))
buf = []
if _utf8_len(line) <= limit:
buf = [line]
else:
encoded = line.encode("utf-8")
start = 0
while start < len(encoded):
piece = encoded[start : start + limit].decode("utf-8", errors="ignore")
chunks.append(piece)
start += len(piece.encode("utf-8"))
if buf:
chunks.append("\n".join(buf))
return chunks
def _split_sections(text: str) -> list[str]:
sections: list[str] = []
current: list[str] = []
for line in text.splitlines():
if _SECTION_START.match(line) and current:
sections.append("\n".join(current))
current = [line]
else:
current.append(line)
if current:
sections.append("\n".join(current))
return sections
def split_wecom_messages(text: str, limit: int = 4096) -> list[str]:
"""超长时拆成多条;每条不超过 limit 字节,按区块边界优先。"""
text = text.strip()
if not text or _utf8_len(text) <= limit:
return [text] if text else []
footer_reserve = 40
pack_limit = max(512, limit - footer_reserve)
sections: list[str] = []
for sec in _split_sections(text):
if _utf8_len(sec) <= pack_limit:
sections.append(sec)
else:
sections.extend(_split_lines_by_budget(sec, pack_limit))
packed: list[str] = []
buf: list[str] = []
for sec in sections:
candidate = "\n\n".join(buf + [sec]) if buf else sec
if _utf8_len(candidate) <= pack_limit:
buf.append(sec)
else:
if buf:
packed.append("\n\n".join(buf))
buf = [sec]
if buf:
packed.append("\n\n".join(buf))
total = len(packed)
if total <= 1:
return packed
result: list[str] = []
for i, chunk in enumerate(packed, 1):
suffix = f"\n\n> 📄 {i}/{total}"
body = chunk
while body and _utf8_len(body + suffix) > limit:
body = body.rsplit("\n", 1)[0] if "\n" in body else body[:-1]
result.append(body + suffix)
return result