项目初始化
This commit is contained in:
6
daily/__init__.py
Normal file
6
daily/__init__.py
Normal file
@@ -0,0 +1,6 @@
|
||||
"""早报生成与企微推送。"""
|
||||
|
||||
from daily.generate import generate_report, main as generate_main
|
||||
from daily.webhook import send_report
|
||||
|
||||
__all__ = ["generate_report", "generate_main", "send_report"]
|
||||
22
daily/__main__.py
Normal file
22
daily/__main__.py
Normal file
@@ -0,0 +1,22 @@
|
||||
"""CLI: python -m daily [generate|push] [report_path]"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import sys
|
||||
|
||||
from daily.generate import main as generate_main
|
||||
from daily.webhook import main as push_main
|
||||
|
||||
|
||||
def main() -> int:
|
||||
cmd = (sys.argv[1] if len(sys.argv) > 1 else "generate").lower()
|
||||
if cmd in {"generate", "gen", "g"}:
|
||||
return generate_main()
|
||||
if cmd in {"push", "send", "webhook"}:
|
||||
return push_main(sys.argv[2:])
|
||||
print(f"未知命令: {cmd}\n用法: python -m daily [generate|push] [report_path]", file=sys.stderr)
|
||||
return 1
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
134
daily/agent_workflow.py
Normal file
134
daily/agent_workflow.py
Normal file
@@ -0,0 +1,134 @@
|
||||
"""Agent 工作流:趋势分析 → 叙事化企微早报。"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import logging
|
||||
import re
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
from daily.config import OUTPUT_DIR, ROOT, env
|
||||
from daily.llm_client import extract_json_object, has_llm_configured, llm_chat
|
||||
from daily.report_data import save_json
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
_SKILL_DIR = ROOT / "skills" / "daily-agent"
|
||||
_MD_BLOCK = re.compile(r"```(?:markdown|md)?\s*([\s\S]*?)```", re.IGNORECASE)
|
||||
_WECOM_NEW_ENTRY_NOTE = re.compile(r"(新入[^)]*)")
|
||||
|
||||
|
||||
def _strip_new_entry_notes(md: str) -> str:
|
||||
md = _WECOM_NEW_ENTRY_NOTE.sub("", md)
|
||||
return re.sub(r"\*\*—", "** —", md)
|
||||
|
||||
|
||||
def report_mode() -> str:
|
||||
return (env("DAILY_REPORT_MODE") or "classic").strip().lower()
|
||||
|
||||
|
||||
def is_agent_mode() -> bool:
|
||||
if report_mode() != "agent":
|
||||
return False
|
||||
if not has_llm_configured():
|
||||
logger.warning("DAILY_REPORT_MODE=agent 但未配置 LLM,回退 classic")
|
||||
return False
|
||||
return True
|
||||
|
||||
|
||||
def trends_json_path(date_str: str) -> Path:
|
||||
return OUTPUT_DIR / f"{date_str}.trends.json"
|
||||
|
||||
|
||||
def _load_skill() -> str:
|
||||
path = _SKILL_DIR / "SKILL.md"
|
||||
if path.exists():
|
||||
return path.read_text(encoding="utf-8").strip()
|
||||
return "你是早报主编 Agent。"
|
||||
|
||||
|
||||
def _extract_markdown(text: str) -> str:
|
||||
text = text.strip()
|
||||
match = _MD_BLOCK.search(text)
|
||||
if match:
|
||||
return match.group(1).strip()
|
||||
if text.startswith("📰"):
|
||||
return text
|
||||
return text
|
||||
|
||||
|
||||
def analyze_trends(llm_input: dict[str, Any], *, date_str: str) -> dict[str, Any] | None:
|
||||
skill = _load_skill()
|
||||
system = (
|
||||
f"{skill}\n\n"
|
||||
"当前执行 **Step 1:趋势分析**。\n"
|
||||
"只输出 trends JSON(headline, opening, themes, top_picks, signals),不要 Markdown。"
|
||||
)
|
||||
user = json.dumps(llm_input, ensure_ascii=False, indent=2)
|
||||
try:
|
||||
raw = llm_chat(system, user)
|
||||
except Exception as exc:
|
||||
logger.warning("Agent Step1 趋势分析失败:%s", exc)
|
||||
return None
|
||||
if not raw:
|
||||
return None
|
||||
parsed = extract_json_object(raw)
|
||||
if not parsed.get("headline") and not parsed.get("opening"):
|
||||
logger.warning("Agent Step1 JSON 无效")
|
||||
return None
|
||||
save_json(trends_json_path(date_str), parsed)
|
||||
logger.info("Agent Step1 完成:%s", parsed.get("headline", "?"))
|
||||
return parsed
|
||||
|
||||
|
||||
def write_wecom_report(
|
||||
llm_input: dict[str, Any],
|
||||
trends: dict[str, Any],
|
||||
*,
|
||||
date_str: str,
|
||||
time_str: str,
|
||||
updated: str,
|
||||
) -> str | None:
|
||||
skill = _load_skill()
|
||||
system = (
|
||||
f"{skill}\n\n"
|
||||
"当前执行 **Step 2:撰写企微早报**。\n"
|
||||
f"日期={date_str},时间={time_str},数据截至={updated}。\n"
|
||||
"只输出企微 Markdown 正文,不要代码块,不要 JSON。"
|
||||
)
|
||||
payload = {"data": llm_input, "trends": trends}
|
||||
user = json.dumps(payload, ensure_ascii=False, indent=2)
|
||||
try:
|
||||
raw = llm_chat(system, user)
|
||||
except Exception as exc:
|
||||
logger.warning("Agent Step2 写稿失败:%s", exc)
|
||||
return None
|
||||
if not raw:
|
||||
return None
|
||||
md = _extract_markdown(raw)
|
||||
if not md.startswith("📰"):
|
||||
md = f"📰 **早报 · {date_str}**\n> ⏱ {time_str} · 数据截至 {updated}\n\n{md}"
|
||||
md = _strip_new_entry_notes(md)
|
||||
logger.info("Agent Step2 完成:%d bytes", len(md.encode("utf-8")))
|
||||
return md
|
||||
|
||||
|
||||
def run_agent_workflow(
|
||||
llm_input: dict[str, Any],
|
||||
*,
|
||||
date_str: str,
|
||||
time_str: str,
|
||||
updated: str,
|
||||
) -> str | None:
|
||||
"""两步 Agent 工作流;成功返回企微 Markdown,失败返回 None。"""
|
||||
trends = analyze_trends(llm_input, date_str=date_str)
|
||||
if not trends:
|
||||
return None
|
||||
return write_wecom_report(
|
||||
llm_input,
|
||||
trends,
|
||||
date_str=date_str,
|
||||
time_str=time_str,
|
||||
updated=updated,
|
||||
)
|
||||
81
daily/config.py
Normal file
81
daily/config.py
Normal file
@@ -0,0 +1,81 @@
|
||||
"""项目路径与环境变量加载。"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import os
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
from dotenv import load_dotenv
|
||||
|
||||
ROOT = Path(__file__).resolve().parent.parent
|
||||
BOT_DIR = ROOT / "bot"
|
||||
OUTPUT_DIR = ROOT / "output"
|
||||
LOG_DIR = ROOT / "logs"
|
||||
CACHE_DIR = ROOT / ".cache"
|
||||
SNAPSHOT_FILE = CACHE_DIR / "last-report.json"
|
||||
|
||||
# 企微 webhook markdown.content 硬上限
|
||||
WECOM_MARKDOWN_LIMIT = 4096
|
||||
|
||||
|
||||
def wecom_chunk_bytes() -> int:
|
||||
"""单条企微 markdown 消息字节上限。"""
|
||||
chunk = env_int("DAILY_WECOM_CHUNK_BYTES", -1)
|
||||
if chunk < 0:
|
||||
chunk = env_int("DAILY_WECOM_MAX_BYTES", WECOM_MARKDOWN_LIMIT)
|
||||
return min(chunk, WECOM_MARKDOWN_LIMIT)
|
||||
|
||||
|
||||
def wecom_skill_desc_limit() -> int:
|
||||
"""企微 Skills 榜单条简介建议字数。"""
|
||||
return env_int("DAILY_WECOM_SKILL_DESC_LIMIT", 56)
|
||||
|
||||
|
||||
def full_desc_limit() -> int:
|
||||
"""完整版早报摘要长度;0 表示不截断。"""
|
||||
return env_int("DAILY_FULL_DESC_LIMIT", 0)
|
||||
|
||||
|
||||
def news_summary_limit() -> int:
|
||||
return env_int("DAILY_FULL_NEWS_SUMMARY_LIMIT", 0)
|
||||
|
||||
|
||||
def wecom_max_bytes() -> int:
|
||||
"""兼容旧配置名。"""
|
||||
return wecom_chunk_bytes()
|
||||
|
||||
load_dotenv(ROOT / ".env")
|
||||
load_dotenv(ROOT / ".env.local", override=True)
|
||||
|
||||
|
||||
def ensure_bot_on_path() -> None:
|
||||
bot = str(BOT_DIR)
|
||||
if bot not in sys.path:
|
||||
sys.path.insert(0, bot)
|
||||
|
||||
|
||||
def _clean_env_value(raw: str | None) -> str | None:
|
||||
if raw is None:
|
||||
return None
|
||||
value = raw.strip()
|
||||
if not value:
|
||||
return None
|
||||
# .env 行内注释(未加引号时 python-dotenv 不会自动去掉)
|
||||
if " #" in value:
|
||||
value = value.split(" #", 1)[0].rstrip()
|
||||
return value or None
|
||||
|
||||
|
||||
def env(key: str, default: str | None = None) -> str | None:
|
||||
return _clean_env_value(os.getenv(key, default))
|
||||
|
||||
|
||||
def env_int(key: str, default: int) -> int:
|
||||
raw = env(key)
|
||||
if not raw:
|
||||
return default
|
||||
try:
|
||||
return int(raw)
|
||||
except ValueError:
|
||||
return default
|
||||
144
daily/cursor_editor.py
Normal file
144
daily/cursor_editor.py
Normal file
@@ -0,0 +1,144 @@
|
||||
"""Cursor 编辑层:JSON 数据 → 主题 / 速览 / 中文描述。"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import logging
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
from daily.config import ROOT, env
|
||||
from daily.llm_client import extract_json_object, has_llm_configured, llm_chat
|
||||
from daily.text_utils import clip_text
|
||||
from daily.report_data import editorial_json_path, save_json, skill_id
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
_SKILL_DIR = ROOT / "skills" / "daily-editor"
|
||||
|
||||
|
||||
def is_enabled() -> bool:
|
||||
raw = (env("DAILY_CURSOR_EDITOR") or "").strip().lower()
|
||||
if raw in {"1", "true", "yes", "on"}:
|
||||
return has_llm_configured()
|
||||
if raw in {"0", "false", "no", "off"}:
|
||||
return False
|
||||
return False
|
||||
|
||||
|
||||
def _load_skill_prompt() -> str:
|
||||
skill_path = _SKILL_DIR / "SKILL.md"
|
||||
if skill_path.exists():
|
||||
return skill_path.read_text(encoding="utf-8").strip()
|
||||
return "你是技术早报编辑。根据输入 JSON 输出编辑结果 JSON。"
|
||||
|
||||
|
||||
def _build_system_prompt() -> str:
|
||||
skill = _load_skill_prompt()
|
||||
return (
|
||||
f"{skill}\n\n"
|
||||
"再次强调:只输出 JSON 对象,包含 theme_line、highlights(3条)、descriptions。"
|
||||
)
|
||||
|
||||
|
||||
def run_editorial(llm_input: dict[str, Any], *, date_str: str) -> dict[str, Any] | None:
|
||||
"""调用 LLM 生成 editorial;失败返回 None。"""
|
||||
if not is_enabled():
|
||||
return None
|
||||
system = _build_system_prompt()
|
||||
user = json.dumps(llm_input, ensure_ascii=False, indent=2)
|
||||
try:
|
||||
raw = llm_chat(system, user)
|
||||
except Exception as exc:
|
||||
logger.warning("Cursor 编辑失败,回退规则模式:%s", exc)
|
||||
return None
|
||||
if not raw:
|
||||
logger.warning("Cursor 编辑无响应,回退规则模式")
|
||||
return None
|
||||
parsed = extract_json_object(raw)
|
||||
if not parsed.get("theme_line") and not parsed.get("descriptions"):
|
||||
logger.warning("Cursor 编辑 JSON 无效,回退规则模式")
|
||||
return None
|
||||
editorial = _normalize_editorial(parsed)
|
||||
save_json(editorial_json_path(date_str), editorial)
|
||||
return editorial
|
||||
|
||||
|
||||
def _normalize_editorial(raw: dict[str, Any]) -> dict[str, Any]:
|
||||
theme = str(raw.get("theme_line") or "").strip()
|
||||
highlights_raw = raw.get("highlights") or []
|
||||
highlights: list[str] = []
|
||||
if isinstance(highlights_raw, list):
|
||||
for item in highlights_raw:
|
||||
if isinstance(item, str) and item.strip():
|
||||
highlights.append(item.strip())
|
||||
descriptions_raw = raw.get("descriptions") or {}
|
||||
descriptions: dict[str, str] = {}
|
||||
if isinstance(descriptions_raw, dict):
|
||||
for key, value in descriptions_raw.items():
|
||||
if isinstance(value, str) and value.strip():
|
||||
limit = 40 if str(key).startswith("github:") else 36
|
||||
descriptions[str(key)] = clip_text(value, limit)
|
||||
return {
|
||||
"theme_line": theme,
|
||||
"highlights": highlights[:3],
|
||||
"descriptions": descriptions,
|
||||
}
|
||||
|
||||
|
||||
def theme_line_from_editorial(editorial: dict[str, Any]) -> str:
|
||||
theme = editorial.get("theme_line", "")
|
||||
if not theme:
|
||||
return ""
|
||||
if "今日主题" in theme:
|
||||
return theme if theme.startswith("**") else f"**{theme}**"
|
||||
return f"**今日主题**:{theme}"
|
||||
|
||||
|
||||
def apply_descriptions(
|
||||
*,
|
||||
trending: list[dict[str, Any]],
|
||||
hot: list[dict[str, Any]],
|
||||
github_trending: list[dict[str, Any]],
|
||||
github_emerging: list[dict[str, Any]],
|
||||
github_topic: list[dict[str, Any]],
|
||||
ai_news: dict[str, Any],
|
||||
descriptions: dict[str, str],
|
||||
) -> None:
|
||||
if not descriptions:
|
||||
return
|
||||
|
||||
for item in trending + hot:
|
||||
key = f"skill:{skill_id(item)}"
|
||||
if key in descriptions:
|
||||
item["description"] = descriptions[key]
|
||||
|
||||
for repo_list in (github_trending, github_emerging, github_topic):
|
||||
for item in repo_list:
|
||||
key = f"github:{item.get('repo', '')}"
|
||||
if key in descriptions:
|
||||
item["description"] = descriptions[key]
|
||||
|
||||
if not ai_news.get("enabled"):
|
||||
return
|
||||
|
||||
def _apply_news_item(item: dict[str, Any]) -> None:
|
||||
key = f"news:{item.get('link', '')}"
|
||||
if key in descriptions:
|
||||
item["summary"] = descriptions[key]
|
||||
|
||||
for cat in ai_news.get("categories") or []:
|
||||
for item in cat.get("items") or []:
|
||||
_apply_news_item(item)
|
||||
for item in ai_news.get("flat") or []:
|
||||
_apply_news_item(item)
|
||||
|
||||
|
||||
def load_cached_editorial(date_str: str) -> dict[str, Any] | None:
|
||||
path = editorial_json_path(date_str)
|
||||
if not path.exists():
|
||||
return None
|
||||
try:
|
||||
return _normalize_editorial(json.loads(path.read_text(encoding="utf-8")))
|
||||
except (OSError, json.JSONDecodeError):
|
||||
return None
|
||||
313
daily/delta.py
Normal file
313
daily/delta.py
Normal file
@@ -0,0 +1,313 @@
|
||||
"""榜单异动:对比昨日 Top N,仅识别「新入榜」条目(分榜、限条)。"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import logging
|
||||
from datetime import datetime, timedelta
|
||||
from pathlib import Path
|
||||
from typing import Any, Callable
|
||||
|
||||
from daily.config import OUTPUT_DIR, env_int
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
KeyFn = Callable[[dict[str, Any]], str]
|
||||
|
||||
|
||||
def compare_depth() -> int:
|
||||
return env_int("DAILY_DELTA_COMPARE_DEPTH", 15)
|
||||
|
||||
|
||||
def wecom_new_limit() -> int:
|
||||
return env_int("DAILY_WECOM_NEW_MAX", 10)
|
||||
|
||||
|
||||
def skill_id(item: dict[str, Any]) -> str:
|
||||
return str(item.get("id") or f"{item.get('source')}/{item.get('title')}")
|
||||
|
||||
|
||||
def _data_json_path(date_str: str) -> Path:
|
||||
return OUTPUT_DIR / f"{date_str}.data.json"
|
||||
|
||||
|
||||
def _load_data_json_file(path: Path) -> dict[str, Any]:
|
||||
return json.loads(path.read_text(encoding="utf-8"))
|
||||
|
||||
|
||||
def _key_set(items: list[dict[str, Any]], key_fn: KeyFn, *, depth: int) -> set[str]:
|
||||
return {key_fn(item) for item in items[:depth] if key_fn(item)}
|
||||
|
||||
|
||||
def find_previous_data(date_str: str) -> tuple[str, dict[str, Any]] | None:
|
||||
"""查找最近一份早于 date_str 的 data.json。"""
|
||||
try:
|
||||
dt = datetime.strptime(date_str, "%Y-%m-%d")
|
||||
except ValueError:
|
||||
return None
|
||||
lookback = env_int("DAILY_DELTA_LOOKBACK_DAYS", 7)
|
||||
for days in range(1, lookback + 1):
|
||||
prev_date = (dt - timedelta(days=days)).strftime("%Y-%m-%d")
|
||||
path = _data_json_path(prev_date)
|
||||
if not path.exists():
|
||||
continue
|
||||
try:
|
||||
payload = _load_data_json_file(path)
|
||||
except (OSError, ValueError) as exc:
|
||||
logger.warning("读取异动基准 %s 失败:%s", path, exc)
|
||||
continue
|
||||
data = payload.get("data")
|
||||
if isinstance(data, dict) and data.get("date"):
|
||||
return prev_date, data
|
||||
return None
|
||||
|
||||
|
||||
def _prev_board_items(prev_data: dict[str, Any], board: str, depth: int) -> list[dict[str, Any]]:
|
||||
baseline = prev_data.get("movement_baseline") or {}
|
||||
if board in baseline and isinstance(baseline[board], list):
|
||||
return baseline[board][:depth]
|
||||
legacy = {
|
||||
"skills_trending": "skills_trending",
|
||||
"skills_hot": "skills_hot",
|
||||
"github_trending": "github_trending",
|
||||
"github_emerging": "github_emerging",
|
||||
"github_topic": "github_topic",
|
||||
}
|
||||
if board == "github_topic":
|
||||
topic = prev_data.get("github_topic") or {}
|
||||
repos = topic.get("repos") if isinstance(topic, dict) else []
|
||||
return (repos or [])[:depth]
|
||||
field = legacy.get(board, board)
|
||||
items = prev_data.get(field) or []
|
||||
return items[:depth] if isinstance(items, list) else []
|
||||
|
||||
|
||||
def _format_new_note(board: str, rank: int, *, topic_name: str = "llm") -> str:
|
||||
labels = {
|
||||
"trending": "Skills Trending",
|
||||
"hot": "Skills Hot",
|
||||
"github_trending": "GitHub Trending",
|
||||
"github_emerging": "GitHub 新兴",
|
||||
"github_topic": f"Topic `{topic_name}`",
|
||||
}
|
||||
return f"新入 {labels.get(board, board)} #{rank}"
|
||||
|
||||
|
||||
def _build_skill_board_moves(
|
||||
*,
|
||||
board: str,
|
||||
items: list[dict[str, Any]],
|
||||
prev_data: dict[str, Any] | None,
|
||||
depth: int,
|
||||
) -> list[dict[str, Any]]:
|
||||
board_key = f"skills_{board}"
|
||||
prev_ids = (
|
||||
_key_set(_prev_board_items(prev_data, board_key, depth), skill_id, depth=depth)
|
||||
if prev_data
|
||||
else set()
|
||||
)
|
||||
moves: list[dict[str, Any]] = []
|
||||
for rank, item in enumerate(items[:depth], 1):
|
||||
sid = skill_id(item)
|
||||
if not sid or not prev_data or sid in prev_ids:
|
||||
continue
|
||||
moves.append(
|
||||
{
|
||||
"kind": "skill",
|
||||
"board": board,
|
||||
"id": sid,
|
||||
"title": item.get("title", ""),
|
||||
"source": item.get("source", ""),
|
||||
"installs": int(item.get("installs") or 0),
|
||||
"link": item.get("link", ""),
|
||||
"description": item.get("description", ""),
|
||||
"rank": rank,
|
||||
"is_new": True,
|
||||
"note": _format_new_note(board, rank),
|
||||
}
|
||||
)
|
||||
return moves
|
||||
|
||||
|
||||
def _repo_key(item: dict[str, Any]) -> str:
|
||||
return str(item.get("repo") or "")
|
||||
|
||||
|
||||
def _build_github_board_moves(
|
||||
*,
|
||||
board: str,
|
||||
items: list[dict[str, Any]],
|
||||
prev_data: dict[str, Any] | None,
|
||||
depth: int,
|
||||
topic_name: str = "llm",
|
||||
) -> list[dict[str, Any]]:
|
||||
prev_ids = (
|
||||
_key_set(_prev_board_items(prev_data, board, depth), _repo_key, depth=depth)
|
||||
if prev_data
|
||||
else set()
|
||||
)
|
||||
moves: list[dict[str, Any]] = []
|
||||
for rank, item in enumerate(items[:depth], 1):
|
||||
repo = _repo_key(item)
|
||||
if not repo or not prev_data or repo in prev_ids:
|
||||
continue
|
||||
moves.append(
|
||||
{
|
||||
"kind": "github",
|
||||
"board": board,
|
||||
"repo": repo,
|
||||
"url": item.get("url", ""),
|
||||
"language": item.get("language", ""),
|
||||
"stars_today_fmt": item.get("stars_today_fmt", ""),
|
||||
"total_stars_fmt": item.get("total_stars_fmt", ""),
|
||||
"created_at": item.get("created_at", ""),
|
||||
"description": item.get("description", ""),
|
||||
"rank": rank,
|
||||
"is_new": True,
|
||||
"note": _format_new_note(board, rank, topic_name=topic_name),
|
||||
}
|
||||
)
|
||||
return moves
|
||||
|
||||
|
||||
def _board_summary(
|
||||
*,
|
||||
label: str,
|
||||
baseline_date: str | None,
|
||||
depth: int,
|
||||
moves: list[dict[str, Any]],
|
||||
capped: list[dict[str, Any]],
|
||||
) -> str:
|
||||
if not baseline_date:
|
||||
return f"无历史基准,无法判断 {label} Top{depth} 新增"
|
||||
if not moves:
|
||||
return f"较 {baseline_date} Top{depth} 无新增 {label} 条目"
|
||||
total = len(moves)
|
||||
shown = len(capped)
|
||||
if shown < total:
|
||||
return f"较 {baseline_date} Top{depth} 新增 {total} 条,企微展示前 {shown} 条"
|
||||
return f"较 {baseline_date} Top{depth} 新增 {total} 条"
|
||||
|
||||
|
||||
def build_movement_baseline(
|
||||
*,
|
||||
trending: list[dict[str, Any]],
|
||||
hot: list[dict[str, Any]],
|
||||
github_trending: list[dict[str, Any]],
|
||||
github_emerging: list[dict[str, Any]],
|
||||
github_topic: list[dict[str, Any]],
|
||||
depth: int | None = None,
|
||||
) -> dict[str, Any]:
|
||||
n = depth if depth is not None else compare_depth()
|
||||
return {
|
||||
"compare_depth": n,
|
||||
"skills_trending": trending[:n],
|
||||
"skills_hot": hot[:n],
|
||||
"github_trending": github_trending[:n],
|
||||
"github_emerging": github_emerging[:n],
|
||||
"github_topic": github_topic[:n],
|
||||
}
|
||||
|
||||
|
||||
def build_movement_context(
|
||||
*,
|
||||
date_str: str,
|
||||
trending: list[dict[str, Any]],
|
||||
hot: list[dict[str, Any]],
|
||||
github_trending: list[dict[str, Any]],
|
||||
github_emerging: list[dict[str, Any]],
|
||||
github_topic: list[dict[str, Any]],
|
||||
topic_name: str = "llm",
|
||||
) -> dict[str, Any]:
|
||||
"""生成 Agent 可用的新增榜上下文:分 Trending/Hot/Topic,企微每榜最多 wecom_new_limit 条。"""
|
||||
depth = compare_depth()
|
||||
cap = wecom_new_limit()
|
||||
baseline = find_previous_data(date_str)
|
||||
baseline_date = baseline[0] if baseline else None
|
||||
prev_data = baseline[1] if baseline else None
|
||||
topic = (topic_name or "llm").strip() or "llm"
|
||||
|
||||
skills_trending_all = _build_skill_board_moves(
|
||||
board="trending", items=trending, prev_data=prev_data, depth=depth
|
||||
)
|
||||
skills_hot_all = _build_skill_board_moves(
|
||||
board="hot", items=hot, prev_data=prev_data, depth=depth
|
||||
)
|
||||
github_trending_all = _build_github_board_moves(
|
||||
board="github_trending", items=github_trending, prev_data=prev_data, depth=depth
|
||||
)
|
||||
github_emerging_all = _build_github_board_moves(
|
||||
board="github_emerging", items=github_emerging, prev_data=prev_data, depth=depth
|
||||
)
|
||||
github_topic_all = _build_github_board_moves(
|
||||
board="github_topic",
|
||||
items=github_topic,
|
||||
prev_data=prev_data,
|
||||
depth=depth,
|
||||
topic_name=topic,
|
||||
)
|
||||
|
||||
skills_trending_moves = skills_trending_all[:cap]
|
||||
skills_hot_moves = skills_hot_all[:cap]
|
||||
github_trending_moves = github_trending_all[:cap]
|
||||
github_emerging_moves = github_emerging_all[:cap]
|
||||
github_topic_moves = github_topic_all[:cap]
|
||||
topic_label = f"Topic `{topic}`"
|
||||
|
||||
return {
|
||||
"baseline_date": baseline_date,
|
||||
"compare_depth": depth,
|
||||
"wecom_new_limit": cap,
|
||||
"selection_mode": "top_n",
|
||||
"topic_name": topic,
|
||||
"skills_trending_moves": skills_trending_moves,
|
||||
"skills_hot_moves": skills_hot_moves,
|
||||
"skills_trending_stable": not skills_trending_all,
|
||||
"skills_hot_stable": not skills_hot_all,
|
||||
"skills_trending_summary": _board_summary(
|
||||
label="Skills Trending",
|
||||
baseline_date=baseline_date,
|
||||
depth=depth,
|
||||
moves=skills_trending_all,
|
||||
capped=skills_trending_moves,
|
||||
),
|
||||
"skills_hot_summary": _board_summary(
|
||||
label="Skills Hot",
|
||||
baseline_date=baseline_date,
|
||||
depth=depth,
|
||||
moves=skills_hot_all,
|
||||
capped=skills_hot_moves,
|
||||
),
|
||||
"github_trending_moves": github_trending_moves,
|
||||
"github_emerging_moves": github_emerging_moves,
|
||||
"github_topic_moves": github_topic_moves,
|
||||
"github_trending_stable": not github_trending_all,
|
||||
"github_emerging_stable": not github_emerging_all,
|
||||
"github_topic_stable": not github_topic_all,
|
||||
"github_trending_summary": _board_summary(
|
||||
label="GitHub Trending",
|
||||
baseline_date=baseline_date,
|
||||
depth=depth,
|
||||
moves=github_trending_all,
|
||||
capped=github_trending_moves,
|
||||
),
|
||||
"github_emerging_summary": _board_summary(
|
||||
label="GitHub 新兴",
|
||||
baseline_date=baseline_date,
|
||||
depth=depth,
|
||||
moves=github_emerging_all,
|
||||
capped=github_emerging_moves,
|
||||
),
|
||||
"github_topic_summary": _board_summary(
|
||||
label=topic_label,
|
||||
baseline_date=baseline_date,
|
||||
depth=depth,
|
||||
moves=github_topic_all,
|
||||
capped=github_topic_moves,
|
||||
),
|
||||
# 兼容旧字段(合并,仅供调试)
|
||||
"skills_moves": skills_trending_moves + skills_hot_moves,
|
||||
"github_moves": github_trending_moves + github_emerging_moves + github_topic_moves,
|
||||
"skills_stable": not skills_trending_all and not skills_hot_all,
|
||||
"github_stable": not github_trending_all and not github_emerging_all and not github_topic_all,
|
||||
}
|
||||
281
daily/format_wecom.py
Normal file
281
daily/format_wecom.py
Normal file
@@ -0,0 +1,281 @@
|
||||
"""企微早报排版。"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import re
|
||||
from typing import Any
|
||||
|
||||
from daily.config import wecom_skill_desc_limit
|
||||
from daily.localize import LocalizeJob, localize_brief_descriptions, needs_chinese
|
||||
from daily.text_utils import trim_brief
|
||||
|
||||
ICONS = {
|
||||
"header": "📰",
|
||||
"highlights": "💡",
|
||||
"trending": "📈",
|
||||
"hot": "🔥",
|
||||
"github": "🐙",
|
||||
"emerging": "🌱",
|
||||
"topic": "🤖",
|
||||
"ainews": "🌍",
|
||||
"pick": "📦",
|
||||
"theme": "🎯",
|
||||
"file": "📄",
|
||||
}
|
||||
|
||||
|
||||
def _skill_line(rank: int, item: dict[str, Any], *, badge: str = "") -> list[str]:
|
||||
source = item.get("source", "?")
|
||||
installs = item.get("installs_fmt", "?")
|
||||
link = item.get("link", "")
|
||||
desc = item.get("desc_short", "")
|
||||
badge_prefix = f"{badge} " if badge else ""
|
||||
if item.get("cluster"):
|
||||
count = int(item.get("cluster_count") or 1)
|
||||
sample = item.get("cluster_titles") or item.get("title", "")
|
||||
label = f"**{source}** · {count} skills · **{installs}**"
|
||||
if link:
|
||||
head = f"{rank}. {badge_prefix}[{label}]({link})"
|
||||
else:
|
||||
head = f"{rank}. {badge_prefix}{label}"
|
||||
lines = [head]
|
||||
if sample or desc:
|
||||
hint = desc or sample
|
||||
lines.append(f" > {hint}")
|
||||
return lines
|
||||
title = item.get("title", "?")
|
||||
if link:
|
||||
head = f"{rank}. {badge_prefix}[**{title}**]({link}) · `{source}` · **{installs}**"
|
||||
else:
|
||||
head = f"{rank}. {badge_prefix}**{title}** · `{source}` · **{installs}**"
|
||||
lines = [head]
|
||||
if desc:
|
||||
lines.append(f" > {desc}")
|
||||
return lines
|
||||
|
||||
|
||||
def _ai_news_lines(items: list[dict[str, Any]]) -> list[str]:
|
||||
lines: list[str] = []
|
||||
for i, item in enumerate(items, 1):
|
||||
title = item.get("title", "?")
|
||||
link = item.get("link", "")
|
||||
source = item.get("source_name", "?")
|
||||
pub = item.get("published_fmt", "")
|
||||
desc = item.get("desc_short", "")
|
||||
pub_suffix = f" · {pub}" if pub else ""
|
||||
if link:
|
||||
head = f"{i}. [**{title}**]({link}) · `{source}`{pub_suffix}"
|
||||
else:
|
||||
head = f"{i}. **{title}** · `{source}`{pub_suffix}"
|
||||
lines.append(head)
|
||||
if desc:
|
||||
lines.append(f" > {desc}")
|
||||
return lines
|
||||
|
||||
|
||||
def _github_repo_lines(repos: list[dict[str, Any]], *, show_created: bool = False) -> list[str]:
|
||||
lines: list[str] = []
|
||||
for i, repo in enumerate(repos, 1):
|
||||
name = repo["repo"]
|
||||
url = repo["url"]
|
||||
lang = repo.get("language", "")
|
||||
stars_today = repo.get("stars_today_fmt", "")
|
||||
total = repo.get("total_stars_fmt", "")
|
||||
created = repo.get("created_at", "")
|
||||
meta_parts: list[str] = []
|
||||
if lang:
|
||||
meta_parts.append(lang)
|
||||
if stars_today:
|
||||
meta_parts.append(f"+{stars_today} today")
|
||||
elif total:
|
||||
meta_parts.append(f"⭐{total}")
|
||||
if show_created and created:
|
||||
meta_parts.append(f"创建于 {created}")
|
||||
meta = f" · {' · '.join(meta_parts)}" if meta_parts else ""
|
||||
lines.append(f"{i}. [{name}]({url}){meta}")
|
||||
desc = repo.get("desc_short") or repo.get("description", "")
|
||||
if desc:
|
||||
lines.append(f" > {desc}")
|
||||
return lines
|
||||
|
||||
|
||||
def _fallback_skill_desc(item: dict[str, Any]) -> str:
|
||||
if item.get("cluster"):
|
||||
count = int(item.get("cluster_count") or 1)
|
||||
source = item.get("source") or "unknown"
|
||||
sample = item.get("cluster_titles") or item.get("title") or ""
|
||||
return f"{count} agent skills from {source}, including {sample}"
|
||||
title = item.get("title") or "skill"
|
||||
source = item.get("source") or "unknown"
|
||||
return f"{title} skill from {source}"
|
||||
|
||||
|
||||
def _skill_group_key(item: dict[str, Any]) -> str:
|
||||
return str(item.get("id") or f"{item.get('source')}/{item.get('title')}")
|
||||
|
||||
|
||||
def finalize_wecom_skill_groups(
|
||||
items: list[dict[str, Any]],
|
||||
*,
|
||||
desc_limit: int | None = None,
|
||||
) -> list[dict[str, Any]]:
|
||||
"""为企微 Skills 榜生成一句简要中文简介(与完整版 .md 长描述分离)。"""
|
||||
if desc_limit is None:
|
||||
desc_limit = wecom_skill_desc_limit()
|
||||
limit = desc_limit if desc_limit > 0 else 48
|
||||
copies: list[tuple[str, dict[str, Any]]] = []
|
||||
jobs: list[LocalizeJob] = []
|
||||
for item in items:
|
||||
copy = dict(item)
|
||||
desc = (copy.get("description") or "").strip()
|
||||
if not desc:
|
||||
desc = _fallback_skill_desc(copy)
|
||||
key = _skill_group_key(copy)
|
||||
job_key = f"wecom:{key}"
|
||||
if needs_chinese(desc) or len(desc) > limit:
|
||||
jobs.append(LocalizeJob(job_key, desc, limit))
|
||||
else:
|
||||
copy["wecom_desc"] = desc
|
||||
copies.append((job_key, copy))
|
||||
|
||||
zh_map = localize_brief_descriptions(jobs, archive=True)
|
||||
out: list[dict[str, Any]] = []
|
||||
for job_key, copy in copies:
|
||||
if job_key in zh_map:
|
||||
copy["wecom_desc"] = zh_map[job_key]
|
||||
elif "wecom_desc" not in copy:
|
||||
copy["wecom_desc"] = _brief_fallback_desc(
|
||||
(copy.get("description") or "").strip() or _fallback_skill_desc(copy),
|
||||
limit,
|
||||
)
|
||||
out.append(copy)
|
||||
return out
|
||||
|
||||
|
||||
def _brief_fallback_desc(text: str, limit: int) -> str:
|
||||
return trim_brief(text, limit)
|
||||
|
||||
|
||||
def _grouped_skill_to_wecom_item(
|
||||
item: dict[str, Any],
|
||||
*,
|
||||
desc_limit: int | None = None,
|
||||
) -> dict[str, Any]:
|
||||
if desc_limit is None:
|
||||
desc_limit = wecom_skill_desc_limit()
|
||||
limit = desc_limit if desc_limit > 0 else 48
|
||||
installs = int(item.get("installs") or 0)
|
||||
installs_fmt = item.get("installs_fmt") or str(installs)
|
||||
desc = (item.get("wecom_desc") or item.get("description") or "").strip()
|
||||
if not desc and item.get("cluster"):
|
||||
desc = item.get("cluster_titles") or ""
|
||||
if not item.get("wecom_desc"):
|
||||
desc = _brief_fallback_desc(desc, limit)
|
||||
return {
|
||||
"title": item.get("title", ""),
|
||||
"source": item.get("source", "?"),
|
||||
"installs_fmt": installs_fmt,
|
||||
"link": item.get("link", ""),
|
||||
"desc_short": desc,
|
||||
"cluster": bool(item.get("cluster")),
|
||||
"cluster_count": item.get("cluster_count"),
|
||||
"cluster_titles": item.get("cluster_titles"),
|
||||
}
|
||||
|
||||
|
||||
def build_skills_board_section(icon_key: str, board_label: str, items: list[dict[str, Any]]) -> str:
|
||||
prepared = finalize_wecom_skill_groups(items)
|
||||
wecom_items = [_grouped_skill_to_wecom_item(x) for x in prepared]
|
||||
lines = [f"{ICONS[icon_key]} **{board_label} Top {len(wecom_items)}**"]
|
||||
for rank, item in enumerate(wecom_items, 1):
|
||||
lines.extend(_skill_line(rank, item))
|
||||
return "\n".join(lines)
|
||||
|
||||
|
||||
_SKILL_SECTIONS = re.compile(
|
||||
r"📈 \*\*Skills Trending.*?(?=🐙 \*\*GitHub Trending)",
|
||||
re.DOTALL,
|
||||
)
|
||||
|
||||
|
||||
def replace_wecom_skill_sections(
|
||||
md: str,
|
||||
*,
|
||||
trending: list[dict[str, Any]],
|
||||
hot: list[dict[str, Any]],
|
||||
) -> str:
|
||||
"""用 Python 合并后的 Skills 榜替换或插入 Agent 早报中的对应区块。"""
|
||||
trending_sec = build_skills_board_section("trending", "Skills Trending", trending)
|
||||
hot_sec = build_skills_board_section("hot", "Skills Hot", hot)
|
||||
replacement = f"{trending_sec}\n\n{hot_sec}\n\n"
|
||||
if _SKILL_SECTIONS.search(md):
|
||||
return _SKILL_SECTIONS.sub(replacement, md)
|
||||
github_marker = "🐙 **GitHub Trending"
|
||||
idx = md.find(github_marker)
|
||||
if idx >= 0:
|
||||
return md[:idx] + replacement + md[idx:]
|
||||
return md.rstrip() + "\n\n" + replacement
|
||||
|
||||
|
||||
def build_wecom_report(
|
||||
*,
|
||||
date_str: str,
|
||||
time_str: str,
|
||||
updated: str,
|
||||
highlights: list[str],
|
||||
theme_line: str,
|
||||
trending: list[dict[str, Any]],
|
||||
hot: list[dict[str, Any]],
|
||||
repos: list[dict[str, Any]],
|
||||
emerging: list[dict[str, Any]],
|
||||
topic_name: str,
|
||||
topic_repos: list[dict[str, Any]],
|
||||
ai_news: list[dict[str, Any]] | None = None,
|
||||
pick_command: str,
|
||||
) -> str:
|
||||
lines = [
|
||||
f"{ICONS['header']} **早报 · {date_str}**",
|
||||
f"> ⏱ {time_str} · 数据截至 {updated}",
|
||||
"",
|
||||
f"{ICONS['highlights']} **今日速览**",
|
||||
]
|
||||
for point in highlights[:3]:
|
||||
lines.append(f"> {point}")
|
||||
lines.append("")
|
||||
lines.append(f"{ICONS['theme']} {theme_line}")
|
||||
lines.append("")
|
||||
|
||||
if ai_news:
|
||||
lines.append(f"{ICONS['ainews']} **国际 AI 时讯 Top {len(ai_news)}**")
|
||||
lines.extend(_ai_news_lines(ai_news))
|
||||
lines.append("")
|
||||
|
||||
lines.append(f"{ICONS['trending']} **Skills Trending Top {len(trending)}**")
|
||||
for rank, item in enumerate(trending, 1):
|
||||
lines.extend(_skill_line(rank, item, badge=item.get("badge", "")))
|
||||
lines.append("")
|
||||
|
||||
lines.append(f"{ICONS['hot']} **Skills Hot Top {len(hot)}**")
|
||||
for rank, item in enumerate(hot, 1):
|
||||
lines.extend(_skill_line(rank, item, badge=item.get("badge", "")))
|
||||
lines.append("")
|
||||
|
||||
if repos:
|
||||
lines.append(f"{ICONS['github']} **GitHub Trending Top {len(repos)}**")
|
||||
lines.extend(_github_repo_lines(repos))
|
||||
lines.append("")
|
||||
|
||||
if emerging:
|
||||
lines.append(f"{ICONS['emerging']} **新兴项目 Top {len(emerging)}**")
|
||||
lines.extend(_github_repo_lines(emerging, show_created=True))
|
||||
lines.append("")
|
||||
|
||||
if topic_repos:
|
||||
lines.append(f"{ICONS['topic']} **Topic `{topic_name}` Top {len(topic_repos)}**")
|
||||
lines.extend(_github_repo_lines(topic_repos))
|
||||
lines.append("")
|
||||
|
||||
lines.append(f"{ICONS['pick']} **今日首推**")
|
||||
lines.append(f"`{pick_command}`")
|
||||
|
||||
return "\n".join(lines)
|
||||
635
daily/generate.py
Normal file
635
daily/generate.py
Normal file
@@ -0,0 +1,635 @@
|
||||
"""生成早报 Markdown(完整版 + 企微短版)。"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import re
|
||||
import sys
|
||||
import xml.etree.ElementTree as ET
|
||||
from collections import defaultdict
|
||||
from datetime import datetime, timezone, timedelta
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
import certifi
|
||||
import httpx
|
||||
import logging
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
from daily.config import (
|
||||
CACHE_DIR,
|
||||
LOG_DIR,
|
||||
OUTPUT_DIR,
|
||||
SNAPSHOT_FILE,
|
||||
ensure_bot_on_path,
|
||||
env,
|
||||
env_int,
|
||||
full_desc_limit,
|
||||
news_summary_limit,
|
||||
wecom_skill_desc_limit,
|
||||
)
|
||||
from daily.format_wecom import build_wecom_report, finalize_wecom_skill_groups, replace_wecom_skill_sections
|
||||
from daily.agent_workflow import is_agent_mode, run_agent_workflow
|
||||
from daily.delta import compare_depth
|
||||
from daily.cursor_editor import (
|
||||
apply_descriptions,
|
||||
is_enabled as cursor_editor_enabled,
|
||||
run_editorial,
|
||||
theme_line_from_editorial,
|
||||
)
|
||||
from daily.github.auth import github_html_headers
|
||||
from daily.github.search import fetch_emerging_repos, fetch_topic_hot_repos
|
||||
from daily.github.trending import fetch_github_trending, trending_data_source_note
|
||||
from daily.localize import LocalizeJob, localize_descriptions, needs_chinese
|
||||
from daily.news.fetch import fetch_ai_news, format_news_section, prepare_wecom_news_items
|
||||
from daily.report_data import (
|
||||
build_full_payload,
|
||||
build_llm_input,
|
||||
data_json_path,
|
||||
save_json,
|
||||
)
|
||||
from daily.skills_board import load_boards
|
||||
from daily.skills_group import group_skills_by_source
|
||||
|
||||
ensure_bot_on_path()
|
||||
from skills_service import _format_installs, load_feed # noqa: E402
|
||||
|
||||
THEME_RULES: list[tuple[str, str, list[str]]] = [
|
||||
("🎬", "AI 多媒体 / 视频", ["runcomfy", "remotion", "video", "seedance", "inpaint", "lipsync"]),
|
||||
("🔧", "工程协作 / Skill 元能力", ["grill", "tdd", "architecture", "find-skills", "to-issues"]),
|
||||
("📱", "飞书 / Lark", ["lark", "feishu"]),
|
||||
("📣", "内容营销", ["viral", "tiktok", "instagram", "reels"]),
|
||||
("🎨", "设计 / 前端", ["frontend", "design", "ui-ux", "tailwind"]),
|
||||
]
|
||||
|
||||
|
||||
def _now_cst() -> datetime:
|
||||
return datetime.now(timezone(timedelta(hours=8)))
|
||||
|
||||
|
||||
def _short_desc(text: str, limit: int = 72) -> str:
|
||||
text = re.sub(r"\s+", " ", text or "").strip()
|
||||
if limit <= 0 or len(text) <= limit:
|
||||
return text
|
||||
return text[: limit - 3] + "..."
|
||||
|
||||
|
||||
def _archive_desc(text: str) -> str:
|
||||
return _short_desc(text, full_desc_limit())
|
||||
|
||||
|
||||
def _wecom_desc(text: str, limit: int = 36) -> str:
|
||||
return _short_desc(text, limit)
|
||||
|
||||
|
||||
def _localize_descriptions_in_place(
|
||||
trending: list[dict[str, Any]],
|
||||
hot: list[dict[str, Any]],
|
||||
github_trending: list[dict[str, Any]],
|
||||
github_emerging: list[dict[str, Any]],
|
||||
github_topic: list[dict[str, Any]],
|
||||
ai_news: dict[str, Any],
|
||||
) -> None:
|
||||
full_limit = full_desc_limit()
|
||||
news_limit = news_summary_limit()
|
||||
jobs: list[LocalizeJob] = []
|
||||
seen_skill: set[str] = set()
|
||||
for item in trending + hot:
|
||||
sid = _skill_id(item)
|
||||
if sid in seen_skill:
|
||||
continue
|
||||
seen_skill.add(sid)
|
||||
desc = (item.get("description") or "").strip()
|
||||
if desc:
|
||||
jobs.append(LocalizeJob(f"skill:{sid}", desc, full_limit))
|
||||
|
||||
seen_repo: set[str] = set()
|
||||
for repo_list in (github_trending, github_emerging, github_topic):
|
||||
for item in repo_list:
|
||||
repo = item.get("repo", "")
|
||||
if not repo or repo in seen_repo:
|
||||
continue
|
||||
seen_repo.add(repo)
|
||||
desc = (item.get("description") or "").strip()
|
||||
if desc:
|
||||
jobs.append(LocalizeJob(f"github:{repo}", desc, full_limit))
|
||||
|
||||
if ai_news.get("enabled"):
|
||||
seen_news: set[str] = set()
|
||||
for item in ai_news.get("flat") or []:
|
||||
link = item.get("link", "")
|
||||
if not link or link in seen_news:
|
||||
continue
|
||||
seen_news.add(link)
|
||||
summary = (item.get("summary") or "").strip()
|
||||
if summary:
|
||||
jobs.append(LocalizeJob(f"news:{link}", summary, news_limit))
|
||||
|
||||
zh_map = localize_descriptions(jobs, archive=True)
|
||||
if not zh_map and not jobs:
|
||||
return
|
||||
|
||||
def _apply_zh(mapping: dict[str, str]) -> None:
|
||||
for item in trending + hot:
|
||||
key = f"skill:{_skill_id(item)}"
|
||||
if key in mapping:
|
||||
item["description"] = mapping[key]
|
||||
for repo_list in (github_trending, github_emerging, github_topic):
|
||||
for item in repo_list:
|
||||
key = f"github:{item.get('repo', '')}"
|
||||
if key in mapping:
|
||||
item["description"] = mapping[key]
|
||||
if ai_news.get("enabled"):
|
||||
for cat in ai_news.get("categories") or []:
|
||||
for item in cat.get("items") or []:
|
||||
key = f"news:{item.get('link', '')}"
|
||||
if key in mapping:
|
||||
item["summary"] = mapping[key]
|
||||
for item in ai_news.get("flat") or []:
|
||||
key = f"news:{item.get('link', '')}"
|
||||
if key in mapping:
|
||||
item["summary"] = mapping[key]
|
||||
|
||||
_apply_zh(zh_map)
|
||||
|
||||
# 仍为英文的条目再译一轮(长描述或批次失败时)
|
||||
retry_jobs: list[LocalizeJob] = []
|
||||
seen_skill.clear()
|
||||
for item in trending + hot:
|
||||
sid = _skill_id(item)
|
||||
if sid in seen_skill:
|
||||
continue
|
||||
seen_skill.add(sid)
|
||||
desc = (item.get("description") or "").strip()
|
||||
if needs_chinese(desc):
|
||||
retry_jobs.append(LocalizeJob(f"skill:{sid}", desc, full_limit))
|
||||
seen_repo.clear()
|
||||
for repo_list in (github_trending, github_emerging, github_topic):
|
||||
for item in repo_list:
|
||||
repo = item.get("repo", "")
|
||||
if not repo or repo in seen_repo:
|
||||
continue
|
||||
seen_repo.add(repo)
|
||||
desc = (item.get("description") or "").strip()
|
||||
if needs_chinese(desc):
|
||||
retry_jobs.append(LocalizeJob(f"github:{repo}", desc, full_limit))
|
||||
if ai_news.get("enabled"):
|
||||
seen_news.clear()
|
||||
for item in ai_news.get("flat") or []:
|
||||
link = item.get("link", "")
|
||||
if not link or link in seen_news:
|
||||
continue
|
||||
seen_news.add(link)
|
||||
summary = (item.get("summary") or "").strip()
|
||||
if needs_chinese(summary):
|
||||
retry_jobs.append(LocalizeJob(f"news:{link}", summary, news_limit))
|
||||
|
||||
if retry_jobs:
|
||||
_apply_zh(localize_descriptions(retry_jobs, archive=True))
|
||||
|
||||
|
||||
def _skill_id(item: dict[str, Any]) -> str:
|
||||
return str(item.get("id") or f"{item.get('source')}/{item.get('title')}")
|
||||
|
||||
|
||||
def _load_snapshot() -> set[str]:
|
||||
if not SNAPSHOT_FILE.exists():
|
||||
return set()
|
||||
try:
|
||||
data = json.loads(SNAPSHOT_FILE.read_text(encoding="utf-8"))
|
||||
return set(str(x) for x in (data.get("skill_ids") or []))
|
||||
except (OSError, json.JSONDecodeError):
|
||||
return set()
|
||||
|
||||
|
||||
def _save_snapshot(feed: dict[str, Any], date_str: str) -> None:
|
||||
CACHE_DIR.mkdir(parents=True, exist_ok=True)
|
||||
ids: list[str] = []
|
||||
for board in ("topTrending", "topHot"):
|
||||
for item in feed.get(board, [])[:20]:
|
||||
sid = _skill_id(item)
|
||||
if sid not in ids:
|
||||
ids.append(sid)
|
||||
SNAPSHOT_FILE.write_text(
|
||||
json.dumps({"date": date_str, "skill_ids": ids}, ensure_ascii=False, indent=2),
|
||||
encoding="utf-8",
|
||||
)
|
||||
|
||||
|
||||
def _prepare_skill_item(item: dict[str, Any], prev_ids: set[str], rank: int) -> dict[str, Any]:
|
||||
badge = ""
|
||||
sid = _skill_id(item)
|
||||
if sid not in prev_ids and prev_ids:
|
||||
badge = "🆕"
|
||||
elif rank == 1:
|
||||
badge = "👑"
|
||||
installs_fmt = item.get("installs_fmt") or _format_installs(item.get("installs", 0))
|
||||
title = item.get("source", "?") if item.get("cluster") else item.get("title", "?")
|
||||
desc = item.get("wecom_desc") or item.get("description") or item.get("cluster_titles") or ""
|
||||
limit = wecom_skill_desc_limit()
|
||||
desc_short = desc if item.get("wecom_desc") or limit <= 0 else _wecom_desc(desc, limit)
|
||||
return {
|
||||
"title": title,
|
||||
"source": item.get("source", "?"),
|
||||
"installs_fmt": installs_fmt,
|
||||
"link": item.get("link", ""),
|
||||
"desc_short": desc_short,
|
||||
"badge": badge,
|
||||
"cluster": bool(item.get("cluster")),
|
||||
"cluster_count": item.get("cluster_count"),
|
||||
"cluster_titles": item.get("cluster_titles"),
|
||||
}
|
||||
|
||||
|
||||
def _detect_theme_line(feed: dict[str, Any]) -> str:
|
||||
scores: dict[str, int] = defaultdict(int)
|
||||
for board in ("topTrending", "topHot"):
|
||||
for rank, item in enumerate(feed.get(board, [])[:10], 1):
|
||||
haystack = " ".join(
|
||||
[item.get("title", ""), item.get("source", ""), item.get("description", "")]
|
||||
).lower()
|
||||
for _icon, label, keywords in THEME_RULES:
|
||||
if any(k in haystack for k in keywords):
|
||||
scores[label] += max(1, 11 - rank)
|
||||
break
|
||||
if not scores:
|
||||
return "**今日主题**:Agent Skills 生态持续活跃"
|
||||
return f"**今日主题**:{max(scores.items(), key=lambda x: x[1])[0]}"
|
||||
|
||||
|
||||
def _build_highlights(
|
||||
trending: list[dict[str, Any]],
|
||||
hot: list[dict[str, Any]],
|
||||
github_trending: list[dict[str, Any]],
|
||||
github_emerging: list[dict[str, Any]],
|
||||
ai_news: dict[str, Any] | None = None,
|
||||
) -> list[str]:
|
||||
points: list[str] = []
|
||||
if ai_news and ai_news.get("enabled"):
|
||||
top_news = prepare_wecom_news_items(ai_news)
|
||||
if top_news:
|
||||
n0 = top_news[0]
|
||||
pub = f" · {n0['published_fmt']}" if n0.get("published_fmt") else ""
|
||||
points.append(
|
||||
f"🌍 AI 时讯 [{n0['title']}]({n0['link']})(`{n0.get('source_name', '?')}`{pub})"
|
||||
)
|
||||
elif ai_news.get("flat"):
|
||||
n0 = ai_news["flat"][0]
|
||||
pub = f" · {n0['published_fmt']}" if n0.get("published_fmt") else ""
|
||||
points.append(f"🌍 AI 时讯 [{n0['title']}]({n0['link']})(`{n0.get('source_name', '?')}`{pub})")
|
||||
if trending:
|
||||
t0 = trending[0]
|
||||
points.append(f"📈 Skills 榜首 **{t0.get('title')}**({_format_installs(t0.get('installs', 0))})")
|
||||
if github_trending:
|
||||
g0 = github_trending[0]
|
||||
stars = g0.get("stars_today_fmt", "")
|
||||
total = g0.get("total_stars_fmt", "")
|
||||
star_hint = f"+{stars} today · " if stars else (f"⭐{total} · " if total else "")
|
||||
points.append(f"🐙 GitHub Trending [{g0['repo']}]({g0['url']})({star_hint}{g0.get('language', '')})")
|
||||
if github_emerging:
|
||||
e0 = github_emerging[0]
|
||||
points.append(f"🌱 新兴 [{e0['repo']}]({e0['url']})(⭐ {e0.get('total_stars_fmt', '?')})")
|
||||
elif hot:
|
||||
h0 = hot[0]
|
||||
points.append(f"🔥 Skills Hot 榜首 **{h0.get('title')}**(1H {_format_installs(h0.get('installs', 0))})")
|
||||
while len(points) < 3 and len(trending) > len(points):
|
||||
item = trending[len(points)]
|
||||
points.append(f"✨ **{item.get('title')}** · `{item.get('source')}`")
|
||||
return points[:3]
|
||||
|
||||
|
||||
def _prepare_github_item(item: dict[str, Any]) -> dict[str, Any]:
|
||||
return {**item, "desc_short": _wecom_desc(item.get("description", ""), 40)}
|
||||
|
||||
|
||||
def _fetch_latest_release_title(repo: str) -> str | None:
|
||||
atom_url = f"https://github.com/{repo}/releases.atom"
|
||||
try:
|
||||
with httpx.Client(
|
||||
timeout=12.0,
|
||||
verify=certifi.where(),
|
||||
follow_redirects=True,
|
||||
headers=github_html_headers(),
|
||||
) as client:
|
||||
resp = client.get(atom_url)
|
||||
if resp.status_code != 200:
|
||||
return None
|
||||
root = ET.fromstring(resp.text)
|
||||
ns = {"a": "http://www.w3.org/2005/Atom"}
|
||||
entry = root.find("a:entry", ns)
|
||||
if entry is None:
|
||||
return None
|
||||
title = entry.find("a:title", ns)
|
||||
return title.text.strip() if title is not None and title.text else None
|
||||
except Exception:
|
||||
return None
|
||||
|
||||
|
||||
def _theme_clusters(feed: dict[str, Any], limit: int = 5) -> list[tuple[str, list[str]]]:
|
||||
buckets: dict[str, list[str]] = defaultdict(list)
|
||||
seen: set[str] = set()
|
||||
for board in ("topTrending", "topHot"):
|
||||
for item in feed.get(board, [])[:20]:
|
||||
item_id = _skill_id(item)
|
||||
if item_id in seen:
|
||||
continue
|
||||
seen.add(item_id)
|
||||
haystack = " ".join(
|
||||
[item.get("title", ""), item.get("source", ""), item.get("description", "")]
|
||||
).lower()
|
||||
for _icon, theme, keywords in THEME_RULES:
|
||||
if any(k in haystack for k in keywords):
|
||||
label = f"**{item.get('title')}** (`{item.get('source')}`)"
|
||||
if label not in buckets[theme]:
|
||||
buckets[theme].append(label)
|
||||
break
|
||||
return [(theme, examples[:limit]) for theme, examples in buckets.items() if examples]
|
||||
|
||||
|
||||
def _format_github_repo_section(repos: list[dict[str, Any]], *, show_created: bool = False) -> list[str]:
|
||||
lines: list[str] = []
|
||||
for i, repo in enumerate(repos, 1):
|
||||
lang = repo.get("language") or "—"
|
||||
stars_today = repo.get("stars_today_fmt") or ""
|
||||
total = repo.get("total_stars_fmt") or ""
|
||||
created = repo.get("created_at") or ""
|
||||
meta_parts = [lang]
|
||||
if stars_today:
|
||||
meta_parts.append(f"+{stars_today} today")
|
||||
if total:
|
||||
meta_parts.append(f"总 ⭐ {total}")
|
||||
if show_created and created:
|
||||
meta_parts.append(f"创建于 {created}")
|
||||
lines.append(f"{i}. **[{repo['repo']}]({repo['url']})** · {' · '.join(meta_parts)}")
|
||||
desc = _archive_desc(repo.get("description", ""))
|
||||
if desc:
|
||||
lines.append(f" - {desc}")
|
||||
lines.append("")
|
||||
return lines
|
||||
|
||||
|
||||
def _format_skill_section(items: list[dict[str, Any]], *, hot: bool = False) -> list[str]:
|
||||
lines: list[str] = []
|
||||
for i, item in enumerate(items, 1):
|
||||
skill_id = item.get("id") or f"{item.get('source', '?')}/{item.get('title', '?')}"
|
||||
link = item.get("link", "")
|
||||
installs = _format_installs(item.get("installs", 0))
|
||||
meta = f"1H {installs}" if hot else f"总安装 {installs}"
|
||||
if link:
|
||||
lines.append(f"{i}. **[{skill_id}]({link})** · {meta}")
|
||||
else:
|
||||
lines.append(f"{i}. **{skill_id}** · {meta}")
|
||||
desc = _archive_desc(item.get("description", ""))
|
||||
if desc:
|
||||
lines.append(f" - {desc}")
|
||||
lines.append("")
|
||||
return lines
|
||||
|
||||
|
||||
def generate_report() -> tuple[str, str, Path, Path]:
|
||||
trending_n = env_int("DAILY_TRENDING_LIMIT", 150)
|
||||
hot_n = max(env_int("DAILY_HOT_LIMIT", 150), compare_depth())
|
||||
compare_n = compare_depth()
|
||||
skill_pool = max(10, env_int("DAILY_WECOM_SKILL_POOL", 200))
|
||||
wecom_trending = env_int("DAILY_WECOM_TRENDING", 10)
|
||||
wecom_hot = env_int("DAILY_WECOM_HOT", 10)
|
||||
github_limit = env_int("DAILY_GITHUB_TRENDING_LIMIT", 10)
|
||||
wecom_github = env_int("DAILY_WECOM_GITHUB_TRENDING", env_int("DAILY_WECOM_REPOS", 10))
|
||||
github_fetch_n = max(github_limit, compare_n, wecom_github)
|
||||
emerging_limit = env_int("DAILY_GITHUB_EMERGING_LIMIT", 10)
|
||||
wecom_emerging = env_int("DAILY_WECOM_GITHUB_EMERGING", 10)
|
||||
emerging_fetch_n = max(emerging_limit, compare_n, wecom_emerging)
|
||||
topic_limit = env_int("DAILY_GITHUB_TOPIC_LIMIT", 10)
|
||||
wecom_topic = env_int("DAILY_WECOM_GITHUB_TOPIC", 10)
|
||||
topic_fetch_n = max(topic_limit, compare_n, wecom_topic)
|
||||
|
||||
feed = load_feed(force=True)
|
||||
prev_ids = _load_snapshot()
|
||||
now = _now_cst()
|
||||
date_str = now.strftime("%Y-%m-%d")
|
||||
time_str = now.strftime("%H:%M") + " (UTC+8)"
|
||||
updated = (feed.get("updatedAt") or "")[:10]
|
||||
|
||||
trending, hot = load_boards(feed, trending_limit=trending_n, hot_limit=hot_n)
|
||||
github_trending = fetch_github_trending(github_fetch_n)
|
||||
seen_repos = {r["repo"] for r in github_trending}
|
||||
github_emerging = fetch_emerging_repos(emerging_fetch_n, exclude=seen_repos)
|
||||
seen_repos.update(r["repo"] for r in github_emerging)
|
||||
topic_name, github_topic = fetch_topic_hot_repos(topic_fetch_n, exclude=seen_repos)
|
||||
ai_news = fetch_ai_news()
|
||||
|
||||
wecom_limits = {
|
||||
"trending": wecom_trending,
|
||||
"hot": wecom_hot,
|
||||
"trending_pool": skill_pool,
|
||||
"hot_pool": skill_pool,
|
||||
"github": wecom_github,
|
||||
"emerging": wecom_emerging,
|
||||
"topic": wecom_topic,
|
||||
"ai_news": env_int("DAILY_WECOM_AI_NEWS", 10),
|
||||
}
|
||||
llm_input = build_llm_input(
|
||||
date_str=date_str,
|
||||
updated=updated,
|
||||
trending=trending,
|
||||
hot=hot,
|
||||
github_trending=github_trending,
|
||||
github_emerging=github_emerging,
|
||||
github_topic=github_topic,
|
||||
topic_name=topic_name,
|
||||
ai_news=ai_news,
|
||||
wecom_limits=wecom_limits,
|
||||
)
|
||||
save_json(
|
||||
data_json_path(date_str),
|
||||
build_full_payload(
|
||||
llm_input,
|
||||
meta={
|
||||
"generated_at": now.isoformat(),
|
||||
"report_mode": "agent" if is_agent_mode() else "classic",
|
||||
"cursor_editor": cursor_editor_enabled() and not is_agent_mode(),
|
||||
},
|
||||
),
|
||||
)
|
||||
|
||||
agent_wecom: str | None = None
|
||||
if is_agent_mode():
|
||||
agent_wecom = run_agent_workflow(
|
||||
llm_input,
|
||||
date_str=date_str,
|
||||
time_str=time_str,
|
||||
updated=updated,
|
||||
)
|
||||
if not agent_wecom:
|
||||
logger.warning("Agent 工作流失败,回退 classic 模式")
|
||||
|
||||
editorial_theme: str | None = None
|
||||
editorial_highlights: list[str] | None = None
|
||||
if agent_wecom is None:
|
||||
editorial = run_editorial(llm_input, date_str=date_str)
|
||||
if editorial:
|
||||
apply_descriptions(
|
||||
trending=trending,
|
||||
hot=hot,
|
||||
github_trending=github_trending,
|
||||
github_emerging=github_emerging,
|
||||
github_topic=github_topic,
|
||||
ai_news=ai_news,
|
||||
descriptions=editorial.get("descriptions") or {},
|
||||
)
|
||||
editorial_theme = theme_line_from_editorial(editorial) or None
|
||||
hl = editorial.get("highlights") or []
|
||||
editorial_highlights = hl if hl else None
|
||||
|
||||
# 完整版归档:中文化 + 加长摘要(已是中文的条目会跳过翻译)
|
||||
_localize_descriptions_in_place(
|
||||
trending, hot, github_trending, github_emerging, github_topic, ai_news
|
||||
)
|
||||
|
||||
themes = _theme_clusters(feed)
|
||||
|
||||
lines = [
|
||||
f"# 早报 · {date_str}",
|
||||
"",
|
||||
f"> 生成时间:{now.strftime('%Y-%m-%d %H:%M')} (UTC+8) ",
|
||||
f"> skills 数据更新:{updated} ",
|
||||
"> 数据来源:[skills.sh/trending](https://skills.sh/trending) · [skills.sh/hot](https://skills.sh/hot) · 国际 AI RSS",
|
||||
"",
|
||||
"---",
|
||||
"",
|
||||
f"## 一、Skills Trending Top {trending_n}",
|
||||
"",
|
||||
*_format_skill_section(trending),
|
||||
"---",
|
||||
"",
|
||||
f"## 二、Skills Hot Top {hot_n}",
|
||||
"",
|
||||
*_format_skill_section(hot, hot=True),
|
||||
"",
|
||||
"---",
|
||||
"",
|
||||
f"## 三、GitHub Trending Top {github_limit}",
|
||||
"",
|
||||
trending_data_source_note(),
|
||||
"",
|
||||
]
|
||||
|
||||
if github_trending:
|
||||
lines.extend(_format_github_repo_section(github_trending))
|
||||
else:
|
||||
lines.append("*GitHub Trending 获取失败,请检查网络或配置 GITHUB_TOKEN。*")
|
||||
lines.append("")
|
||||
|
||||
lines.extend(["---", "", f"## 四、新兴项目 Top {emerging_limit}", "", "> 数据来源:GitHub Search API(需 `GITHUB_TOKEN`)", ""])
|
||||
if github_emerging:
|
||||
lines.extend(_format_github_repo_section(github_emerging, show_created=True))
|
||||
else:
|
||||
lines.append("*新兴项目获取失败或未配置 GITHUB_TOKEN。*")
|
||||
lines.append("")
|
||||
|
||||
lines.extend(["---", "", f"## 五、Topic `{topic_name}` Top {topic_limit}", "", "> 数据来源:GitHub Search API(需 `GITHUB_TOKEN`)", ""])
|
||||
if github_topic:
|
||||
lines.extend(_format_github_repo_section(github_topic))
|
||||
else:
|
||||
lines.append(f"*Topic `{topic_name}` 热点获取失败或未配置 GITHUB_TOKEN。*")
|
||||
lines.append("")
|
||||
|
||||
section_no = 6
|
||||
lines.extend(format_news_section(ai_news, section_no=section_no))
|
||||
section_no += 1
|
||||
|
||||
watch = (env("GITHUB_REPOS") or "").strip()
|
||||
if watch:
|
||||
lines.extend(["---", "", f"## {section_no}、关注仓库 Release", ""])
|
||||
section_no += 1
|
||||
for repo in [r.strip() for r in watch.split(",") if r.strip()]:
|
||||
release = _fetch_latest_release_title(repo)
|
||||
lines.append(f"- **{repo}**:{release or '暂无 release'}")
|
||||
lines.append("")
|
||||
|
||||
lines.extend(["---", "", f"## {section_no}、主题聚类", ""])
|
||||
for theme, examples in themes:
|
||||
lines.append(f"### {theme}")
|
||||
for ex in examples:
|
||||
lines.append(f"- {ex}")
|
||||
lines.append("")
|
||||
|
||||
pick_src = trending[0].get("source", "") if trending else ""
|
||||
pick_name = trending[0].get("title", "") if trending else ""
|
||||
pick_command = (
|
||||
f"npx skills add {pick_src}/{pick_name}"
|
||||
if pick_src and pick_name
|
||||
else "npx skills add vercel-labs/skills/find-skills"
|
||||
)
|
||||
|
||||
lines.extend(["---", "", "## 安装示例", "", "```bash"])
|
||||
for item in trending[:4]:
|
||||
src, name = item.get("source", ""), item.get("title", "")
|
||||
if src and name:
|
||||
lines.append(f"npx skills add {src}/{name}")
|
||||
lines.extend(["```", "", f"*企微短版见 `output/{date_str}.wecom.md`*"])
|
||||
|
||||
markdown = "\n".join(lines)
|
||||
if agent_wecom:
|
||||
gt = group_skills_by_source(trending, limit=wecom_trending, pool_size=skill_pool)
|
||||
gh = group_skills_by_source(hot, limit=wecom_hot, pool_size=skill_pool)
|
||||
wecom_md = replace_wecom_skill_sections(agent_wecom, trending=gt, hot=gh)
|
||||
else:
|
||||
wecom_md = build_wecom_report(
|
||||
date_str=date_str,
|
||||
time_str=time_str,
|
||||
updated=updated,
|
||||
highlights=editorial_highlights
|
||||
or _build_highlights(trending, hot, github_trending, github_emerging, ai_news),
|
||||
theme_line=editorial_theme or _detect_theme_line(feed),
|
||||
ai_news=prepare_wecom_news_items(ai_news),
|
||||
trending=[
|
||||
_prepare_skill_item(item, prev_ids, r)
|
||||
for r, item in enumerate(
|
||||
finalize_wecom_skill_groups(
|
||||
group_skills_by_source(trending, limit=wecom_trending, pool_size=skill_pool)
|
||||
),
|
||||
1,
|
||||
)
|
||||
],
|
||||
hot=[
|
||||
_prepare_skill_item(item, prev_ids, r)
|
||||
for r, item in enumerate(
|
||||
finalize_wecom_skill_groups(
|
||||
group_skills_by_source(hot, limit=wecom_hot, pool_size=skill_pool)
|
||||
),
|
||||
1,
|
||||
)
|
||||
],
|
||||
repos=[_prepare_github_item(item) for item in github_trending[:wecom_github]],
|
||||
emerging=[_prepare_github_item(item) for item in github_emerging[:wecom_emerging]],
|
||||
topic_name=topic_name,
|
||||
topic_repos=[_prepare_github_item(item) for item in github_topic[:wecom_topic]],
|
||||
pick_command=pick_command,
|
||||
)
|
||||
|
||||
_save_snapshot(feed, date_str)
|
||||
OUTPUT_DIR.mkdir(parents=True, exist_ok=True)
|
||||
out_md = OUTPUT_DIR / f"{date_str}.md"
|
||||
out_wecom = OUTPUT_DIR / f"{date_str}.wecom.md"
|
||||
out_md.write_text(markdown, encoding="utf-8")
|
||||
out_wecom.write_text(wecom_md, encoding="utf-8")
|
||||
return markdown, wecom_md, out_md, out_wecom
|
||||
|
||||
|
||||
def main() -> int:
|
||||
LOG_DIR.mkdir(parents=True, exist_ok=True)
|
||||
log_file = LOG_DIR / f"{_now_cst():%Y-%m-%d}.log"
|
||||
try:
|
||||
_, wecom_md, out_md, out_wecom = generate_report()
|
||||
nbytes = len(wecom_md.encode("utf-8"))
|
||||
msg = f"[{_now_cst():%H:%M:%S}] OK -> {out_md}, {out_wecom} ({nbytes} bytes)\n"
|
||||
log_file.write_text(msg, encoding="utf-8")
|
||||
print(msg.strip())
|
||||
return 0
|
||||
except Exception as exc:
|
||||
msg = f"[{_now_cst():%H:%M:%S}] FAIL: {exc}\n"
|
||||
log_file.write_text(msg, encoding="utf-8")
|
||||
print(msg.strip(), file=sys.stderr)
|
||||
return 1
|
||||
9
daily/github/__init__.py
Normal file
9
daily/github/__init__.py
Normal file
@@ -0,0 +1,9 @@
|
||||
from daily.github.search import fetch_emerging_repos, fetch_topic_hot_repos
|
||||
from daily.github.trending import fetch_github_trending, trending_data_source_note
|
||||
|
||||
__all__ = [
|
||||
"fetch_emerging_repos",
|
||||
"fetch_topic_hot_repos",
|
||||
"fetch_github_trending",
|
||||
"trending_data_source_note",
|
||||
]
|
||||
81
daily/github/auth.py
Normal file
81
daily/github/auth.py
Normal file
@@ -0,0 +1,81 @@
|
||||
"""GitHub 请求共用:GITHUB_TOKEN、请求头、仓库 API。"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
from typing import Any
|
||||
|
||||
import certifi
|
||||
import httpx
|
||||
|
||||
from daily.config import env
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
DEFAULT_USER_AGENT = (
|
||||
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 "
|
||||
"(KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36"
|
||||
)
|
||||
|
||||
|
||||
def github_token() -> str | None:
|
||||
raw = (env("GITHUB_TOKEN") or "").strip()
|
||||
return raw or None
|
||||
|
||||
|
||||
def github_api_headers() -> dict[str, str]:
|
||||
headers = {
|
||||
"User-Agent": env("GITHUB_TRENDING_USER_AGENT", DEFAULT_USER_AGENT),
|
||||
"Accept": "application/vnd.github+json",
|
||||
"X-GitHub-Api-Version": "2022-11-28",
|
||||
}
|
||||
token = github_token()
|
||||
if token:
|
||||
headers["Authorization"] = f"Bearer {token}"
|
||||
return headers
|
||||
|
||||
|
||||
def github_html_headers() -> dict[str, str]:
|
||||
headers = {
|
||||
"User-Agent": env("GITHUB_TRENDING_USER_AGENT", DEFAULT_USER_AGENT),
|
||||
"Accept": "text/html,application/xhtml+xml",
|
||||
"Accept-Language": "en-US,en;q=0.9",
|
||||
}
|
||||
token = github_token()
|
||||
if token:
|
||||
headers["Authorization"] = f"Bearer {token}"
|
||||
return headers
|
||||
|
||||
|
||||
def fetch_repo_api(repo: str) -> dict[str, Any] | None:
|
||||
url = f"https://api.github.com/repos/{repo}"
|
||||
try:
|
||||
with httpx.Client(
|
||||
timeout=12.0,
|
||||
verify=certifi.where(),
|
||||
headers=github_api_headers(),
|
||||
) as client:
|
||||
resp = client.get(url)
|
||||
if resp.status_code != 200:
|
||||
return None
|
||||
data = resp.json()
|
||||
return {
|
||||
"description": data.get("description") or "",
|
||||
"language": data.get("language") or "",
|
||||
"stars": data.get("stargazers_count", 0),
|
||||
}
|
||||
except Exception as exc:
|
||||
logger.debug("GitHub API repo %s 失败: %s", repo, exc)
|
||||
return None
|
||||
|
||||
|
||||
def format_star_count(value: int | float | str) -> str:
|
||||
try:
|
||||
n = int(value)
|
||||
except (TypeError, ValueError):
|
||||
return str(value)
|
||||
if n >= 1_000_000:
|
||||
return f"{n / 1_000_000:.1f}M".replace(".0M", "M")
|
||||
if n >= 1_000:
|
||||
return f"{n / 1_000:.1f}K".replace(".0K", "K")
|
||||
return f"{n:,}"
|
||||
121
daily/github/search.py
Normal file
121
daily/github/search.py
Normal file
@@ -0,0 +1,121 @@
|
||||
"""GitHub Search API:新兴项目、Topic 热点。"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
from datetime import datetime, timedelta, timezone
|
||||
from typing import Any
|
||||
|
||||
import certifi
|
||||
import httpx
|
||||
|
||||
from daily.config import env, env_int
|
||||
from daily.github.auth import format_star_count, github_api_headers, github_token
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def _repo_from_api_item(item: dict[str, Any], *, source: str) -> dict[str, Any]:
|
||||
full_name = item.get("full_name") or ""
|
||||
stars = item.get("stargazers_count", 0)
|
||||
created = (item.get("created_at") or "")[:10]
|
||||
return {
|
||||
"repo": full_name,
|
||||
"url": item.get("html_url") or f"https://github.com/{full_name}",
|
||||
"description": item.get("description") or "",
|
||||
"language": item.get("language") or "",
|
||||
"stars_today": None,
|
||||
"stars_today_fmt": "",
|
||||
"total_stars_fmt": format_star_count(stars),
|
||||
"created_at": created,
|
||||
"source": source,
|
||||
}
|
||||
|
||||
|
||||
def search_github_repos(
|
||||
query: str,
|
||||
limit: int,
|
||||
*,
|
||||
sort: str = "stars",
|
||||
require_token: bool = True,
|
||||
) -> list[dict[str, Any]]:
|
||||
if require_token and not github_token():
|
||||
logger.warning("GitHub Search 需要 GITHUB_TOKEN: %s", query[:80])
|
||||
return []
|
||||
|
||||
try:
|
||||
with httpx.Client(
|
||||
timeout=20.0,
|
||||
verify=certifi.where(),
|
||||
headers=github_api_headers(),
|
||||
) as client:
|
||||
resp = client.get(
|
||||
"https://api.github.com/search/repositories",
|
||||
params={
|
||||
"q": query,
|
||||
"sort": sort,
|
||||
"order": "desc",
|
||||
"per_page": min(max(limit, 1), 30),
|
||||
},
|
||||
)
|
||||
if resp.status_code != 200:
|
||||
logger.warning("GitHub Search 失败 (%s): %s", resp.status_code, query[:80])
|
||||
return []
|
||||
items = resp.json().get("items") or []
|
||||
except Exception as exc:
|
||||
logger.warning("GitHub Search 异常: %s", exc)
|
||||
return []
|
||||
|
||||
repos: list[dict[str, Any]] = []
|
||||
for item in items:
|
||||
full_name = item.get("full_name") or ""
|
||||
if not full_name:
|
||||
continue
|
||||
repos.append(_repo_from_api_item(item, source="api-search"))
|
||||
if len(repos) >= limit:
|
||||
break
|
||||
return repos
|
||||
|
||||
|
||||
def _date_days_ago(days: int) -> str:
|
||||
return (datetime.now(timezone.utc) - timedelta(days=days)).strftime("%Y-%m-%d")
|
||||
|
||||
|
||||
def fetch_emerging_repos(
|
||||
limit: int = 3,
|
||||
*,
|
||||
days: int | None = None,
|
||||
min_stars: int | None = None,
|
||||
exclude: set[str] | None = None,
|
||||
) -> list[dict[str, Any]]:
|
||||
days = days if days is not None else env_int("GITHUB_EMERGING_DAYS", 14)
|
||||
min_stars = min_stars if min_stars is not None else env_int("GITHUB_EMERGING_MIN_STARS", 200)
|
||||
created_after = _date_days_ago(days)
|
||||
query = f"created:>{created_after} stars:>{min_stars} fork:false"
|
||||
repos = search_github_repos(query, limit + len(exclude or set()))
|
||||
if exclude:
|
||||
repos = [r for r in repos if r["repo"] not in exclude]
|
||||
for item in repos:
|
||||
item["source"] = "api-emerging"
|
||||
return repos[:limit]
|
||||
|
||||
|
||||
def fetch_topic_hot_repos(
|
||||
limit: int = 3,
|
||||
*,
|
||||
topic: str | None = None,
|
||||
pushed_days: int | None = None,
|
||||
min_stars: int | None = None,
|
||||
exclude: set[str] | None = None,
|
||||
) -> tuple[str, list[dict[str, Any]]]:
|
||||
topic = (topic or env("GITHUB_TOPIC") or "llm").strip()
|
||||
pushed_days = pushed_days if pushed_days is not None else env_int("GITHUB_TOPIC_PUSHED_DAYS", 7)
|
||||
min_stars = min_stars if min_stars is not None else env_int("GITHUB_TOPIC_MIN_STARS", 50)
|
||||
pushed_after = _date_days_ago(pushed_days)
|
||||
query = f"topic:{topic} pushed:>{pushed_after} stars:>{min_stars} fork:false"
|
||||
repos = search_github_repos(query, limit + len(exclude or set()))
|
||||
if exclude:
|
||||
repos = [r for r in repos if r["repo"] not in exclude]
|
||||
for item in repos:
|
||||
item["source"] = "api-topic"
|
||||
return topic, repos[:limit]
|
||||
206
daily/github/trending.py
Normal file
206
daily/github/trending.py
Normal file
@@ -0,0 +1,206 @@
|
||||
"""GitHub Trending:页面爬取或 Search API。"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
import re
|
||||
from datetime import datetime, timedelta, timezone
|
||||
from html import unescape
|
||||
from typing import Any, Literal
|
||||
from urllib.parse import urlencode
|
||||
|
||||
import certifi
|
||||
import httpx
|
||||
|
||||
from daily.config import env
|
||||
from daily.github.auth import (
|
||||
fetch_repo_api,
|
||||
format_star_count,
|
||||
github_html_headers,
|
||||
github_token,
|
||||
)
|
||||
from daily.github.search import search_github_repos
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
TrendingSince = Literal["daily", "weekly", "monthly"]
|
||||
TrendingMode = Literal["scrape", "api"]
|
||||
|
||||
DEFAULT_LIMIT = 5
|
||||
DEFAULT_SINCE: TrendingSince = "daily"
|
||||
DEFAULT_MODE: TrendingMode = "scrape"
|
||||
TRENDING_URL = "https://github.com/trending"
|
||||
|
||||
_ARTICLE_RE = re.compile(r'<article class="Box-row">.*?</article>', re.S)
|
||||
_REPO_HREF_RE = re.compile(r'h2[^>]*>\s*<a[^>]+href="([^"]+)"')
|
||||
_DESC_RE = re.compile(r'<p class="col-9[^"]*"[^>]*>([^<]*)</p>')
|
||||
_STARS_TODAY_RE = re.compile(r"([\d,]+)\s+stars?\s+today", re.I)
|
||||
_LANG_RE = re.compile(r'itemprop="programmingLanguage"[^>]*>([^<]+)<')
|
||||
_TOTAL_STARS_RE = re.compile(
|
||||
r'href="/[^/]+/[^/]+/stargazers"[^>]*>\s*<svg[^>]*octicon-star[^>]*>.*?</svg>\s*([\d.,kKmM]+)',
|
||||
re.S,
|
||||
)
|
||||
|
||||
|
||||
def _strip_html(text: str) -> str:
|
||||
return unescape(re.sub(r"\s+", " ", text or "")).strip()
|
||||
|
||||
|
||||
def trending_mode() -> TrendingMode:
|
||||
raw = (env("GITHUB_TRENDING_MODE") or DEFAULT_MODE).strip().lower()
|
||||
if raw in {"api", "token", "search"}:
|
||||
return "api"
|
||||
return "scrape"
|
||||
|
||||
|
||||
def trending_data_source_note() -> str:
|
||||
if trending_mode() == "api":
|
||||
return "> 数据来源:GitHub Search API(`GITHUB_TRENDING_MODE=api`,需 `GITHUB_TOKEN`)"
|
||||
return "> 数据来源:[github.com/trending](https://github.com/trending?since=daily)(页面爬取,失败时 API 降级)"
|
||||
|
||||
|
||||
def _parse_article(article_html: str) -> dict[str, Any] | None:
|
||||
href_match = _REPO_HREF_RE.search(article_html)
|
||||
if not href_match:
|
||||
return None
|
||||
href = href_match.group(1).strip("/")
|
||||
if href.count("/") != 1:
|
||||
return None
|
||||
owner, name = href.split("/", 1)
|
||||
repo = f"{owner}/{name}"
|
||||
desc_match = _DESC_RE.search(article_html)
|
||||
stars_today_match = _STARS_TODAY_RE.search(article_html)
|
||||
lang_match = _LANG_RE.search(article_html)
|
||||
total_stars_match = _TOTAL_STARS_RE.search(article_html)
|
||||
stars_today_raw = stars_today_match.group(1).replace(",", "") if stars_today_match else ""
|
||||
stars_today = int(stars_today_raw) if stars_today_raw.isdigit() else None
|
||||
return {
|
||||
"repo": repo,
|
||||
"url": f"https://github.com/{repo}",
|
||||
"description": _strip_html(desc_match.group(1)) if desc_match else "",
|
||||
"language": _strip_html(lang_match.group(1)) if lang_match else "",
|
||||
"stars_today": stars_today,
|
||||
"stars_today_fmt": stars_today_match.group(1) if stars_today_match else "",
|
||||
"total_stars_fmt": _strip_html(total_stars_match.group(1)) if total_stars_match else "",
|
||||
"source": "scrape",
|
||||
}
|
||||
|
||||
|
||||
def _build_trending_url(*, since: TrendingSince = DEFAULT_SINCE, language: str = "") -> str:
|
||||
if language:
|
||||
return f"{TRENDING_URL}/{language}?{urlencode({'since': since})}"
|
||||
return f"{TRENDING_URL}?{urlencode({'since': since})}"
|
||||
|
||||
|
||||
def _since_push_date(since: TrendingSince) -> str:
|
||||
now = datetime.now(timezone.utc)
|
||||
if since == "weekly":
|
||||
delta = timedelta(days=7)
|
||||
elif since == "monthly":
|
||||
delta = timedelta(days=30)
|
||||
else:
|
||||
delta = timedelta(days=1)
|
||||
return (now - delta).strftime("%Y-%m-%d")
|
||||
|
||||
|
||||
def _enrich_repo_from_api(item: dict[str, Any]) -> dict[str, Any]:
|
||||
if not github_token() and env("GITHUB_API_ENRICH", "1") != "1":
|
||||
return item
|
||||
meta = fetch_repo_api(item["repo"])
|
||||
if not meta:
|
||||
return item
|
||||
enriched = dict(item)
|
||||
if not enriched.get("description"):
|
||||
enriched["description"] = meta["description"]
|
||||
if not enriched.get("language"):
|
||||
enriched["language"] = meta["language"]
|
||||
if not enriched.get("total_stars_fmt") and meta["stars"]:
|
||||
enriched["total_stars_fmt"] = format_star_count(meta["stars"])
|
||||
enriched["source"] = enriched.get("source", "scrape") + "+api"
|
||||
return enriched
|
||||
|
||||
|
||||
def _fetch_trending_html(url: str) -> str | None:
|
||||
try:
|
||||
with httpx.Client(
|
||||
timeout=20.0,
|
||||
verify=certifi.where(),
|
||||
follow_redirects=True,
|
||||
headers=github_html_headers(),
|
||||
) as client:
|
||||
resp = client.get(url)
|
||||
if resp.status_code in {403, 429} and not github_token():
|
||||
logger.warning("GitHub Trending %s(匿名可能被限),可配置 GITHUB_TOKEN", resp.status_code)
|
||||
resp.raise_for_status()
|
||||
return resp.text
|
||||
except Exception as exc:
|
||||
logger.warning("GitHub Trending 页面抓取失败: %s", exc)
|
||||
return None
|
||||
|
||||
|
||||
def _parse_trending_html(html: str, limit: int) -> list[dict[str, Any]]:
|
||||
repos: list[dict[str, Any]] = []
|
||||
for article_html in _ARTICLE_RE.findall(html):
|
||||
item = _parse_article(article_html)
|
||||
if item:
|
||||
repos.append(_enrich_repo_from_api(item))
|
||||
if len(repos) >= limit:
|
||||
break
|
||||
return repos
|
||||
|
||||
|
||||
def _fetch_trending_via_api(limit: int, since: TrendingSince, language: str) -> list[dict[str, Any]]:
|
||||
pushed_after = _since_push_date(since)
|
||||
parts = [f"pushed:>{pushed_after}", "stars:>50", "fork:false"]
|
||||
if language:
|
||||
parts.append(f"language:{language}")
|
||||
query = " ".join(parts)
|
||||
repos = search_github_repos(query, limit, require_token=True)
|
||||
for item in repos:
|
||||
item["source"] = "api-search"
|
||||
return repos
|
||||
|
||||
|
||||
def fetch_github_trending(
|
||||
limit: int = DEFAULT_LIMIT,
|
||||
*,
|
||||
since: TrendingSince | None = None,
|
||||
language: str = "",
|
||||
) -> list[dict[str, Any]]:
|
||||
since = since or env("GITHUB_TRENDING_SINCE", DEFAULT_SINCE) # type: ignore[assignment]
|
||||
if since not in ("daily", "weekly", "monthly"):
|
||||
since = DEFAULT_SINCE
|
||||
lang = (language or env("GITHUB_TRENDING_LANGUAGE") or "").strip()
|
||||
mode = trending_mode()
|
||||
|
||||
if mode == "api":
|
||||
repos = _fetch_trending_via_api(limit, since, lang)
|
||||
if not repos and not github_token():
|
||||
logger.warning("GITHUB_TRENDING_MODE=api 需要配置 GITHUB_TOKEN")
|
||||
elif repos:
|
||||
logger.info("GitHub Trending 使用 API 模式,共 %d 条", len(repos))
|
||||
return repos[:limit]
|
||||
|
||||
url = _build_trending_url(since=since, language=lang)
|
||||
html = _fetch_trending_html(url)
|
||||
repos: list[dict[str, Any]] = []
|
||||
if html:
|
||||
repos = _parse_trending_html(html, limit)
|
||||
if not repos:
|
||||
logger.warning("GitHub Trending 页面解析为空: %s", url)
|
||||
|
||||
if len(repos) < limit:
|
||||
before = len(repos)
|
||||
fallback = _fetch_trending_via_api(limit, since, lang)
|
||||
seen = {r["repo"] for r in repos}
|
||||
for item in fallback:
|
||||
if item["repo"] in seen:
|
||||
continue
|
||||
repos.append(item)
|
||||
seen.add(item["repo"])
|
||||
if len(repos) >= limit:
|
||||
break
|
||||
if len(repos) > before:
|
||||
logger.info("已用 GitHub API 补充 %d 条 Trending 数据", len(repos) - before)
|
||||
|
||||
return repos[:limit]
|
||||
119
daily/llm_client.py
Normal file
119
daily/llm_client.py
Normal file
@@ -0,0 +1,119 @@
|
||||
"""LLM 调用共享工具(OpenAI 兼容 API / Cursor SDK)。"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import logging
|
||||
import os
|
||||
import re
|
||||
from typing import Any
|
||||
|
||||
import certifi
|
||||
import httpx
|
||||
|
||||
from daily.config import ROOT, env, env_int
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
_JSON_BLOCK = re.compile(r"```(?:json)?\s*([\s\S]*?)```", re.IGNORECASE)
|
||||
|
||||
|
||||
def extract_json_object(text: str) -> dict[str, Any]:
|
||||
text = text.strip()
|
||||
if not text:
|
||||
return {}
|
||||
try:
|
||||
data = json.loads(text)
|
||||
return data if isinstance(data, dict) else {}
|
||||
except json.JSONDecodeError:
|
||||
pass
|
||||
match = _JSON_BLOCK.search(text)
|
||||
if match:
|
||||
try:
|
||||
data = json.loads(match.group(1).strip())
|
||||
return data if isinstance(data, dict) else {}
|
||||
except json.JSONDecodeError:
|
||||
pass
|
||||
start, end = text.find("{"), text.rfind("}")
|
||||
if start >= 0 and end > start:
|
||||
try:
|
||||
data = json.loads(text[start : end + 1])
|
||||
return data if isinstance(data, dict) else {}
|
||||
except json.JSONDecodeError:
|
||||
pass
|
||||
return {}
|
||||
|
||||
|
||||
def _openai_chat(system: str, user: str) -> str:
|
||||
api_key = (env("DAILY_LLM_API_KEY") or env("OPENAI_API_KEY") or "").strip()
|
||||
if not api_key:
|
||||
return ""
|
||||
base = (env("DAILY_LLM_API_BASE") or env("OPENAI_API_BASE") or "https://api.openai.com/v1").rstrip("/")
|
||||
model = env("DAILY_LLM_MODEL") or env("OPENAI_MODEL") or "gpt-4o-mini"
|
||||
timeout = env_int("DAILY_LLM_TIMEOUT", 120)
|
||||
payload = {
|
||||
"model": model,
|
||||
"temperature": 0.2,
|
||||
"messages": [
|
||||
{"role": "system", "content": system},
|
||||
{"role": "user", "content": user},
|
||||
],
|
||||
}
|
||||
with httpx.Client(timeout=timeout, verify=certifi.where()) as client:
|
||||
resp = client.post(
|
||||
f"{base}/chat/completions",
|
||||
headers={"Authorization": f"Bearer {api_key}", "Content-Type": "application/json"},
|
||||
json=payload,
|
||||
)
|
||||
resp.raise_for_status()
|
||||
data = resp.json()
|
||||
return str(data["choices"][0]["message"]["content"] or "").strip()
|
||||
|
||||
|
||||
def _cursor_chat(system: str, user: str) -> str:
|
||||
api_key = (env("CURSOR_API_KEY") or "").strip()
|
||||
if not api_key:
|
||||
return ""
|
||||
from cursor_sdk import Agent, AgentOptions, CursorAgentError, LocalAgentOptions
|
||||
|
||||
from daily.config import ensure_bot_on_path
|
||||
|
||||
ensure_bot_on_path()
|
||||
try:
|
||||
from bridge_manager import warm_cursor_bridge
|
||||
except ImportError:
|
||||
warm_cursor_bridge = lambda: None # noqa: E731
|
||||
|
||||
cwd = env("DAILY_CURSOR_CWD") or str(ROOT)
|
||||
# bridge_manager 读 bot env_config 的 CURSOR_CWD,早报侧须先对齐工作目录
|
||||
os.environ["CURSOR_CWD"] = cwd
|
||||
warm_cursor_bridge()
|
||||
model = env("CURSOR_MODEL") or "composer-2.5"
|
||||
prompt = f"{system}\n\n{user}"
|
||||
try:
|
||||
result = Agent.prompt(
|
||||
prompt,
|
||||
AgentOptions(
|
||||
api_key=api_key,
|
||||
model=model,
|
||||
local=LocalAgentOptions(cwd=cwd),
|
||||
),
|
||||
)
|
||||
except CursorAgentError as exc:
|
||||
raise RuntimeError(f"LLM 调用失败:{exc.message}") from exc
|
||||
if result.status == "error":
|
||||
raise RuntimeError(f"LLM 调用失败:{result.result or '未知错误'}")
|
||||
return (result.result or "").strip()
|
||||
|
||||
|
||||
def llm_chat(system: str, user: str) -> str:
|
||||
"""优先 OpenAI 兼容 API,否则 Cursor SDK。"""
|
||||
if env("DAILY_LLM_API_KEY") or env("OPENAI_API_KEY"):
|
||||
return _openai_chat(system, user)
|
||||
if env("CURSOR_API_KEY"):
|
||||
return _cursor_chat(system, user)
|
||||
return ""
|
||||
|
||||
|
||||
def has_llm_configured() -> bool:
|
||||
return bool(env("DAILY_LLM_API_KEY") or env("OPENAI_API_KEY") or env("CURSOR_API_KEY"))
|
||||
236
daily/localize.py
Normal file
236
daily/localize.py
Normal file
@@ -0,0 +1,236 @@
|
||||
"""将英文描述批量改写为简短中文(大模型 + 本地缓存)。"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import hashlib
|
||||
import json
|
||||
import logging
|
||||
from dataclasses import dataclass
|
||||
|
||||
from daily.config import CACHE_DIR, env, env_int
|
||||
from daily.cursor_editor import is_enabled as cursor_editor_enabled
|
||||
from daily.llm_client import extract_json_object, llm_chat
|
||||
from daily.text_utils import clip_text, trim_brief
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
_CACHE_FILE = CACHE_DIR / "zh-desc-cache.json"
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class LocalizeJob:
|
||||
key: str
|
||||
text: str
|
||||
limit: int
|
||||
|
||||
|
||||
def _enabled(*, archive: bool = False) -> bool:
|
||||
if not archive and cursor_editor_enabled():
|
||||
return False
|
||||
raw = (env("DAILY_ZH_DESC") or "1").strip().lower()
|
||||
return raw not in {"0", "false", "no", "off"}
|
||||
|
||||
|
||||
def _is_mostly_chinese(text: str) -> bool:
|
||||
text = text.strip()
|
||||
if not text:
|
||||
return True
|
||||
cjk = sum(1 for c in text if "\u4e00" <= c <= "\u9fff")
|
||||
latin = sum(1 for c in text if c.isascii() and c.isalpha())
|
||||
return cjk >= max(latin, 1)
|
||||
|
||||
|
||||
def needs_chinese(text: str) -> bool:
|
||||
"""文本非空且尚未以中文为主。"""
|
||||
text = (text or "").strip()
|
||||
if not text:
|
||||
return False
|
||||
return not _is_mostly_chinese(text)
|
||||
|
||||
|
||||
def _cache_key(text: str) -> str:
|
||||
return hashlib.sha1(text.encode("utf-8")).hexdigest()[:16]
|
||||
|
||||
|
||||
def _legacy_cache_key(text: str, limit: int) -> str:
|
||||
return hashlib.sha1(f"{limit}:{text}".encode("utf-8")).hexdigest()[:16]
|
||||
|
||||
|
||||
def _cache_is_truncated(text: str) -> bool:
|
||||
t = (text or "").rstrip()
|
||||
return t.endswith("…") or t.endswith("...")
|
||||
|
||||
|
||||
def _lookup_cached(cache: dict[str, str], text: str, limit: int) -> str | None:
|
||||
hit = cache.get(_cache_key(text))
|
||||
if hit and not (limit <= 0 and _cache_is_truncated(hit)):
|
||||
return hit
|
||||
for legacy_limit in (72, 120, 200):
|
||||
legacy = cache.get(_legacy_cache_key(text, legacy_limit))
|
||||
if legacy and not (limit <= 0 and _cache_is_truncated(legacy)):
|
||||
cache[_cache_key(text)] = legacy
|
||||
return legacy
|
||||
return None
|
||||
|
||||
|
||||
def _load_cache() -> dict[str, str]:
|
||||
if not _CACHE_FILE.exists():
|
||||
return {}
|
||||
try:
|
||||
data = json.loads(_CACHE_FILE.read_text(encoding="utf-8"))
|
||||
return {str(k): str(v) for k, v in (data.get("entries") or {}).items()}
|
||||
except (OSError, json.JSONDecodeError, TypeError):
|
||||
return {}
|
||||
|
||||
|
||||
def _save_cache(entries: dict[str, str]) -> None:
|
||||
CACHE_DIR.mkdir(parents=True, exist_ok=True)
|
||||
_CACHE_FILE.write_text(
|
||||
json.dumps({"entries": entries}, ensure_ascii=False, indent=2),
|
||||
encoding="utf-8",
|
||||
)
|
||||
|
||||
|
||||
def _build_prompt(jobs: list[LocalizeJob], *, brief: bool = False) -> tuple[str, str]:
|
||||
if brief:
|
||||
system = (
|
||||
"你是技术早报编辑。把输入 JSON 中每条描述改写为**一句**中文简要介绍。"
|
||||
"要求:输出必须是中文;只保留核心能力与典型场景;不要逐字翻译;不要加引号或编号;"
|
||||
"每条必须语义完整、可独立阅读;limit 为建议最大字数,请控制在 limit 以内且不要用省略号截断;"
|
||||
"已是中文且足够简短时可适度精简;"
|
||||
"只输出 JSON 对象,key 与输入一致,value 为中文简介字符串。"
|
||||
)
|
||||
else:
|
||||
system = (
|
||||
"你是技术早报编辑。把输入 JSON 中每条英文描述改写为**完整**中文介绍。"
|
||||
"要求:输出必须是中文;保留关键能力与使用场景;不要逐字翻译;不要加引号或编号;"
|
||||
"不要以省略号截断;已是中文则原样或适度精简;"
|
||||
"只输出 JSON 对象,key 与输入一致,value 为中文简介字符串。"
|
||||
)
|
||||
payload = {job.key: {"text": job.text, "limit": job.limit} for job in jobs}
|
||||
user = json.dumps(payload, ensure_ascii=False, indent=2)
|
||||
return system, user
|
||||
|
||||
|
||||
def _brief_cache_key(text: str, limit: int) -> str:
|
||||
return hashlib.sha1(f"brief:{limit}:{text}".encode("utf-8")).hexdigest()[:16]
|
||||
|
||||
|
||||
def _lookup_brief_cached(cache: dict[str, str], text: str, limit: int) -> str | None:
|
||||
hit = cache.get(_brief_cache_key(text, limit))
|
||||
if hit and not _cache_is_truncated(hit):
|
||||
return hit
|
||||
return None
|
||||
|
||||
|
||||
def _translate_batch(jobs: list[LocalizeJob], *, brief: bool = False) -> dict[str, str]:
|
||||
if not jobs:
|
||||
return {}
|
||||
system, user = _build_prompt(jobs, brief=brief)
|
||||
raw = llm_chat(system, user)
|
||||
if not raw:
|
||||
return {}
|
||||
parsed = extract_json_object(raw)
|
||||
out: dict[str, str] = {}
|
||||
for job in jobs:
|
||||
value = parsed.get(job.key)
|
||||
if isinstance(value, str) and value.strip():
|
||||
trimmed = value.strip()
|
||||
if brief and job.limit > 0:
|
||||
out[job.key] = trim_brief(trimmed, job.limit)
|
||||
else:
|
||||
out[job.key] = clip_text(trimmed, job.limit) if job.limit > 0 else trimmed
|
||||
return out
|
||||
|
||||
|
||||
def localize_brief_descriptions(jobs: list[LocalizeJob], *, archive: bool = False) -> dict[str, str]:
|
||||
"""企微用:将描述改写为一句简要中文。"""
|
||||
if not _enabled(archive=archive) or not jobs:
|
||||
return {}
|
||||
|
||||
cache = _load_cache()
|
||||
result: dict[str, str] = {}
|
||||
pending: list[LocalizeJob] = []
|
||||
|
||||
for job in jobs:
|
||||
if not job.text.strip():
|
||||
continue
|
||||
limit = job.limit if job.limit > 0 else 48
|
||||
text = job.text.strip()
|
||||
if _is_mostly_chinese(text) and (limit <= 0 or len(text) <= limit):
|
||||
result[job.key] = text
|
||||
continue
|
||||
cached = _lookup_brief_cached(cache, text, limit)
|
||||
if cached:
|
||||
result[job.key] = trim_brief(cached, limit) if limit > 0 else cached
|
||||
else:
|
||||
pending.append(LocalizeJob(job.key, text, limit))
|
||||
|
||||
if not pending:
|
||||
return result
|
||||
|
||||
batch_size = max(3, min(10, env_int("DAILY_ZH_DESC_BATCH", 20)))
|
||||
for i in range(0, len(pending), batch_size):
|
||||
chunk = pending[i : i + batch_size]
|
||||
try:
|
||||
translated = _translate_batch(chunk, brief=True)
|
||||
except Exception as exc:
|
||||
logger.warning("企微简要摘要批次失败,保留原文:%s", exc)
|
||||
continue
|
||||
for job in chunk:
|
||||
zh = translated.get(job.key)
|
||||
if not zh:
|
||||
continue
|
||||
ck = _brief_cache_key(job.text, job.limit if job.limit > 0 else 48)
|
||||
cache[ck] = zh
|
||||
result[job.key] = zh
|
||||
|
||||
if cache:
|
||||
_save_cache(cache)
|
||||
return result
|
||||
|
||||
|
||||
def localize_descriptions(jobs: list[LocalizeJob], *, archive: bool = False) -> dict[str, str]:
|
||||
"""返回 job.key -> 中文简介。archive=True 时用于完整版 .md,不受 Cursor 编辑层开关影响。"""
|
||||
if not _enabled(archive=archive) or not jobs:
|
||||
return {}
|
||||
|
||||
cache = _load_cache()
|
||||
result: dict[str, str] = {}
|
||||
pending: list[LocalizeJob] = []
|
||||
|
||||
for job in jobs:
|
||||
if not job.text.strip():
|
||||
continue
|
||||
if _is_mostly_chinese(job.text):
|
||||
result[job.key] = clip_text(job.text, job.limit)
|
||||
continue
|
||||
ck = _cache_key(job.text)
|
||||
cached = _lookup_cached(cache, job.text, job.limit)
|
||||
if cached:
|
||||
result[job.key] = clip_text(cached, job.limit)
|
||||
else:
|
||||
pending.append(job)
|
||||
|
||||
if not pending:
|
||||
return result
|
||||
|
||||
batch_size = max(5, env_int("DAILY_ZH_DESC_BATCH", 20))
|
||||
for i in range(0, len(pending), batch_size):
|
||||
chunk = pending[i : i + batch_size]
|
||||
try:
|
||||
translated = _translate_batch(chunk, brief=False)
|
||||
except Exception as exc:
|
||||
logger.warning("中文摘要批次失败,保留英文:%s", exc)
|
||||
continue
|
||||
for job in chunk:
|
||||
zh = translated.get(job.key)
|
||||
if not zh:
|
||||
continue
|
||||
ck = _cache_key(job.text)
|
||||
cache[ck] = zh
|
||||
result[job.key] = clip_text(zh, job.limit)
|
||||
|
||||
if cache:
|
||||
_save_cache(cache)
|
||||
return result
|
||||
3
daily/news/__init__.py
Normal file
3
daily/news/__init__.py
Normal file
@@ -0,0 +1,3 @@
|
||||
from daily.news.fetch import fetch_ai_news, format_news_section
|
||||
|
||||
__all__ = ["fetch_ai_news", "format_news_section"]
|
||||
124
daily/news/feeds.py
Normal file
124
daily/news/feeds.py
Normal file
@@ -0,0 +1,124 @@
|
||||
"""国际 AI 时讯 RSS 源定义(按类别分组)。"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from dataclasses import dataclass
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class NewsFeed:
|
||||
name: str
|
||||
url: str
|
||||
slow: bool = False # 限速源(如 Reddit)串行抓取
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class NewsCategory:
|
||||
id: str
|
||||
name: str
|
||||
icon: str
|
||||
feeds: tuple[NewsFeed, ...]
|
||||
|
||||
|
||||
NEWS_CATEGORIES: tuple[NewsCategory, ...] = (
|
||||
NewsCategory(
|
||||
id="official",
|
||||
name="厂商官方",
|
||||
icon="🏢",
|
||||
feeds=(
|
||||
NewsFeed("Anthropic Claude 更新", "https://docs.anthropic.com/en/release-notes/feed"),
|
||||
NewsFeed("OpenAI", "https://openai.com/news/rss.xml"),
|
||||
NewsFeed("Google AI", "https://blog.google/technology/ai/rss/"),
|
||||
NewsFeed("DeepMind", "https://deepmind.google/blog/rss.xml"),
|
||||
NewsFeed("Meta Engineering", "https://engineering.fb.com/feed/"),
|
||||
NewsFeed("Microsoft Research", "https://www.microsoft.com/en-us/research/feed/"),
|
||||
NewsFeed("Microsoft Blog", "https://blogs.microsoft.com/feed/"),
|
||||
NewsFeed("Cohere", "https://cohere.com/blog/rss.xml"),
|
||||
NewsFeed("Cursor Changelog", "https://cursor.com/changelog/rss.xml"),
|
||||
),
|
||||
),
|
||||
NewsCategory(
|
||||
id="developer",
|
||||
name="Agent / LLM 开发者",
|
||||
icon="🛠",
|
||||
feeds=(
|
||||
NewsFeed("LangChain", "https://blog.langchain.dev/rss/"),
|
||||
NewsFeed("Hugging Face", "https://huggingface.co/blog/feed.xml"),
|
||||
NewsFeed("Vercel Changelog", "https://vercel.com/changelog/rss.xml"),
|
||||
NewsFeed("GitHub Copilot", "https://github.blog/changelog/label/copilot/feed/"),
|
||||
),
|
||||
),
|
||||
NewsCategory(
|
||||
id="media",
|
||||
name="综合科技媒体",
|
||||
icon="📰",
|
||||
feeds=(
|
||||
NewsFeed("The Verge AI", "https://www.theverge.com/rss/ai-artificial-intelligence/index.xml"),
|
||||
NewsFeed("TechCrunch AI", "https://techcrunch.com/category/artificial-intelligence/feed/"),
|
||||
NewsFeed("Ars Technica AI", "https://arstechnica.com/ai/feed/"),
|
||||
NewsFeed("Wired AI", "https://www.wired.com/feed/tag/ai/latest/rss"),
|
||||
NewsFeed("MIT Tech Review", "https://www.technologyreview.com/feed/"),
|
||||
NewsFeed("VentureBeat AI", "https://venturebeat.com/category/ai/feed/"),
|
||||
),
|
||||
),
|
||||
NewsCategory(
|
||||
id="newsletter",
|
||||
name="Newsletter 日报",
|
||||
icon="✉️",
|
||||
feeds=(
|
||||
NewsFeed("Ben's Bites", "https://bensbites.substack.com/feed"),
|
||||
NewsFeed("The Rundown AI", "https://therundown.substack.com/feed"),
|
||||
NewsFeed("Latent Space", "https://www.latent.space/feed"),
|
||||
NewsFeed("Simon Willison", "https://simonwillison.net/atom/everything/"),
|
||||
NewsFeed("Import AI", "https://importai.substack.com/feed"),
|
||||
NewsFeed("Last Week in AI", "https://lastweekin.ai/feed"),
|
||||
NewsFeed("The Neuron", "https://www.theneuron.ai/feed"),
|
||||
),
|
||||
),
|
||||
NewsCategory(
|
||||
id="research",
|
||||
name="研究 / 论文",
|
||||
icon="📚",
|
||||
feeds=(
|
||||
NewsFeed("arXiv cs.CL", "https://arxiv.org/rss/cs.CL"),
|
||||
NewsFeed("arXiv cs.AI", "https://arxiv.org/rss/cs.AI"),
|
||||
NewsFeed("arXiv cs.LG", "https://arxiv.org/rss/cs.LG"),
|
||||
),
|
||||
),
|
||||
NewsCategory(
|
||||
id="trending",
|
||||
name="热点 / 趋势",
|
||||
icon="🔥",
|
||||
feeds=(
|
||||
NewsFeed(
|
||||
"Google News · AI",
|
||||
"https://news.google.com/rss/search?q=artificial+intelligence+OR+LLM+OR+Claude+OR+GPT&hl=en-US&gl=US&ceid=US:en",
|
||||
),
|
||||
NewsFeed(
|
||||
"Google News · Technology",
|
||||
"https://news.google.com/rss/headlines/section/topic/TECHNOLOGY?hl=en-US&gl=US&ceid=US:en",
|
||||
),
|
||||
NewsFeed("Techmeme", "https://www.techmeme.com/feed.xml"),
|
||||
NewsFeed("HN · Front Page", "https://hnrss.org/frontpage"),
|
||||
NewsFeed("HN · 100+ Points", "https://hnrss.org/newest?points=100"),
|
||||
NewsFeed("Dev.to · AI", "https://dev.to/feed/tag/ai"),
|
||||
NewsFeed("Lobsters", "https://lobste.rs/rss"),
|
||||
),
|
||||
),
|
||||
NewsCategory(
|
||||
id="community",
|
||||
name="社区讨论",
|
||||
icon="💬",
|
||||
feeds=(
|
||||
NewsFeed(
|
||||
"HN · AI/LLM/Agent",
|
||||
"https://hnrss.org/newest?q=AI+OR+LLM+OR+Claude+OR+agent+OR+GPT+OR+Gemini",
|
||||
),
|
||||
NewsFeed(
|
||||
"Reddit · LLM/Claude/ML",
|
||||
"https://old.reddit.com/r/LocalLLaMA+ClaudeAI+MachineLearning+OpenAI/.rss?limit=25",
|
||||
slow=True,
|
||||
),
|
||||
),
|
||||
),
|
||||
)
|
||||
462
daily/news/fetch.py
Normal file
462
daily/news/fetch.py
Normal file
@@ -0,0 +1,462 @@
|
||||
"""抓取并整理国际 AI 时讯 RSS。"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
import re
|
||||
import time
|
||||
import html
|
||||
import xml.etree.ElementTree as ET
|
||||
from concurrent.futures import ThreadPoolExecutor, as_completed
|
||||
from datetime import datetime, timezone, timedelta
|
||||
from email.utils import parsedate_to_datetime
|
||||
from typing import Any
|
||||
from urllib.parse import parse_qs, urlencode, urlparse, urlunparse
|
||||
|
||||
import certifi
|
||||
import httpx
|
||||
|
||||
from daily.config import env, env_int, news_summary_limit
|
||||
from daily.news.feeds import NEWS_CATEGORIES, NewsCategory
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
USER_AGENT = "Mozilla/5.0 (compatible; skills-hot-daily/1.0; +https://skills.sh)"
|
||||
BROWSER_USER_AGENT = (
|
||||
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) "
|
||||
"AppleWebKit/537.36 (KHTML, like Gecko) "
|
||||
"Chrome/131.0.0.0 Safari/537.36"
|
||||
)
|
||||
STRIP_HTML = re.compile(r"<[^>]+>")
|
||||
WS = re.compile(r"\s+")
|
||||
|
||||
ATOM_NS = {"a": "http://www.w3.org/2005/Atom"}
|
||||
RSS_NS = {"r": "http://purl.org/rss/1.0/modules/content/"}
|
||||
|
||||
|
||||
def _enabled() -> bool:
|
||||
raw = (env("DAILY_AI_NEWS") or "1").strip().lower()
|
||||
return raw not in {"0", "false", "no", "off"}
|
||||
|
||||
|
||||
def _hours_window() -> int:
|
||||
return max(1, env_int("DAILY_AI_NEWS_HOURS", 72))
|
||||
|
||||
|
||||
def _per_feed_limit() -> int:
|
||||
return max(1, env_int("DAILY_AI_NEWS_PER_FEED", 3))
|
||||
|
||||
|
||||
def _per_category_limit() -> int:
|
||||
return max(1, env_int("DAILY_AI_NEWS_PER_CATEGORY", 5))
|
||||
|
||||
|
||||
def _wecom_limit() -> int:
|
||||
return max(1, env_int("DAILY_WECOM_AI_NEWS", 10))
|
||||
|
||||
|
||||
def _now_utc() -> datetime:
|
||||
return datetime.now(timezone.utc)
|
||||
|
||||
|
||||
def _parse_datetime(value: str | None) -> datetime | None:
|
||||
if not value:
|
||||
return None
|
||||
text = value.strip()
|
||||
if not text:
|
||||
return None
|
||||
try:
|
||||
dt = parsedate_to_datetime(text)
|
||||
if dt.tzinfo is None:
|
||||
dt = dt.replace(tzinfo=timezone.utc)
|
||||
return dt.astimezone(timezone.utc)
|
||||
except (TypeError, ValueError, OverflowError):
|
||||
pass
|
||||
for fmt in (
|
||||
"%Y-%m-%dT%H:%M:%SZ",
|
||||
"%Y-%m-%dT%H:%M:%S%z",
|
||||
"%Y-%m-%d",
|
||||
):
|
||||
try:
|
||||
dt = datetime.strptime(text[: len(fmt.replace("%z", "+0000"))], fmt.replace("%z", ""))
|
||||
if dt.tzinfo is None:
|
||||
dt = dt.replace(tzinfo=timezone.utc)
|
||||
return dt.astimezone(timezone.utc)
|
||||
except ValueError:
|
||||
continue
|
||||
return None
|
||||
|
||||
|
||||
def _clean_text(text: str | None, limit: int = 200) -> str:
|
||||
if not text:
|
||||
return ""
|
||||
plain = STRIP_HTML.sub(" ", html.unescape(text))
|
||||
plain = WS.sub(" ", plain).strip()
|
||||
if limit <= 0 or len(plain) <= limit:
|
||||
return plain
|
||||
return plain[: limit - 3] + "..."
|
||||
|
||||
|
||||
def _normalize_link(link: str) -> str:
|
||||
parsed = urlparse(link.strip())
|
||||
query = parse_qs(parsed.query, keep_blank_values=False)
|
||||
for key in list(query.keys()):
|
||||
if key.lower().startswith("utm_") or key.lower() in {"ref", "source"}:
|
||||
query.pop(key, None)
|
||||
clean_query = urlencode({k: v[0] for k, v in query.items() if v}, doseq=False)
|
||||
return urlunparse((parsed.scheme, parsed.netloc, parsed.path.rstrip("/"), "", clean_query, ""))
|
||||
|
||||
|
||||
def _normalize_title(title: str) -> str:
|
||||
return WS.sub(" ", title.strip().lower())
|
||||
|
||||
|
||||
def _entry_datetime(entry: dict[str, Any]) -> datetime | None:
|
||||
for key in ("published", "updated"):
|
||||
dt = _parse_datetime(entry.get(key))
|
||||
if dt:
|
||||
return dt
|
||||
return None
|
||||
|
||||
|
||||
def _parse_atom(content: str, feed_name: str, category: NewsCategory) -> list[dict[str, Any]]:
|
||||
items: list[dict[str, Any]] = []
|
||||
try:
|
||||
root = ET.fromstring(content)
|
||||
except ET.ParseError:
|
||||
return items
|
||||
|
||||
for entry in root.findall("a:entry", ATOM_NS):
|
||||
title_el = entry.find("a:title", ATOM_NS)
|
||||
link_el = entry.find("a:link", ATOM_NS)
|
||||
summary_el = entry.find("a:summary", ATOM_NS) or entry.find("a:content", ATOM_NS)
|
||||
updated_el = entry.find("a:updated", ATOM_NS) or entry.find("a:published", ATOM_NS)
|
||||
title = title_el.text.strip() if title_el is not None and title_el.text else ""
|
||||
link = ""
|
||||
if link_el is not None:
|
||||
link = link_el.get("href") or (link_el.text or "").strip()
|
||||
if not title or not link:
|
||||
continue
|
||||
items.append(
|
||||
{
|
||||
"title": title,
|
||||
"link": link,
|
||||
"summary": _clean_text(summary_el.text if summary_el is not None else ""),
|
||||
"published": updated_el.text.strip() if updated_el is not None and updated_el.text else "",
|
||||
"source_name": feed_name,
|
||||
"category_id": category.id,
|
||||
"category_name": category.name,
|
||||
"category_icon": category.icon,
|
||||
}
|
||||
)
|
||||
return items
|
||||
|
||||
|
||||
def _parse_rss(content: str, feed_name: str, category: NewsCategory) -> list[dict[str, Any]]:
|
||||
items: list[dict[str, Any]] = []
|
||||
try:
|
||||
root = ET.fromstring(content)
|
||||
except ET.ParseError:
|
||||
return items
|
||||
|
||||
channel = root.find("channel")
|
||||
if channel is None:
|
||||
return items
|
||||
|
||||
for item in channel.findall("item"):
|
||||
title_el = item.find("title")
|
||||
link_el = item.find("link")
|
||||
desc_el = item.find("description")
|
||||
pub_el = item.find("pubDate")
|
||||
title = title_el.text.strip() if title_el is not None and title_el.text else ""
|
||||
link = link_el.text.strip() if link_el is not None and link_el.text else ""
|
||||
if not title or not link:
|
||||
continue
|
||||
items.append(
|
||||
{
|
||||
"title": title,
|
||||
"link": link,
|
||||
"summary": _clean_text(desc_el.text if desc_el is not None else ""),
|
||||
"published": pub_el.text.strip() if pub_el is not None and pub_el.text else "",
|
||||
"source_name": feed_name,
|
||||
"category_id": category.id,
|
||||
"category_name": category.name,
|
||||
"category_icon": category.icon,
|
||||
}
|
||||
)
|
||||
return items
|
||||
|
||||
|
||||
def _parse_feed(content: str, feed_name: str, category: NewsCategory) -> list[dict[str, Any]]:
|
||||
text = content.lstrip("\ufeff").strip()
|
||||
if not text:
|
||||
return []
|
||||
if text.startswith("<rss") or "<channel>" in text[:500]:
|
||||
return _parse_rss(text, feed_name, category)
|
||||
if text.startswith("<feed") or "<entry>" in text[:500]:
|
||||
return _parse_atom(text, feed_name, category)
|
||||
if "<item>" in text[:2000]:
|
||||
return _parse_rss(text, feed_name, category)
|
||||
return _parse_atom(text, feed_name, category)
|
||||
|
||||
|
||||
def _is_reddit_url(url: str) -> bool:
|
||||
host = urlparse(url).netloc.lower()
|
||||
return host.endswith("reddit.com")
|
||||
|
||||
|
||||
def _reddit_auth_params() -> dict[str, str]:
|
||||
user = (env("REDDIT_RSS_USER") or "").strip()
|
||||
feed = (env("REDDIT_RSS_FEED") or "").strip()
|
||||
if user and feed:
|
||||
return {"user": user, "feed": feed}
|
||||
return {}
|
||||
|
||||
|
||||
def _with_query_params(url: str, extra: dict[str, str]) -> str:
|
||||
if not extra:
|
||||
return url
|
||||
parsed = urlparse(url)
|
||||
query = parse_qs(parsed.query, keep_blank_values=True)
|
||||
for key, value in extra.items():
|
||||
if value and key not in query:
|
||||
query[key] = [value]
|
||||
clean_query = urlencode({k: v[0] for k, v in query.items() if v and v[0]}, doseq=False)
|
||||
return urlunparse((parsed.scheme, parsed.netloc, parsed.path, "", clean_query, ""))
|
||||
|
||||
|
||||
def _reddit_old_url(url: str) -> str:
|
||||
parsed = urlparse(url)
|
||||
host = parsed.netloc.lower()
|
||||
if host.startswith("old."):
|
||||
return url
|
||||
if host in {"www.reddit.com", "reddit.com"}:
|
||||
return urlunparse((parsed.scheme, "old.reddit.com", parsed.path, "", parsed.query, ""))
|
||||
return url
|
||||
|
||||
|
||||
def _request_headers(base: dict[str, str], url: str) -> dict[str, str]:
|
||||
if not _is_reddit_url(url):
|
||||
return base
|
||||
return {
|
||||
**base,
|
||||
"User-Agent": BROWSER_USER_AGENT,
|
||||
"Accept-Language": "en-US,en;q=0.9",
|
||||
}
|
||||
|
||||
|
||||
def _reddit_fetch_urls(feed_url: str) -> list[str]:
|
||||
primary = _with_query_params(feed_url, _reddit_auth_params())
|
||||
if not _is_reddit_url(primary):
|
||||
return [primary]
|
||||
fallback = _reddit_old_url(primary)
|
||||
if fallback == primary:
|
||||
return [primary]
|
||||
return [primary, fallback]
|
||||
|
||||
|
||||
def _fetch_one(client: httpx.Client, category: NewsCategory, feed_url: str, feed_name: str) -> list[dict[str, Any]]:
|
||||
last_exc: Exception | None = None
|
||||
for url in _reddit_fetch_urls(feed_url):
|
||||
try:
|
||||
headers = _request_headers(dict(client.headers), url)
|
||||
resp = client.get(url, headers=headers)
|
||||
resp.raise_for_status()
|
||||
return _parse_feed(resp.text, feed_name, category)
|
||||
except Exception as exc:
|
||||
last_exc = exc
|
||||
continue
|
||||
logger.warning("RSS fetch failed [%s] %s: %s", feed_name, feed_url, last_exc)
|
||||
return []
|
||||
|
||||
|
||||
def _dedupe_items(items: list[dict[str, Any]]) -> list[dict[str, Any]]:
|
||||
seen_links: set[str] = set()
|
||||
seen_titles: set[str] = set()
|
||||
result: list[dict[str, Any]] = []
|
||||
for item in items:
|
||||
link_key = _normalize_link(item["link"])
|
||||
title_key = _normalize_title(item["title"])
|
||||
if link_key in seen_links or title_key in seen_titles:
|
||||
continue
|
||||
seen_links.add(link_key)
|
||||
seen_titles.add(title_key)
|
||||
result.append(item)
|
||||
return result
|
||||
|
||||
|
||||
def _within_window(item: dict[str, Any], cutoff: datetime) -> bool:
|
||||
dt = _entry_datetime(item)
|
||||
if dt is None:
|
||||
return True
|
||||
return dt >= cutoff
|
||||
|
||||
|
||||
def _sort_key(item: dict[str, Any]) -> tuple[int, datetime]:
|
||||
dt = _entry_datetime(item)
|
||||
if dt is None:
|
||||
return (1, datetime.min.replace(tzinfo=timezone.utc))
|
||||
return (0, dt)
|
||||
|
||||
|
||||
def fetch_ai_news() -> dict[str, Any]:
|
||||
"""按类别抓取 AI 时讯,返回 {enabled, hours, categories, flat, stats}。"""
|
||||
if not _enabled():
|
||||
return {"enabled": False, "categories": [], "flat": [], "stats": {}}
|
||||
|
||||
hours = _hours_window()
|
||||
per_feed = _per_feed_limit()
|
||||
per_category = _per_category_limit()
|
||||
cutoff = _now_utc() - timedelta(hours=hours)
|
||||
|
||||
headers = {"User-Agent": USER_AGENT, "Accept": "application/rss+xml, application/atom+xml, application/xml, text/xml, */*"}
|
||||
tasks: list[tuple[NewsCategory, str, str, bool]] = []
|
||||
for category in NEWS_CATEGORIES:
|
||||
for feed in category.feeds:
|
||||
tasks.append((category, feed.url, feed.name, feed.slow))
|
||||
|
||||
raw_by_category: dict[str, list[dict[str, Any]]] = {c.id: [] for c in NEWS_CATEGORIES}
|
||||
stats = {"feeds_total": len(tasks), "feeds_ok": 0, "items_raw": 0}
|
||||
|
||||
with httpx.Client(timeout=15.0, verify=certifi.where(), follow_redirects=True, headers=headers) as client:
|
||||
fast_tasks = [t for t in tasks if not t[3]]
|
||||
slow_tasks = [t for t in tasks if t[3]]
|
||||
|
||||
with ThreadPoolExecutor(max_workers=8) as pool:
|
||||
futures = {
|
||||
pool.submit(_fetch_one, client, cat, url, name): (cat.id, name)
|
||||
for cat, url, name, _slow in fast_tasks
|
||||
}
|
||||
for future in as_completed(futures):
|
||||
cat_id, feed_name = futures[future]
|
||||
try:
|
||||
entries = future.result()
|
||||
except Exception as exc:
|
||||
logger.warning("RSS 任务异常 [%s]: %s", feed_name, exc)
|
||||
continue
|
||||
if entries:
|
||||
stats["feeds_ok"] += 1
|
||||
stats["items_raw"] += len(entries)
|
||||
raw_by_category[cat_id].extend(entries[:per_feed])
|
||||
|
||||
for cat, url, name, _slow in slow_tasks:
|
||||
entries = _fetch_one(client, cat, url, name)
|
||||
if entries:
|
||||
stats["feeds_ok"] += 1
|
||||
stats["items_raw"] += len(entries)
|
||||
raw_by_category[cat.id].extend(entries[:per_feed])
|
||||
if _is_reddit_url(url):
|
||||
time.sleep(2.0)
|
||||
else:
|
||||
time.sleep(1.0)
|
||||
|
||||
categories_out: list[dict[str, Any]] = []
|
||||
flat: list[dict[str, Any]] = []
|
||||
|
||||
for category in NEWS_CATEGORIES:
|
||||
items = raw_by_category[category.id]
|
||||
items = [i for i in items if _within_window(i, cutoff)]
|
||||
items.sort(key=_sort_key, reverse=True)
|
||||
items = _dedupe_items(items)[:per_category]
|
||||
for item in items:
|
||||
dt = _entry_datetime(item)
|
||||
item["published_fmt"] = dt.astimezone(timezone(timedelta(hours=8))).strftime("%m-%d %H:%M") if dt else ""
|
||||
if items:
|
||||
categories_out.append(
|
||||
{
|
||||
"id": category.id,
|
||||
"name": category.name,
|
||||
"icon": category.icon,
|
||||
"items": items,
|
||||
}
|
||||
)
|
||||
flat.extend(items)
|
||||
|
||||
flat.sort(key=_sort_key, reverse=True)
|
||||
flat = _dedupe_items(flat)
|
||||
|
||||
return {
|
||||
"enabled": True,
|
||||
"hours": hours,
|
||||
"categories": categories_out,
|
||||
"flat": flat,
|
||||
"stats": stats,
|
||||
}
|
||||
|
||||
|
||||
def format_news_section(news: dict[str, Any], *, section_no: int, wecom_limit: int | None = None) -> list[str]:
|
||||
if not news.get("enabled"):
|
||||
return ["---", "", f"## {section_no}、国际 AI 时讯", "", "*AI 时讯已关闭(`DAILY_AI_NEWS=0`)。*", ""]
|
||||
|
||||
categories = news.get("categories") or []
|
||||
hours = news.get("hours", 72)
|
||||
lines = [
|
||||
"---",
|
||||
"",
|
||||
f"## {section_no}、国际 AI 时讯",
|
||||
"",
|
||||
f"> 近 **{hours}h** · {news.get('stats', {}).get('feeds_ok', 0)}/{news.get('stats', {}).get('feeds_total', 0)} 源可用",
|
||||
"",
|
||||
]
|
||||
|
||||
if not categories:
|
||||
lines.append("*暂无可用条目(网络/RSS 源异常或时间窗口内无更新)。*")
|
||||
lines.append("")
|
||||
return lines
|
||||
|
||||
if wecom_limit is not None:
|
||||
flat = (news.get("flat") or [])[:wecom_limit]
|
||||
for i, item in enumerate(flat, 1):
|
||||
pub = f" · {item['published_fmt']}" if item.get("published_fmt") else ""
|
||||
lines.append(
|
||||
f"{i}. [{item['title']}]({item['link']}) · `{item['source_name']}`{pub}"
|
||||
)
|
||||
lines.append("")
|
||||
return lines
|
||||
|
||||
for cat in categories:
|
||||
lines.append(f"### {cat['icon']} {cat['name']}")
|
||||
lines.append("")
|
||||
for i, item in enumerate(cat["items"], 1):
|
||||
pub = f" · {item['published_fmt']}" if item.get("published_fmt") else ""
|
||||
lines.append(f"{i}. **[{item['title']}]({item['link']})** · `{item['source_name']}`{pub}")
|
||||
summary = item.get("summary", "")
|
||||
if summary:
|
||||
lines.append(f" - {_clean_text(summary, news_summary_limit())}")
|
||||
lines.append("")
|
||||
|
||||
return lines
|
||||
|
||||
|
||||
def prepare_wecom_news_items(news: dict[str, Any]) -> list[dict[str, Any]]:
|
||||
if not news.get("enabled"):
|
||||
return []
|
||||
limit = _wecom_limit()
|
||||
flat = _dedupe_items(news.get("flat") or [])
|
||||
flat.sort(key=_sort_key, reverse=True)
|
||||
preferred = ("media", "newsletter", "official", "community", "research", "developer")
|
||||
picked: list[dict[str, Any]] = []
|
||||
seen: set[str] = set()
|
||||
for cat in preferred:
|
||||
for item in flat:
|
||||
link = _normalize_link(item.get("link", ""))
|
||||
if item.get("category_id") != cat or link in seen:
|
||||
continue
|
||||
picked.append(item)
|
||||
seen.add(link)
|
||||
if len(picked) >= limit:
|
||||
break
|
||||
if len(picked) >= limit:
|
||||
break
|
||||
items: list[dict[str, Any]] = []
|
||||
for item in picked[:limit]:
|
||||
items.append(
|
||||
{
|
||||
"title": item.get("title", "?"),
|
||||
"link": item.get("link", ""),
|
||||
"source_name": item.get("source_name", "?"),
|
||||
"published_fmt": item.get("published_fmt", ""),
|
||||
"desc_short": _clean_text(item.get("summary", ""), 36),
|
||||
}
|
||||
)
|
||||
return items
|
||||
183
daily/report_data.py
Normal file
183
daily/report_data.py
Normal file
@@ -0,0 +1,183 @@
|
||||
"""早报结构化数据:抓取结果 → JSON 中间层。"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
from daily.config import OUTPUT_DIR, env_int
|
||||
from daily.delta import build_movement_baseline, build_movement_context, compare_depth
|
||||
from daily.news.fetch import prepare_wecom_news_items
|
||||
from daily.skills_group import group_skills_by_source
|
||||
|
||||
|
||||
def skill_id(item: dict[str, Any]) -> str:
|
||||
return str(item.get("id") or f"{item.get('source')}/{item.get('title')}")
|
||||
|
||||
|
||||
def _slim_skill(item: dict[str, Any]) -> dict[str, Any]:
|
||||
payload = {
|
||||
"id": skill_id(item),
|
||||
"title": item.get("title", ""),
|
||||
"source": item.get("source", ""),
|
||||
"installs": item.get("installs", 0),
|
||||
"link": item.get("link", ""),
|
||||
"description": item.get("description", ""),
|
||||
}
|
||||
if item.get("cluster"):
|
||||
payload.update(
|
||||
{
|
||||
"cluster": True,
|
||||
"cluster_count": item.get("cluster_count", 1),
|
||||
"cluster_skills": item.get("cluster_skills", []),
|
||||
"cluster_titles": item.get("cluster_titles", ""),
|
||||
"installs_fmt": item.get("installs_fmt", ""),
|
||||
"installs_min": item.get("installs_min"),
|
||||
"installs_max": item.get("installs_max"),
|
||||
}
|
||||
)
|
||||
elif item.get("installs_fmt"):
|
||||
payload["installs_fmt"] = item.get("installs_fmt")
|
||||
return payload
|
||||
|
||||
|
||||
def _slim_github(item: dict[str, Any]) -> dict[str, Any]:
|
||||
return {
|
||||
"repo": item.get("repo", ""),
|
||||
"url": item.get("url", ""),
|
||||
"language": item.get("language", ""),
|
||||
"stars_today_fmt": item.get("stars_today_fmt", ""),
|
||||
"total_stars_fmt": item.get("total_stars_fmt", ""),
|
||||
"created_at": item.get("created_at", ""),
|
||||
"description": item.get("description", ""),
|
||||
}
|
||||
|
||||
|
||||
def _slim_news_items(ai_news: dict[str, Any], limit: int) -> list[dict[str, Any]]:
|
||||
items: list[dict[str, Any]] = []
|
||||
for item in prepare_wecom_news_items(ai_news):
|
||||
items.append(
|
||||
{
|
||||
"link": item.get("link", ""),
|
||||
"title": item.get("title", ""),
|
||||
"source_name": item.get("source_name", ""),
|
||||
"published_fmt": item.get("published_fmt", ""),
|
||||
"summary": item.get("desc_short") or "",
|
||||
}
|
||||
)
|
||||
if len(items) >= limit:
|
||||
break
|
||||
if items:
|
||||
return items
|
||||
for item in (ai_news.get("flat") or [])[:limit]:
|
||||
items.append(
|
||||
{
|
||||
"link": item.get("link", ""),
|
||||
"title": item.get("title", ""),
|
||||
"source_name": item.get("source_name", ""),
|
||||
"published_fmt": item.get("published_fmt", ""),
|
||||
"summary": item.get("summary", ""),
|
||||
}
|
||||
)
|
||||
return items
|
||||
|
||||
|
||||
def _wecom_skill_pool() -> int:
|
||||
return max(10, env_int("DAILY_WECOM_SKILL_POOL", 200))
|
||||
|
||||
|
||||
def build_llm_input(
|
||||
*,
|
||||
date_str: str,
|
||||
updated: str,
|
||||
trending: list[dict[str, Any]],
|
||||
hot: list[dict[str, Any]],
|
||||
github_trending: list[dict[str, Any]],
|
||||
github_emerging: list[dict[str, Any]],
|
||||
github_topic: list[dict[str, Any]],
|
||||
topic_name: str,
|
||||
ai_news: dict[str, Any],
|
||||
wecom_limits: dict[str, int],
|
||||
) -> dict[str, Any]:
|
||||
"""供 Cursor 编辑的精简 JSON(不含完整 markdown)。"""
|
||||
news_limit = wecom_limits.get("ai_news", 10)
|
||||
depth = compare_depth()
|
||||
trend_cmp = trending[:depth]
|
||||
hot_cmp = hot[:depth]
|
||||
github_cmp = github_trending[:depth]
|
||||
emerging_cmp = github_emerging[:depth]
|
||||
topic_cmp = github_topic[:depth]
|
||||
|
||||
trending_slice = group_skills_by_source(
|
||||
trending,
|
||||
limit=wecom_limits.get("trending", 10),
|
||||
pool_size=wecom_limits.get("trending_pool", _wecom_skill_pool()),
|
||||
)
|
||||
hot_slice = group_skills_by_source(
|
||||
hot,
|
||||
limit=wecom_limits.get("hot", 10),
|
||||
pool_size=wecom_limits.get("hot_pool", _wecom_skill_pool()),
|
||||
)
|
||||
github_slice = github_trending[: wecom_limits.get("github", 5)]
|
||||
emerging_slice = github_emerging[: wecom_limits.get("emerging", 3)]
|
||||
topic_slice = github_topic[: wecom_limits.get("topic", 3)]
|
||||
|
||||
movement = build_movement_context(
|
||||
date_str=date_str,
|
||||
trending=[_slim_skill(x) for x in trend_cmp],
|
||||
hot=[_slim_skill(x) for x in hot_cmp],
|
||||
github_trending=[_slim_github(x) for x in github_cmp],
|
||||
github_emerging=[_slim_github(x) for x in emerging_cmp],
|
||||
github_topic=[_slim_github(x) for x in topic_cmp],
|
||||
topic_name=topic_name,
|
||||
)
|
||||
movement_baseline = build_movement_baseline(
|
||||
trending=[_slim_skill(x) for x in trend_cmp],
|
||||
hot=[_slim_skill(x) for x in hot_cmp],
|
||||
github_trending=[_slim_github(x) for x in github_cmp],
|
||||
github_emerging=[_slim_github(x) for x in emerging_cmp],
|
||||
github_topic=[_slim_github(x) for x in topic_cmp],
|
||||
depth=depth,
|
||||
)
|
||||
|
||||
return {
|
||||
"date": date_str,
|
||||
"data_updated": updated,
|
||||
"skills_trending": [_slim_skill(x) for x in trending_slice],
|
||||
"skills_hot": [_slim_skill(x) for x in hot_slice],
|
||||
"github_trending": [_slim_github(x) for x in github_slice],
|
||||
"github_emerging": [_slim_github(x) for x in emerging_slice],
|
||||
"github_topic": {
|
||||
"topic": topic_name,
|
||||
"repos": [_slim_github(x) for x in topic_slice],
|
||||
},
|
||||
"ai_news": _slim_news_items(ai_news, news_limit) if ai_news.get("enabled") else [],
|
||||
"movement": movement,
|
||||
"movement_baseline": movement_baseline,
|
||||
}
|
||||
|
||||
|
||||
def build_full_payload(
|
||||
llm_input: dict[str, Any],
|
||||
*,
|
||||
meta: dict[str, Any],
|
||||
) -> dict[str, Any]:
|
||||
return {"meta": meta, "data": llm_input}
|
||||
|
||||
|
||||
def data_json_path(date_str: str) -> Path:
|
||||
return OUTPUT_DIR / f"{date_str}.data.json"
|
||||
|
||||
|
||||
def editorial_json_path(date_str: str) -> Path:
|
||||
return OUTPUT_DIR / f"{date_str}.editorial.json"
|
||||
|
||||
|
||||
def save_json(path: Path, data: dict[str, Any]) -> None:
|
||||
path.parent.mkdir(parents=True, exist_ok=True)
|
||||
path.write_text(json.dumps(data, ensure_ascii=False, indent=2), encoding="utf-8")
|
||||
|
||||
|
||||
def load_json(path: Path) -> dict[str, Any]:
|
||||
return json.loads(path.read_text(encoding="utf-8"))
|
||||
120
daily/skills_board.py
Normal file
120
daily/skills_board.py
Normal file
@@ -0,0 +1,120 @@
|
||||
"""从 skills.sh 官网抓取 Trending / Hot 完整榜单(突破 feed.json 50 条限制)。"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
import re
|
||||
from typing import Any, Literal
|
||||
|
||||
import certifi
|
||||
import httpx
|
||||
|
||||
from daily.config import env
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
Board = Literal["trending", "hot"]
|
||||
SKILLS_SITE = "https://www.skills.sh"
|
||||
USER_AGENT = "Mozilla/5.0 (compatible; skills-hot-daily/1.0; +https://skills.sh)"
|
||||
|
||||
_SKILL_RE = re.compile(
|
||||
r'\{"source":"(?P<source>[^"]+)","skillId":"(?P<skill_id>[^"]+)",'
|
||||
r'"name":"(?P<name>[^"]+)","installs":(?P<installs>\d+)'
|
||||
)
|
||||
_RSC_CHUNK_RE = re.compile(r"self\.__next_f\.push\(\[1,\"(.*?)\"\]\)", re.DOTALL)
|
||||
|
||||
|
||||
def board_source() -> str:
|
||||
return (env("SKILLS_BOARD_SOURCE") or "website").strip().lower()
|
||||
|
||||
|
||||
def _fetch_html(path: str) -> str:
|
||||
url = f"{SKILLS_SITE}{path}"
|
||||
headers = {"User-Agent": USER_AGENT, "Accept": "text/html"}
|
||||
with httpx.Client(timeout=30.0, verify=certifi.where(), follow_redirects=True) as client:
|
||||
resp = client.get(url, headers=headers)
|
||||
resp.raise_for_status()
|
||||
return resp.text
|
||||
|
||||
|
||||
def _rsc_blob(html: str) -> str:
|
||||
chunks = _RSC_CHUNK_RE.findall(html)
|
||||
blob = "\n".join(chunks)
|
||||
return blob.encode("utf-8").decode("unicode_escape", errors="ignore")
|
||||
|
||||
|
||||
def _parse_initial_skills(blob: str, *, limit: int) -> list[dict[str, Any]]:
|
||||
items: list[dict[str, Any]] = []
|
||||
seen: set[str] = set()
|
||||
for match in _SKILL_RE.finditer(blob):
|
||||
source = match.group("source")
|
||||
skill_id = match.group("skill_id")
|
||||
uid = f"{source}/{skill_id}"
|
||||
if uid in seen:
|
||||
continue
|
||||
seen.add(uid)
|
||||
items.append(
|
||||
{
|
||||
"id": uid,
|
||||
"title": skill_id,
|
||||
"source": source,
|
||||
"installs": int(match.group("installs")),
|
||||
"link": f"{SKILLS_SITE}/{source}/{skill_id}",
|
||||
"description": "",
|
||||
}
|
||||
)
|
||||
if len(items) >= limit:
|
||||
break
|
||||
return items
|
||||
|
||||
|
||||
def fetch_board(board: Board, *, limit: int) -> list[dict[str, Any]]:
|
||||
path = "/trending" if board == "trending" else "/hot"
|
||||
try:
|
||||
html = _fetch_html(path)
|
||||
items = _parse_initial_skills(_rsc_blob(html), limit=limit)
|
||||
if items:
|
||||
logger.info("skills.sh %s: %d items (limit=%d)", board, len(items), limit)
|
||||
return items
|
||||
except Exception as exc:
|
||||
logger.warning("skills.sh %s fetch failed, fallback to feed.json: %s", board, exc)
|
||||
return []
|
||||
|
||||
|
||||
def enrich_from_feed(items: list[dict[str, Any]], feed: dict[str, Any]) -> None:
|
||||
desc_by_id: dict[str, str] = {}
|
||||
for key in ("topTrending", "topHot", "topAllTime"):
|
||||
for row in feed.get(key, []):
|
||||
uid = str(row.get("id") or f"{row.get('source')}/{row.get('title')}")
|
||||
desc = (row.get("description") or "").strip()
|
||||
if desc:
|
||||
desc_by_id[uid] = desc
|
||||
for item in items:
|
||||
if not item.get("description"):
|
||||
item["description"] = desc_by_id.get(item["id"], "")
|
||||
|
||||
|
||||
def load_boards(
|
||||
feed: dict[str, Any],
|
||||
*,
|
||||
trending_limit: int,
|
||||
hot_limit: int,
|
||||
) -> tuple[list[dict[str, Any]], list[dict[str, Any]]]:
|
||||
"""优先 skills.sh 官网;失败时回退 feed.json。"""
|
||||
if board_source() == "feed":
|
||||
return (
|
||||
list(feed.get("topTrending", [])[:trending_limit]),
|
||||
list(feed.get("topHot", [])[:hot_limit]),
|
||||
)
|
||||
|
||||
trending = fetch_board("trending", limit=trending_limit)
|
||||
hot = fetch_board("hot", limit=hot_limit)
|
||||
if not trending:
|
||||
trending = list(feed.get("topTrending", [])[:trending_limit])
|
||||
else:
|
||||
enrich_from_feed(trending, feed)
|
||||
if not hot:
|
||||
hot = list(feed.get("topHot", [])[:hot_limit])
|
||||
else:
|
||||
enrich_from_feed(hot, feed)
|
||||
return trending, hot
|
||||
103
daily/skills_group.py
Normal file
103
daily/skills_group.py
Normal file
@@ -0,0 +1,103 @@
|
||||
"""Skills 榜单:同 source 合并为一条(企微 Top N)。"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import Any
|
||||
|
||||
|
||||
def skill_id(item: dict[str, Any]) -> str:
|
||||
return str(item.get("id") or f"{item.get('source')}/{item.get('title')}")
|
||||
|
||||
|
||||
def format_installs(n: int | float) -> str:
|
||||
if n >= 1_000_000:
|
||||
return f"{n / 1_000_000:.1f}M"
|
||||
if n >= 1_000:
|
||||
return f"{n / 1_000:.1f}K"
|
||||
return str(int(n))
|
||||
|
||||
|
||||
def _installs_range(items: list[dict[str, Any]]) -> tuple[int, int, str]:
|
||||
values = [int(item.get("installs") or 0) for item in items]
|
||||
lo, hi = min(values), max(values)
|
||||
if lo == hi:
|
||||
return lo, hi, format_installs(hi)
|
||||
return lo, hi, f"{format_installs(lo)}–{format_installs(hi)}"
|
||||
|
||||
|
||||
def _best_description(items: list[dict[str, Any]]) -> str:
|
||||
for item in items:
|
||||
desc = (item.get("description") or "").strip()
|
||||
if desc:
|
||||
return desc
|
||||
return ""
|
||||
|
||||
|
||||
def _cluster_skill(items: list[dict[str, Any]]) -> dict[str, Any]:
|
||||
ranked = sorted(items, key=lambda x: int(x.get("installs") or 0), reverse=True)
|
||||
top = ranked[0]
|
||||
_lo, _hi, installs_fmt = _installs_range(ranked)
|
||||
titles = [str(x.get("title") or "") for x in ranked if x.get("title")]
|
||||
sample = ", ".join(titles[:4])
|
||||
if len(titles) > 4:
|
||||
sample = f"{sample}…"
|
||||
return {
|
||||
"id": skill_id(top),
|
||||
"title": top.get("title", ""),
|
||||
"source": top.get("source", ""),
|
||||
"installs": int(top.get("installs") or 0),
|
||||
"installs_min": _lo,
|
||||
"installs_max": _hi,
|
||||
"installs_fmt": installs_fmt,
|
||||
"link": top.get("link", ""),
|
||||
"description": _best_description(ranked),
|
||||
"cluster": True,
|
||||
"cluster_count": len(ranked),
|
||||
"cluster_skills": titles,
|
||||
"cluster_titles": sample,
|
||||
}
|
||||
|
||||
|
||||
def _single_skill(item: dict[str, Any]) -> dict[str, Any]:
|
||||
installs = int(item.get("installs") or 0)
|
||||
return {
|
||||
"id": skill_id(item),
|
||||
"title": item.get("title", ""),
|
||||
"source": item.get("source", ""),
|
||||
"installs": installs,
|
||||
"installs_fmt": format_installs(installs),
|
||||
"link": item.get("link", ""),
|
||||
"description": item.get("description", ""),
|
||||
"cluster": False,
|
||||
}
|
||||
|
||||
|
||||
def group_skills_by_source(
|
||||
items: list[dict[str, Any]],
|
||||
*,
|
||||
limit: int = 10,
|
||||
pool_size: int = 50,
|
||||
) -> list[dict[str, Any]]:
|
||||
"""按 source 去重合并;保留各 source 在榜内的最佳名次顺序。"""
|
||||
if not items or limit <= 0:
|
||||
return []
|
||||
|
||||
pool = items[: max(pool_size, limit)]
|
||||
by_source: dict[str, list[dict[str, Any]]] = {}
|
||||
first_rank: dict[str, int] = {}
|
||||
for rank, item in enumerate(pool, 1):
|
||||
source = (item.get("source") or "?").strip() or "?"
|
||||
by_source.setdefault(source, []).append(item)
|
||||
first_rank.setdefault(source, rank)
|
||||
|
||||
ordered_sources = sorted(by_source.keys(), key=lambda s: first_rank[s])
|
||||
result: list[dict[str, Any]] = []
|
||||
for source in ordered_sources:
|
||||
group = by_source[source]
|
||||
if len(group) == 1:
|
||||
result.append(_single_skill(group[0]))
|
||||
else:
|
||||
result.append(_cluster_skill(group))
|
||||
if len(result) >= limit:
|
||||
break
|
||||
return result
|
||||
30
daily/text_utils.py
Normal file
30
daily/text_utils.py
Normal file
@@ -0,0 +1,30 @@
|
||||
"""文本裁剪等轻量工具。"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import re
|
||||
|
||||
_WS = re.compile(r"\s+")
|
||||
|
||||
|
||||
def clip_text(text: str, limit: int) -> str:
|
||||
text = _WS.sub(" ", (text or "").strip())
|
||||
if limit <= 0 or len(text) <= limit:
|
||||
return text
|
||||
return text[: max(1, limit - 1)] + "…"
|
||||
|
||||
|
||||
def trim_brief(text: str, limit: int) -> str:
|
||||
"""企微简要:控制在 limit 内,优先在句读处截断,不加省略号。"""
|
||||
text = _WS.sub(" ", (text or "").strip())
|
||||
if not text or limit <= 0 or len(text) <= limit:
|
||||
return text
|
||||
for sep in ("。", "!", "?", ";"):
|
||||
pos = text.find(sep)
|
||||
if pos != -1 and pos + 1 <= limit + 8:
|
||||
return text[: pos + 1]
|
||||
for sep in (",", "、"):
|
||||
pos = text.find(sep)
|
||||
if pos != -1 and pos + 1 <= limit:
|
||||
return text[: pos + 1]
|
||||
return text[:limit].rstrip(",、;: ")
|
||||
96
daily/webhook.py
Normal file
96
daily/webhook.py
Normal file
@@ -0,0 +1,96 @@
|
||||
"""推送早报至企业微信群 webhook(超长自动分多条)。"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import sys
|
||||
import time
|
||||
from pathlib import Path
|
||||
|
||||
import certifi
|
||||
import httpx
|
||||
|
||||
from daily.config import OUTPUT_DIR, ROOT, env_int, wecom_chunk_bytes
|
||||
from daily.wecom_split import split_wecom_messages
|
||||
|
||||
_PUSH_GAP_MS = 300
|
||||
|
||||
|
||||
def _load_webhook_key() -> str:
|
||||
from daily.config import env
|
||||
|
||||
key = (env("WECOM_WEBHOOK_KEY") or "").strip()
|
||||
if not key:
|
||||
raise RuntimeError("请设置 WECOM_WEBHOOK_KEY(项目根 .env)")
|
||||
return key
|
||||
|
||||
|
||||
def _resolve_report_path(arg: str | None) -> Path:
|
||||
if arg:
|
||||
path = Path(arg)
|
||||
if not path.is_absolute():
|
||||
path = ROOT / path
|
||||
return path
|
||||
candidates = sorted(
|
||||
OUTPUT_DIR.glob("*.wecom.md"),
|
||||
key=lambda p: p.stat().st_mtime,
|
||||
reverse=True,
|
||||
)
|
||||
if candidates:
|
||||
return candidates[0]
|
||||
legacy = sorted(ROOT.glob("*.wecom.md"), key=lambda p: p.stat().st_mtime, reverse=True)
|
||||
if legacy:
|
||||
return legacy[0]
|
||||
raise RuntimeError("未找到 .wecom.md 报告,请先运行 python -m daily")
|
||||
|
||||
|
||||
def _post_markdown(client: httpx.Client, url: str, content: str) -> None:
|
||||
payload = {"msgtype": "markdown", "markdown": {"content": content}}
|
||||
resp = client.post(url, json=payload)
|
||||
resp.raise_for_status()
|
||||
data = resp.json()
|
||||
if data.get("errcode", 0) != 0:
|
||||
raise RuntimeError(f"推送失败: errcode={data.get('errcode')} errmsg={data.get('errmsg')}")
|
||||
|
||||
|
||||
def send_report(report_path: Path | None = None) -> None:
|
||||
path = _resolve_report_path(str(report_path) if report_path else None)
|
||||
if not path.exists():
|
||||
raise RuntimeError(f"报告文件不存在: {path}")
|
||||
|
||||
content = path.read_text(encoding="utf-8").strip()
|
||||
if not content:
|
||||
raise RuntimeError(f"报告内容为空: {path}")
|
||||
|
||||
chunk_limit = wecom_chunk_bytes()
|
||||
max_parts = env_int("DAILY_WECOM_MAX_PARTS", 5)
|
||||
parts = split_wecom_messages(content, chunk_limit)
|
||||
if len(parts) > max_parts:
|
||||
raise RuntimeError(
|
||||
f"早报需 {len(parts)} 条消息,超过 DAILY_WECOM_MAX_PARTS={max_parts},请调小各区块条数"
|
||||
)
|
||||
|
||||
key = _load_webhook_key()
|
||||
url = f"https://qyapi.weixin.qq.com/cgi-bin/webhook/send?key={key}"
|
||||
|
||||
with httpx.Client(timeout=20.0, verify=certifi.where()) as client:
|
||||
for i, part in enumerate(parts, 1):
|
||||
if i > 1:
|
||||
time.sleep(_PUSH_GAP_MS / 1000.0)
|
||||
_post_markdown(client, url, part)
|
||||
|
||||
total_bytes = len(content.encode("utf-8"))
|
||||
if len(parts) == 1:
|
||||
print(f"已推送至企业微信: {path.name} ({total_bytes} bytes)")
|
||||
else:
|
||||
sizes = ", ".join(str(len(p.encode("utf-8"))) for p in parts)
|
||||
print(f"已推送至企业微信: {path.name} ({total_bytes} bytes → {len(parts)} 条: {sizes})")
|
||||
|
||||
|
||||
def main(argv: list[str] | None = None) -> int:
|
||||
args = argv if argv is not None else sys.argv[1:]
|
||||
try:
|
||||
send_report(Path(args[0]) if args else None)
|
||||
return 0
|
||||
except Exception as exc:
|
||||
print(f"ERROR: {exc}", file=sys.stderr)
|
||||
return 1
|
||||
94
daily/wecom_split.py
Normal file
94
daily/wecom_split.py
Normal file
@@ -0,0 +1,94 @@
|
||||
"""企微 markdown 按字节上限拆分为多条消息(按区块,不截断正文)。"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import re
|
||||
|
||||
_SECTION_START = re.compile(r"^[📰💡🎯🌍📈🔥🐙🌱🤖📦📄]")
|
||||
|
||||
|
||||
def _utf8_len(text: str) -> int:
|
||||
return len(text.encode("utf-8"))
|
||||
|
||||
|
||||
def _split_lines_by_budget(text: str, limit: int) -> list[str]:
|
||||
lines = text.splitlines()
|
||||
chunks: list[str] = []
|
||||
buf: list[str] = []
|
||||
for line in lines:
|
||||
candidate = "\n".join(buf + [line]) if buf else line
|
||||
if _utf8_len(candidate) <= limit:
|
||||
buf.append(line)
|
||||
continue
|
||||
if buf:
|
||||
chunks.append("\n".join(buf))
|
||||
buf = []
|
||||
if _utf8_len(line) <= limit:
|
||||
buf = [line]
|
||||
else:
|
||||
encoded = line.encode("utf-8")
|
||||
start = 0
|
||||
while start < len(encoded):
|
||||
piece = encoded[start : start + limit].decode("utf-8", errors="ignore")
|
||||
chunks.append(piece)
|
||||
start += len(piece.encode("utf-8"))
|
||||
if buf:
|
||||
chunks.append("\n".join(buf))
|
||||
return chunks
|
||||
|
||||
|
||||
def _split_sections(text: str) -> list[str]:
|
||||
sections: list[str] = []
|
||||
current: list[str] = []
|
||||
for line in text.splitlines():
|
||||
if _SECTION_START.match(line) and current:
|
||||
sections.append("\n".join(current))
|
||||
current = [line]
|
||||
else:
|
||||
current.append(line)
|
||||
if current:
|
||||
sections.append("\n".join(current))
|
||||
return sections
|
||||
|
||||
|
||||
def split_wecom_messages(text: str, limit: int = 4096) -> list[str]:
|
||||
"""超长时拆成多条;每条不超过 limit 字节,按区块边界优先。"""
|
||||
text = text.strip()
|
||||
if not text or _utf8_len(text) <= limit:
|
||||
return [text] if text else []
|
||||
|
||||
footer_reserve = 40
|
||||
pack_limit = max(512, limit - footer_reserve)
|
||||
|
||||
sections: list[str] = []
|
||||
for sec in _split_sections(text):
|
||||
if _utf8_len(sec) <= pack_limit:
|
||||
sections.append(sec)
|
||||
else:
|
||||
sections.extend(_split_lines_by_budget(sec, pack_limit))
|
||||
|
||||
packed: list[str] = []
|
||||
buf: list[str] = []
|
||||
for sec in sections:
|
||||
candidate = "\n\n".join(buf + [sec]) if buf else sec
|
||||
if _utf8_len(candidate) <= pack_limit:
|
||||
buf.append(sec)
|
||||
else:
|
||||
if buf:
|
||||
packed.append("\n\n".join(buf))
|
||||
buf = [sec]
|
||||
if buf:
|
||||
packed.append("\n\n".join(buf))
|
||||
|
||||
total = len(packed)
|
||||
if total <= 1:
|
||||
return packed
|
||||
|
||||
result: list[str] = []
|
||||
for i, chunk in enumerate(packed, 1):
|
||||
suffix = f"\n\n> 📄 {i}/{total}"
|
||||
body = chunk
|
||||
while body and _utf8_len(body + suffix) > limit:
|
||||
body = body.rsplit("\n", 1)[0] if "\n" in body else body[:-1]
|
||||
result.append(body + suffix)
|
||||
return result
|
||||
Reference in New Issue
Block a user