"""早报结构化数据：抓取结果 → JSON 中间层。""" from __future__ import annotations import json from pathlib import Path from typing import Any from daily.config import OUTPUT_DIR, env_int from daily.delta import build_movement_baseline, build_movement_context, compare_depth from daily.news.fetch import prepare_wecom_cn_news_items, prepare_wecom_news_items from daily.skills_group import group_skills_by_source def skill_id(item: dict[str, Any]) -> str: return str(item.get("id") or f"{item.get('source')}/{item.get('title')}") def _slim_skill(item: dict[str, Any]) -> dict[str, Any]: payload = { "id": skill_id(item), "title": item.get("title", ""), "source": item.get("source", ""), "installs": item.get("installs", 0), "link": item.get("link", ""), "description": item.get("description", ""), } if item.get("cluster"): payload.update( { "cluster": True, "cluster_count": item.get("cluster_count", 1), "cluster_skills": item.get("cluster_skills", []), "cluster_titles": item.get("cluster_titles", ""), "installs_fmt": item.get("installs_fmt", ""), "installs_min": item.get("installs_min"), "installs_max": item.get("installs_max"), } ) elif item.get("installs_fmt"): payload["installs_fmt"] = item.get("installs_fmt") return payload def _slim_github(item: dict[str, Any]) -> dict[str, Any]: return { "repo": item.get("repo", ""), "url": item.get("url", ""), "language": item.get("language", ""), "stars_today_fmt": item.get("stars_today_fmt", ""), "total_stars_fmt": item.get("total_stars_fmt", ""), "created_at": item.get("created_at", ""), "description": item.get("description", ""), } def _slim_news_items( ai_news: dict[str, Any], limit: int, *, prepare=prepare_wecom_news_items, ) -> list[dict[str, Any]]: items: list[dict[str, Any]] = [] for item in prepare(ai_news, limit=limit): payload = { "link": item.get("link", ""), "title": item.get("title", ""), "source_name": item.get("source_name", ""), "published_fmt": item.get("published_fmt", ""), "summary": item.get("desc_short") or "", } if item.get("score") is not None: payload["score"] = item.get("score") items.append(payload) if len(items) >= limit: break if items: return items flat = sorted( ai_news.get("flat") or [], key=lambda row: float(row.get("score") or 0), reverse=True, ) for item in flat[:limit]: payload = { "link": item.get("link", ""), "title": item.get("title", ""), "source_name": item.get("source_name", ""), "published_fmt": item.get("published_fmt", ""), "summary": item.get("summary", ""), } if item.get("score") is not None: payload["score"] = item.get("score") items.append(payload) return items def _news_pool_limit(wecom_limit: int, *, env_key: str, default_pool: int, agent_mode: bool) -> int: if not agent_mode: return wecom_limit pool = env_int(env_key, default_pool) return max(wecom_limit, pool) def _wecom_skill_pool() -> int: return max(10, env_int("DAILY_WECOM_SKILL_POOL", 200)) def build_llm_input( *, date_str: str, updated: str, trending: list[dict[str, Any]], hot: list[dict[str, Any]], github_trending: list[dict[str, Any]], github_emerging: list[dict[str, Any]], github_topic: list[dict[str, Any]], topic_name: str, ai_news: dict[str, Any], cn_ai_news: dict[str, Any], wecom_limits: dict[str, int], agent_mode: bool = False, ) -> dict[str, Any]: """供 Cursor 编辑的精简 JSON（不含完整 markdown）。""" news_limit = _news_pool_limit( wecom_limits.get("ai_news", 10), env_key="DAILY_AGENT_NEWS_POOL", default_pool=40, agent_mode=agent_mode, ) cn_news_limit = _news_pool_limit( wecom_limits.get("cn_ai_news", 8), env_key="DAILY_AGENT_CN_NEWS_POOL", default_pool=30, agent_mode=agent_mode, ) depth = compare_depth() trend_cmp = trending[:depth] hot_cmp = hot[:depth] github_cmp = github_trending[:depth] emerging_cmp = github_emerging[:depth] topic_cmp = github_topic[:depth] trending_slice = group_skills_by_source( trending, limit=wecom_limits.get("trending", 10), pool_size=wecom_limits.get("trending_pool", _wecom_skill_pool()), ) hot_slice = group_skills_by_source( hot, limit=wecom_limits.get("hot", 10), pool_size=wecom_limits.get("hot_pool", _wecom_skill_pool()), ) github_slice = github_trending[: wecom_limits.get("github", 5)] emerging_slice = github_emerging[: wecom_limits.get("emerging", 3)] topic_slice = github_topic[: wecom_limits.get("topic", 3)] movement = build_movement_context( date_str=date_str, trending=[_slim_skill(x) for x in trend_cmp], hot=[_slim_skill(x) for x in hot_cmp], github_trending=[_slim_github(x) for x in github_cmp], github_emerging=[_slim_github(x) for x in emerging_cmp], github_topic=[_slim_github(x) for x in topic_cmp], topic_name=topic_name, ) movement_baseline = build_movement_baseline( trending=[_slim_skill(x) for x in trend_cmp], hot=[_slim_skill(x) for x in hot_cmp], github_trending=[_slim_github(x) for x in github_cmp], github_emerging=[_slim_github(x) for x in emerging_cmp], github_topic=[_slim_github(x) for x in topic_cmp], depth=depth, ) return { "date": date_str, "data_updated": updated, "skills_trending": [_slim_skill(x) for x in trending_slice], "skills_hot": [_slim_skill(x) for x in hot_slice], "github_trending": [_slim_github(x) for x in github_slice], "github_emerging": [_slim_github(x) for x in emerging_slice], "github_topic": { "topic": topic_name, "repos": [_slim_github(x) for x in topic_slice], }, "ai_news": _slim_news_items(ai_news, news_limit) if ai_news.get("enabled") else [], "cn_ai_news": _slim_news_items( cn_ai_news, cn_news_limit, prepare=prepare_wecom_cn_news_items ) if cn_ai_news.get("enabled") else [], "movement": movement, "movement_baseline": movement_baseline, } def build_full_payload( llm_input: dict[str, Any], *, meta: dict[str, Any], ) -> dict[str, Any]: return {"meta": meta, "data": llm_input} def data_json_path(date_str: str) -> Path: return OUTPUT_DIR / f"{date_str}.data.json" def editorial_json_path(date_str: str) -> Path: return OUTPUT_DIR / f"{date_str}.editorial.json" def save_json(path: Path, data: dict[str, Any]) -> None: path.parent.mkdir(parents=True, exist_ok=True) path.write_text(json.dumps(data, ensure_ascii=False, indent=2), encoding="utf-8") def load_json(path: Path) -> dict[str, Any]: return json.loads(path.read_text(encoding="utf-8"))