yumao
391f887d73
标题相似度合并、信源/时效/昨日重复加权排序，Agent 模式扩大 LLM 新闻候选池并记录抓取失败源。 Co-authored-by: Cursor <cursoragent@cursor.com>
221 lines
7.2 KiB
Python
221 lines
7.2 KiB
Python
"""早报结构化数据：抓取结果 → JSON 中间层。"""
|
||
|
||
from __future__ import annotations
|
||
|
||
import json
|
||
from pathlib import Path
|
||
from typing import Any
|
||
|
||
from daily.config import OUTPUT_DIR, env_int
|
||
from daily.delta import build_movement_baseline, build_movement_context, compare_depth
|
||
from daily.news.fetch import prepare_wecom_cn_news_items, prepare_wecom_news_items
|
||
from daily.skills_group import group_skills_by_source
|
||
|
||
|
||
def skill_id(item: dict[str, Any]) -> str:
|
||
return str(item.get("id") or f"{item.get('source')}/{item.get('title')}")
|
||
|
||
|
||
def _slim_skill(item: dict[str, Any]) -> dict[str, Any]:
|
||
payload = {
|
||
"id": skill_id(item),
|
||
"title": item.get("title", ""),
|
||
"source": item.get("source", ""),
|
||
"installs": item.get("installs", 0),
|
||
"link": item.get("link", ""),
|
||
"description": item.get("description", ""),
|
||
}
|
||
if item.get("cluster"):
|
||
payload.update(
|
||
{
|
||
"cluster": True,
|
||
"cluster_count": item.get("cluster_count", 1),
|
||
"cluster_skills": item.get("cluster_skills", []),
|
||
"cluster_titles": item.get("cluster_titles", ""),
|
||
"installs_fmt": item.get("installs_fmt", ""),
|
||
"installs_min": item.get("installs_min"),
|
||
"installs_max": item.get("installs_max"),
|
||
}
|
||
)
|
||
elif item.get("installs_fmt"):
|
||
payload["installs_fmt"] = item.get("installs_fmt")
|
||
return payload
|
||
|
||
|
||
def _slim_github(item: dict[str, Any]) -> dict[str, Any]:
|
||
return {
|
||
"repo": item.get("repo", ""),
|
||
"url": item.get("url", ""),
|
||
"language": item.get("language", ""),
|
||
"stars_today_fmt": item.get("stars_today_fmt", ""),
|
||
"total_stars_fmt": item.get("total_stars_fmt", ""),
|
||
"created_at": item.get("created_at", ""),
|
||
"description": item.get("description", ""),
|
||
}
|
||
|
||
|
||
def _slim_news_items(
|
||
ai_news: dict[str, Any],
|
||
limit: int,
|
||
*,
|
||
prepare=prepare_wecom_news_items,
|
||
) -> list[dict[str, Any]]:
|
||
items: list[dict[str, Any]] = []
|
||
for item in prepare(ai_news, limit=limit):
|
||
payload = {
|
||
"link": item.get("link", ""),
|
||
"title": item.get("title", ""),
|
||
"source_name": item.get("source_name", ""),
|
||
"published_fmt": item.get("published_fmt", ""),
|
||
"summary": item.get("desc_short") or "",
|
||
}
|
||
if item.get("score") is not None:
|
||
payload["score"] = item.get("score")
|
||
items.append(payload)
|
||
if len(items) >= limit:
|
||
break
|
||
if items:
|
||
return items
|
||
flat = sorted(
|
||
ai_news.get("flat") or [],
|
||
key=lambda row: float(row.get("score") or 0),
|
||
reverse=True,
|
||
)
|
||
for item in flat[:limit]:
|
||
payload = {
|
||
"link": item.get("link", ""),
|
||
"title": item.get("title", ""),
|
||
"source_name": item.get("source_name", ""),
|
||
"published_fmt": item.get("published_fmt", ""),
|
||
"summary": item.get("summary", ""),
|
||
}
|
||
if item.get("score") is not None:
|
||
payload["score"] = item.get("score")
|
||
items.append(payload)
|
||
return items
|
||
|
||
|
||
def _news_pool_limit(wecom_limit: int, *, env_key: str, default_pool: int, agent_mode: bool) -> int:
|
||
if not agent_mode:
|
||
return wecom_limit
|
||
pool = env_int(env_key, default_pool)
|
||
return max(wecom_limit, pool)
|
||
|
||
|
||
def _wecom_skill_pool() -> int:
|
||
return max(10, env_int("DAILY_WECOM_SKILL_POOL", 200))
|
||
|
||
|
||
def build_llm_input(
|
||
*,
|
||
date_str: str,
|
||
updated: str,
|
||
trending: list[dict[str, Any]],
|
||
hot: list[dict[str, Any]],
|
||
github_trending: list[dict[str, Any]],
|
||
github_emerging: list[dict[str, Any]],
|
||
github_topic: list[dict[str, Any]],
|
||
topic_name: str,
|
||
ai_news: dict[str, Any],
|
||
cn_ai_news: dict[str, Any],
|
||
wecom_limits: dict[str, int],
|
||
agent_mode: bool = False,
|
||
) -> dict[str, Any]:
|
||
"""供 Cursor 编辑的精简 JSON（不含完整 markdown）。"""
|
||
news_limit = _news_pool_limit(
|
||
wecom_limits.get("ai_news", 10),
|
||
env_key="DAILY_AGENT_NEWS_POOL",
|
||
default_pool=40,
|
||
agent_mode=agent_mode,
|
||
)
|
||
cn_news_limit = _news_pool_limit(
|
||
wecom_limits.get("cn_ai_news", 8),
|
||
env_key="DAILY_AGENT_CN_NEWS_POOL",
|
||
default_pool=30,
|
||
agent_mode=agent_mode,
|
||
)
|
||
depth = compare_depth()
|
||
trend_cmp = trending[:depth]
|
||
hot_cmp = hot[:depth]
|
||
github_cmp = github_trending[:depth]
|
||
emerging_cmp = github_emerging[:depth]
|
||
topic_cmp = github_topic[:depth]
|
||
|
||
trending_slice = group_skills_by_source(
|
||
trending,
|
||
limit=wecom_limits.get("trending", 10),
|
||
pool_size=wecom_limits.get("trending_pool", _wecom_skill_pool()),
|
||
)
|
||
hot_slice = group_skills_by_source(
|
||
hot,
|
||
limit=wecom_limits.get("hot", 10),
|
||
pool_size=wecom_limits.get("hot_pool", _wecom_skill_pool()),
|
||
)
|
||
github_slice = github_trending[: wecom_limits.get("github", 5)]
|
||
emerging_slice = github_emerging[: wecom_limits.get("emerging", 3)]
|
||
topic_slice = github_topic[: wecom_limits.get("topic", 3)]
|
||
|
||
movement = build_movement_context(
|
||
date_str=date_str,
|
||
trending=[_slim_skill(x) for x in trend_cmp],
|
||
hot=[_slim_skill(x) for x in hot_cmp],
|
||
github_trending=[_slim_github(x) for x in github_cmp],
|
||
github_emerging=[_slim_github(x) for x in emerging_cmp],
|
||
github_topic=[_slim_github(x) for x in topic_cmp],
|
||
topic_name=topic_name,
|
||
)
|
||
movement_baseline = build_movement_baseline(
|
||
trending=[_slim_skill(x) for x in trend_cmp],
|
||
hot=[_slim_skill(x) for x in hot_cmp],
|
||
github_trending=[_slim_github(x) for x in github_cmp],
|
||
github_emerging=[_slim_github(x) for x in emerging_cmp],
|
||
github_topic=[_slim_github(x) for x in topic_cmp],
|
||
depth=depth,
|
||
)
|
||
|
||
return {
|
||
"date": date_str,
|
||
"data_updated": updated,
|
||
"skills_trending": [_slim_skill(x) for x in trending_slice],
|
||
"skills_hot": [_slim_skill(x) for x in hot_slice],
|
||
"github_trending": [_slim_github(x) for x in github_slice],
|
||
"github_emerging": [_slim_github(x) for x in emerging_slice],
|
||
"github_topic": {
|
||
"topic": topic_name,
|
||
"repos": [_slim_github(x) for x in topic_slice],
|
||
},
|
||
"ai_news": _slim_news_items(ai_news, news_limit) if ai_news.get("enabled") else [],
|
||
"cn_ai_news": _slim_news_items(
|
||
cn_ai_news, cn_news_limit, prepare=prepare_wecom_cn_news_items
|
||
)
|
||
if cn_ai_news.get("enabled")
|
||
else [],
|
||
"movement": movement,
|
||
"movement_baseline": movement_baseline,
|
||
}
|
||
|
||
|
||
def build_full_payload(
|
||
llm_input: dict[str, Any],
|
||
*,
|
||
meta: dict[str, Any],
|
||
) -> dict[str, Any]:
|
||
return {"meta": meta, "data": llm_input}
|
||
|
||
|
||
def data_json_path(date_str: str) -> Path:
|
||
return OUTPUT_DIR / f"{date_str}.data.json"
|
||
|
||
|
||
def editorial_json_path(date_str: str) -> Path:
|
||
return OUTPUT_DIR / f"{date_str}.editorial.json"
|
||
|
||
|
||
def save_json(path: Path, data: dict[str, Any]) -> None:
|
||
path.parent.mkdir(parents=True, exist_ok=True)
|
||
path.write_text(json.dumps(data, ensure_ascii=False, indent=2), encoding="utf-8")
|
||
|
||
|
||
def load_json(path: Path) -> dict[str, Any]:
|
||
return json.loads(path.read_text(encoding="utf-8"))
|