yumao
e2f609a027
候选池按独立事件拉取，白名单过滤低质源，同事件与 tech 主题去重后按展示上限打包。 Co-authored-by: Cursor <cursoragent@cursor.com>
387 lines
13 KiB
Python
387 lines
13 KiB
Python
"""Cursor SDK + deep-research 工作流：采集 AI 时讯（方案 A，内置 WebSearch）。"""
|
||
|
||
from __future__ import annotations
|
||
|
||
import json
|
||
import logging
|
||
from datetime import datetime, timezone, timedelta
|
||
from pathlib import Path
|
||
from typing import Any
|
||
from urllib.parse import urlparse
|
||
|
||
from daily.config import OUTPUT_DIR, ROOT, env, env_int, wecom_ai_news_tech_limit
|
||
from daily.llm_client import cursor_agent_prompt, extract_json_object, has_cursor_configured
|
||
from daily.news.fetch import brief_news_summary, _normalize_link
|
||
from daily.news.pushed_links import filter_unpushed_items
|
||
from daily.news.research_quality import post_process_research_news, research_cn_min
|
||
|
||
logger = logging.getLogger(__name__)
|
||
|
||
_SKILL_DIR = ROOT / "skills" / "daily-ai-news-research"
|
||
_DEEP_RESEARCH_CANDIDATES = (
|
||
ROOT / "skills" / "deep-research" / "SKILL.md",
|
||
Path.home() / ".agents" / "skills" / "deep-research" / "SKILL.md",
|
||
Path.home() / ".cursor" / "skills" / "deep-research" / "SKILL.md",
|
||
)
|
||
|
||
|
||
def ai_news_mode() -> str:
|
||
return (env("DAILY_AI_NEWS_MODE") or "rss").strip().lower()
|
||
|
||
|
||
def is_research_mode() -> bool:
|
||
return ai_news_mode() == "research"
|
||
|
||
|
||
def research_hours() -> int:
|
||
return max(1, env_int("DAILY_AI_NEWS_HOURS", 24))
|
||
|
||
|
||
def research_limit() -> int:
|
||
return max(1, env_int("DAILY_WECOM_AI_NEWS", 10))
|
||
|
||
|
||
def research_pool_limit(display_limit: int | None = None) -> int:
|
||
"""Agent 原始候选条数（展示上限之上多拉，供可信/去重筛）。"""
|
||
lim = display_limit if display_limit is not None else research_limit()
|
||
explicit = env_int("DAILY_AI_NEWS_RESEARCH_POOL", 0)
|
||
if explicit > 0:
|
||
return max(lim, explicit)
|
||
return max(lim * 2, lim + 8)
|
||
|
||
|
||
def research_tech_pool_limit(display_limit: int | None = None) -> int:
|
||
tech = display_limit if display_limit is not None else research_tech_limit()
|
||
if tech <= 0:
|
||
return 0
|
||
explicit = env_int("DAILY_AI_NEWS_RESEARCH_TECH_POOL", 0)
|
||
if explicit > 0:
|
||
return max(tech, explicit)
|
||
return max(tech * 2, tech + 4)
|
||
|
||
|
||
def research_json_path(date_str: str) -> Path:
|
||
return OUTPUT_DIR / f"{date_str}.ai-news-research.json"
|
||
|
||
|
||
def _save_research_json(path: Path, data: dict[str, Any]) -> None:
|
||
path.parent.mkdir(parents=True, exist_ok=True)
|
||
path.write_text(json.dumps(data, ensure_ascii=False, indent=2), encoding="utf-8")
|
||
|
||
|
||
def _load_skill() -> str:
|
||
parts: list[str] = []
|
||
for path in _DEEP_RESEARCH_CANDIDATES:
|
||
if path.exists():
|
||
parts.append(path.read_text(encoding="utf-8").strip())
|
||
break
|
||
local = _SKILL_DIR / "SKILL.md"
|
||
if local.exists():
|
||
parts.append(local.read_text(encoding="utf-8").strip())
|
||
if not parts:
|
||
return "你是 AI 时讯调研员，只输出 JSON。"
|
||
return "\n\n---\n\n".join(parts)
|
||
|
||
|
||
def _guess_source_name(link: str, explicit: str) -> str:
|
||
name = (explicit or "").strip()
|
||
if name:
|
||
return name
|
||
host = urlparse(link).netloc.lower().removeprefix("www.")
|
||
mapping = {
|
||
"techcrunch.com": "TechCrunch",
|
||
"theverge.com": "The Verge",
|
||
"openai.com": "OpenAI",
|
||
"anthropic.com": "Anthropic",
|
||
"arxiv.org": "arXiv",
|
||
"qbitai.com": "量子位",
|
||
"36kr.com": "36氪",
|
||
"leiphone.com": "雷锋网",
|
||
}
|
||
for key, label in mapping.items():
|
||
if host.endswith(key) or key in host:
|
||
return label
|
||
return host.split(".")[0].capitalize() if host else "?"
|
||
|
||
|
||
def _normalize_research_item(raw: dict[str, Any]) -> dict[str, Any] | None:
|
||
title = str(raw.get("title") or "").strip()
|
||
link = _normalize_link(str(raw.get("link") or ""))
|
||
if not title or not link or not link.startswith("http"):
|
||
return None
|
||
desc = brief_news_summary(str(raw.get("desc_short") or raw.get("summary") or ""))
|
||
item: dict[str, Any] = {
|
||
"title": title,
|
||
"link": link,
|
||
"source_name": _guess_source_name(link, str(raw.get("source_name") or "")),
|
||
"published_fmt": str(raw.get("published_fmt") or "").strip(),
|
||
"desc_short": desc,
|
||
"summary_plain": desc,
|
||
}
|
||
region = str(raw.get("region") or "").strip().lower()
|
||
if region:
|
||
item["region"] = region
|
||
return item
|
||
|
||
|
||
def research_tech_limit() -> int:
|
||
return wecom_ai_news_tech_limit()
|
||
|
||
|
||
def _parse_items_array(
|
||
items_raw: Any,
|
||
*,
|
||
limit: int,
|
||
seen: set[str],
|
||
) -> list[dict[str, Any]]:
|
||
if not isinstance(items_raw, list):
|
||
return []
|
||
out: list[dict[str, Any]] = []
|
||
for row in items_raw:
|
||
if not isinstance(row, dict):
|
||
continue
|
||
item = _normalize_research_item(row)
|
||
if not item:
|
||
continue
|
||
if item["link"] in seen:
|
||
continue
|
||
seen.add(item["link"])
|
||
out.append(item)
|
||
if len(out) >= limit:
|
||
break
|
||
return out
|
||
|
||
|
||
def parse_research_response(
|
||
raw: str,
|
||
*,
|
||
limit: int,
|
||
tech_limit: int = 0,
|
||
pool_limit: int | None = None,
|
||
tech_pool_limit: int | None = None,
|
||
) -> tuple[list[dict[str, Any]], list[dict[str, Any]]]:
|
||
parsed = extract_json_object(raw)
|
||
item_cap = pool_limit if pool_limit is not None else limit
|
||
tech_cap = tech_pool_limit if tech_pool_limit is not None else tech_limit
|
||
seen: set[str] = set()
|
||
items = _parse_items_array(parsed.get("items"), limit=item_cap, seen=seen)
|
||
tech_items = (
|
||
_parse_items_array(parsed.get("tech_items"), limit=tech_cap, seen=seen) if tech_cap else []
|
||
)
|
||
return items, tech_items
|
||
|
||
|
||
def _apply_pushed_dedup(items: list[dict[str, Any]], *, date_str: str, limit: int) -> list[dict[str, Any]]:
|
||
from daily.config import news_backfill_enabled
|
||
from daily.news.sanitize import strip_relax_window_prefix
|
||
|
||
for item in items:
|
||
if item.get("desc_short"):
|
||
item["desc_short"] = strip_relax_window_prefix(str(item.get("desc_short") or ""))
|
||
fresh = filter_unpushed_items(items, date_str=date_str)
|
||
if len(fresh) >= limit:
|
||
return fresh[:limit]
|
||
if not news_backfill_enabled():
|
||
if len(fresh) < limit:
|
||
logger.info("news_short:%s", len(fresh))
|
||
return fresh[:limit]
|
||
seen = {i.get("link") for i in fresh}
|
||
for item in items:
|
||
if len(fresh) >= limit:
|
||
break
|
||
if item.get("link") not in seen:
|
||
fresh.append(item)
|
||
seen.add(item.get("link"))
|
||
return fresh[:limit]
|
||
|
||
|
||
def fetch_ai_news_research(
|
||
*,
|
||
date_str: str,
|
||
hours: int | None = None,
|
||
limit: int | None = None,
|
||
) -> dict[str, Any]:
|
||
"""Cursor Agent 调研 AI 时讯；返回 {enabled, mode, hours, items, flat, stats}。"""
|
||
h = hours if hours is not None else research_hours()
|
||
lim = limit if limit is not None else research_limit()
|
||
tech_lim = research_tech_limit()
|
||
|
||
if not has_cursor_configured():
|
||
logger.warning("DAILY_AI_NEWS_MODE=research 但未配置 CURSOR_API_KEY")
|
||
return {
|
||
"enabled": False,
|
||
"mode": "research",
|
||
"items": [],
|
||
"tech_items": [],
|
||
"flat": [],
|
||
"stats": {"error": "no_cursor_key"},
|
||
}
|
||
|
||
skill = _load_skill()
|
||
now_cst = datetime.now(timezone(timedelta(hours=8)))
|
||
cn_min = research_cn_min(lim)
|
||
pool = research_pool_limit(lim)
|
||
tech_pool = research_tech_pool_limit(tech_lim)
|
||
# 候选池内国内目标略高于展示配额，避免筛完国内不足
|
||
cn_pool_target = max(cn_min * 2, cn_min + 2)
|
||
# 解析多收一点原始行，输出前/后处理再压成「去重后候选池」
|
||
raw_cap = max(pool + 10, (pool * 3) // 2)
|
||
tech_raw_cap = max(tech_pool + 4, (tech_pool * 3) // 2) if tech_pool else 0
|
||
tech_clause = ""
|
||
if tech_pool:
|
||
tech_clause = (
|
||
f"\n另输出 **去重后** 约 **{tech_pool} 条** tech_items 候选（最终展示约 {tech_lim} 条），聚焦工程技术："
|
||
"模型/框架发布、开源项目、芯片算力、开发者工具、推理与工程实践。"
|
||
"不得与 items 重复 link/同事件；输出前自行去重，候选池内每条应为独立事件。"
|
||
)
|
||
system = (
|
||
f"{skill}\n\n"
|
||
"当前执行 **早报 AI 时讯调研**。\n"
|
||
f"时间窗口：近 **{h}** 小时（截至 {now_cst.strftime('%Y-%m-%d %H:%M')} UTC+8）。\n"
|
||
f"输出 **同事件去重后** 约 **{pool} 条** items 候选（按重要性排序；最终展示约 {lim} 条）。\n"
|
||
"候选池条数 = 独立事件数：同一事件多源报道只留一条最权威源，禁止用换源重复充数。\n"
|
||
f"去重后的候选中国内可信源尽量不少于 **{cn_pool_target}** 条（展示侧至少 {cn_min} 条）。\n"
|
||
f"禁止用低质源凑数；可信独立事件不足才少返回。{tech_clause}\n"
|
||
"只采用官方博客/新闻稿、政府监管原文、一线权威媒体、学术官方；"
|
||
"禁止二手搬运、标题党、营销号。使用 WebSearch 检索；不要读取本项目文档或 RSS 配置。"
|
||
)
|
||
user = (
|
||
f"/deep-research 获取近 {h} 小时的 AI 人工智能新闻资讯，"
|
||
f"国内与国际合并；items 去重后约 {pool} 条独立事件（国内可信尽量 ≥{cn_pool_target}）；"
|
||
"输出前完成同事件去重；可信度不足则不写。"
|
||
f"只输出 JSON，items 去重后目标约 {pool} 条"
|
||
+ (f"，tech_items 去重后目标约 {tech_pool} 条" if tech_pool else "")
|
||
+ "。"
|
||
)
|
||
|
||
try:
|
||
raw = cursor_agent_prompt(system, user)
|
||
except Exception as exc:
|
||
logger.warning("AI 时讯 research 失败：%s", exc)
|
||
return {
|
||
"enabled": False,
|
||
"mode": "research",
|
||
"items": [],
|
||
"tech_items": [],
|
||
"flat": [],
|
||
"stats": {"error": str(exc)},
|
||
}
|
||
|
||
if not raw:
|
||
return {
|
||
"enabled": False,
|
||
"mode": "research",
|
||
"items": [],
|
||
"tech_items": [],
|
||
"flat": [],
|
||
"stats": {"error": "empty_response"},
|
||
}
|
||
|
||
items, tech_items = parse_research_response(
|
||
raw,
|
||
limit=lim,
|
||
tech_limit=tech_lim,
|
||
pool_limit=raw_cap,
|
||
tech_pool_limit=tech_raw_cap,
|
||
)
|
||
payload = extract_json_object(raw)
|
||
if payload:
|
||
_save_research_json(research_json_path(date_str), payload)
|
||
|
||
if not items and not tech_items:
|
||
logger.warning("AI 时讯 research JSON 无效或无条目")
|
||
return {
|
||
"enabled": False,
|
||
"mode": "research",
|
||
"items": [],
|
||
"tech_items": [],
|
||
"flat": [],
|
||
"stats": {"error": "invalid_json"},
|
||
}
|
||
|
||
items, tech_items = post_process_research_news(
|
||
items,
|
||
tech_items,
|
||
limit=lim,
|
||
tech_limit=tech_lim,
|
||
min_cn=cn_min,
|
||
)
|
||
items = _apply_pushed_dedup(items, date_str=date_str, limit=lim)
|
||
if tech_items:
|
||
tech_items = _apply_pushed_dedup(tech_items, date_str=date_str, limit=tech_lim)
|
||
logger.info(
|
||
"AI 时讯 research 完成：%d 条 + %d 技术（候选池 %d/%d）",
|
||
len(items),
|
||
len(tech_items),
|
||
pool,
|
||
tech_pool,
|
||
)
|
||
|
||
flat = [
|
||
{
|
||
"title": i["title"],
|
||
"link": i["link"],
|
||
"summary": i.get("summary_plain") or i.get("desc_short") or "",
|
||
"source_name": i["source_name"],
|
||
"published_fmt": i.get("published_fmt") or "",
|
||
"category_id": "research",
|
||
"category_name": "Deep Research",
|
||
"category_icon": "🔍",
|
||
}
|
||
for i in items + tech_items
|
||
]
|
||
|
||
return {
|
||
"enabled": True,
|
||
"mode": "research",
|
||
"hours": h,
|
||
"items": items,
|
||
"tech_items": tech_items,
|
||
"flat": flat,
|
||
"stats": {"source": "cursor_research", "items": len(items), "tech_items": len(tech_items)},
|
||
}
|
||
|
||
|
||
def format_research_news_section(
|
||
research: dict[str, Any],
|
||
*,
|
||
section_no: int,
|
||
wecom_limit: int | None = None,
|
||
) -> list[str]:
|
||
if not research.get("enabled"):
|
||
hint = research.get("stats", {}).get("error", "调研失败或未配置 CURSOR_API_KEY")
|
||
return [
|
||
"---",
|
||
"",
|
||
f"## {section_no}、AI 时讯精选（Deep Research）",
|
||
"",
|
||
f"*不可用：{hint}*",
|
||
"",
|
||
]
|
||
|
||
hours = research.get("hours", 24)
|
||
items = (research.get("flat") or [])[: wecom_limit or research_limit()]
|
||
lines = [
|
||
"---",
|
||
"",
|
||
f"## {section_no}、AI 时讯精选（Deep Research）",
|
||
"",
|
||
f"> 近 **{hours}h** · Cursor Agent WebSearch · {len(items)} 条",
|
||
"",
|
||
]
|
||
if not items:
|
||
lines.append("*暂无可用条目。*")
|
||
lines.append("")
|
||
return lines
|
||
|
||
for i, item in enumerate(items, 1):
|
||
pub = f" · {item['published_fmt']}" if item.get("published_fmt") else ""
|
||
lines.append(
|
||
f"{i}. **[{item['title']}]({item['link']})** · `{item['source_name']}`{pub}"
|
||
)
|
||
summary = item.get("summary") or ""
|
||
if summary:
|
||
lines.append(f" - {summary}")
|
||
lines.append("")
|
||
return lines
|