ftb-diy/daily-robots
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat(news): research 时讯加可信过滤、去重与国内配额

Browse Source 
候选池按独立事件拉取，白名单过滤低质源，同事件与 tech 主题去重后按展示上限打包。

Co-authored-by: Cursor <cursoragent@cursor.com>
This commit is contained in:
王礼
2026-07-31 15:47:50 +08:00
parent 0384e2c7a9
commit e2f609a027
6 changed files with 848 additions and 23 deletions
Download Patch File Download Diff File Expand all files Collapse all files

88
daily/news/research.py
View File

@@ -13,6 +13,7 @@ from daily.config import OUTPUT_DIR, ROOT, env, env_int, wecom_ai_news_tech_limi
from daily.llm_client import cursor_agent_prompt, extract_json_object, has_cursor_configured
from daily.news.fetch import brief_news_summary, _normalize_link
from daily.news.pushed_links import filter_unpushed_items
from daily.news.research_quality import post_process_research_news, research_cn_min
logger = logging.getLogger(__name__)
@@ -40,6 +41,25 @@ def research_limit() -> int:
return max(1, env_int("DAILY_WECOM_AI_NEWS", 10))
def research_pool_limit(display_limit: int | None = None) -> int:
"""Agent 原始候选条数（展示上限之上多拉，供可信/去重筛）。"""
lim = display_limit if display_limit is not None else research_limit()
explicit = env_int("DAILY_AI_NEWS_RESEARCH_POOL", 0)
if explicit > 0:
return max(lim, explicit)
return max(lim * 2, lim + 8)
def research_tech_pool_limit(display_limit: int | None = None) -> int:
tech = display_limit if display_limit is not None else research_tech_limit()
if tech <= 0:
return 0
explicit = env_int("DAILY_AI_NEWS_RESEARCH_TECH_POOL", 0)
if explicit > 0:
return max(tech, explicit)
return max(tech * 2, tech + 4)
def research_json_path(date_str: str) -> Path:
return OUTPUT_DIR / f"{date_str}.ai-news-research.json"
@@ -90,7 +110,7 @@ def _normalize_research_item(raw: dict[str, Any]) -> dict[str, Any] | None:
if not title or not link or not link.startswith("http"):
return None
desc = brief_news_summary(str(raw.get("desc_short") or raw.get("summary") or ""))
return {
item: dict[str, Any] = {
"title": title,
"link": link,
"source_name": _guess_source_name(link, str(raw.get("source_name") or "")),
@@ -98,6 +118,10 @@ def _normalize_research_item(raw: dict[str, Any]) -> dict[str, Any] | None:
"desc_short": desc,
"summary_plain": desc,
}
region = str(raw.get("region") or "").strip().lower()
if region:
item["region"] = region
return item
def research_tech_limit() -> int:
@@ -133,11 +157,17 @@ def parse_research_response(
*,
limit: int,
tech_limit: int = 0,
pool_limit: int | None = None,
tech_pool_limit: int | None = None,
) -> tuple[list[dict[str, Any]], list[dict[str, Any]]]:
parsed = extract_json_object(raw)
item_cap = pool_limit if pool_limit is not None else limit
tech_cap = tech_pool_limit if tech_pool_limit is not None else tech_limit
seen: set[str] = set()
items = _parse_items_array(parsed.get("items"), limit=limit, seen=seen)
tech_items = _parse_items_array(parsed.get("tech_items"), limit=tech_limit, seen=seen) if tech_limit else []
items = _parse_items_array(parsed.get("items"), limit=item_cap, seen=seen)
tech_items = (
_parse_items_array(parsed.get("tech_items"), limit=tech_cap, seen=seen) if tech_cap else []
)
return items, tech_items
@@ -189,25 +219,38 @@ def fetch_ai_news_research(
skill = _load_skill()
now_cst = datetime.now(timezone(timedelta(hours=8)))
cn_min = research_cn_min(lim)
pool = research_pool_limit(lim)
tech_pool = research_tech_pool_limit(tech_lim)
# 候选池内国内目标略高于展示配额，避免筛完国内不足
cn_pool_target = max(cn_min * 2, cn_min + 2)
# 解析多收一点原始行，输出前/后处理再压成「去重后候选池」
raw_cap = max(pool + 10, (pool * 3) // 2)
tech_raw_cap = max(tech_pool + 4, (tech_pool * 3) // 2) if tech_pool else 0
tech_clause = ""
if tech_lim:
if tech_pool:
tech_clause = (
f"\n另输出 **tech_items 恰好 {tech_lim} 条**，聚焦工程技术："
f"\n另输出 **去重后** 约 **{tech_pool} 条** tech_items 候选（最终展示约 {tech_lim} 条），聚焦工程技术："
"模型/框架发布、开源项目、芯片算力、开发者工具、推理与工程实践。"
"与 items 不得重复 link。"
"不得与 items 重复 link/同事件；输出前自行去重，候选池内每条应为独立事件。"
)
system = (
f"{skill}\n\n"
"当前执行 **早报 AI 时讯调研**。\n"
f"时间窗口：近 **{h}** 小时（截至 {now_cst.strftime('%Y-%m-%d %H:%M')} UTC+8）。\n"
f"输出 **恰好 {lim} 条** items，按重要性排序。{tech_clause}\n"
"使用 WebSearch 检索；不要读取本项目文档或 RSS 配置。"
f"输出 **同事件去重后** 约 **{pool} 条** items 候选（按重要性排序；最终展示约 {lim} 条）。\n"
"候选池条数 = 独立事件数：同一事件多源报道只留一条最权威源，禁止用换源重复充数。\n"
f"去重后的候选中国内可信源尽量不少于 **{cn_pool_target}** 条（展示侧至少 {cn_min} 条）。\n"
f"禁止用低质源凑数；可信独立事件不足才少返回。{tech_clause}\n"
"只采用官方博客/新闻稿、政府监管原文、一线权威媒体、学术官方；"
"禁止二手搬运、标题党、营销号。使用 WebSearch 检索；不要读取本项目文档或 RSS 配置。"
)
user = (
f"/deep-research 获取近 {h} 小时的 AI 人工智能新闻资讯，"
"不区分国内国外，合并精选。"
f"只输出 JSON，items 长度={lim}"
+ (f"，tech_items 长度={tech_lim}" if tech_lim else "")
f"国内与国际合并；items 去重后约 {pool} 条独立事件（国内可信尽量 ≥{cn_pool_target}）；"
"输出前完成同事件去重；可信度不足则不写。"
f"只输出 JSON，items 去重后目标约 {pool} 条"
+ (f"，tech_items 去重后目标约 {tech_pool} 条" if tech_pool else "")
+ "。"
)
@@ -234,7 +277,13 @@ def fetch_ai_news_research(
"stats": {"error": "empty_response"},
}
items, tech_items = parse_research_response(raw, limit=lim, tech_limit=tech_lim)
items, tech_items = parse_research_response(
raw,
limit=lim,
tech_limit=tech_lim,
pool_limit=raw_cap,
tech_pool_limit=tech_raw_cap,
)
payload = extract_json_object(raw)
if payload:
_save_research_json(research_json_path(date_str), payload)
@@ -250,10 +299,23 @@ def fetch_ai_news_research(
"stats": {"error": "invalid_json"},
}
items, tech_items = post_process_research_news(
items,
tech_items,
limit=lim,
tech_limit=tech_lim,
min_cn=cn_min,
)
items = _apply_pushed_dedup(items, date_str=date_str, limit=lim)
if tech_items:
tech_items = _apply_pushed_dedup(tech_items, date_str=date_str, limit=tech_lim)
logger.info("AI 时讯 research 完成：%d 条 + %d 技术", len(items), len(tech_items))
logger.info(
"AI 时讯 research 完成：%d 条 + %d 技术（候选池 %d/%d）",
len(items),
len(tech_items),
pool,
tech_pool,
)
flat = [
{