From 4a128b0fa632b4c56ed0f34f3709528e65011c21 Mon Sep 17 00:00:00 2001 From: yumao Date: Tue, 14 Jul 2026 10:42:00 +0800 Subject: [PATCH] =?UTF-8?q?fix:=20=E5=85=B3=E9=97=AD=E6=96=B0=E9=97=BB?= =?UTF-8?q?=E6=94=BE=E5=AE=BD=E5=87=91=E6=95=B0=E5=B9=B6=E5=89=A5=E7=A6=BB?= =?UTF-8?q?=E6=94=BE=E5=AE=BD=E7=AA=97=E5=8F=A3=E6=96=87=E6=A1=88?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Cursor --- daily/news/fetch.py | 342 ++++++++++++++++++------- daily/news/research.py | 324 +++++++++++++++++++++++ daily/news/sanitize.py | 18 ++ skills/daily-ai-news-research/SKILL.md | 58 +++++ tests/test_news_relax.py | 62 +++++ 5 files changed, 712 insertions(+), 92 deletions(-) create mode 100644 daily/news/research.py create mode 100644 daily/news/sanitize.py create mode 100644 skills/daily-ai-news-research/SKILL.md create mode 100644 tests/test_news_relax.py diff --git a/daily/news/fetch.py b/daily/news/fetch.py index 237eadc..669b857 100644 --- a/daily/news/fetch.py +++ b/daily/news/fetch.py @@ -8,17 +8,19 @@ import time import html import xml.etree.ElementTree as ET from concurrent.futures import ThreadPoolExecutor, as_completed -from datetime import datetime, timezone, timedelta +from datetime import datetime, time as dt_time, timezone, timedelta from email.utils import parsedate_to_datetime from typing import Any from urllib.parse import parse_qs, urlencode, urlparse, urlunparse +from zoneinfo import ZoneInfo import certifi import httpx -from daily.config import env, env_int, news_summary_limit +from daily.config import env, env_int, news_summary_limit, wecom_news_desc_limit from daily.news.feeds import NEWS_CATEGORIES, NewsCategory, NewsFeed from daily.news.feeds_cn import CN_AI_TITLE_KEYWORDS, CN_NEWS_CATEGORIES +from daily.text_utils import trim_brief logger = logging.getLogger(__name__) @@ -49,12 +51,37 @@ def _hours_window() -> int: return max(1, env_int("DAILY_AI_NEWS_HOURS", 24)) +def _news_tz_name() -> str: + return (env("DAILY_AI_NEWS_TZ") or env("DAILY_SCHEDULE_TZ") or "Asia/Shanghai").strip() + + +def _floor_today_enabled() -> bool: + raw = env("DAILY_AI_NEWS_FLOOR_TODAY") + if raw is None: + return True + return raw.strip().lower() not in {"0", "false", "no", "off"} + + +def _cutoff_datetime(*, floor_today: bool) -> datetime: + """滚动 N 小时窗口；国际新闻可叠加「不早于今日 0 点（本地时区）」。""" + now = _now_utc() + rolling = now - timedelta(hours=_hours_window()) + if not floor_today: + return rolling + tz = ZoneInfo(_news_tz_name()) + local = now.astimezone(tz) + start_today = local.replace(hour=0, minute=0, second=0, microsecond=0).astimezone(timezone.utc) + return max(rolling, start_today) + + def _per_feed_limit() -> int: - return max(1, env_int("DAILY_AI_NEWS_PER_FEED", 3)) + want = max(_wecom_limit(), _wecom_cn_limit()) + return max(want // 2, env_int("DAILY_AI_NEWS_PER_FEED", 5)) def _per_category_limit() -> int: - return max(1, env_int("DAILY_AI_NEWS_PER_CATEGORY", 5)) + want = max(_wecom_limit(), _wecom_cn_limit()) + return max(want, env_int("DAILY_AI_NEWS_PER_CATEGORY", 10)) def _wecom_limit() -> int: @@ -62,7 +89,7 @@ def _wecom_limit() -> int: def _wecom_cn_limit() -> int: - return max(1, env_int("DAILY_WECOM_CN_AI_NEWS", 8)) + return max(1, env_int("DAILY_WECOM_CN_AI_NEWS", 10)) def _matches_cn_ai_title(title: str) -> bool: @@ -102,7 +129,6 @@ def _parse_datetime(value: str | None) -> datetime | None: for fmt in ( "%Y-%m-%dT%H:%M:%SZ", "%Y-%m-%dT%H:%M:%S%z", - "%Y-%m-%d", ): try: dt = datetime.strptime(text[: len(fmt.replace("%z", "+0000"))], fmt.replace("%z", "")) @@ -111,17 +137,103 @@ def _parse_datetime(value: str | None) -> datetime | None: return dt.astimezone(timezone.utc) except ValueError: continue + if re.match(r"^\d{4}-\d{2}-\d{2}$", text): + try: + tz = ZoneInfo(_news_tz_name()) + day = datetime.strptime(text, "%Y-%m-%d").date() + # 仅日期时按本地中午估算，避免 UTC 0 点误判为「前一天」 + dt = datetime.combine(day, dt_time(12, 0), tzinfo=tz) + return dt.astimezone(timezone.utc) + except ValueError: + pass return None def _clean_text(text: str | None, limit: int = 200) -> str: if not text: return "" - plain = STRIP_HTML.sub(" ", html.unescape(text)) - plain = WS.sub(" ", plain).strip() + plain = _strip_summary_plain(text) if limit <= 0 or len(plain) <= limit: return plain - return plain[: limit - 3] + "..." + return trim_brief(plain, limit) + + +def _strip_summary_plain(text: str | None) -> str: + if not text: + return "" + plain = STRIP_HTML.sub(" ", html.unescape(text)) + return WS.sub(" ", plain).strip() + + +_JUNK_SUMMARY_RE = re.compile( + r"^(点击查看原文|article url:|comments url:|discussion on hn|read more)", + re.IGNORECASE, +) + + +def _is_junk_news_summary(text: str) -> bool: + if not text: + return True + if _JUNK_SUMMARY_RE.match(text.strip()): + return True + if text.strip().endswith(">") and "点击" in text: + return True + return False + + +def brief_news_summary(text: str | None, limit: int | None = None) -> str: + """企微新闻一句摘要：去 HTML、过滤占位文案、句读处截断。""" + plain = _strip_summary_plain(text) + if _is_junk_news_summary(plain): + return "" + lim = wecom_news_desc_limit() if limit is None else limit + return trim_brief(plain, lim) + + +def sync_wecom_news_rows(items: list[dict[str, Any]], flat: list[dict[str, Any]]) -> None: + """中文化后，用 flat 最新 summary 刷新企微 desc_short。""" + by_link = {_normalize_link(str(i.get("link") or "")): i for i in flat if i.get("link")} + for row in items: + link = _normalize_link(str(row.get("link") or "")) + src = by_link.get(link) + if src: + row["desc_short"] = brief_news_summary(src.get("summary")) + + +def finalize_wecom_news_items( + items: list[dict[str, Any]], + *, + force_chinese: bool = False, +) -> None: + """企微新闻摘要：确保 desc_short 为中文（国际源 force_chinese=True）。""" + from daily.localize import LocalizeJob, localize_brief_descriptions, needs_chinese + from daily.news.sanitize import strip_relax_window_prefix + + limit = wecom_news_desc_limit() + jobs: list[LocalizeJob] = [] + keyed: list[tuple[str, dict[str, Any]]] = [] + for idx, item in enumerate(items): + text = strip_relax_window_prefix( + (item.get("desc_short") or item.get("summary_plain") or "").strip() + ) + if text: + item["desc_short"] = text + if not text or _is_junk_news_summary(text): + item["desc_short"] = "" + continue + if force_chinese or needs_chinese(text): + key = f"wecom-news:{item.get('link') or idx}" + jobs.append(LocalizeJob(key, text, limit)) + keyed.append((key, item)) + elif not item.get("desc_short"): + item["desc_short"] = brief_news_summary(text, limit) + + if not jobs: + return + zh_map = localize_brief_descriptions(jobs, archive=True) + for key, item in keyed: + if key in zh_map: + item["desc_short"] = strip_relax_window_prefix(zh_map[key]) def _normalize_link(link: str) -> str: @@ -317,10 +429,119 @@ def _dedupe_items(items: list[dict[str, Any]]) -> list[dict[str, Any]]: return result +def _filter_flat_in_window( + flat: list[dict[str, Any]], + *, + floor_today: bool, +) -> list[dict[str, Any]]: + cutoff = _cutoff_datetime(floor_today=floor_today) + items = [i for i in _dedupe_items(flat) if _within_window(i, cutoff)] + items.sort(key=_sort_key, reverse=True) + return items + + +def _pick_news_items( + flat: list[dict[str, Any]], + limit: int, + preferred: tuple[str, ...], + *, + one_per_source: bool = False, +) -> list[dict[str, Any]]: + picked: list[dict[str, Any]] = [] + seen_links: set[str] = set() + seen_sources: set[str] = set() + + def _try_take(item: dict[str, Any]) -> bool: + link = _normalize_link(item.get("link", "")) + if not link or link in seen_links: + return False + if one_per_source: + source = item.get("source_name", "?") + if source in seen_sources: + return False + seen_sources.add(source) + seen_links.add(link) + picked.append(item) + return True + + for cat in preferred: + for item in flat: + if item.get("category_id") != cat: + continue + if _try_take(item) and len(picked) >= limit: + return picked[:limit] + + for item in flat: + if _try_take(item) and len(picked) >= limit: + break + return picked[:limit] + + +def _fill_picked_to_limit( + picked: list[dict[str, Any]], + pools: list[list[dict[str, Any]]], + limit: int, +) -> list[dict[str, Any]]: + seen_links = {_normalize_link(i.get("link", "")) for i in picked} + for pool in pools: + for item in pool: + if len(picked) >= limit: + return picked[:limit] + link = _normalize_link(item.get("link", "")) + if not link or link in seen_links: + continue + picked.append(item) + seen_links.add(link) + return picked[:limit] + + +def _to_wecom_news_row(item: dict[str, Any]) -> dict[str, Any]: + plain = _strip_summary_plain(item.get("summary", "")) + return { + "title": item.get("title", "?"), + "link": item.get("link", ""), + "source_name": item.get("source_name", "?"), + "published_fmt": item.get("published_fmt", ""), + "desc_short": brief_news_summary(plain), + "summary_plain": plain, + } + + +def _apply_pushed_dedup_with_backfill( + items: list[dict[str, Any]], + picked: list[dict[str, Any]], + *, + date_str: str | None, + limit: int, +) -> list[dict[str, Any]]: + if not date_str: + return items[:limit] + from daily.config import news_backfill_enabled + from daily.news.pushed_links import filter_unpushed_items + + fresh = filter_unpushed_items(items, date_str=date_str) + if len(fresh) >= limit: + return fresh[:limit] + if not news_backfill_enabled(): + if len(fresh) < limit: + logger.info("news_short:%s", len(fresh)) + return fresh[:limit] + seen = {_normalize_link(i.get("link", "")) for i in fresh if i.get("link")} + for item in picked: + if len(fresh) >= limit: + break + link = _normalize_link(item.get("link", "")) + if not link or link in seen: + continue + fresh.append(_to_wecom_news_row(item)) + seen.add(link) + return fresh[:limit] + + def _within_window(item: dict[str, Any], cutoff: datetime) -> bool: dt = _entry_datetime(item) if dt is None: - return True + return False return dt >= cutoff @@ -331,11 +552,11 @@ def _sort_key(item: dict[str, Any]) -> tuple[int, datetime]: return (0, dt) -def _fetch_news(categories: tuple[NewsCategory, ...]) -> dict[str, Any]: +def _fetch_news(categories: tuple[NewsCategory, ...], *, floor_today: bool = False) -> dict[str, Any]: hours = _hours_window() per_feed = _per_feed_limit() per_category = _per_category_limit() - cutoff = _now_utc() - timedelta(hours=hours) + cutoff = _cutoff_datetime(floor_today=floor_today) headers = {"User-Agent": USER_AGENT, "Accept": "application/rss+xml, application/atom+xml, application/xml, text/xml, */*"} tasks: list[tuple[NewsCategory, NewsFeed]] = [] @@ -406,6 +627,7 @@ def _fetch_news(categories: tuple[NewsCategory, ...]) -> dict[str, Any]: return { "enabled": True, "hours": hours, + "floor_today": floor_today, "categories": categories_out, "flat": flat, "stats": stats, @@ -416,7 +638,7 @@ def fetch_ai_news() -> dict[str, Any]: """按类别抓取国际 AI 时讯，返回 {enabled, hours, categories, flat, stats}。""" if not _enabled(): return {"enabled": False, "categories": [], "flat": [], "stats": {}} - return _fetch_news(NEWS_CATEGORIES) + return _fetch_news(NEWS_CATEGORIES, floor_today=_floor_today_enabled()) def fetch_cn_ai_news() -> dict[str, Any]: @@ -459,12 +681,13 @@ def _format_news_section( categories = news.get("categories") or [] hours = news.get("hours", 72) + floor_note = " · 仅今日" if news.get("floor_today") else "" lines = [ "---", "", f"## {section_no}、{title}", "", - f"> 近 **{hours}h** · {news.get('stats', {}).get('feeds_ok', 0)}/{news.get('stats', {}).get('feeds_total', 0)} 源可用", + f"> 近 **{hours}h**{floor_note} · {news.get('stats', {}).get('feeds_ok', 0)}/{news.get('stats', {}).get('feeds_total', 0)} 源可用", "", ] @@ -501,88 +724,23 @@ def prepare_wecom_news_items(news: dict[str, Any], *, date_str: str | None = Non if not news.get("enabled"): return [] limit = _wecom_limit() - flat = _dedupe_items(news.get("flat") or []) - flat.sort(key=_sort_key, reverse=True) + floor = bool(news.get("floor_today", _floor_today_enabled())) + flat_strict = _filter_flat_in_window(news.get("flat") or [], floor_today=floor) + flat_relaxed = _filter_flat_in_window(news.get("flat") or [], floor_today=False) preferred = ("media", "newsletter", "official", "community", "research", "developer") - picked: list[dict[str, Any]] = [] - seen: set[str] = set() - for cat in preferred: - for item in flat: - link = _normalize_link(item.get("link", "")) - if item.get("category_id") != cat or link in seen: - continue - picked.append(item) - seen.add(link) - if len(picked) >= limit: - break - if len(picked) >= limit: - break - items: list[dict[str, Any]] = [] - for item in picked[:limit]: - items.append( - { - "title": item.get("title", "?"), - "link": item.get("link", ""), - "source_name": item.get("source_name", "?"), - "published_fmt": item.get("published_fmt", ""), - "desc_short": _clean_text(item.get("summary", ""), 36), - } - ) - if date_str: - from daily.news.pushed_links import filter_unpushed_items - - items = filter_unpushed_items(items, date_str=date_str) - return items + picked = _pick_news_items(flat_strict, limit, preferred) + picked = _fill_picked_to_limit(picked, [flat_relaxed, news.get("flat") or []], limit) + items = [_to_wecom_news_row(item) for item in picked[:limit]] + return _apply_pushed_dedup_with_backfill(items, picked, date_str=date_str, limit=limit) def prepare_wecom_cn_news_items(news: dict[str, Any], *, date_str: str | None = None) -> list[dict[str, Any]]: if not news.get("enabled"): return [] limit = _wecom_cn_limit() - flat = _dedupe_items(news.get("flat") or []) - flat.sort(key=_sort_key, reverse=True) - preferred = ("media", "tech", "dev") - picked: list[dict[str, Any]] = [] - seen_links: set[str] = set() - seen_sources: set[str] = set() - - for cat in preferred: - for item in flat: - link = _normalize_link(item.get("link", "")) - source = item.get("source_name", "?") - if item.get("category_id") != cat or not link or link in seen_links or source in seen_sources: - continue - picked.append(item) - seen_links.add(link) - seen_sources.add(source) - if len(picked) >= limit: - break - if len(picked) >= limit: - break - - if len(picked) < limit: - for item in flat: - link = _normalize_link(item.get("link", "")) - if not link or link in seen_links: - continue - picked.append(item) - seen_links.add(link) - if len(picked) >= limit: - break - - items: list[dict[str, Any]] = [] - for item in picked[:limit]: - items.append( - { - "title": item.get("title", "?"), - "link": item.get("link", ""), - "source_name": item.get("source_name", "?"), - "published_fmt": item.get("published_fmt", ""), - "desc_short": _clean_text(item.get("summary", ""), 36), - } - ) - if date_str: - from daily.news.pushed_links import filter_unpushed_items - - items = filter_unpushed_items(items, date_str=date_str) - return items + flat = _filter_flat_in_window(news.get("flat") or [], floor_today=False) + preferred = ("media", "tech") + picked = _pick_news_items(flat, limit, preferred, one_per_source=True) + picked = _fill_picked_to_limit(picked, [news.get("flat") or []], limit) + items = [_to_wecom_news_row(item) for item in picked[:limit]] + return _apply_pushed_dedup_with_backfill(items, picked, date_str=date_str, limit=limit) diff --git a/daily/news/research.py b/daily/news/research.py new file mode 100644 index 0000000..5fdfad1 --- /dev/null +++ b/daily/news/research.py @@ -0,0 +1,324 @@ +"""Cursor SDK + deep-research 工作流：采集 AI 时讯（方案 A，内置 WebSearch）。""" + +from __future__ import annotations + +import json +import logging +from datetime import datetime, timezone, timedelta +from pathlib import Path +from typing import Any +from urllib.parse import urlparse + +from daily.config import OUTPUT_DIR, ROOT, env, env_int, wecom_ai_news_tech_limit +from daily.llm_client import cursor_agent_prompt, extract_json_object, has_cursor_configured +from daily.news.fetch import brief_news_summary, _normalize_link +from daily.news.pushed_links import filter_unpushed_items + +logger = logging.getLogger(__name__) + +_SKILL_DIR = ROOT / "skills" / "daily-ai-news-research" +_DEEP_RESEARCH_CANDIDATES = ( + ROOT / "skills" / "deep-research" / "SKILL.md", + Path.home() / ".agents" / "skills" / "deep-research" / "SKILL.md", + Path.home() / ".cursor" / "skills" / "deep-research" / "SKILL.md", +) + + +def ai_news_mode() -> str: + return (env("DAILY_AI_NEWS_MODE") or "rss").strip().lower() + + +def is_research_mode() -> bool: + return ai_news_mode() == "research" + + +def research_hours() -> int: + return max(1, env_int("DAILY_AI_NEWS_HOURS", 24)) + + +def research_limit() -> int: + return max(1, env_int("DAILY_WECOM_AI_NEWS", 10)) + + +def research_json_path(date_str: str) -> Path: + return OUTPUT_DIR / f"{date_str}.ai-news-research.json" + + +def _save_research_json(path: Path, data: dict[str, Any]) -> None: + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(json.dumps(data, ensure_ascii=False, indent=2), encoding="utf-8") + + +def _load_skill() -> str: + parts: list[str] = [] + for path in _DEEP_RESEARCH_CANDIDATES: + if path.exists(): + parts.append(path.read_text(encoding="utf-8").strip()) + break + local = _SKILL_DIR / "SKILL.md" + if local.exists(): + parts.append(local.read_text(encoding="utf-8").strip()) + if not parts: + return "你是 AI 时讯调研员，只输出 JSON。" + return "\n\n---\n\n".join(parts) + + +def _guess_source_name(link: str, explicit: str) -> str: + name = (explicit or "").strip() + if name: + return name + host = urlparse(link).netloc.lower().removeprefix("www.") + mapping = { + "techcrunch.com": "TechCrunch", + "theverge.com": "The Verge", + "openai.com": "OpenAI", + "anthropic.com": "Anthropic", + "arxiv.org": "arXiv", + "qbitai.com": "量子位", + "36kr.com": "36氪", + "leiphone.com": "雷锋网", + } + for key, label in mapping.items(): + if host.endswith(key) or key in host: + return label + return host.split(".")[0].capitalize() if host else "?" + + +def _normalize_research_item(raw: dict[str, Any]) -> dict[str, Any] | None: + title = str(raw.get("title") or "").strip() + link = _normalize_link(str(raw.get("link") or "")) + if not title or not link or not link.startswith("http"): + return None + desc = brief_news_summary(str(raw.get("desc_short") or raw.get("summary") or "")) + return { + "title": title, + "link": link, + "source_name": _guess_source_name(link, str(raw.get("source_name") or "")), + "published_fmt": str(raw.get("published_fmt") or "").strip(), + "desc_short": desc, + "summary_plain": desc, + } + + +def research_tech_limit() -> int: + return wecom_ai_news_tech_limit() + + +def _parse_items_array( + items_raw: Any, + *, + limit: int, + seen: set[str], +) -> list[dict[str, Any]]: + if not isinstance(items_raw, list): + return [] + out: list[dict[str, Any]] = [] + for row in items_raw: + if not isinstance(row, dict): + continue + item = _normalize_research_item(row) + if not item: + continue + if item["link"] in seen: + continue + seen.add(item["link"]) + out.append(item) + if len(out) >= limit: + break + return out + + +def parse_research_response( + raw: str, + *, + limit: int, + tech_limit: int = 0, +) -> tuple[list[dict[str, Any]], list[dict[str, Any]]]: + parsed = extract_json_object(raw) + seen: set[str] = set() + items = _parse_items_array(parsed.get("items"), limit=limit, seen=seen) + tech_items = _parse_items_array(parsed.get("tech_items"), limit=tech_limit, seen=seen) if tech_limit else [] + return items, tech_items + + +def _apply_pushed_dedup(items: list[dict[str, Any]], *, date_str: str, limit: int) -> list[dict[str, Any]]: + from daily.config import news_backfill_enabled + from daily.news.sanitize import strip_relax_window_prefix + + for item in items: + if item.get("desc_short"): + item["desc_short"] = strip_relax_window_prefix(str(item.get("desc_short") or "")) + fresh = filter_unpushed_items(items, date_str=date_str) + if len(fresh) >= limit: + return fresh[:limit] + if not news_backfill_enabled(): + if len(fresh) < limit: + logger.info("news_short:%s", len(fresh)) + return fresh[:limit] + seen = {i.get("link") for i in fresh} + for item in items: + if len(fresh) >= limit: + break + if item.get("link") not in seen: + fresh.append(item) + seen.add(item.get("link")) + return fresh[:limit] + + +def fetch_ai_news_research( + *, + date_str: str, + hours: int | None = None, + limit: int | None = None, +) -> dict[str, Any]: + """Cursor Agent 调研 AI 时讯；返回 {enabled, mode, hours, items, flat, stats}。""" + h = hours if hours is not None else research_hours() + lim = limit if limit is not None else research_limit() + tech_lim = research_tech_limit() + + if not has_cursor_configured(): + logger.warning("DAILY_AI_NEWS_MODE=research 但未配置 CURSOR_API_KEY") + return { + "enabled": False, + "mode": "research", + "items": [], + "tech_items": [], + "flat": [], + "stats": {"error": "no_cursor_key"}, + } + + skill = _load_skill() + now_cst = datetime.now(timezone(timedelta(hours=8))) + tech_clause = "" + if tech_lim: + tech_clause = ( + f"\n另输出 **tech_items 恰好 {tech_lim} 条**，聚焦工程技术：" + "模型/框架发布、开源项目、芯片算力、开发者工具、推理与工程实践。" + "与 items 不得重复 link。" + ) + system = ( + f"{skill}\n\n" + "当前执行 **早报 AI 时讯调研**。\n" + f"时间窗口：近 **{h}** 小时（截至 {now_cst.strftime('%Y-%m-%d %H:%M')} UTC+8）。\n" + f"输出 **恰好 {lim} 条** items，按重要性排序。{tech_clause}\n" + "使用 WebSearch 检索；不要读取本项目文档或 RSS 配置。" + ) + user = ( + f"/deep-research 获取近 {h} 小时的 AI 人工智能新闻资讯，" + "不区分国内国外，合并精选。" + f"只输出 JSON，items 长度={lim}" + + (f"，tech_items 长度={tech_lim}" if tech_lim else "") + + "。" + ) + + try: + raw = cursor_agent_prompt(system, user) + except Exception as exc: + logger.warning("AI 时讯 research 失败：%s", exc) + return { + "enabled": False, + "mode": "research", + "items": [], + "tech_items": [], + "flat": [], + "stats": {"error": str(exc)}, + } + + if not raw: + return { + "enabled": False, + "mode": "research", + "items": [], + "tech_items": [], + "flat": [], + "stats": {"error": "empty_response"}, + } + + items, tech_items = parse_research_response(raw, limit=lim, tech_limit=tech_lim) + payload = extract_json_object(raw) + if payload: + _save_research_json(research_json_path(date_str), payload) + + if not items and not tech_items: + logger.warning("AI 时讯 research JSON 无效或无条目") + return { + "enabled": False, + "mode": "research", + "items": [], + "tech_items": [], + "flat": [], + "stats": {"error": "invalid_json"}, + } + + items = _apply_pushed_dedup(items, date_str=date_str, limit=lim) + if tech_items: + tech_items = _apply_pushed_dedup(tech_items, date_str=date_str, limit=tech_lim) + logger.info("AI 时讯 research 完成：%d 条 + %d 技术", len(items), len(tech_items)) + + flat = [ + { + "title": i["title"], + "link": i["link"], + "summary": i.get("summary_plain") or i.get("desc_short") or "", + "source_name": i["source_name"], + "published_fmt": i.get("published_fmt") or "", + "category_id": "research", + "category_name": "Deep Research", + "category_icon": "🔍", + } + for i in items + tech_items + ] + + return { + "enabled": True, + "mode": "research", + "hours": h, + "items": items, + "tech_items": tech_items, + "flat": flat, + "stats": {"source": "cursor_research", "items": len(items), "tech_items": len(tech_items)}, + } + + +def format_research_news_section( + research: dict[str, Any], + *, + section_no: int, + wecom_limit: int | None = None, +) -> list[str]: + if not research.get("enabled"): + hint = research.get("stats", {}).get("error", "调研失败或未配置 CURSOR_API_KEY") + return [ + "---", + "", + f"## {section_no}、AI 时讯精选（Deep Research）", + "", + f"*不可用：{hint}*", + "", + ] + + hours = research.get("hours", 24) + items = (research.get("flat") or [])[: wecom_limit or research_limit()] + lines = [ + "---", + "", + f"## {section_no}、AI 时讯精选（Deep Research）", + "", + f"> 近 **{hours}h** · Cursor Agent WebSearch · {len(items)} 条", + "", + ] + if not items: + lines.append("*暂无可用条目。*") + lines.append("") + return lines + + for i, item in enumerate(items, 1): + pub = f" · {item['published_fmt']}" if item.get("published_fmt") else "" + lines.append( + f"{i}. **[{item['title']}]({item['link']})** · `{item['source_name']}`{pub}" + ) + summary = item.get("summary") or "" + if summary: + lines.append(f" - {summary}") + lines.append("") + return lines diff --git a/daily/news/sanitize.py b/daily/news/sanitize.py new file mode 100644 index 0000000..970f2cc --- /dev/null +++ b/daily/news/sanitize.py @@ -0,0 +1,18 @@ +"""新闻文案清洗：剥离「放宽窗口」类凑数前缀。""" + +from __future__ import annotations + +import re + +_RELAX_PREFIX = re.compile( + r"^(?:放宽窗口|放宽至[^：:]*)\s*[:：]\s*", + re.UNICODE, +) + + +def strip_relax_window_prefix(text: str) -> str: + """去掉开头的「放宽窗口：」/「放宽至…：」前缀。""" + raw = (text or "").strip() + if not raw: + return "" + return _RELAX_PREFIX.sub("", raw, count=1).strip() diff --git a/skills/daily-ai-news-research/SKILL.md b/skills/daily-ai-news-research/SKILL.md new file mode 100644 index 0000000..cd03f9c --- /dev/null +++ b/skills/daily-ai-news-research/SKILL.md @@ -0,0 +1,58 @@ +# AI 时讯 Deep Research（早报专用） + +你是 **AI 时讯调研员**。使用 **WebSearch** 与网页抓取工具，收集近 N 小时全球 AI 新闻（不区分国内/国外），输出供企微早报使用的结构化 JSON。 + +## 工作流 + +1. 将任务拆成 3–5 个子问题（模型发布、监管政策、大厂动态、芯片算力、研究突破等） +2. 每个子问题用 WebSearch 检索 2–3 组关键词（中英文混合） +3. 交叉验证：优先权威媒体 / 官方博客 / 学术来源 +4. 精选最多 **10 条**最重要、可核实的新闻（`items`）；窗口内不足则少返回，勿凑数 +5. 另精选最多 **5 条**工程技术向新闻（`tech_items`）：模型/框架发布、开源、芯片算力、开发者工具、推理与工程实践；不得与 `items` 重复 link；不足则少返回 +6. **只输出 JSON**，不要 Markdown 报告，不要代码块 + +## 质量规则 + +1. 每条必须有可访问的 `link`（https://） +2. 禁止编造未在搜索结果中出现的事实 +3. 优先近 N 小时内的新闻；若不足目标条数，**少返回**即可，禁止放宽至 48 小时凑数，禁止在 `desc_short` 标注「放宽窗口」 +4. `desc_short` 用中文一句话摘要（≤72 字） +5. `title` 保留原文标题；中文源可用中文标题 +6. `source_name` 为媒体/站点简称（如 TechCrunch、量子位、OpenAI Blog） + +## 输出格式（严格 JSON） + +```json +{ + "items": [ + { + "title": "Apple sues OpenAI over trade secret theft", + "link": "https://techcrunch.com/...", + "source_name": "TechCrunch", + "desc_short": "苹果起诉 OpenAI 涉嫌窃取硬件商业机密", + "published_fmt": "07-11 05:00" + } + ], + "tech_items": [ + { + "title": "Meta Iris AI chip enters production", + "link": "https://example.com/...", + "source_name": "TechCrunch", + "desc_short": "Meta 自研 Iris 芯片 9 月量产", + "published_fmt": "" + } + ], + "methodology": "检索 6 组 query，分析 12 源，子问题：诉讼、模型安全、监管" +} +``` + +- `items` 数组长度 **必须等于** 请求的 limit（默认 10） +- `tech_items` 数组长度 **必须等于** 请求的 tech limit（默认 5）；聚焦工程技术，可与 `items` 主题重叠但 link 不得重复 +- `published_fmt` 格式 `MM-DD HH:MM`（UTC+8），无法确定则留空字符串 +- 不要输出 `items` 以外的长文；`methodology` 可选，一行即可 + +## 禁止 + +- 不要输出 ```json 代码块包裹（直接输出 JSON 对象） +- 不要输出 Executive Summary / Key Takeaways 等报告章节 +- 不要使用本项目 RSS 或本地文档作为来源 diff --git a/tests/test_news_relax.py b/tests/test_news_relax.py new file mode 100644 index 0000000..6df64c6 --- /dev/null +++ b/tests/test_news_relax.py @@ -0,0 +1,62 @@ +# tests/test_news_relax.py +from __future__ import annotations + +import os +import unittest +from unittest.mock import patch + + +class NewsRelaxTests(unittest.TestCase): + def test_strip_relax_prefix(self): + from daily.news.sanitize import strip_relax_window_prefix + + self.assertEqual( + strip_relax_window_prefix("放宽窗口：苹果起诉 OpenAI"), + "苹果起诉 OpenAI", + ) + self.assertEqual( + strip_relax_window_prefix("放宽至48小时：某新闻"), + "某新闻", + ) + self.assertEqual( + strip_relax_window_prefix("正常摘要无前缀"), + "正常摘要无前缀", + ) + + def test_backfill_disabled_does_not_reinsert_pushed(self): + from daily.news.fetch import _apply_pushed_dedup_with_backfill + + fresh_only = [ + { + "link": "https://example.com/fresh", + "title": "fresh", + "source_name": "S", + "published_fmt": "07-14", + "desc_short": "新", + "summary_plain": "新", + } + ] + picked = [ + { + "link": "https://example.com/old", + "title": "old", + "source_name": "S", + "published": "2026-07-13T10:00:00+00:00", + "summary": "旧闻", + } + ] + with patch("daily.news.pushed_links.filter_unpushed_items", return_value=list(fresh_only)): + with patch.dict(os.environ, {"DAILY_NEWS_BACKFILL": "0"}, clear=False): + out = _apply_pushed_dedup_with_backfill( + fresh_only + [{"link": "https://example.com/old", "title": "old"}], + picked, + date_str="2026-07-14", + limit=5, + ) + links = [x.get("link") for x in out] + self.assertIn("https://example.com/fresh", links) + self.assertNotIn("https://example.com/old", links) + + +if __name__ == "__main__": + unittest.main()