@@ -8,17 +8,19 @@ import time
|
|||||||
import html
|
import html
|
||||||
import xml.etree.ElementTree as ET
|
import xml.etree.ElementTree as ET
|
||||||
from concurrent.futures import ThreadPoolExecutor, as_completed
|
from concurrent.futures import ThreadPoolExecutor, as_completed
|
||||||
from datetime import datetime, timezone, timedelta
|
from datetime import datetime, time as dt_time, timezone, timedelta
|
||||||
from email.utils import parsedate_to_datetime
|
from email.utils import parsedate_to_datetime
|
||||||
from typing import Any
|
from typing import Any
|
||||||
from urllib.parse import parse_qs, urlencode, urlparse, urlunparse
|
from urllib.parse import parse_qs, urlencode, urlparse, urlunparse
|
||||||
|
from zoneinfo import ZoneInfo
|
||||||
|
|
||||||
import certifi
|
import certifi
|
||||||
import httpx
|
import httpx
|
||||||
|
|
||||||
from daily.config import env, env_int, news_summary_limit
|
from daily.config import env, env_int, news_summary_limit, wecom_news_desc_limit
|
||||||
from daily.news.feeds import NEWS_CATEGORIES, NewsCategory, NewsFeed
|
from daily.news.feeds import NEWS_CATEGORIES, NewsCategory, NewsFeed
|
||||||
from daily.news.feeds_cn import CN_AI_TITLE_KEYWORDS, CN_NEWS_CATEGORIES
|
from daily.news.feeds_cn import CN_AI_TITLE_KEYWORDS, CN_NEWS_CATEGORIES
|
||||||
|
from daily.text_utils import trim_brief
|
||||||
|
|
||||||
logger = logging.getLogger(__name__)
|
logger = logging.getLogger(__name__)
|
||||||
|
|
||||||
@@ -49,12 +51,37 @@ def _hours_window() -> int:
|
|||||||
return max(1, env_int("DAILY_AI_NEWS_HOURS", 24))
|
return max(1, env_int("DAILY_AI_NEWS_HOURS", 24))
|
||||||
|
|
||||||
|
|
||||||
|
def _news_tz_name() -> str:
|
||||||
|
return (env("DAILY_AI_NEWS_TZ") or env("DAILY_SCHEDULE_TZ") or "Asia/Shanghai").strip()
|
||||||
|
|
||||||
|
|
||||||
|
def _floor_today_enabled() -> bool:
|
||||||
|
raw = env("DAILY_AI_NEWS_FLOOR_TODAY")
|
||||||
|
if raw is None:
|
||||||
|
return True
|
||||||
|
return raw.strip().lower() not in {"0", "false", "no", "off"}
|
||||||
|
|
||||||
|
|
||||||
|
def _cutoff_datetime(*, floor_today: bool) -> datetime:
|
||||||
|
"""滚动 N 小时窗口;国际新闻可叠加「不早于今日 0 点(本地时区)」。"""
|
||||||
|
now = _now_utc()
|
||||||
|
rolling = now - timedelta(hours=_hours_window())
|
||||||
|
if not floor_today:
|
||||||
|
return rolling
|
||||||
|
tz = ZoneInfo(_news_tz_name())
|
||||||
|
local = now.astimezone(tz)
|
||||||
|
start_today = local.replace(hour=0, minute=0, second=0, microsecond=0).astimezone(timezone.utc)
|
||||||
|
return max(rolling, start_today)
|
||||||
|
|
||||||
|
|
||||||
def _per_feed_limit() -> int:
|
def _per_feed_limit() -> int:
|
||||||
return max(1, env_int("DAILY_AI_NEWS_PER_FEED", 3))
|
want = max(_wecom_limit(), _wecom_cn_limit())
|
||||||
|
return max(want // 2, env_int("DAILY_AI_NEWS_PER_FEED", 5))
|
||||||
|
|
||||||
|
|
||||||
def _per_category_limit() -> int:
|
def _per_category_limit() -> int:
|
||||||
return max(1, env_int("DAILY_AI_NEWS_PER_CATEGORY", 5))
|
want = max(_wecom_limit(), _wecom_cn_limit())
|
||||||
|
return max(want, env_int("DAILY_AI_NEWS_PER_CATEGORY", 10))
|
||||||
|
|
||||||
|
|
||||||
def _wecom_limit() -> int:
|
def _wecom_limit() -> int:
|
||||||
@@ -62,7 +89,7 @@ def _wecom_limit() -> int:
|
|||||||
|
|
||||||
|
|
||||||
def _wecom_cn_limit() -> int:
|
def _wecom_cn_limit() -> int:
|
||||||
return max(1, env_int("DAILY_WECOM_CN_AI_NEWS", 8))
|
return max(1, env_int("DAILY_WECOM_CN_AI_NEWS", 10))
|
||||||
|
|
||||||
|
|
||||||
def _matches_cn_ai_title(title: str) -> bool:
|
def _matches_cn_ai_title(title: str) -> bool:
|
||||||
@@ -102,7 +129,6 @@ def _parse_datetime(value: str | None) -> datetime | None:
|
|||||||
for fmt in (
|
for fmt in (
|
||||||
"%Y-%m-%dT%H:%M:%SZ",
|
"%Y-%m-%dT%H:%M:%SZ",
|
||||||
"%Y-%m-%dT%H:%M:%S%z",
|
"%Y-%m-%dT%H:%M:%S%z",
|
||||||
"%Y-%m-%d",
|
|
||||||
):
|
):
|
||||||
try:
|
try:
|
||||||
dt = datetime.strptime(text[: len(fmt.replace("%z", "+0000"))], fmt.replace("%z", ""))
|
dt = datetime.strptime(text[: len(fmt.replace("%z", "+0000"))], fmt.replace("%z", ""))
|
||||||
@@ -111,17 +137,103 @@ def _parse_datetime(value: str | None) -> datetime | None:
|
|||||||
return dt.astimezone(timezone.utc)
|
return dt.astimezone(timezone.utc)
|
||||||
except ValueError:
|
except ValueError:
|
||||||
continue
|
continue
|
||||||
|
if re.match(r"^\d{4}-\d{2}-\d{2}$", text):
|
||||||
|
try:
|
||||||
|
tz = ZoneInfo(_news_tz_name())
|
||||||
|
day = datetime.strptime(text, "%Y-%m-%d").date()
|
||||||
|
# 仅日期时按本地中午估算,避免 UTC 0 点误判为「前一天」
|
||||||
|
dt = datetime.combine(day, dt_time(12, 0), tzinfo=tz)
|
||||||
|
return dt.astimezone(timezone.utc)
|
||||||
|
except ValueError:
|
||||||
|
pass
|
||||||
return None
|
return None
|
||||||
|
|
||||||
|
|
||||||
def _clean_text(text: str | None, limit: int = 200) -> str:
|
def _clean_text(text: str | None, limit: int = 200) -> str:
|
||||||
if not text:
|
if not text:
|
||||||
return ""
|
return ""
|
||||||
plain = STRIP_HTML.sub(" ", html.unescape(text))
|
plain = _strip_summary_plain(text)
|
||||||
plain = WS.sub(" ", plain).strip()
|
|
||||||
if limit <= 0 or len(plain) <= limit:
|
if limit <= 0 or len(plain) <= limit:
|
||||||
return plain
|
return plain
|
||||||
return plain[: limit - 3] + "..."
|
return trim_brief(plain, limit)
|
||||||
|
|
||||||
|
|
||||||
|
def _strip_summary_plain(text: str | None) -> str:
|
||||||
|
if not text:
|
||||||
|
return ""
|
||||||
|
plain = STRIP_HTML.sub(" ", html.unescape(text))
|
||||||
|
return WS.sub(" ", plain).strip()
|
||||||
|
|
||||||
|
|
||||||
|
_JUNK_SUMMARY_RE = re.compile(
|
||||||
|
r"^(点击查看原文|article url:|comments url:|discussion on hn|read more)",
|
||||||
|
re.IGNORECASE,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def _is_junk_news_summary(text: str) -> bool:
|
||||||
|
if not text:
|
||||||
|
return True
|
||||||
|
if _JUNK_SUMMARY_RE.match(text.strip()):
|
||||||
|
return True
|
||||||
|
if text.strip().endswith(">") and "点击" in text:
|
||||||
|
return True
|
||||||
|
return False
|
||||||
|
|
||||||
|
|
||||||
|
def brief_news_summary(text: str | None, limit: int | None = None) -> str:
|
||||||
|
"""企微新闻一句摘要:去 HTML、过滤占位文案、句读处截断。"""
|
||||||
|
plain = _strip_summary_plain(text)
|
||||||
|
if _is_junk_news_summary(plain):
|
||||||
|
return ""
|
||||||
|
lim = wecom_news_desc_limit() if limit is None else limit
|
||||||
|
return trim_brief(plain, lim)
|
||||||
|
|
||||||
|
|
||||||
|
def sync_wecom_news_rows(items: list[dict[str, Any]], flat: list[dict[str, Any]]) -> None:
|
||||||
|
"""中文化后,用 flat 最新 summary 刷新企微 desc_short。"""
|
||||||
|
by_link = {_normalize_link(str(i.get("link") or "")): i for i in flat if i.get("link")}
|
||||||
|
for row in items:
|
||||||
|
link = _normalize_link(str(row.get("link") or ""))
|
||||||
|
src = by_link.get(link)
|
||||||
|
if src:
|
||||||
|
row["desc_short"] = brief_news_summary(src.get("summary"))
|
||||||
|
|
||||||
|
|
||||||
|
def finalize_wecom_news_items(
|
||||||
|
items: list[dict[str, Any]],
|
||||||
|
*,
|
||||||
|
force_chinese: bool = False,
|
||||||
|
) -> None:
|
||||||
|
"""企微新闻摘要:确保 desc_short 为中文(国际源 force_chinese=True)。"""
|
||||||
|
from daily.localize import LocalizeJob, localize_brief_descriptions, needs_chinese
|
||||||
|
from daily.news.sanitize import strip_relax_window_prefix
|
||||||
|
|
||||||
|
limit = wecom_news_desc_limit()
|
||||||
|
jobs: list[LocalizeJob] = []
|
||||||
|
keyed: list[tuple[str, dict[str, Any]]] = []
|
||||||
|
for idx, item in enumerate(items):
|
||||||
|
text = strip_relax_window_prefix(
|
||||||
|
(item.get("desc_short") or item.get("summary_plain") or "").strip()
|
||||||
|
)
|
||||||
|
if text:
|
||||||
|
item["desc_short"] = text
|
||||||
|
if not text or _is_junk_news_summary(text):
|
||||||
|
item["desc_short"] = ""
|
||||||
|
continue
|
||||||
|
if force_chinese or needs_chinese(text):
|
||||||
|
key = f"wecom-news:{item.get('link') or idx}"
|
||||||
|
jobs.append(LocalizeJob(key, text, limit))
|
||||||
|
keyed.append((key, item))
|
||||||
|
elif not item.get("desc_short"):
|
||||||
|
item["desc_short"] = brief_news_summary(text, limit)
|
||||||
|
|
||||||
|
if not jobs:
|
||||||
|
return
|
||||||
|
zh_map = localize_brief_descriptions(jobs, archive=True)
|
||||||
|
for key, item in keyed:
|
||||||
|
if key in zh_map:
|
||||||
|
item["desc_short"] = strip_relax_window_prefix(zh_map[key])
|
||||||
|
|
||||||
|
|
||||||
def _normalize_link(link: str) -> str:
|
def _normalize_link(link: str) -> str:
|
||||||
@@ -317,10 +429,119 @@ def _dedupe_items(items: list[dict[str, Any]]) -> list[dict[str, Any]]:
|
|||||||
return result
|
return result
|
||||||
|
|
||||||
|
|
||||||
|
def _filter_flat_in_window(
|
||||||
|
flat: list[dict[str, Any]],
|
||||||
|
*,
|
||||||
|
floor_today: bool,
|
||||||
|
) -> list[dict[str, Any]]:
|
||||||
|
cutoff = _cutoff_datetime(floor_today=floor_today)
|
||||||
|
items = [i for i in _dedupe_items(flat) if _within_window(i, cutoff)]
|
||||||
|
items.sort(key=_sort_key, reverse=True)
|
||||||
|
return items
|
||||||
|
|
||||||
|
|
||||||
|
def _pick_news_items(
|
||||||
|
flat: list[dict[str, Any]],
|
||||||
|
limit: int,
|
||||||
|
preferred: tuple[str, ...],
|
||||||
|
*,
|
||||||
|
one_per_source: bool = False,
|
||||||
|
) -> list[dict[str, Any]]:
|
||||||
|
picked: list[dict[str, Any]] = []
|
||||||
|
seen_links: set[str] = set()
|
||||||
|
seen_sources: set[str] = set()
|
||||||
|
|
||||||
|
def _try_take(item: dict[str, Any]) -> bool:
|
||||||
|
link = _normalize_link(item.get("link", ""))
|
||||||
|
if not link or link in seen_links:
|
||||||
|
return False
|
||||||
|
if one_per_source:
|
||||||
|
source = item.get("source_name", "?")
|
||||||
|
if source in seen_sources:
|
||||||
|
return False
|
||||||
|
seen_sources.add(source)
|
||||||
|
seen_links.add(link)
|
||||||
|
picked.append(item)
|
||||||
|
return True
|
||||||
|
|
||||||
|
for cat in preferred:
|
||||||
|
for item in flat:
|
||||||
|
if item.get("category_id") != cat:
|
||||||
|
continue
|
||||||
|
if _try_take(item) and len(picked) >= limit:
|
||||||
|
return picked[:limit]
|
||||||
|
|
||||||
|
for item in flat:
|
||||||
|
if _try_take(item) and len(picked) >= limit:
|
||||||
|
break
|
||||||
|
return picked[:limit]
|
||||||
|
|
||||||
|
|
||||||
|
def _fill_picked_to_limit(
|
||||||
|
picked: list[dict[str, Any]],
|
||||||
|
pools: list[list[dict[str, Any]]],
|
||||||
|
limit: int,
|
||||||
|
) -> list[dict[str, Any]]:
|
||||||
|
seen_links = {_normalize_link(i.get("link", "")) for i in picked}
|
||||||
|
for pool in pools:
|
||||||
|
for item in pool:
|
||||||
|
if len(picked) >= limit:
|
||||||
|
return picked[:limit]
|
||||||
|
link = _normalize_link(item.get("link", ""))
|
||||||
|
if not link or link in seen_links:
|
||||||
|
continue
|
||||||
|
picked.append(item)
|
||||||
|
seen_links.add(link)
|
||||||
|
return picked[:limit]
|
||||||
|
|
||||||
|
|
||||||
|
def _to_wecom_news_row(item: dict[str, Any]) -> dict[str, Any]:
|
||||||
|
plain = _strip_summary_plain(item.get("summary", ""))
|
||||||
|
return {
|
||||||
|
"title": item.get("title", "?"),
|
||||||
|
"link": item.get("link", ""),
|
||||||
|
"source_name": item.get("source_name", "?"),
|
||||||
|
"published_fmt": item.get("published_fmt", ""),
|
||||||
|
"desc_short": brief_news_summary(plain),
|
||||||
|
"summary_plain": plain,
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
def _apply_pushed_dedup_with_backfill(
|
||||||
|
items: list[dict[str, Any]],
|
||||||
|
picked: list[dict[str, Any]],
|
||||||
|
*,
|
||||||
|
date_str: str | None,
|
||||||
|
limit: int,
|
||||||
|
) -> list[dict[str, Any]]:
|
||||||
|
if not date_str:
|
||||||
|
return items[:limit]
|
||||||
|
from daily.config import news_backfill_enabled
|
||||||
|
from daily.news.pushed_links import filter_unpushed_items
|
||||||
|
|
||||||
|
fresh = filter_unpushed_items(items, date_str=date_str)
|
||||||
|
if len(fresh) >= limit:
|
||||||
|
return fresh[:limit]
|
||||||
|
if not news_backfill_enabled():
|
||||||
|
if len(fresh) < limit:
|
||||||
|
logger.info("news_short:%s", len(fresh))
|
||||||
|
return fresh[:limit]
|
||||||
|
seen = {_normalize_link(i.get("link", "")) for i in fresh if i.get("link")}
|
||||||
|
for item in picked:
|
||||||
|
if len(fresh) >= limit:
|
||||||
|
break
|
||||||
|
link = _normalize_link(item.get("link", ""))
|
||||||
|
if not link or link in seen:
|
||||||
|
continue
|
||||||
|
fresh.append(_to_wecom_news_row(item))
|
||||||
|
seen.add(link)
|
||||||
|
return fresh[:limit]
|
||||||
|
|
||||||
|
|
||||||
def _within_window(item: dict[str, Any], cutoff: datetime) -> bool:
|
def _within_window(item: dict[str, Any], cutoff: datetime) -> bool:
|
||||||
dt = _entry_datetime(item)
|
dt = _entry_datetime(item)
|
||||||
if dt is None:
|
if dt is None:
|
||||||
return True
|
return False
|
||||||
return dt >= cutoff
|
return dt >= cutoff
|
||||||
|
|
||||||
|
|
||||||
@@ -331,11 +552,11 @@ def _sort_key(item: dict[str, Any]) -> tuple[int, datetime]:
|
|||||||
return (0, dt)
|
return (0, dt)
|
||||||
|
|
||||||
|
|
||||||
def _fetch_news(categories: tuple[NewsCategory, ...]) -> dict[str, Any]:
|
def _fetch_news(categories: tuple[NewsCategory, ...], *, floor_today: bool = False) -> dict[str, Any]:
|
||||||
hours = _hours_window()
|
hours = _hours_window()
|
||||||
per_feed = _per_feed_limit()
|
per_feed = _per_feed_limit()
|
||||||
per_category = _per_category_limit()
|
per_category = _per_category_limit()
|
||||||
cutoff = _now_utc() - timedelta(hours=hours)
|
cutoff = _cutoff_datetime(floor_today=floor_today)
|
||||||
|
|
||||||
headers = {"User-Agent": USER_AGENT, "Accept": "application/rss+xml, application/atom+xml, application/xml, text/xml, */*"}
|
headers = {"User-Agent": USER_AGENT, "Accept": "application/rss+xml, application/atom+xml, application/xml, text/xml, */*"}
|
||||||
tasks: list[tuple[NewsCategory, NewsFeed]] = []
|
tasks: list[tuple[NewsCategory, NewsFeed]] = []
|
||||||
@@ -406,6 +627,7 @@ def _fetch_news(categories: tuple[NewsCategory, ...]) -> dict[str, Any]:
|
|||||||
return {
|
return {
|
||||||
"enabled": True,
|
"enabled": True,
|
||||||
"hours": hours,
|
"hours": hours,
|
||||||
|
"floor_today": floor_today,
|
||||||
"categories": categories_out,
|
"categories": categories_out,
|
||||||
"flat": flat,
|
"flat": flat,
|
||||||
"stats": stats,
|
"stats": stats,
|
||||||
@@ -416,7 +638,7 @@ def fetch_ai_news() -> dict[str, Any]:
|
|||||||
"""按类别抓取国际 AI 时讯,返回 {enabled, hours, categories, flat, stats}。"""
|
"""按类别抓取国际 AI 时讯,返回 {enabled, hours, categories, flat, stats}。"""
|
||||||
if not _enabled():
|
if not _enabled():
|
||||||
return {"enabled": False, "categories": [], "flat": [], "stats": {}}
|
return {"enabled": False, "categories": [], "flat": [], "stats": {}}
|
||||||
return _fetch_news(NEWS_CATEGORIES)
|
return _fetch_news(NEWS_CATEGORIES, floor_today=_floor_today_enabled())
|
||||||
|
|
||||||
|
|
||||||
def fetch_cn_ai_news() -> dict[str, Any]:
|
def fetch_cn_ai_news() -> dict[str, Any]:
|
||||||
@@ -459,12 +681,13 @@ def _format_news_section(
|
|||||||
|
|
||||||
categories = news.get("categories") or []
|
categories = news.get("categories") or []
|
||||||
hours = news.get("hours", 72)
|
hours = news.get("hours", 72)
|
||||||
|
floor_note = " · 仅今日" if news.get("floor_today") else ""
|
||||||
lines = [
|
lines = [
|
||||||
"---",
|
"---",
|
||||||
"",
|
"",
|
||||||
f"## {section_no}、{title}",
|
f"## {section_no}、{title}",
|
||||||
"",
|
"",
|
||||||
f"> 近 **{hours}h** · {news.get('stats', {}).get('feeds_ok', 0)}/{news.get('stats', {}).get('feeds_total', 0)} 源可用",
|
f"> 近 **{hours}h**{floor_note} · {news.get('stats', {}).get('feeds_ok', 0)}/{news.get('stats', {}).get('feeds_total', 0)} 源可用",
|
||||||
"",
|
"",
|
||||||
]
|
]
|
||||||
|
|
||||||
@@ -501,88 +724,23 @@ def prepare_wecom_news_items(news: dict[str, Any], *, date_str: str | None = Non
|
|||||||
if not news.get("enabled"):
|
if not news.get("enabled"):
|
||||||
return []
|
return []
|
||||||
limit = _wecom_limit()
|
limit = _wecom_limit()
|
||||||
flat = _dedupe_items(news.get("flat") or [])
|
floor = bool(news.get("floor_today", _floor_today_enabled()))
|
||||||
flat.sort(key=_sort_key, reverse=True)
|
flat_strict = _filter_flat_in_window(news.get("flat") or [], floor_today=floor)
|
||||||
|
flat_relaxed = _filter_flat_in_window(news.get("flat") or [], floor_today=False)
|
||||||
preferred = ("media", "newsletter", "official", "community", "research", "developer")
|
preferred = ("media", "newsletter", "official", "community", "research", "developer")
|
||||||
picked: list[dict[str, Any]] = []
|
picked = _pick_news_items(flat_strict, limit, preferred)
|
||||||
seen: set[str] = set()
|
picked = _fill_picked_to_limit(picked, [flat_relaxed, news.get("flat") or []], limit)
|
||||||
for cat in preferred:
|
items = [_to_wecom_news_row(item) for item in picked[:limit]]
|
||||||
for item in flat:
|
return _apply_pushed_dedup_with_backfill(items, picked, date_str=date_str, limit=limit)
|
||||||
link = _normalize_link(item.get("link", ""))
|
|
||||||
if item.get("category_id") != cat or link in seen:
|
|
||||||
continue
|
|
||||||
picked.append(item)
|
|
||||||
seen.add(link)
|
|
||||||
if len(picked) >= limit:
|
|
||||||
break
|
|
||||||
if len(picked) >= limit:
|
|
||||||
break
|
|
||||||
items: list[dict[str, Any]] = []
|
|
||||||
for item in picked[:limit]:
|
|
||||||
items.append(
|
|
||||||
{
|
|
||||||
"title": item.get("title", "?"),
|
|
||||||
"link": item.get("link", ""),
|
|
||||||
"source_name": item.get("source_name", "?"),
|
|
||||||
"published_fmt": item.get("published_fmt", ""),
|
|
||||||
"desc_short": _clean_text(item.get("summary", ""), 36),
|
|
||||||
}
|
|
||||||
)
|
|
||||||
if date_str:
|
|
||||||
from daily.news.pushed_links import filter_unpushed_items
|
|
||||||
|
|
||||||
items = filter_unpushed_items(items, date_str=date_str)
|
|
||||||
return items
|
|
||||||
|
|
||||||
|
|
||||||
def prepare_wecom_cn_news_items(news: dict[str, Any], *, date_str: str | None = None) -> list[dict[str, Any]]:
|
def prepare_wecom_cn_news_items(news: dict[str, Any], *, date_str: str | None = None) -> list[dict[str, Any]]:
|
||||||
if not news.get("enabled"):
|
if not news.get("enabled"):
|
||||||
return []
|
return []
|
||||||
limit = _wecom_cn_limit()
|
limit = _wecom_cn_limit()
|
||||||
flat = _dedupe_items(news.get("flat") or [])
|
flat = _filter_flat_in_window(news.get("flat") or [], floor_today=False)
|
||||||
flat.sort(key=_sort_key, reverse=True)
|
preferred = ("media", "tech")
|
||||||
preferred = ("media", "tech", "dev")
|
picked = _pick_news_items(flat, limit, preferred, one_per_source=True)
|
||||||
picked: list[dict[str, Any]] = []
|
picked = _fill_picked_to_limit(picked, [news.get("flat") or []], limit)
|
||||||
seen_links: set[str] = set()
|
items = [_to_wecom_news_row(item) for item in picked[:limit]]
|
||||||
seen_sources: set[str] = set()
|
return _apply_pushed_dedup_with_backfill(items, picked, date_str=date_str, limit=limit)
|
||||||
|
|
||||||
for cat in preferred:
|
|
||||||
for item in flat:
|
|
||||||
link = _normalize_link(item.get("link", ""))
|
|
||||||
source = item.get("source_name", "?")
|
|
||||||
if item.get("category_id") != cat or not link or link in seen_links or source in seen_sources:
|
|
||||||
continue
|
|
||||||
picked.append(item)
|
|
||||||
seen_links.add(link)
|
|
||||||
seen_sources.add(source)
|
|
||||||
if len(picked) >= limit:
|
|
||||||
break
|
|
||||||
if len(picked) >= limit:
|
|
||||||
break
|
|
||||||
|
|
||||||
if len(picked) < limit:
|
|
||||||
for item in flat:
|
|
||||||
link = _normalize_link(item.get("link", ""))
|
|
||||||
if not link or link in seen_links:
|
|
||||||
continue
|
|
||||||
picked.append(item)
|
|
||||||
seen_links.add(link)
|
|
||||||
if len(picked) >= limit:
|
|
||||||
break
|
|
||||||
|
|
||||||
items: list[dict[str, Any]] = []
|
|
||||||
for item in picked[:limit]:
|
|
||||||
items.append(
|
|
||||||
{
|
|
||||||
"title": item.get("title", "?"),
|
|
||||||
"link": item.get("link", ""),
|
|
||||||
"source_name": item.get("source_name", "?"),
|
|
||||||
"published_fmt": item.get("published_fmt", ""),
|
|
||||||
"desc_short": _clean_text(item.get("summary", ""), 36),
|
|
||||||
}
|
|
||||||
)
|
|
||||||
if date_str:
|
|
||||||
from daily.news.pushed_links import filter_unpushed_items
|
|
||||||
|
|
||||||
items = filter_unpushed_items(items, date_str=date_str)
|
|
||||||
return items
|
|
||||||
|
|||||||
324
daily/news/research.py
Normal file
324
daily/news/research.py
Normal file
@@ -0,0 +1,324 @@
|
|||||||
|
"""Cursor SDK + deep-research 工作流:采集 AI 时讯(方案 A,内置 WebSearch)。"""
|
||||||
|
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import json
|
||||||
|
import logging
|
||||||
|
from datetime import datetime, timezone, timedelta
|
||||||
|
from pathlib import Path
|
||||||
|
from typing import Any
|
||||||
|
from urllib.parse import urlparse
|
||||||
|
|
||||||
|
from daily.config import OUTPUT_DIR, ROOT, env, env_int, wecom_ai_news_tech_limit
|
||||||
|
from daily.llm_client import cursor_agent_prompt, extract_json_object, has_cursor_configured
|
||||||
|
from daily.news.fetch import brief_news_summary, _normalize_link
|
||||||
|
from daily.news.pushed_links import filter_unpushed_items
|
||||||
|
|
||||||
|
logger = logging.getLogger(__name__)
|
||||||
|
|
||||||
|
_SKILL_DIR = ROOT / "skills" / "daily-ai-news-research"
|
||||||
|
_DEEP_RESEARCH_CANDIDATES = (
|
||||||
|
ROOT / "skills" / "deep-research" / "SKILL.md",
|
||||||
|
Path.home() / ".agents" / "skills" / "deep-research" / "SKILL.md",
|
||||||
|
Path.home() / ".cursor" / "skills" / "deep-research" / "SKILL.md",
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def ai_news_mode() -> str:
|
||||||
|
return (env("DAILY_AI_NEWS_MODE") or "rss").strip().lower()
|
||||||
|
|
||||||
|
|
||||||
|
def is_research_mode() -> bool:
|
||||||
|
return ai_news_mode() == "research"
|
||||||
|
|
||||||
|
|
||||||
|
def research_hours() -> int:
|
||||||
|
return max(1, env_int("DAILY_AI_NEWS_HOURS", 24))
|
||||||
|
|
||||||
|
|
||||||
|
def research_limit() -> int:
|
||||||
|
return max(1, env_int("DAILY_WECOM_AI_NEWS", 10))
|
||||||
|
|
||||||
|
|
||||||
|
def research_json_path(date_str: str) -> Path:
|
||||||
|
return OUTPUT_DIR / f"{date_str}.ai-news-research.json"
|
||||||
|
|
||||||
|
|
||||||
|
def _save_research_json(path: Path, data: dict[str, Any]) -> None:
|
||||||
|
path.parent.mkdir(parents=True, exist_ok=True)
|
||||||
|
path.write_text(json.dumps(data, ensure_ascii=False, indent=2), encoding="utf-8")
|
||||||
|
|
||||||
|
|
||||||
|
def _load_skill() -> str:
|
||||||
|
parts: list[str] = []
|
||||||
|
for path in _DEEP_RESEARCH_CANDIDATES:
|
||||||
|
if path.exists():
|
||||||
|
parts.append(path.read_text(encoding="utf-8").strip())
|
||||||
|
break
|
||||||
|
local = _SKILL_DIR / "SKILL.md"
|
||||||
|
if local.exists():
|
||||||
|
parts.append(local.read_text(encoding="utf-8").strip())
|
||||||
|
if not parts:
|
||||||
|
return "你是 AI 时讯调研员,只输出 JSON。"
|
||||||
|
return "\n\n---\n\n".join(parts)
|
||||||
|
|
||||||
|
|
||||||
|
def _guess_source_name(link: str, explicit: str) -> str:
|
||||||
|
name = (explicit or "").strip()
|
||||||
|
if name:
|
||||||
|
return name
|
||||||
|
host = urlparse(link).netloc.lower().removeprefix("www.")
|
||||||
|
mapping = {
|
||||||
|
"techcrunch.com": "TechCrunch",
|
||||||
|
"theverge.com": "The Verge",
|
||||||
|
"openai.com": "OpenAI",
|
||||||
|
"anthropic.com": "Anthropic",
|
||||||
|
"arxiv.org": "arXiv",
|
||||||
|
"qbitai.com": "量子位",
|
||||||
|
"36kr.com": "36氪",
|
||||||
|
"leiphone.com": "雷锋网",
|
||||||
|
}
|
||||||
|
for key, label in mapping.items():
|
||||||
|
if host.endswith(key) or key in host:
|
||||||
|
return label
|
||||||
|
return host.split(".")[0].capitalize() if host else "?"
|
||||||
|
|
||||||
|
|
||||||
|
def _normalize_research_item(raw: dict[str, Any]) -> dict[str, Any] | None:
|
||||||
|
title = str(raw.get("title") or "").strip()
|
||||||
|
link = _normalize_link(str(raw.get("link") or ""))
|
||||||
|
if not title or not link or not link.startswith("http"):
|
||||||
|
return None
|
||||||
|
desc = brief_news_summary(str(raw.get("desc_short") or raw.get("summary") or ""))
|
||||||
|
return {
|
||||||
|
"title": title,
|
||||||
|
"link": link,
|
||||||
|
"source_name": _guess_source_name(link, str(raw.get("source_name") or "")),
|
||||||
|
"published_fmt": str(raw.get("published_fmt") or "").strip(),
|
||||||
|
"desc_short": desc,
|
||||||
|
"summary_plain": desc,
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
def research_tech_limit() -> int:
|
||||||
|
return wecom_ai_news_tech_limit()
|
||||||
|
|
||||||
|
|
||||||
|
def _parse_items_array(
|
||||||
|
items_raw: Any,
|
||||||
|
*,
|
||||||
|
limit: int,
|
||||||
|
seen: set[str],
|
||||||
|
) -> list[dict[str, Any]]:
|
||||||
|
if not isinstance(items_raw, list):
|
||||||
|
return []
|
||||||
|
out: list[dict[str, Any]] = []
|
||||||
|
for row in items_raw:
|
||||||
|
if not isinstance(row, dict):
|
||||||
|
continue
|
||||||
|
item = _normalize_research_item(row)
|
||||||
|
if not item:
|
||||||
|
continue
|
||||||
|
if item["link"] in seen:
|
||||||
|
continue
|
||||||
|
seen.add(item["link"])
|
||||||
|
out.append(item)
|
||||||
|
if len(out) >= limit:
|
||||||
|
break
|
||||||
|
return out
|
||||||
|
|
||||||
|
|
||||||
|
def parse_research_response(
|
||||||
|
raw: str,
|
||||||
|
*,
|
||||||
|
limit: int,
|
||||||
|
tech_limit: int = 0,
|
||||||
|
) -> tuple[list[dict[str, Any]], list[dict[str, Any]]]:
|
||||||
|
parsed = extract_json_object(raw)
|
||||||
|
seen: set[str] = set()
|
||||||
|
items = _parse_items_array(parsed.get("items"), limit=limit, seen=seen)
|
||||||
|
tech_items = _parse_items_array(parsed.get("tech_items"), limit=tech_limit, seen=seen) if tech_limit else []
|
||||||
|
return items, tech_items
|
||||||
|
|
||||||
|
|
||||||
|
def _apply_pushed_dedup(items: list[dict[str, Any]], *, date_str: str, limit: int) -> list[dict[str, Any]]:
|
||||||
|
from daily.config import news_backfill_enabled
|
||||||
|
from daily.news.sanitize import strip_relax_window_prefix
|
||||||
|
|
||||||
|
for item in items:
|
||||||
|
if item.get("desc_short"):
|
||||||
|
item["desc_short"] = strip_relax_window_prefix(str(item.get("desc_short") or ""))
|
||||||
|
fresh = filter_unpushed_items(items, date_str=date_str)
|
||||||
|
if len(fresh) >= limit:
|
||||||
|
return fresh[:limit]
|
||||||
|
if not news_backfill_enabled():
|
||||||
|
if len(fresh) < limit:
|
||||||
|
logger.info("news_short:%s", len(fresh))
|
||||||
|
return fresh[:limit]
|
||||||
|
seen = {i.get("link") for i in fresh}
|
||||||
|
for item in items:
|
||||||
|
if len(fresh) >= limit:
|
||||||
|
break
|
||||||
|
if item.get("link") not in seen:
|
||||||
|
fresh.append(item)
|
||||||
|
seen.add(item.get("link"))
|
||||||
|
return fresh[:limit]
|
||||||
|
|
||||||
|
|
||||||
|
def fetch_ai_news_research(
|
||||||
|
*,
|
||||||
|
date_str: str,
|
||||||
|
hours: int | None = None,
|
||||||
|
limit: int | None = None,
|
||||||
|
) -> dict[str, Any]:
|
||||||
|
"""Cursor Agent 调研 AI 时讯;返回 {enabled, mode, hours, items, flat, stats}。"""
|
||||||
|
h = hours if hours is not None else research_hours()
|
||||||
|
lim = limit if limit is not None else research_limit()
|
||||||
|
tech_lim = research_tech_limit()
|
||||||
|
|
||||||
|
if not has_cursor_configured():
|
||||||
|
logger.warning("DAILY_AI_NEWS_MODE=research 但未配置 CURSOR_API_KEY")
|
||||||
|
return {
|
||||||
|
"enabled": False,
|
||||||
|
"mode": "research",
|
||||||
|
"items": [],
|
||||||
|
"tech_items": [],
|
||||||
|
"flat": [],
|
||||||
|
"stats": {"error": "no_cursor_key"},
|
||||||
|
}
|
||||||
|
|
||||||
|
skill = _load_skill()
|
||||||
|
now_cst = datetime.now(timezone(timedelta(hours=8)))
|
||||||
|
tech_clause = ""
|
||||||
|
if tech_lim:
|
||||||
|
tech_clause = (
|
||||||
|
f"\n另输出 **tech_items 恰好 {tech_lim} 条**,聚焦工程技术:"
|
||||||
|
"模型/框架发布、开源项目、芯片算力、开发者工具、推理与工程实践。"
|
||||||
|
"与 items 不得重复 link。"
|
||||||
|
)
|
||||||
|
system = (
|
||||||
|
f"{skill}\n\n"
|
||||||
|
"当前执行 **早报 AI 时讯调研**。\n"
|
||||||
|
f"时间窗口:近 **{h}** 小时(截至 {now_cst.strftime('%Y-%m-%d %H:%M')} UTC+8)。\n"
|
||||||
|
f"输出 **恰好 {lim} 条** items,按重要性排序。{tech_clause}\n"
|
||||||
|
"使用 WebSearch 检索;不要读取本项目文档或 RSS 配置。"
|
||||||
|
)
|
||||||
|
user = (
|
||||||
|
f"/deep-research 获取近 {h} 小时的 AI 人工智能新闻资讯,"
|
||||||
|
"不区分国内国外,合并精选。"
|
||||||
|
f"只输出 JSON,items 长度={lim}"
|
||||||
|
+ (f",tech_items 长度={tech_lim}" if tech_lim else "")
|
||||||
|
+ "。"
|
||||||
|
)
|
||||||
|
|
||||||
|
try:
|
||||||
|
raw = cursor_agent_prompt(system, user)
|
||||||
|
except Exception as exc:
|
||||||
|
logger.warning("AI 时讯 research 失败:%s", exc)
|
||||||
|
return {
|
||||||
|
"enabled": False,
|
||||||
|
"mode": "research",
|
||||||
|
"items": [],
|
||||||
|
"tech_items": [],
|
||||||
|
"flat": [],
|
||||||
|
"stats": {"error": str(exc)},
|
||||||
|
}
|
||||||
|
|
||||||
|
if not raw:
|
||||||
|
return {
|
||||||
|
"enabled": False,
|
||||||
|
"mode": "research",
|
||||||
|
"items": [],
|
||||||
|
"tech_items": [],
|
||||||
|
"flat": [],
|
||||||
|
"stats": {"error": "empty_response"},
|
||||||
|
}
|
||||||
|
|
||||||
|
items, tech_items = parse_research_response(raw, limit=lim, tech_limit=tech_lim)
|
||||||
|
payload = extract_json_object(raw)
|
||||||
|
if payload:
|
||||||
|
_save_research_json(research_json_path(date_str), payload)
|
||||||
|
|
||||||
|
if not items and not tech_items:
|
||||||
|
logger.warning("AI 时讯 research JSON 无效或无条目")
|
||||||
|
return {
|
||||||
|
"enabled": False,
|
||||||
|
"mode": "research",
|
||||||
|
"items": [],
|
||||||
|
"tech_items": [],
|
||||||
|
"flat": [],
|
||||||
|
"stats": {"error": "invalid_json"},
|
||||||
|
}
|
||||||
|
|
||||||
|
items = _apply_pushed_dedup(items, date_str=date_str, limit=lim)
|
||||||
|
if tech_items:
|
||||||
|
tech_items = _apply_pushed_dedup(tech_items, date_str=date_str, limit=tech_lim)
|
||||||
|
logger.info("AI 时讯 research 完成:%d 条 + %d 技术", len(items), len(tech_items))
|
||||||
|
|
||||||
|
flat = [
|
||||||
|
{
|
||||||
|
"title": i["title"],
|
||||||
|
"link": i["link"],
|
||||||
|
"summary": i.get("summary_plain") or i.get("desc_short") or "",
|
||||||
|
"source_name": i["source_name"],
|
||||||
|
"published_fmt": i.get("published_fmt") or "",
|
||||||
|
"category_id": "research",
|
||||||
|
"category_name": "Deep Research",
|
||||||
|
"category_icon": "🔍",
|
||||||
|
}
|
||||||
|
for i in items + tech_items
|
||||||
|
]
|
||||||
|
|
||||||
|
return {
|
||||||
|
"enabled": True,
|
||||||
|
"mode": "research",
|
||||||
|
"hours": h,
|
||||||
|
"items": items,
|
||||||
|
"tech_items": tech_items,
|
||||||
|
"flat": flat,
|
||||||
|
"stats": {"source": "cursor_research", "items": len(items), "tech_items": len(tech_items)},
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
def format_research_news_section(
|
||||||
|
research: dict[str, Any],
|
||||||
|
*,
|
||||||
|
section_no: int,
|
||||||
|
wecom_limit: int | None = None,
|
||||||
|
) -> list[str]:
|
||||||
|
if not research.get("enabled"):
|
||||||
|
hint = research.get("stats", {}).get("error", "调研失败或未配置 CURSOR_API_KEY")
|
||||||
|
return [
|
||||||
|
"---",
|
||||||
|
"",
|
||||||
|
f"## {section_no}、AI 时讯精选(Deep Research)",
|
||||||
|
"",
|
||||||
|
f"*不可用:{hint}*",
|
||||||
|
"",
|
||||||
|
]
|
||||||
|
|
||||||
|
hours = research.get("hours", 24)
|
||||||
|
items = (research.get("flat") or [])[: wecom_limit or research_limit()]
|
||||||
|
lines = [
|
||||||
|
"---",
|
||||||
|
"",
|
||||||
|
f"## {section_no}、AI 时讯精选(Deep Research)",
|
||||||
|
"",
|
||||||
|
f"> 近 **{hours}h** · Cursor Agent WebSearch · {len(items)} 条",
|
||||||
|
"",
|
||||||
|
]
|
||||||
|
if not items:
|
||||||
|
lines.append("*暂无可用条目。*")
|
||||||
|
lines.append("")
|
||||||
|
return lines
|
||||||
|
|
||||||
|
for i, item in enumerate(items, 1):
|
||||||
|
pub = f" · {item['published_fmt']}" if item.get("published_fmt") else ""
|
||||||
|
lines.append(
|
||||||
|
f"{i}. **[{item['title']}]({item['link']})** · `{item['source_name']}`{pub}"
|
||||||
|
)
|
||||||
|
summary = item.get("summary") or ""
|
||||||
|
if summary:
|
||||||
|
lines.append(f" - {summary}")
|
||||||
|
lines.append("")
|
||||||
|
return lines
|
||||||
18
daily/news/sanitize.py
Normal file
18
daily/news/sanitize.py
Normal file
@@ -0,0 +1,18 @@
|
|||||||
|
"""新闻文案清洗:剥离「放宽窗口」类凑数前缀。"""
|
||||||
|
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import re
|
||||||
|
|
||||||
|
_RELAX_PREFIX = re.compile(
|
||||||
|
r"^(?:放宽窗口|放宽至[^::]*)\s*[::]\s*",
|
||||||
|
re.UNICODE,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def strip_relax_window_prefix(text: str) -> str:
|
||||||
|
"""去掉开头的「放宽窗口:」/「放宽至…:」前缀。"""
|
||||||
|
raw = (text or "").strip()
|
||||||
|
if not raw:
|
||||||
|
return ""
|
||||||
|
return _RELAX_PREFIX.sub("", raw, count=1).strip()
|
||||||
58
skills/daily-ai-news-research/SKILL.md
Normal file
58
skills/daily-ai-news-research/SKILL.md
Normal file
@@ -0,0 +1,58 @@
|
|||||||
|
# AI 时讯 Deep Research(早报专用)
|
||||||
|
|
||||||
|
你是 **AI 时讯调研员**。使用 **WebSearch** 与网页抓取工具,收集近 N 小时全球 AI 新闻(不区分国内/国外),输出供企微早报使用的结构化 JSON。
|
||||||
|
|
||||||
|
## 工作流
|
||||||
|
|
||||||
|
1. 将任务拆成 3–5 个子问题(模型发布、监管政策、大厂动态、芯片算力、研究突破等)
|
||||||
|
2. 每个子问题用 WebSearch 检索 2–3 组关键词(中英文混合)
|
||||||
|
3. 交叉验证:优先权威媒体 / 官方博客 / 学术来源
|
||||||
|
4. 精选最多 **10 条**最重要、可核实的新闻(`items`);窗口内不足则少返回,勿凑数
|
||||||
|
5. 另精选最多 **5 条**工程技术向新闻(`tech_items`):模型/框架发布、开源、芯片算力、开发者工具、推理与工程实践;不得与 `items` 重复 link;不足则少返回
|
||||||
|
6. **只输出 JSON**,不要 Markdown 报告,不要代码块
|
||||||
|
|
||||||
|
## 质量规则
|
||||||
|
|
||||||
|
1. 每条必须有可访问的 `link`(https://)
|
||||||
|
2. 禁止编造未在搜索结果中出现的事实
|
||||||
|
3. 优先近 N 小时内的新闻;若不足目标条数,**少返回**即可,禁止放宽至 48 小时凑数,禁止在 `desc_short` 标注「放宽窗口」
|
||||||
|
4. `desc_short` 用中文一句话摘要(≤72 字)
|
||||||
|
5. `title` 保留原文标题;中文源可用中文标题
|
||||||
|
6. `source_name` 为媒体/站点简称(如 TechCrunch、量子位、OpenAI Blog)
|
||||||
|
|
||||||
|
## 输出格式(严格 JSON)
|
||||||
|
|
||||||
|
```json
|
||||||
|
{
|
||||||
|
"items": [
|
||||||
|
{
|
||||||
|
"title": "Apple sues OpenAI over trade secret theft",
|
||||||
|
"link": "https://techcrunch.com/...",
|
||||||
|
"source_name": "TechCrunch",
|
||||||
|
"desc_short": "苹果起诉 OpenAI 涉嫌窃取硬件商业机密",
|
||||||
|
"published_fmt": "07-11 05:00"
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"tech_items": [
|
||||||
|
{
|
||||||
|
"title": "Meta Iris AI chip enters production",
|
||||||
|
"link": "https://example.com/...",
|
||||||
|
"source_name": "TechCrunch",
|
||||||
|
"desc_short": "Meta 自研 Iris 芯片 9 月量产",
|
||||||
|
"published_fmt": ""
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"methodology": "检索 6 组 query,分析 12 源,子问题:诉讼、模型安全、监管"
|
||||||
|
}
|
||||||
|
```
|
||||||
|
|
||||||
|
- `items` 数组长度 **必须等于** 请求的 limit(默认 10)
|
||||||
|
- `tech_items` 数组长度 **必须等于** 请求的 tech limit(默认 5);聚焦工程技术,可与 `items` 主题重叠但 link 不得重复
|
||||||
|
- `published_fmt` 格式 `MM-DD HH:MM`(UTC+8),无法确定则留空字符串
|
||||||
|
- 不要输出 `items` 以外的长文;`methodology` 可选,一行即可
|
||||||
|
|
||||||
|
## 禁止
|
||||||
|
|
||||||
|
- 不要输出 ```json 代码块包裹(直接输出 JSON 对象)
|
||||||
|
- 不要输出 Executive Summary / Key Takeaways 等报告章节
|
||||||
|
- 不要使用本项目 RSS 或本地文档作为来源
|
||||||
62
tests/test_news_relax.py
Normal file
62
tests/test_news_relax.py
Normal file
@@ -0,0 +1,62 @@
|
|||||||
|
# tests/test_news_relax.py
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import os
|
||||||
|
import unittest
|
||||||
|
from unittest.mock import patch
|
||||||
|
|
||||||
|
|
||||||
|
class NewsRelaxTests(unittest.TestCase):
|
||||||
|
def test_strip_relax_prefix(self):
|
||||||
|
from daily.news.sanitize import strip_relax_window_prefix
|
||||||
|
|
||||||
|
self.assertEqual(
|
||||||
|
strip_relax_window_prefix("放宽窗口:苹果起诉 OpenAI"),
|
||||||
|
"苹果起诉 OpenAI",
|
||||||
|
)
|
||||||
|
self.assertEqual(
|
||||||
|
strip_relax_window_prefix("放宽至48小时:某新闻"),
|
||||||
|
"某新闻",
|
||||||
|
)
|
||||||
|
self.assertEqual(
|
||||||
|
strip_relax_window_prefix("正常摘要无前缀"),
|
||||||
|
"正常摘要无前缀",
|
||||||
|
)
|
||||||
|
|
||||||
|
def test_backfill_disabled_does_not_reinsert_pushed(self):
|
||||||
|
from daily.news.fetch import _apply_pushed_dedup_with_backfill
|
||||||
|
|
||||||
|
fresh_only = [
|
||||||
|
{
|
||||||
|
"link": "https://example.com/fresh",
|
||||||
|
"title": "fresh",
|
||||||
|
"source_name": "S",
|
||||||
|
"published_fmt": "07-14",
|
||||||
|
"desc_short": "新",
|
||||||
|
"summary_plain": "新",
|
||||||
|
}
|
||||||
|
]
|
||||||
|
picked = [
|
||||||
|
{
|
||||||
|
"link": "https://example.com/old",
|
||||||
|
"title": "old",
|
||||||
|
"source_name": "S",
|
||||||
|
"published": "2026-07-13T10:00:00+00:00",
|
||||||
|
"summary": "旧闻",
|
||||||
|
}
|
||||||
|
]
|
||||||
|
with patch("daily.news.pushed_links.filter_unpushed_items", return_value=list(fresh_only)):
|
||||||
|
with patch.dict(os.environ, {"DAILY_NEWS_BACKFILL": "0"}, clear=False):
|
||||||
|
out = _apply_pushed_dedup_with_backfill(
|
||||||
|
fresh_only + [{"link": "https://example.com/old", "title": "old"}],
|
||||||
|
picked,
|
||||||
|
date_str="2026-07-14",
|
||||||
|
limit=5,
|
||||||
|
)
|
||||||
|
links = [x.get("link") for x in out]
|
||||||
|
self.assertIn("https://example.com/fresh", links)
|
||||||
|
self.assertNotIn("https://example.com/old", links)
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
unittest.main()
|
||||||
Reference in New Issue
Block a user