fix: 关闭新闻放宽凑数并剥离放宽窗口文案

Co-authored-by: Cursor <cursoragent@cursor.com>

daily/news/fetch.py

@@ -8,17 +8,19 @@ import time
 import html
 import xml.etree.ElementTree as ET
 from concurrent.futures import ThreadPoolExecutor, as_completed
-from datetime import datetime, timezone, timedelta
+from datetime import datetime, time as dt_time, timezone, timedelta
 from email.utils import parsedate_to_datetime
 from typing import Any
 from urllib.parse import parse_qs, urlencode, urlparse, urlunparse
+from zoneinfo import ZoneInfo
 
 import certifi
 import httpx
 
-from daily.config import env, env_int, news_summary_limit
+from daily.config import env, env_int, news_summary_limit, wecom_news_desc_limit
 from daily.news.feeds import NEWS_CATEGORIES, NewsCategory, NewsFeed
 from daily.news.feeds_cn import CN_AI_TITLE_KEYWORDS, CN_NEWS_CATEGORIES
+from daily.text_utils import trim_brief
 
 logger = logging.getLogger(__name__)
 
@@ -49,12 +51,37 @@ def _hours_window() -> int:
     return max(1, env_int("DAILY_AI_NEWS_HOURS", 24))
 
 
+def _news_tz_name() -> str:
+    return (env("DAILY_AI_NEWS_TZ") or env("DAILY_SCHEDULE_TZ") or "Asia/Shanghai").strip()
+
+
+def _floor_today_enabled() -> bool:
+    raw = env("DAILY_AI_NEWS_FLOOR_TODAY")
+    if raw is None:
+        return True
+    return raw.strip().lower() not in {"0", "false", "no", "off"}
+
+
+def _cutoff_datetime(*, floor_today: bool) -> datetime:
+    """滚动 N 小时窗口；国际新闻可叠加「不早于今日 0 点（本地时区）」。"""
+    now = _now_utc()
+    rolling = now - timedelta(hours=_hours_window())
+    if not floor_today:
+        return rolling
+    tz = ZoneInfo(_news_tz_name())
+    local = now.astimezone(tz)
+    start_today = local.replace(hour=0, minute=0, second=0, microsecond=0).astimezone(timezone.utc)
+    return max(rolling, start_today)
+
+
 def _per_feed_limit() -> int:
-    return max(1, env_int("DAILY_AI_NEWS_PER_FEED", 3))
+    want = max(_wecom_limit(), _wecom_cn_limit())
+    return max(want // 2, env_int("DAILY_AI_NEWS_PER_FEED", 5))
 
 
 def _per_category_limit() -> int:
-    return max(1, env_int("DAILY_AI_NEWS_PER_CATEGORY", 5))
+    want = max(_wecom_limit(), _wecom_cn_limit())
+    return max(want, env_int("DAILY_AI_NEWS_PER_CATEGORY", 10))
 
 
 def _wecom_limit() -> int:
@@ -62,7 +89,7 @@ def _wecom_limit() -> int:
 
 
 def _wecom_cn_limit() -> int:
-    return max(1, env_int("DAILY_WECOM_CN_AI_NEWS", 8))
+    return max(1, env_int("DAILY_WECOM_CN_AI_NEWS", 10))
 
 
 def _matches_cn_ai_title(title: str) -> bool:
@@ -102,7 +129,6 @@ def _parse_datetime(value: str | None) -> datetime | None:
     for fmt in (
         "%Y-%m-%dT%H:%M:%SZ",
         "%Y-%m-%dT%H:%M:%S%z",
-        "%Y-%m-%d",
     ):
         try:
             dt = datetime.strptime(text[: len(fmt.replace("%z", "+0000"))], fmt.replace("%z", ""))
@@ -111,17 +137,103 @@ def _parse_datetime(value: str | None) -> datetime | None:
             return dt.astimezone(timezone.utc)
         except ValueError:
             continue
+    if re.match(r"^\d{4}-\d{2}-\d{2}$", text):
+        try:
+            tz = ZoneInfo(_news_tz_name())
+            day = datetime.strptime(text, "%Y-%m-%d").date()
+            # 仅日期时按本地中午估算，避免 UTC 0 点误判为「前一天」
+            dt = datetime.combine(day, dt_time(12, 0), tzinfo=tz)
+            return dt.astimezone(timezone.utc)
+        except ValueError:
+            pass
     return None
 
 
 def _clean_text(text: str | None, limit: int = 200) -> str:
     if not text:
         return ""
-    plain = STRIP_HTML.sub(" ", html.unescape(text))
-    plain = WS.sub(" ", plain).strip()
+    plain = _strip_summary_plain(text)
     if limit <= 0 or len(plain) <= limit:
         return plain
-    return plain[: limit - 3] + "..."
+    return trim_brief(plain, limit)
+
+
+def _strip_summary_plain(text: str | None) -> str:
+    if not text:
+        return ""
+    plain = STRIP_HTML.sub(" ", html.unescape(text))
+    return WS.sub(" ", plain).strip()
+
+
+_JUNK_SUMMARY_RE = re.compile(
+    r"^(点击查看原文|article url:|comments url:|discussion on hn|read more)",
+    re.IGNORECASE,
+)
+
+
+def _is_junk_news_summary(text: str) -> bool:
+    if not text:
+        return True
+    if _JUNK_SUMMARY_RE.match(text.strip()):
+        return True
+    if text.strip().endswith(">") and "点击" in text:
+        return True
+    return False
+
+
+def brief_news_summary(text: str | None, limit: int | None = None) -> str:
+    """企微新闻一句摘要：去 HTML、过滤占位文案、句读处截断。"""
+    plain = _strip_summary_plain(text)
+    if _is_junk_news_summary(plain):
+        return ""
+    lim = wecom_news_desc_limit() if limit is None else limit
+    return trim_brief(plain, lim)
+
+
+def sync_wecom_news_rows(items: list[dict[str, Any]], flat: list[dict[str, Any]]) -> None:
+    """中文化后，用 flat 最新 summary 刷新企微 desc_short。"""
+    by_link = {_normalize_link(str(i.get("link") or "")): i for i in flat if i.get("link")}
+    for row in items:
+        link = _normalize_link(str(row.get("link") or ""))
+        src = by_link.get(link)
+        if src:
+            row["desc_short"] = brief_news_summary(src.get("summary"))
+
+
+def finalize_wecom_news_items(
+    items: list[dict[str, Any]],
+    *,
+    force_chinese: bool = False,
+) -> None:
+    """企微新闻摘要：确保 desc_short 为中文（国际源 force_chinese=True）。"""
+    from daily.localize import LocalizeJob, localize_brief_descriptions, needs_chinese
+    from daily.news.sanitize import strip_relax_window_prefix
+
+    limit = wecom_news_desc_limit()
+    jobs: list[LocalizeJob] = []
+    keyed: list[tuple[str, dict[str, Any]]] = []
+    for idx, item in enumerate(items):
+        text = strip_relax_window_prefix(
+            (item.get("desc_short") or item.get("summary_plain") or "").strip()
+        )
+        if text:
+            item["desc_short"] = text
+        if not text or _is_junk_news_summary(text):
+            item["desc_short"] = ""
+            continue
+        if force_chinese or needs_chinese(text):
+            key = f"wecom-news:{item.get('link') or idx}"
+            jobs.append(LocalizeJob(key, text, limit))
+            keyed.append((key, item))
+        elif not item.get("desc_short"):
+            item["desc_short"] = brief_news_summary(text, limit)
+
+    if not jobs:
+        return
+    zh_map = localize_brief_descriptions(jobs, archive=True)
+    for key, item in keyed:
+        if key in zh_map:
+            item["desc_short"] = strip_relax_window_prefix(zh_map[key])
 
 
 def _normalize_link(link: str) -> str:
@@ -317,10 +429,119 @@ def _dedupe_items(items: list[dict[str, Any]]) -> list[dict[str, Any]]:
     return result
 
 
+def _filter_flat_in_window(
+    flat: list[dict[str, Any]],
+    *,
+    floor_today: bool,
+) -> list[dict[str, Any]]:
+    cutoff = _cutoff_datetime(floor_today=floor_today)
+    items = [i for i in _dedupe_items(flat) if _within_window(i, cutoff)]
+    items.sort(key=_sort_key, reverse=True)
+    return items
+
+
+def _pick_news_items(
+    flat: list[dict[str, Any]],
+    limit: int,
+    preferred: tuple[str, ...],
+    *,
+    one_per_source: bool = False,
+) -> list[dict[str, Any]]:
+    picked: list[dict[str, Any]] = []
+    seen_links: set[str] = set()
+    seen_sources: set[str] = set()
+
+    def _try_take(item: dict[str, Any]) -> bool:
+        link = _normalize_link(item.get("link", ""))
+        if not link or link in seen_links:
+            return False
+        if one_per_source:
+            source = item.get("source_name", "?")
+            if source in seen_sources:
+                return False
+            seen_sources.add(source)
+        seen_links.add(link)
+        picked.append(item)
+        return True
+
+    for cat in preferred:
+        for item in flat:
+            if item.get("category_id") != cat:
+                continue
+            if _try_take(item) and len(picked) >= limit:
+                return picked[:limit]
+
+    for item in flat:
+        if _try_take(item) and len(picked) >= limit:
+            break
+    return picked[:limit]
+
+
+def _fill_picked_to_limit(
+    picked: list[dict[str, Any]],
+    pools: list[list[dict[str, Any]]],
+    limit: int,
+) -> list[dict[str, Any]]:
+    seen_links = {_normalize_link(i.get("link", "")) for i in picked}
+    for pool in pools:
+        for item in pool:
+            if len(picked) >= limit:
+                return picked[:limit]
+            link = _normalize_link(item.get("link", ""))
+            if not link or link in seen_links:
+                continue
+            picked.append(item)
+            seen_links.add(link)
+    return picked[:limit]
+
+
+def _to_wecom_news_row(item: dict[str, Any]) -> dict[str, Any]:
+    plain = _strip_summary_plain(item.get("summary", ""))
+    return {
+        "title": item.get("title", "?"),
+        "link": item.get("link", ""),
+        "source_name": item.get("source_name", "?"),
+        "published_fmt": item.get("published_fmt", ""),
+        "desc_short": brief_news_summary(plain),
+        "summary_plain": plain,
+    }
+
+
+def _apply_pushed_dedup_with_backfill(
+    items: list[dict[str, Any]],
+    picked: list[dict[str, Any]],
+    *,
+    date_str: str | None,
+    limit: int,
+) -> list[dict[str, Any]]:
+    if not date_str:
+        return items[:limit]
+    from daily.config import news_backfill_enabled
+    from daily.news.pushed_links import filter_unpushed_items
+
+    fresh = filter_unpushed_items(items, date_str=date_str)
+    if len(fresh) >= limit:
+        return fresh[:limit]
+    if not news_backfill_enabled():
+        if len(fresh) < limit:
+            logger.info("news_short:%s", len(fresh))
+        return fresh[:limit]
+    seen = {_normalize_link(i.get("link", "")) for i in fresh if i.get("link")}
+    for item in picked:
+        if len(fresh) >= limit:
+            break
+        link = _normalize_link(item.get("link", ""))
+        if not link or link in seen:
+            continue
+        fresh.append(_to_wecom_news_row(item))
+        seen.add(link)
+    return fresh[:limit]
+
+
 def _within_window(item: dict[str, Any], cutoff: datetime) -> bool:
     dt = _entry_datetime(item)
     if dt is None:
-        return True
+        return False
     return dt >= cutoff
 
 
@@ -331,11 +552,11 @@ def _sort_key(item: dict[str, Any]) -> tuple[int, datetime]:
     return (0, dt)
 
 
-def _fetch_news(categories: tuple[NewsCategory, ...]) -> dict[str, Any]:
+def _fetch_news(categories: tuple[NewsCategory, ...], *, floor_today: bool = False) -> dict[str, Any]:
     hours = _hours_window()
     per_feed = _per_feed_limit()
     per_category = _per_category_limit()
-    cutoff = _now_utc() - timedelta(hours=hours)
+    cutoff = _cutoff_datetime(floor_today=floor_today)
 
     headers = {"User-Agent": USER_AGENT, "Accept": "application/rss+xml, application/atom+xml, application/xml, text/xml, */*"}
     tasks: list[tuple[NewsCategory, NewsFeed]] = []
@@ -406,6 +627,7 @@ def _fetch_news(categories: tuple[NewsCategory, ...]) -> dict[str, Any]:
     return {
         "enabled": True,
         "hours": hours,
+        "floor_today": floor_today,
         "categories": categories_out,
         "flat": flat,
         "stats": stats,
@@ -416,7 +638,7 @@ def fetch_ai_news() -> dict[str, Any]:
     """按类别抓取国际 AI 时讯，返回 {enabled, hours, categories, flat, stats}。"""
     if not _enabled():
         return {"enabled": False, "categories": [], "flat": [], "stats": {}}
-    return _fetch_news(NEWS_CATEGORIES)
+    return _fetch_news(NEWS_CATEGORIES, floor_today=_floor_today_enabled())
 
 
 def fetch_cn_ai_news() -> dict[str, Any]:
@@ -459,12 +681,13 @@ def _format_news_section(
 
     categories = news.get("categories") or []
     hours = news.get("hours", 72)
+    floor_note = " · 仅今日" if news.get("floor_today") else ""
     lines = [
         "---",
         "",
         f"## {section_no}、{title}",
         "",
-        f"> 近 **{hours}h** · {news.get('stats', {}).get('feeds_ok', 0)}/{news.get('stats', {}).get('feeds_total', 0)} 源可用",
+        f"> 近 **{hours}h**{floor_note} · {news.get('stats', {}).get('feeds_ok', 0)}/{news.get('stats', {}).get('feeds_total', 0)} 源可用",
         "",
     ]
 
@@ -501,88 +724,23 @@ def prepare_wecom_news_items(news: dict[str, Any], *, date_str: str | None = Non
     if not news.get("enabled"):
         return []
     limit = _wecom_limit()
-    flat = _dedupe_items(news.get("flat") or [])
-    flat.sort(key=_sort_key, reverse=True)
+    floor = bool(news.get("floor_today", _floor_today_enabled()))
+    flat_strict = _filter_flat_in_window(news.get("flat") or [], floor_today=floor)
+    flat_relaxed = _filter_flat_in_window(news.get("flat") or [], floor_today=False)
     preferred = ("media", "newsletter", "official", "community", "research", "developer")
-    picked: list[dict[str, Any]] = []
-    seen: set[str] = set()
-    for cat in preferred:
-        for item in flat:
-            link = _normalize_link(item.get("link", ""))
-            if item.get("category_id") != cat or link in seen:
-                continue
-            picked.append(item)
-            seen.add(link)
-            if len(picked) >= limit:
-                break
-        if len(picked) >= limit:
-            break
-    items: list[dict[str, Any]] = []
-    for item in picked[:limit]:
-        items.append(
-            {
-                "title": item.get("title", "?"),
-                "link": item.get("link", ""),
-                "source_name": item.get("source_name", "?"),
-                "published_fmt": item.get("published_fmt", ""),
-                "desc_short": _clean_text(item.get("summary", ""), 36),
-            }
-        )
-    if date_str:
-        from daily.news.pushed_links import filter_unpushed_items
-
-        items = filter_unpushed_items(items, date_str=date_str)
-    return items
+    picked = _pick_news_items(flat_strict, limit, preferred)
+    picked = _fill_picked_to_limit(picked, [flat_relaxed, news.get("flat") or []], limit)
+    items = [_to_wecom_news_row(item) for item in picked[:limit]]
+    return _apply_pushed_dedup_with_backfill(items, picked, date_str=date_str, limit=limit)
 
 
 def prepare_wecom_cn_news_items(news: dict[str, Any], *, date_str: str | None = None) -> list[dict[str, Any]]:
     if not news.get("enabled"):
         return []
     limit = _wecom_cn_limit()
-    flat = _dedupe_items(news.get("flat") or [])
-    flat.sort(key=_sort_key, reverse=True)
-    preferred = ("media", "tech", "dev")
-    picked: list[dict[str, Any]] = []
-    seen_links: set[str] = set()
-    seen_sources: set[str] = set()
-
-    for cat in preferred:
-        for item in flat:
-            link = _normalize_link(item.get("link", ""))
-            source = item.get("source_name", "?")
-            if item.get("category_id") != cat or not link or link in seen_links or source in seen_sources:
-                continue
-            picked.append(item)
-            seen_links.add(link)
-            seen_sources.add(source)
-            if len(picked) >= limit:
-                break
-        if len(picked) >= limit:
-            break
-
-    if len(picked) < limit:
-        for item in flat:
-            link = _normalize_link(item.get("link", ""))
-            if not link or link in seen_links:
-                continue
-            picked.append(item)
-            seen_links.add(link)
-            if len(picked) >= limit:
-                break
-
-    items: list[dict[str, Any]] = []
-    for item in picked[:limit]:
-        items.append(
-            {
-                "title": item.get("title", "?"),
-                "link": item.get("link", ""),
-                "source_name": item.get("source_name", "?"),
-                "published_fmt": item.get("published_fmt", ""),
-                "desc_short": _clean_text(item.get("summary", ""), 36),
-            }
-        )
-    if date_str:
-        from daily.news.pushed_links import filter_unpushed_items
-
-        items = filter_unpushed_items(items, date_str=date_str)
-    return items
+    flat = _filter_flat_in_window(news.get("flat") or [], floor_today=False)
+    preferred = ("media", "tech")
+    picked = _pick_news_items(flat, limit, preferred, one_per_source=True)
+    picked = _fill_picked_to_limit(picked, [news.get("flat") or []], limit)
+    items = [_to_wecom_news_row(item) for item in picked[:limit]]
+    return _apply_pushed_dedup_with_backfill(items, picked, date_str=date_str, limit=limit)