88 lines
2.4 KiB
Python
88 lines
2.4 KiB
Python
"""已推送企微早报的新闻 link 去重缓存。"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import json
|
|
from datetime import datetime, timedelta
|
|
from pathlib import Path
|
|
from typing import Any
|
|
|
|
from daily.config import CACHE_DIR, news_dedup_days
|
|
from daily.news.fetch import _normalize_link
|
|
|
|
|
|
def _cache_path() -> Path:
|
|
return CACHE_DIR / "pushed-news-links.json"
|
|
|
|
|
|
def _load_raw() -> dict[str, Any]:
|
|
path = _cache_path()
|
|
if not path.exists():
|
|
return {"dates": {}}
|
|
try:
|
|
data = json.loads(path.read_text(encoding="utf-8"))
|
|
except (OSError, ValueError):
|
|
return {"dates": {}}
|
|
if not isinstance(data.get("dates"), dict):
|
|
return {"dates": {}}
|
|
return data
|
|
|
|
|
|
def _save_raw(data: dict[str, Any]) -> None:
|
|
path = _cache_path()
|
|
path.parent.mkdir(parents=True, exist_ok=True)
|
|
path.write_text(json.dumps(data, ensure_ascii=False, indent=2), encoding="utf-8")
|
|
|
|
|
|
def _prune(data: dict[str, Any], *, keep_days: int) -> None:
|
|
dates: dict[str, list[str]] = data.setdefault("dates", {})
|
|
try:
|
|
anchor = max(datetime.strptime(d, "%Y-%m-%d") for d in dates)
|
|
except ValueError:
|
|
return
|
|
cutoff = anchor - timedelta(days=keep_days)
|
|
for key in list(dates.keys()):
|
|
try:
|
|
if datetime.strptime(key, "%Y-%m-%d") < cutoff:
|
|
dates.pop(key, None)
|
|
except ValueError:
|
|
dates.pop(key, None)
|
|
|
|
|
|
def load_pushed_link_set() -> set[str]:
|
|
data = _load_raw()
|
|
out: set[str] = set()
|
|
for links in (data.get("dates") or {}).values():
|
|
if isinstance(links, list):
|
|
out.update(str(x) for x in links if x)
|
|
return out
|
|
|
|
|
|
def filter_unpushed_items(
|
|
items: list[dict[str, Any]],
|
|
*,
|
|
date_str: str,
|
|
) -> list[dict[str, Any]]:
|
|
del date_str # reserved for per-day scoping if needed later
|
|
seen = load_pushed_link_set()
|
|
out: list[dict[str, Any]] = []
|
|
for item in items:
|
|
link = _normalize_link(str(item.get("link") or ""))
|
|
if not link or link in seen:
|
|
continue
|
|
out.append(item)
|
|
return out
|
|
|
|
|
|
def record_pushed_links(date_str: str, links: list[str]) -> None:
|
|
data = _load_raw()
|
|
dates: dict[str, list[str]] = data.setdefault("dates", {})
|
|
normalized: list[str] = []
|
|
for link in links:
|
|
clean = _normalize_link(link)
|
|
if clean:
|
|
normalized.append(clean)
|
|
dates[date_str] = sorted(set(normalized))
|
|
_prune(data, keep_days=news_dedup_days())
|
|
_save_raw(data)
|