617 lines
20 KiB
Python
617 lines
20 KiB
Python
"""抓取并整理国际 / 国内 AI 时讯 RSS。"""
|
||
|
||
from __future__ import annotations
|
||
|
||
import logging
|
||
import re
|
||
import time
|
||
import html
|
||
import xml.etree.ElementTree as ET
|
||
from concurrent.futures import ThreadPoolExecutor, as_completed
|
||
from datetime import datetime, timezone, timedelta
|
||
from email.utils import parsedate_to_datetime
|
||
from typing import Any
|
||
from urllib.parse import parse_qs, urlencode, urlparse, urlunparse
|
||
|
||
import certifi
|
||
import httpx
|
||
|
||
from daily.config import env, env_int, news_summary_limit
|
||
from daily.news.content_filter import filter_news_items
|
||
from daily.news.feeds import NEWS_CATEGORIES, NewsCategory, NewsFeed
|
||
from daily.news.feeds_cn import CN_AI_TITLE_KEYWORDS, CN_NEWS_CATEGORIES
|
||
|
||
logger = logging.getLogger(__name__)
|
||
|
||
# 部分站点（如 InfoQ）会拦截含 bot 标识的 UA，RSS 抓取统一用浏览器 UA
|
||
RSS_USER_AGENT = (
|
||
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) "
|
||
"AppleWebKit/537.36 (KHTML, like Gecko) "
|
||
"Chrome/131.0.0.0 Safari/537.36"
|
||
)
|
||
BROWSER_USER_AGENT = RSS_USER_AGENT
|
||
STRIP_HTML = re.compile(r"<[^>]+>")
|
||
WS = re.compile(r"\s+")
|
||
|
||
ATOM_NS = {"a": "http://www.w3.org/2005/Atom"}
|
||
RSS_NS = {"r": "http://purl.org/rss/1.0/modules/content/"}
|
||
|
||
|
||
def _enabled() -> bool:
|
||
raw = (env("DAILY_AI_NEWS") or "1").strip().lower()
|
||
return raw not in {"0", "false", "no", "off"}
|
||
|
||
|
||
def _cn_enabled() -> bool:
|
||
raw = (env("DAILY_CN_AI_NEWS") or "1").strip().lower()
|
||
return raw not in {"0", "false", "no", "off"}
|
||
|
||
|
||
def _hours_window() -> int:
|
||
return max(1, env_int("DAILY_AI_NEWS_HOURS", 72))
|
||
|
||
|
||
def _per_feed_limit() -> int:
|
||
return max(1, env_int("DAILY_AI_NEWS_PER_FEED", 3))
|
||
|
||
|
||
def _per_category_limit() -> int:
|
||
return max(1, env_int("DAILY_AI_NEWS_PER_CATEGORY", 5))
|
||
|
||
|
||
def _wecom_limit() -> int:
|
||
return max(1, env_int("DAILY_WECOM_AI_NEWS", 10))
|
||
|
||
|
||
def _wecom_cn_limit() -> int:
|
||
return max(1, env_int("DAILY_WECOM_CN_AI_NEWS", 8))
|
||
|
||
|
||
def _matches_cn_ai_title(title: str) -> bool:
|
||
text = title.strip()
|
||
if not text:
|
||
return False
|
||
lower = text.lower()
|
||
for keyword in CN_AI_TITLE_KEYWORDS:
|
||
if keyword.lower() in lower:
|
||
return True
|
||
return False
|
||
|
||
|
||
def _filter_ai_entries(entries: list[dict[str, Any]], *, ai_filter: bool) -> list[dict[str, Any]]:
|
||
if not ai_filter:
|
||
return entries
|
||
return [item for item in entries if _matches_cn_ai_title(item.get("title", ""))]
|
||
|
||
|
||
def _now_utc() -> datetime:
|
||
return datetime.now(timezone.utc)
|
||
|
||
|
||
def _parse_datetime(value: str | None) -> datetime | None:
|
||
if not value:
|
||
return None
|
||
text = value.strip()
|
||
if not text:
|
||
return None
|
||
try:
|
||
dt = parsedate_to_datetime(text)
|
||
if dt.tzinfo is None:
|
||
dt = dt.replace(tzinfo=timezone.utc)
|
||
return dt.astimezone(timezone.utc)
|
||
except (TypeError, ValueError, OverflowError):
|
||
pass
|
||
for fmt in (
|
||
"%Y-%m-%dT%H:%M:%SZ",
|
||
"%Y-%m-%dT%H:%M:%S%z",
|
||
"%Y-%m-%d",
|
||
):
|
||
try:
|
||
dt = datetime.strptime(text[: len(fmt.replace("%z", "+0000"))], fmt.replace("%z", ""))
|
||
if dt.tzinfo is None:
|
||
dt = dt.replace(tzinfo=timezone.utc)
|
||
return dt.astimezone(timezone.utc)
|
||
except ValueError:
|
||
continue
|
||
return None
|
||
|
||
|
||
def _clean_text(text: str | None, limit: int = 200) -> str:
|
||
if not text:
|
||
return ""
|
||
plain = STRIP_HTML.sub(" ", html.unescape(text))
|
||
plain = WS.sub(" ", plain).strip()
|
||
if limit <= 0 or len(plain) <= limit:
|
||
return plain
|
||
return plain[: limit - 3] + "..."
|
||
|
||
|
||
def _normalize_link(link: str) -> str:
|
||
parsed = urlparse(link.strip())
|
||
query = parse_qs(parsed.query, keep_blank_values=False)
|
||
for key in list(query.keys()):
|
||
if key.lower().startswith("utm_") or key.lower() in {"ref", "source"}:
|
||
query.pop(key, None)
|
||
clean_query = urlencode({k: v[0] for k, v in query.items() if v}, doseq=False)
|
||
return urlunparse((parsed.scheme, parsed.netloc, parsed.path.rstrip("/"), "", clean_query, ""))
|
||
|
||
|
||
def _normalize_title(title: str) -> str:
|
||
return WS.sub(" ", title.strip().lower())
|
||
|
||
|
||
def _entry_datetime(entry: dict[str, Any]) -> datetime | None:
|
||
for key in ("published", "updated"):
|
||
dt = _parse_datetime(entry.get(key))
|
||
if dt:
|
||
return dt
|
||
return None
|
||
|
||
|
||
def _parse_atom(content: str, feed_name: str, category: NewsCategory) -> list[dict[str, Any]]:
|
||
items: list[dict[str, Any]] = []
|
||
try:
|
||
root = ET.fromstring(content)
|
||
except ET.ParseError:
|
||
return items
|
||
|
||
for entry in root.findall("a:entry", ATOM_NS):
|
||
title_el = entry.find("a:title", ATOM_NS)
|
||
link_el = entry.find("a:link", ATOM_NS)
|
||
summary_el = entry.find("a:summary", ATOM_NS) or entry.find("a:content", ATOM_NS)
|
||
updated_el = entry.find("a:updated", ATOM_NS) or entry.find("a:published", ATOM_NS)
|
||
title = title_el.text.strip() if title_el is not None and title_el.text else ""
|
||
link = ""
|
||
if link_el is not None:
|
||
link = link_el.get("href") or (link_el.text or "").strip()
|
||
if not title or not link:
|
||
continue
|
||
items.append(
|
||
{
|
||
"title": title,
|
||
"link": link,
|
||
"summary": _clean_text(summary_el.text if summary_el is not None else ""),
|
||
"published": updated_el.text.strip() if updated_el is not None and updated_el.text else "",
|
||
"source_name": feed_name,
|
||
"category_id": category.id,
|
||
"category_name": category.name,
|
||
"category_icon": category.icon,
|
||
}
|
||
)
|
||
return items
|
||
|
||
|
||
def _parse_rss(content: str, feed_name: str, category: NewsCategory) -> list[dict[str, Any]]:
|
||
items: list[dict[str, Any]] = []
|
||
try:
|
||
root = ET.fromstring(content)
|
||
except ET.ParseError:
|
||
return items
|
||
|
||
channel = root.find("channel")
|
||
if channel is None:
|
||
return items
|
||
|
||
for item in channel.findall("item"):
|
||
title_el = item.find("title")
|
||
link_el = item.find("link")
|
||
desc_el = item.find("description")
|
||
pub_el = item.find("pubDate")
|
||
title = title_el.text.strip() if title_el is not None and title_el.text else ""
|
||
link = link_el.text.strip() if link_el is not None and link_el.text else ""
|
||
if not title or not link:
|
||
continue
|
||
items.append(
|
||
{
|
||
"title": title,
|
||
"link": link,
|
||
"summary": _clean_text(desc_el.text if desc_el is not None else ""),
|
||
"published": pub_el.text.strip() if pub_el is not None and pub_el.text else "",
|
||
"source_name": feed_name,
|
||
"category_id": category.id,
|
||
"category_name": category.name,
|
||
"category_icon": category.icon,
|
||
}
|
||
)
|
||
return items
|
||
|
||
|
||
def _parse_feed(content: str, feed_name: str, category: NewsCategory) -> list[dict[str, Any]]:
|
||
text = content.lstrip("\ufeff").strip()
|
||
if not text:
|
||
return []
|
||
if text.startswith("<rss") or "<channel>" in text[:500]:
|
||
return _parse_rss(text, feed_name, category)
|
||
if text.startswith("<feed") or "<entry>" in text[:500]:
|
||
return _parse_atom(text, feed_name, category)
|
||
if "<item>" in text[:2000]:
|
||
return _parse_rss(text, feed_name, category)
|
||
return _parse_atom(text, feed_name, category)
|
||
|
||
|
||
def _is_reddit_url(url: str) -> bool:
|
||
host = urlparse(url).netloc.lower()
|
||
return host.endswith("reddit.com")
|
||
|
||
|
||
def _reddit_auth_params() -> dict[str, str]:
|
||
user = (env("REDDIT_RSS_USER") or "").strip()
|
||
feed = (env("REDDIT_RSS_FEED") or "").strip()
|
||
if user and feed:
|
||
return {"user": user, "feed": feed}
|
||
return {}
|
||
|
||
|
||
def _with_query_params(url: str, extra: dict[str, str]) -> str:
|
||
if not extra:
|
||
return url
|
||
parsed = urlparse(url)
|
||
query = parse_qs(parsed.query, keep_blank_values=True)
|
||
for key, value in extra.items():
|
||
if value and key not in query:
|
||
query[key] = [value]
|
||
clean_query = urlencode({k: v[0] for k, v in query.items() if v and v[0]}, doseq=False)
|
||
return urlunparse((parsed.scheme, parsed.netloc, parsed.path, "", clean_query, ""))
|
||
|
||
|
||
def _reddit_old_url(url: str) -> str:
|
||
parsed = urlparse(url)
|
||
host = parsed.netloc.lower()
|
||
if host.startswith("old."):
|
||
return url
|
||
if host in {"www.reddit.com", "reddit.com"}:
|
||
return urlunparse((parsed.scheme, "old.reddit.com", parsed.path, "", parsed.query, ""))
|
||
return url
|
||
|
||
|
||
def _request_headers(base: dict[str, str], url: str) -> dict[str, str]:
|
||
if not _is_reddit_url(url):
|
||
return base
|
||
return {
|
||
**base,
|
||
"User-Agent": BROWSER_USER_AGENT,
|
||
"Accept-Language": "en-US,en;q=0.9",
|
||
}
|
||
|
||
|
||
def _reddit_fetch_urls(feed_url: str) -> list[str]:
|
||
primary = _with_query_params(feed_url, _reddit_auth_params())
|
||
if not _is_reddit_url(primary):
|
||
return [primary]
|
||
fallback = _reddit_old_url(primary)
|
||
if fallback == primary:
|
||
return [primary]
|
||
return [primary, fallback]
|
||
|
||
|
||
def _fetch_one(
|
||
client: httpx.Client,
|
||
category: NewsCategory,
|
||
feed: NewsFeed,
|
||
) -> tuple[list[dict[str, Any]], bool]:
|
||
max_attempts = max(1, env_int("DAILY_RSS_RETRY", 3))
|
||
last_exc: Exception | None = None
|
||
for url in _reddit_fetch_urls(feed.url):
|
||
for attempt in range(max_attempts):
|
||
try:
|
||
headers = _request_headers(dict(client.headers), url)
|
||
resp = client.get(url, headers=headers)
|
||
resp.raise_for_status()
|
||
entries = _parse_feed(resp.text, feed.name, category)
|
||
return _filter_ai_entries(entries, ai_filter=feed.ai_filter), True
|
||
except Exception as exc:
|
||
last_exc = exc
|
||
if attempt + 1 < max_attempts:
|
||
time.sleep(min(2.0 * (attempt + 1), 5.0))
|
||
continue
|
||
break
|
||
logger.warning("RSS fetch failed [%s] %s: %s", feed.name, feed.url, last_exc)
|
||
return [], False
|
||
|
||
|
||
def _dedupe_items(items: list[dict[str, Any]]) -> list[dict[str, Any]]:
|
||
seen_links: set[str] = set()
|
||
seen_titles: set[str] = set()
|
||
result: list[dict[str, Any]] = []
|
||
for item in items:
|
||
link_key = _normalize_link(item["link"])
|
||
title_key = _normalize_title(item["title"])
|
||
if link_key in seen_links or title_key in seen_titles:
|
||
continue
|
||
seen_links.add(link_key)
|
||
seen_titles.add(title_key)
|
||
result.append(item)
|
||
return result
|
||
|
||
|
||
def _within_window(item: dict[str, Any], cutoff: datetime) -> bool:
|
||
dt = _entry_datetime(item)
|
||
if dt is None:
|
||
return True
|
||
return dt >= cutoff
|
||
|
||
|
||
def _sort_key(item: dict[str, Any]) -> tuple[int, datetime]:
|
||
dt = _entry_datetime(item)
|
||
if dt is None:
|
||
return (1, datetime.min.replace(tzinfo=timezone.utc))
|
||
return (0, dt)
|
||
|
||
|
||
def _fetch_news(categories: tuple[NewsCategory, ...]) -> dict[str, Any]:
|
||
hours = _hours_window()
|
||
per_feed = _per_feed_limit()
|
||
per_category = _per_category_limit()
|
||
cutoff = _now_utc() - timedelta(hours=hours)
|
||
|
||
headers = {"User-Agent": RSS_USER_AGENT, "Accept": "application/rss+xml, application/atom+xml, application/xml, text/xml, */*"}
|
||
tasks: list[tuple[NewsCategory, NewsFeed]] = []
|
||
for category in categories:
|
||
for feed in category.feeds:
|
||
tasks.append((category, feed))
|
||
|
||
raw_by_category: dict[str, list[dict[str, Any]]] = {c.id: [] for c in categories}
|
||
stats: dict[str, Any] = {
|
||
"feeds_total": len(tasks),
|
||
"feeds_ok": 0,
|
||
"items_raw": 0,
|
||
"feeds_failed": [],
|
||
"content_filtered": 0,
|
||
}
|
||
|
||
with httpx.Client(timeout=15.0, verify=certifi.where(), follow_redirects=True, headers=headers) as client:
|
||
fast_tasks = [t for t in tasks if not t[1].slow]
|
||
slow_tasks = [t for t in tasks if t[1].slow]
|
||
|
||
with ThreadPoolExecutor(max_workers=8) as pool:
|
||
futures = {
|
||
pool.submit(_fetch_one, client, cat, feed): (cat.id, feed.name)
|
||
for cat, feed in fast_tasks
|
||
}
|
||
for future in as_completed(futures):
|
||
cat_id, feed_name = futures[future]
|
||
try:
|
||
entries, ok = future.result()
|
||
except Exception as exc:
|
||
logger.warning("RSS 任务异常 [%s]: %s", feed_name, exc)
|
||
stats["feeds_failed"].append(feed_name)
|
||
continue
|
||
if ok:
|
||
stats["feeds_ok"] += 1
|
||
else:
|
||
stats["feeds_failed"].append(feed_name)
|
||
stats["items_raw"] += len(entries)
|
||
raw_by_category[cat_id].extend(entries[:per_feed])
|
||
|
||
for cat, feed in slow_tasks:
|
||
entries, ok = _fetch_one(client, cat, feed)
|
||
if ok:
|
||
stats["feeds_ok"] += 1
|
||
else:
|
||
stats["feeds_failed"].append(feed.name)
|
||
stats["items_raw"] += len(entries)
|
||
raw_by_category[cat.id].extend(entries[:per_feed])
|
||
if _is_reddit_url(feed.url):
|
||
time.sleep(2.0)
|
||
else:
|
||
time.sleep(1.0)
|
||
|
||
categories_out: list[dict[str, Any]] = []
|
||
flat: list[dict[str, Any]] = []
|
||
|
||
for category in categories:
|
||
items = raw_by_category[category.id]
|
||
items = [i for i in items if _within_window(i, cutoff)]
|
||
items, filtered_count = filter_news_items(items)
|
||
if filtered_count:
|
||
stats["content_filtered"] = stats.get("content_filtered", 0) + filtered_count
|
||
items.sort(key=_sort_key, reverse=True)
|
||
items = _dedupe_items(items)[:per_category]
|
||
for item in items:
|
||
dt = _entry_datetime(item)
|
||
item["published_fmt"] = dt.astimezone(timezone(timedelta(hours=8))).strftime("%m-%d %H:%M") if dt else ""
|
||
if items:
|
||
categories_out.append(
|
||
{
|
||
"id": category.id,
|
||
"name": category.name,
|
||
"icon": category.icon,
|
||
"items": items,
|
||
}
|
||
)
|
||
flat.extend(items)
|
||
|
||
flat.sort(key=_sort_key, reverse=True)
|
||
flat = _dedupe_items(flat)
|
||
|
||
return {
|
||
"enabled": True,
|
||
"hours": hours,
|
||
"categories": categories_out,
|
||
"flat": flat,
|
||
"stats": stats,
|
||
}
|
||
|
||
|
||
def fetch_ai_news() -> dict[str, Any]:
|
||
"""按类别抓取国际 AI 时讯，返回 {enabled, hours, categories, flat, stats}。"""
|
||
if not _enabled():
|
||
return {"enabled": False, "categories": [], "flat": [], "stats": {}}
|
||
return _fetch_news(NEWS_CATEGORIES)
|
||
|
||
|
||
def fetch_cn_ai_news() -> dict[str, Any]:
|
||
"""按类别抓取国内 AI 时讯，返回 {enabled, hours, categories, flat, stats}。"""
|
||
if not _cn_enabled():
|
||
return {"enabled": False, "categories": [], "flat": [], "stats": {}}
|
||
return _fetch_news(CN_NEWS_CATEGORIES)
|
||
|
||
|
||
def format_news_section(news: dict[str, Any], *, section_no: int, wecom_limit: int | None = None) -> list[str]:
|
||
return _format_news_section(
|
||
news,
|
||
section_no=section_no,
|
||
title="国际 AI 时讯",
|
||
disabled_hint="AI 时讯已关闭（`DAILY_AI_NEWS=0`）。",
|
||
wecom_limit=wecom_limit,
|
||
)
|
||
|
||
|
||
def format_cn_news_section(news: dict[str, Any], *, section_no: int, wecom_limit: int | None = None) -> list[str]:
|
||
return _format_news_section(
|
||
news,
|
||
section_no=section_no,
|
||
title="国内 AI 时讯",
|
||
disabled_hint="国内 AI 时讯已关闭（`DAILY_CN_AI_NEWS=0`）。",
|
||
wecom_limit=wecom_limit,
|
||
)
|
||
|
||
|
||
def _format_news_section(
|
||
news: dict[str, Any],
|
||
*,
|
||
section_no: int,
|
||
title: str,
|
||
disabled_hint: str,
|
||
wecom_limit: int | None = None,
|
||
) -> list[str]:
|
||
if not news.get("enabled"):
|
||
return ["---", "", f"## {section_no}、{title}", "", f"*{disabled_hint}*", ""]
|
||
|
||
categories = news.get("categories") or []
|
||
hours = news.get("hours", 72)
|
||
lines = [
|
||
"---",
|
||
"",
|
||
f"## {section_no}、{title}",
|
||
"",
|
||
f"> 近 **{hours}h** · {news.get('stats', {}).get('feeds_ok', 0)}/{news.get('stats', {}).get('feeds_total', 0)} 源可用",
|
||
"",
|
||
]
|
||
failed = news.get("stats", {}).get("feeds_failed") or []
|
||
if failed:
|
||
preview = "、".join(failed[:5])
|
||
suffix = "…" if len(failed) > 5 else ""
|
||
lines.append(f"> ⚠️ {len(failed)} 个源抓取失败：{preview}{suffix}")
|
||
lines.append("")
|
||
|
||
if not categories:
|
||
lines.append("*暂无可用条目（网络/RSS 源异常或时间窗口内无更新）。*")
|
||
lines.append("")
|
||
return lines
|
||
|
||
if wecom_limit is not None:
|
||
flat = (news.get("flat") or [])[:wecom_limit]
|
||
for i, item in enumerate(flat, 1):
|
||
pub = f" · {item['published_fmt']}" if item.get("published_fmt") else ""
|
||
lines.append(
|
||
f"{i}. [{item['title']}]({item['link']}) · `{item['source_name']}`{pub}"
|
||
)
|
||
lines.append("")
|
||
return lines
|
||
|
||
for cat in categories:
|
||
lines.append(f"### {cat['icon']} {cat['name']}")
|
||
lines.append("")
|
||
for i, item in enumerate(cat["items"], 1):
|
||
pub = f" · {item['published_fmt']}" if item.get("published_fmt") else ""
|
||
lines.append(f"{i}. **[{item['title']}]({item['link']})** · `{item['source_name']}`{pub}")
|
||
summary = item.get("summary", "")
|
||
if summary:
|
||
lines.append(f" - {_clean_text(summary, news_summary_limit())}")
|
||
lines.append("")
|
||
|
||
return lines
|
||
|
||
|
||
def _sorted_flat(news: dict[str, Any]) -> list[dict[str, Any]]:
|
||
flat = list(news.get("flat") or [])
|
||
flat.sort(
|
||
key=lambda item: (float(item.get("score") or 0), _sort_key(item)[1]),
|
||
reverse=True,
|
||
)
|
||
return flat
|
||
|
||
|
||
def prepare_wecom_news_items(news: dict[str, Any], *, limit: int | None = None) -> list[dict[str, Any]]:
|
||
if not news.get("enabled"):
|
||
return []
|
||
pick_limit = limit or _wecom_limit()
|
||
flat = _sorted_flat(news)
|
||
preferred = ("media", "newsletter", "official", "community", "research", "developer")
|
||
picked: list[dict[str, Any]] = []
|
||
seen: set[str] = set()
|
||
for cat in preferred:
|
||
for item in flat:
|
||
link = _normalize_link(item.get("link", ""))
|
||
if item.get("category_id") != cat or link in seen:
|
||
continue
|
||
picked.append(item)
|
||
seen.add(link)
|
||
if len(picked) >= pick_limit:
|
||
break
|
||
if len(picked) >= pick_limit:
|
||
break
|
||
items: list[dict[str, Any]] = []
|
||
for item in picked[:pick_limit]:
|
||
items.append(
|
||
{
|
||
"title": item.get("title", "?"),
|
||
"link": item.get("link", ""),
|
||
"source_name": item.get("source_name", "?"),
|
||
"published_fmt": item.get("published_fmt", ""),
|
||
"desc_short": _clean_text(item.get("summary", ""), 36),
|
||
"score": item.get("score"),
|
||
}
|
||
)
|
||
return items
|
||
|
||
|
||
def prepare_wecom_cn_news_items(news: dict[str, Any], *, limit: int | None = None) -> list[dict[str, Any]]:
|
||
if not news.get("enabled"):
|
||
return []
|
||
pick_limit = limit or _wecom_cn_limit()
|
||
flat = _sorted_flat(news)
|
||
preferred = ("media", "tech", "dev")
|
||
picked: list[dict[str, Any]] = []
|
||
seen_links: set[str] = set()
|
||
seen_sources: set[str] = set()
|
||
|
||
for cat in preferred:
|
||
for item in flat:
|
||
link = _normalize_link(item.get("link", ""))
|
||
source = item.get("source_name", "?")
|
||
if item.get("category_id") != cat or not link or link in seen_links or source in seen_sources:
|
||
continue
|
||
picked.append(item)
|
||
seen_links.add(link)
|
||
seen_sources.add(source)
|
||
if len(picked) >= pick_limit:
|
||
break
|
||
if len(picked) >= pick_limit:
|
||
break
|
||
|
||
if len(picked) < pick_limit:
|
||
for item in flat:
|
||
link = _normalize_link(item.get("link", ""))
|
||
if not link or link in seen_links:
|
||
continue
|
||
picked.append(item)
|
||
seen_links.add(link)
|
||
if len(picked) >= pick_limit:
|
||
break
|
||
|
||
items: list[dict[str, Any]] = []
|
||
for item in picked[:pick_limit]:
|
||
items.append(
|
||
{
|
||
"title": item.get("title", "?"),
|
||
"link": item.get("link", ""),
|
||
"source_name": item.get("source_name", "?"),
|
||
"published_fmt": item.get("published_fmt", ""),
|
||
"desc_short": _clean_text(item.get("summary", ""), 36),
|
||
"score": item.get("score"),
|
||
}
|
||
)
|
||
return items
|