国内AI时讯
This commit is contained in:
@@ -32,6 +32,7 @@ DAILY_WECOM_GITHUB_TRENDING=10
|
|||||||
DAILY_WECOM_GITHUB_EMERGING=10
|
DAILY_WECOM_GITHUB_EMERGING=10
|
||||||
DAILY_WECOM_GITHUB_TOPIC=10
|
DAILY_WECOM_GITHUB_TOPIC=10
|
||||||
DAILY_WECOM_AI_NEWS=10
|
DAILY_WECOM_AI_NEWS=10
|
||||||
|
DAILY_WECOM_CN_AI_NEWS=8
|
||||||
# 企微 Skills 合并前扫描池大小(同 source 合并后仍凑满 Top N)
|
# 企微 Skills 合并前扫描池大小(同 source 合并后仍凑满 Top N)
|
||||||
# DAILY_WECOM_SKILL_POOL=200
|
# DAILY_WECOM_SKILL_POOL=200
|
||||||
|
|
||||||
@@ -43,6 +44,8 @@ DAILY_WECOM_CHUNK_BYTES=4096
|
|||||||
|
|
||||||
# 国际 AI 时讯(RSS,见 daily/news/feeds.py)
|
# 国际 AI 时讯(RSS,见 daily/news/feeds.py)
|
||||||
DAILY_AI_NEWS=1
|
DAILY_AI_NEWS=1
|
||||||
|
# 国内 AI 时讯(RSS,见 daily/news/feeds_cn.py)
|
||||||
|
DAILY_CN_AI_NEWS=1
|
||||||
# 英文描述 → 简短中文(DAILY_CURSOR_EDITOR=0 时生效)
|
# 英文描述 → 简短中文(DAILY_CURSOR_EDITOR=0 时生效)
|
||||||
# DAILY_ZH_DESC=1
|
# DAILY_ZH_DESC=1
|
||||||
# DAILY_ZH_DESC_BATCH=20
|
# DAILY_ZH_DESC_BATCH=20
|
||||||
|
|||||||
11
README.md
11
README.md
@@ -84,7 +84,16 @@ WECOM_WEBHOOK_KEY=your-key
|
|||||||
| 研究 / 论文 | arXiv cs.CL/AI/LG、HF Papers |
|
| 研究 / 论文 | arXiv cs.CL/AI/LG、HF Papers |
|
||||||
| 社区讨论 | HN、Reddit r/LocalLLaMA / ClaudeAI / ML 等 |
|
| 社区讨论 | HN、Reddit r/LocalLLaMA / ClaudeAI / ML 等 |
|
||||||
|
|
||||||
环境变量:`DAILY_AI_NEWS=1` · `DAILY_AI_NEWS_HOURS=72` · `DAILY_WECOM_AI_NEWS=8`
|
环境变量:`DAILY_AI_NEWS=1` · `DAILY_CN_AI_NEWS=1` · `DAILY_AI_NEWS_HOURS=72` · `DAILY_WECOM_AI_NEWS=10` · `DAILY_WECOM_CN_AI_NEWS=8`
|
||||||
|
|
||||||
|
**国内 AI 时讯**(RSS,见 `daily/news/feeds_cn.py`):
|
||||||
|
|
||||||
|
| 类别 | 覆盖 |
|
||||||
|
|------|------|
|
||||||
|
| AI 专业媒体 | 量子位、InfoQ 中文 |
|
||||||
|
| 综合科技 | 36氪、雷锋网、Google News 中文 |
|
||||||
|
| 开发者社区 | 掘金(标题 AI 关键词过滤) |
|
||||||
|
|
||||||
|
|
||||||
### 生成架构(Tier B · Cursor 编辑层)
|
### 生成架构(Tier B · Cursor 编辑层)
|
||||||
|
|
||||||
|
|||||||
@@ -18,6 +18,7 @@ ICONS = {
|
|||||||
"emerging": "🌱",
|
"emerging": "🌱",
|
||||||
"topic": "🤖",
|
"topic": "🤖",
|
||||||
"ainews": "🌍",
|
"ainews": "🌍",
|
||||||
|
"cnainews": "🇨🇳",
|
||||||
"pick": "📦",
|
"pick": "📦",
|
||||||
"theme": "🎯",
|
"theme": "🎯",
|
||||||
"file": "📄",
|
"file": "📄",
|
||||||
@@ -231,6 +232,7 @@ def build_wecom_report(
|
|||||||
topic_name: str,
|
topic_name: str,
|
||||||
topic_repos: list[dict[str, Any]],
|
topic_repos: list[dict[str, Any]],
|
||||||
ai_news: list[dict[str, Any]] | None = None,
|
ai_news: list[dict[str, Any]] | None = None,
|
||||||
|
cn_ai_news: list[dict[str, Any]] | None = None,
|
||||||
pick_command: str,
|
pick_command: str,
|
||||||
) -> str:
|
) -> str:
|
||||||
lines = [
|
lines = [
|
||||||
@@ -250,6 +252,11 @@ def build_wecom_report(
|
|||||||
lines.extend(_ai_news_lines(ai_news))
|
lines.extend(_ai_news_lines(ai_news))
|
||||||
lines.append("")
|
lines.append("")
|
||||||
|
|
||||||
|
if cn_ai_news:
|
||||||
|
lines.append(f"{ICONS['cnainews']} **国内 AI 时讯 Top {len(cn_ai_news)}**")
|
||||||
|
lines.extend(_ai_news_lines(cn_ai_news))
|
||||||
|
lines.append("")
|
||||||
|
|
||||||
lines.append(f"{ICONS['trending']} **Skills Trending Top {len(trending)}**")
|
lines.append(f"{ICONS['trending']} **Skills Trending Top {len(trending)}**")
|
||||||
for rank, item in enumerate(trending, 1):
|
for rank, item in enumerate(trending, 1):
|
||||||
lines.extend(_skill_line(rank, item, badge=item.get("badge", "")))
|
lines.extend(_skill_line(rank, item, badge=item.get("badge", "")))
|
||||||
|
|||||||
@@ -42,7 +42,14 @@ from daily.github.auth import github_html_headers
|
|||||||
from daily.github.search import fetch_emerging_repos, fetch_topic_hot_repos
|
from daily.github.search import fetch_emerging_repos, fetch_topic_hot_repos
|
||||||
from daily.github.trending import fetch_github_trending, trending_data_source_note
|
from daily.github.trending import fetch_github_trending, trending_data_source_note
|
||||||
from daily.localize import LocalizeJob, localize_descriptions, needs_chinese
|
from daily.localize import LocalizeJob, localize_descriptions, needs_chinese
|
||||||
from daily.news.fetch import fetch_ai_news, format_news_section, prepare_wecom_news_items
|
from daily.news.fetch import (
|
||||||
|
fetch_ai_news,
|
||||||
|
fetch_cn_ai_news,
|
||||||
|
format_cn_news_section,
|
||||||
|
format_news_section,
|
||||||
|
prepare_wecom_cn_news_items,
|
||||||
|
prepare_wecom_news_items,
|
||||||
|
)
|
||||||
from daily.report_data import (
|
from daily.report_data import (
|
||||||
build_full_payload,
|
build_full_payload,
|
||||||
build_llm_input,
|
build_llm_input,
|
||||||
@@ -90,6 +97,7 @@ def _localize_descriptions_in_place(
|
|||||||
github_emerging: list[dict[str, Any]],
|
github_emerging: list[dict[str, Any]],
|
||||||
github_topic: list[dict[str, Any]],
|
github_topic: list[dict[str, Any]],
|
||||||
ai_news: dict[str, Any],
|
ai_news: dict[str, Any],
|
||||||
|
cn_ai_news: dict[str, Any] | None = None,
|
||||||
) -> None:
|
) -> None:
|
||||||
full_limit = full_desc_limit()
|
full_limit = full_desc_limit()
|
||||||
news_limit = news_summary_limit()
|
news_limit = news_summary_limit()
|
||||||
@@ -264,6 +272,7 @@ def _build_highlights(
|
|||||||
github_trending: list[dict[str, Any]],
|
github_trending: list[dict[str, Any]],
|
||||||
github_emerging: list[dict[str, Any]],
|
github_emerging: list[dict[str, Any]],
|
||||||
ai_news: dict[str, Any] | None = None,
|
ai_news: dict[str, Any] | None = None,
|
||||||
|
cn_ai_news: dict[str, Any] | None = None,
|
||||||
) -> list[str]:
|
) -> list[str]:
|
||||||
points: list[str] = []
|
points: list[str] = []
|
||||||
if ai_news and ai_news.get("enabled"):
|
if ai_news and ai_news.get("enabled"):
|
||||||
@@ -278,6 +287,20 @@ def _build_highlights(
|
|||||||
n0 = ai_news["flat"][0]
|
n0 = ai_news["flat"][0]
|
||||||
pub = f" · {n0['published_fmt']}" if n0.get("published_fmt") else ""
|
pub = f" · {n0['published_fmt']}" if n0.get("published_fmt") else ""
|
||||||
points.append(f"🌍 AI 时讯 [{n0['title']}]({n0['link']})(`{n0.get('source_name', '?')}`{pub})")
|
points.append(f"🌍 AI 时讯 [{n0['title']}]({n0['link']})(`{n0.get('source_name', '?')}`{pub})")
|
||||||
|
if cn_ai_news and cn_ai_news.get("enabled"):
|
||||||
|
top_cn = prepare_wecom_cn_news_items(cn_ai_news)
|
||||||
|
if top_cn:
|
||||||
|
n0 = top_cn[0]
|
||||||
|
pub = f" · {n0['published_fmt']}" if n0.get("published_fmt") else ""
|
||||||
|
points.append(
|
||||||
|
f"🇨🇳 国内 AI [{n0['title']}]({n0['link']})(`{n0.get('source_name', '?')}`{pub})"
|
||||||
|
)
|
||||||
|
elif cn_ai_news.get("flat"):
|
||||||
|
n0 = cn_ai_news["flat"][0]
|
||||||
|
pub = f" · {n0['published_fmt']}" if n0.get("published_fmt") else ""
|
||||||
|
points.append(
|
||||||
|
f"🇨🇳 国内 AI [{n0['title']}]({n0['link']})(`{n0.get('source_name', '?')}`{pub})"
|
||||||
|
)
|
||||||
if trending:
|
if trending:
|
||||||
t0 = trending[0]
|
t0 = trending[0]
|
||||||
points.append(f"📈 Skills 榜首 **{t0.get('title')}**({_format_installs(t0.get('installs', 0))})")
|
points.append(f"📈 Skills 榜首 **{t0.get('title')}**({_format_installs(t0.get('installs', 0))})")
|
||||||
@@ -418,6 +441,7 @@ def generate_report() -> tuple[str, str, Path, Path]:
|
|||||||
seen_repos.update(r["repo"] for r in github_emerging)
|
seen_repos.update(r["repo"] for r in github_emerging)
|
||||||
topic_name, github_topic = fetch_topic_hot_repos(topic_fetch_n, exclude=seen_repos)
|
topic_name, github_topic = fetch_topic_hot_repos(topic_fetch_n, exclude=seen_repos)
|
||||||
ai_news = fetch_ai_news()
|
ai_news = fetch_ai_news()
|
||||||
|
cn_ai_news = fetch_cn_ai_news()
|
||||||
|
|
||||||
wecom_limits = {
|
wecom_limits = {
|
||||||
"trending": wecom_trending,
|
"trending": wecom_trending,
|
||||||
@@ -428,6 +452,7 @@ def generate_report() -> tuple[str, str, Path, Path]:
|
|||||||
"emerging": wecom_emerging,
|
"emerging": wecom_emerging,
|
||||||
"topic": wecom_topic,
|
"topic": wecom_topic,
|
||||||
"ai_news": env_int("DAILY_WECOM_AI_NEWS", 10),
|
"ai_news": env_int("DAILY_WECOM_AI_NEWS", 10),
|
||||||
|
"cn_ai_news": env_int("DAILY_WECOM_CN_AI_NEWS", 8),
|
||||||
}
|
}
|
||||||
llm_input = build_llm_input(
|
llm_input = build_llm_input(
|
||||||
date_str=date_str,
|
date_str=date_str,
|
||||||
@@ -439,6 +464,7 @@ def generate_report() -> tuple[str, str, Path, Path]:
|
|||||||
github_topic=github_topic,
|
github_topic=github_topic,
|
||||||
topic_name=topic_name,
|
topic_name=topic_name,
|
||||||
ai_news=ai_news,
|
ai_news=ai_news,
|
||||||
|
cn_ai_news=cn_ai_news,
|
||||||
wecom_limits=wecom_limits,
|
wecom_limits=wecom_limits,
|
||||||
)
|
)
|
||||||
save_json(
|
save_json(
|
||||||
@@ -476,6 +502,7 @@ def generate_report() -> tuple[str, str, Path, Path]:
|
|||||||
github_emerging=github_emerging,
|
github_emerging=github_emerging,
|
||||||
github_topic=github_topic,
|
github_topic=github_topic,
|
||||||
ai_news=ai_news,
|
ai_news=ai_news,
|
||||||
|
cn_ai_news=cn_ai_news,
|
||||||
descriptions=editorial.get("descriptions") or {},
|
descriptions=editorial.get("descriptions") or {},
|
||||||
)
|
)
|
||||||
editorial_theme = theme_line_from_editorial(editorial) or None
|
editorial_theme = theme_line_from_editorial(editorial) or None
|
||||||
@@ -484,7 +511,7 @@ def generate_report() -> tuple[str, str, Path, Path]:
|
|||||||
|
|
||||||
# 完整版归档:中文化 + 加长摘要(已是中文的条目会跳过翻译)
|
# 完整版归档:中文化 + 加长摘要(已是中文的条目会跳过翻译)
|
||||||
_localize_descriptions_in_place(
|
_localize_descriptions_in_place(
|
||||||
trending, hot, github_trending, github_emerging, github_topic, ai_news
|
trending, hot, github_trending, github_emerging, github_topic, ai_news, cn_ai_news
|
||||||
)
|
)
|
||||||
|
|
||||||
themes = _theme_clusters(feed)
|
themes = _theme_clusters(feed)
|
||||||
@@ -494,7 +521,7 @@ def generate_report() -> tuple[str, str, Path, Path]:
|
|||||||
"",
|
"",
|
||||||
f"> 生成时间:{now.strftime('%Y-%m-%d %H:%M')} (UTC+8) ",
|
f"> 生成时间:{now.strftime('%Y-%m-%d %H:%M')} (UTC+8) ",
|
||||||
f"> skills 数据更新:{updated} ",
|
f"> skills 数据更新:{updated} ",
|
||||||
"> 数据来源:[skills.sh/trending](https://skills.sh/trending) · [skills.sh/hot](https://skills.sh/hot) · 国际 AI RSS",
|
"> 数据来源:[skills.sh/trending](https://skills.sh/trending) · [skills.sh/hot](https://skills.sh/hot) · 国际/国内 AI RSS",
|
||||||
"",
|
"",
|
||||||
"---",
|
"---",
|
||||||
"",
|
"",
|
||||||
@@ -538,6 +565,8 @@ def generate_report() -> tuple[str, str, Path, Path]:
|
|||||||
section_no = 6
|
section_no = 6
|
||||||
lines.extend(format_news_section(ai_news, section_no=section_no))
|
lines.extend(format_news_section(ai_news, section_no=section_no))
|
||||||
section_no += 1
|
section_no += 1
|
||||||
|
lines.extend(format_cn_news_section(cn_ai_news, section_no=section_no))
|
||||||
|
section_no += 1
|
||||||
|
|
||||||
watch = (env("GITHUB_REPOS") or "").strip()
|
watch = (env("GITHUB_REPOS") or "").strip()
|
||||||
if watch:
|
if watch:
|
||||||
@@ -581,9 +610,10 @@ def generate_report() -> tuple[str, str, Path, Path]:
|
|||||||
time_str=time_str,
|
time_str=time_str,
|
||||||
updated=updated,
|
updated=updated,
|
||||||
highlights=editorial_highlights
|
highlights=editorial_highlights
|
||||||
or _build_highlights(trending, hot, github_trending, github_emerging, ai_news),
|
or _build_highlights(trending, hot, github_trending, github_emerging, ai_news, cn_ai_news),
|
||||||
theme_line=editorial_theme or _detect_theme_line(feed),
|
theme_line=editorial_theme or _detect_theme_line(feed),
|
||||||
ai_news=prepare_wecom_news_items(ai_news),
|
ai_news=prepare_wecom_news_items(ai_news),
|
||||||
|
cn_ai_news=prepare_wecom_cn_news_items(cn_ai_news),
|
||||||
trending=[
|
trending=[
|
||||||
_prepare_skill_item(item, prev_ids, r)
|
_prepare_skill_item(item, prev_ids, r)
|
||||||
for r, item in enumerate(
|
for r, item in enumerate(
|
||||||
|
|||||||
@@ -1,3 +1,13 @@
|
|||||||
from daily.news.fetch import fetch_ai_news, format_news_section
|
from daily.news.fetch import (
|
||||||
|
fetch_ai_news,
|
||||||
|
fetch_cn_ai_news,
|
||||||
|
format_cn_news_section,
|
||||||
|
format_news_section,
|
||||||
|
)
|
||||||
|
|
||||||
__all__ = ["fetch_ai_news", "format_news_section"]
|
__all__ = [
|
||||||
|
"fetch_ai_news",
|
||||||
|
"fetch_cn_ai_news",
|
||||||
|
"format_news_section",
|
||||||
|
"format_cn_news_section",
|
||||||
|
]
|
||||||
|
|||||||
@@ -10,6 +10,7 @@ class NewsFeed:
|
|||||||
name: str
|
name: str
|
||||||
url: str
|
url: str
|
||||||
slow: bool = False # 限速源(如 Reddit)串行抓取
|
slow: bool = False # 限速源(如 Reddit)串行抓取
|
||||||
|
ai_filter: bool = False # 综合源仅保留标题命中 AI 关键词的条目
|
||||||
|
|
||||||
|
|
||||||
@dataclass(frozen=True)
|
@dataclass(frozen=True)
|
||||||
|
|||||||
71
daily/news/feeds_cn.py
Normal file
71
daily/news/feeds_cn.py
Normal file
@@ -0,0 +1,71 @@
|
|||||||
|
"""国内 AI 时讯 RSS 源定义(按类别分组)。"""
|
||||||
|
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
from daily.news.feeds import NewsCategory, NewsFeed
|
||||||
|
|
||||||
|
# 综合源 ai_filter=True 时,仅保留标题命中以下词之一的条目
|
||||||
|
CN_AI_TITLE_KEYWORDS: tuple[str, ...] = (
|
||||||
|
"人工智能",
|
||||||
|
"大模型",
|
||||||
|
"智能体",
|
||||||
|
"多模态",
|
||||||
|
"AIGC",
|
||||||
|
"LLM",
|
||||||
|
"GPT",
|
||||||
|
"Claude",
|
||||||
|
"Gemini",
|
||||||
|
"ChatGPT",
|
||||||
|
"OpenAI",
|
||||||
|
"Anthropic",
|
||||||
|
"Copilot",
|
||||||
|
"Agent",
|
||||||
|
"AI ",
|
||||||
|
" AI",
|
||||||
|
"AI·",
|
||||||
|
"AI业务",
|
||||||
|
"AI模型",
|
||||||
|
"AI助手",
|
||||||
|
"AI工具",
|
||||||
|
"AI编程",
|
||||||
|
"AI 编程",
|
||||||
|
"AI版",
|
||||||
|
"AI Agent",
|
||||||
|
"推理模型",
|
||||||
|
"深度学习",
|
||||||
|
"机器学习",
|
||||||
|
"Function Calling",
|
||||||
|
)
|
||||||
|
|
||||||
|
CN_NEWS_CATEGORIES: tuple[NewsCategory, ...] = (
|
||||||
|
NewsCategory(
|
||||||
|
id="media",
|
||||||
|
name="AI 专业媒体",
|
||||||
|
icon="📰",
|
||||||
|
feeds=(
|
||||||
|
NewsFeed("量子位", "https://www.qbitai.com/feed"),
|
||||||
|
NewsFeed("InfoQ 中文", "https://www.infoq.cn/feed/AI"),
|
||||||
|
),
|
||||||
|
),
|
||||||
|
NewsCategory(
|
||||||
|
id="tech",
|
||||||
|
name="综合科技",
|
||||||
|
icon="📱",
|
||||||
|
feeds=(
|
||||||
|
NewsFeed("36氪", "https://36kr.com/feed", ai_filter=True),
|
||||||
|
NewsFeed("雷锋网", "https://www.leiphone.com/feed"),
|
||||||
|
NewsFeed(
|
||||||
|
"Google News · AI",
|
||||||
|
"https://news.google.com/rss/search?q=人工智能+OR+大模型+OR+Agent+OR+LLM&hl=zh-CN&gl=CN&ceid=CN:zh-Hans",
|
||||||
|
),
|
||||||
|
),
|
||||||
|
),
|
||||||
|
NewsCategory(
|
||||||
|
id="dev",
|
||||||
|
name="开发者社区",
|
||||||
|
icon="💻",
|
||||||
|
feeds=(
|
||||||
|
NewsFeed("掘金", "https://juejin.cn/rss", ai_filter=True),
|
||||||
|
),
|
||||||
|
),
|
||||||
|
)
|
||||||
@@ -1,4 +1,4 @@
|
|||||||
"""抓取并整理国际 AI 时讯 RSS。"""
|
"""抓取并整理国际 / 国内 AI 时讯 RSS。"""
|
||||||
|
|
||||||
from __future__ import annotations
|
from __future__ import annotations
|
||||||
|
|
||||||
@@ -17,7 +17,8 @@ import certifi
|
|||||||
import httpx
|
import httpx
|
||||||
|
|
||||||
from daily.config import env, env_int, news_summary_limit
|
from daily.config import env, env_int, news_summary_limit
|
||||||
from daily.news.feeds import NEWS_CATEGORIES, NewsCategory
|
from daily.news.feeds import NEWS_CATEGORIES, NewsCategory, NewsFeed
|
||||||
|
from daily.news.feeds_cn import CN_AI_TITLE_KEYWORDS, CN_NEWS_CATEGORIES
|
||||||
|
|
||||||
logger = logging.getLogger(__name__)
|
logger = logging.getLogger(__name__)
|
||||||
|
|
||||||
@@ -39,6 +40,11 @@ def _enabled() -> bool:
|
|||||||
return raw not in {"0", "false", "no", "off"}
|
return raw not in {"0", "false", "no", "off"}
|
||||||
|
|
||||||
|
|
||||||
|
def _cn_enabled() -> bool:
|
||||||
|
raw = (env("DAILY_CN_AI_NEWS") or "1").strip().lower()
|
||||||
|
return raw not in {"0", "false", "no", "off"}
|
||||||
|
|
||||||
|
|
||||||
def _hours_window() -> int:
|
def _hours_window() -> int:
|
||||||
return max(1, env_int("DAILY_AI_NEWS_HOURS", 72))
|
return max(1, env_int("DAILY_AI_NEWS_HOURS", 72))
|
||||||
|
|
||||||
@@ -55,6 +61,27 @@ def _wecom_limit() -> int:
|
|||||||
return max(1, env_int("DAILY_WECOM_AI_NEWS", 10))
|
return max(1, env_int("DAILY_WECOM_AI_NEWS", 10))
|
||||||
|
|
||||||
|
|
||||||
|
def _wecom_cn_limit() -> int:
|
||||||
|
return max(1, env_int("DAILY_WECOM_CN_AI_NEWS", 8))
|
||||||
|
|
||||||
|
|
||||||
|
def _matches_cn_ai_title(title: str) -> bool:
|
||||||
|
text = title.strip()
|
||||||
|
if not text:
|
||||||
|
return False
|
||||||
|
lower = text.lower()
|
||||||
|
for keyword in CN_AI_TITLE_KEYWORDS:
|
||||||
|
if keyword.lower() in lower:
|
||||||
|
return True
|
||||||
|
return False
|
||||||
|
|
||||||
|
|
||||||
|
def _filter_ai_entries(entries: list[dict[str, Any]], *, ai_filter: bool) -> list[dict[str, Any]]:
|
||||||
|
if not ai_filter:
|
||||||
|
return entries
|
||||||
|
return [item for item in entries if _matches_cn_ai_title(item.get("title", ""))]
|
||||||
|
|
||||||
|
|
||||||
def _now_utc() -> datetime:
|
def _now_utc() -> datetime:
|
||||||
return datetime.now(timezone.utc)
|
return datetime.now(timezone.utc)
|
||||||
|
|
||||||
@@ -255,18 +282,23 @@ def _reddit_fetch_urls(feed_url: str) -> list[str]:
|
|||||||
return [primary, fallback]
|
return [primary, fallback]
|
||||||
|
|
||||||
|
|
||||||
def _fetch_one(client: httpx.Client, category: NewsCategory, feed_url: str, feed_name: str) -> list[dict[str, Any]]:
|
def _fetch_one(
|
||||||
|
client: httpx.Client,
|
||||||
|
category: NewsCategory,
|
||||||
|
feed: NewsFeed,
|
||||||
|
) -> list[dict[str, Any]]:
|
||||||
last_exc: Exception | None = None
|
last_exc: Exception | None = None
|
||||||
for url in _reddit_fetch_urls(feed_url):
|
for url in _reddit_fetch_urls(feed.url):
|
||||||
try:
|
try:
|
||||||
headers = _request_headers(dict(client.headers), url)
|
headers = _request_headers(dict(client.headers), url)
|
||||||
resp = client.get(url, headers=headers)
|
resp = client.get(url, headers=headers)
|
||||||
resp.raise_for_status()
|
resp.raise_for_status()
|
||||||
return _parse_feed(resp.text, feed_name, category)
|
entries = _parse_feed(resp.text, feed.name, category)
|
||||||
|
return _filter_ai_entries(entries, ai_filter=feed.ai_filter)
|
||||||
except Exception as exc:
|
except Exception as exc:
|
||||||
last_exc = exc
|
last_exc = exc
|
||||||
continue
|
continue
|
||||||
logger.warning("RSS fetch failed [%s] %s: %s", feed_name, feed_url, last_exc)
|
logger.warning("RSS fetch failed [%s] %s: %s", feed.name, feed.url, last_exc)
|
||||||
return []
|
return []
|
||||||
|
|
||||||
|
|
||||||
@@ -299,33 +331,29 @@ def _sort_key(item: dict[str, Any]) -> tuple[int, datetime]:
|
|||||||
return (0, dt)
|
return (0, dt)
|
||||||
|
|
||||||
|
|
||||||
def fetch_ai_news() -> dict[str, Any]:
|
def _fetch_news(categories: tuple[NewsCategory, ...]) -> dict[str, Any]:
|
||||||
"""按类别抓取 AI 时讯,返回 {enabled, hours, categories, flat, stats}。"""
|
|
||||||
if not _enabled():
|
|
||||||
return {"enabled": False, "categories": [], "flat": [], "stats": {}}
|
|
||||||
|
|
||||||
hours = _hours_window()
|
hours = _hours_window()
|
||||||
per_feed = _per_feed_limit()
|
per_feed = _per_feed_limit()
|
||||||
per_category = _per_category_limit()
|
per_category = _per_category_limit()
|
||||||
cutoff = _now_utc() - timedelta(hours=hours)
|
cutoff = _now_utc() - timedelta(hours=hours)
|
||||||
|
|
||||||
headers = {"User-Agent": USER_AGENT, "Accept": "application/rss+xml, application/atom+xml, application/xml, text/xml, */*"}
|
headers = {"User-Agent": USER_AGENT, "Accept": "application/rss+xml, application/atom+xml, application/xml, text/xml, */*"}
|
||||||
tasks: list[tuple[NewsCategory, str, str, bool]] = []
|
tasks: list[tuple[NewsCategory, NewsFeed]] = []
|
||||||
for category in NEWS_CATEGORIES:
|
for category in categories:
|
||||||
for feed in category.feeds:
|
for feed in category.feeds:
|
||||||
tasks.append((category, feed.url, feed.name, feed.slow))
|
tasks.append((category, feed))
|
||||||
|
|
||||||
raw_by_category: dict[str, list[dict[str, Any]]] = {c.id: [] for c in NEWS_CATEGORIES}
|
raw_by_category: dict[str, list[dict[str, Any]]] = {c.id: [] for c in categories}
|
||||||
stats = {"feeds_total": len(tasks), "feeds_ok": 0, "items_raw": 0}
|
stats = {"feeds_total": len(tasks), "feeds_ok": 0, "items_raw": 0}
|
||||||
|
|
||||||
with httpx.Client(timeout=15.0, verify=certifi.where(), follow_redirects=True, headers=headers) as client:
|
with httpx.Client(timeout=15.0, verify=certifi.where(), follow_redirects=True, headers=headers) as client:
|
||||||
fast_tasks = [t for t in tasks if not t[3]]
|
fast_tasks = [t for t in tasks if not t[1].slow]
|
||||||
slow_tasks = [t for t in tasks if t[3]]
|
slow_tasks = [t for t in tasks if t[1].slow]
|
||||||
|
|
||||||
with ThreadPoolExecutor(max_workers=8) as pool:
|
with ThreadPoolExecutor(max_workers=8) as pool:
|
||||||
futures = {
|
futures = {
|
||||||
pool.submit(_fetch_one, client, cat, url, name): (cat.id, name)
|
pool.submit(_fetch_one, client, cat, feed): (cat.id, feed.name)
|
||||||
for cat, url, name, _slow in fast_tasks
|
for cat, feed in fast_tasks
|
||||||
}
|
}
|
||||||
for future in as_completed(futures):
|
for future in as_completed(futures):
|
||||||
cat_id, feed_name = futures[future]
|
cat_id, feed_name = futures[future]
|
||||||
@@ -339,13 +367,13 @@ def fetch_ai_news() -> dict[str, Any]:
|
|||||||
stats["items_raw"] += len(entries)
|
stats["items_raw"] += len(entries)
|
||||||
raw_by_category[cat_id].extend(entries[:per_feed])
|
raw_by_category[cat_id].extend(entries[:per_feed])
|
||||||
|
|
||||||
for cat, url, name, _slow in slow_tasks:
|
for cat, feed in slow_tasks:
|
||||||
entries = _fetch_one(client, cat, url, name)
|
entries = _fetch_one(client, cat, feed)
|
||||||
if entries:
|
if entries:
|
||||||
stats["feeds_ok"] += 1
|
stats["feeds_ok"] += 1
|
||||||
stats["items_raw"] += len(entries)
|
stats["items_raw"] += len(entries)
|
||||||
raw_by_category[cat.id].extend(entries[:per_feed])
|
raw_by_category[cat.id].extend(entries[:per_feed])
|
||||||
if _is_reddit_url(url):
|
if _is_reddit_url(feed.url):
|
||||||
time.sleep(2.0)
|
time.sleep(2.0)
|
||||||
else:
|
else:
|
||||||
time.sleep(1.0)
|
time.sleep(1.0)
|
||||||
@@ -353,7 +381,7 @@ def fetch_ai_news() -> dict[str, Any]:
|
|||||||
categories_out: list[dict[str, Any]] = []
|
categories_out: list[dict[str, Any]] = []
|
||||||
flat: list[dict[str, Any]] = []
|
flat: list[dict[str, Any]] = []
|
||||||
|
|
||||||
for category in NEWS_CATEGORIES:
|
for category in categories:
|
||||||
items = raw_by_category[category.id]
|
items = raw_by_category[category.id]
|
||||||
items = [i for i in items if _within_window(i, cutoff)]
|
items = [i for i in items if _within_window(i, cutoff)]
|
||||||
items.sort(key=_sort_key, reverse=True)
|
items.sort(key=_sort_key, reverse=True)
|
||||||
@@ -384,16 +412,57 @@ def fetch_ai_news() -> dict[str, Any]:
|
|||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
|
def fetch_ai_news() -> dict[str, Any]:
|
||||||
|
"""按类别抓取国际 AI 时讯,返回 {enabled, hours, categories, flat, stats}。"""
|
||||||
|
if not _enabled():
|
||||||
|
return {"enabled": False, "categories": [], "flat": [], "stats": {}}
|
||||||
|
return _fetch_news(NEWS_CATEGORIES)
|
||||||
|
|
||||||
|
|
||||||
|
def fetch_cn_ai_news() -> dict[str, Any]:
|
||||||
|
"""按类别抓取国内 AI 时讯,返回 {enabled, hours, categories, flat, stats}。"""
|
||||||
|
if not _cn_enabled():
|
||||||
|
return {"enabled": False, "categories": [], "flat": [], "stats": {}}
|
||||||
|
return _fetch_news(CN_NEWS_CATEGORIES)
|
||||||
|
|
||||||
|
|
||||||
def format_news_section(news: dict[str, Any], *, section_no: int, wecom_limit: int | None = None) -> list[str]:
|
def format_news_section(news: dict[str, Any], *, section_no: int, wecom_limit: int | None = None) -> list[str]:
|
||||||
|
return _format_news_section(
|
||||||
|
news,
|
||||||
|
section_no=section_no,
|
||||||
|
title="国际 AI 时讯",
|
||||||
|
disabled_hint="AI 时讯已关闭(`DAILY_AI_NEWS=0`)。",
|
||||||
|
wecom_limit=wecom_limit,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def format_cn_news_section(news: dict[str, Any], *, section_no: int, wecom_limit: int | None = None) -> list[str]:
|
||||||
|
return _format_news_section(
|
||||||
|
news,
|
||||||
|
section_no=section_no,
|
||||||
|
title="国内 AI 时讯",
|
||||||
|
disabled_hint="国内 AI 时讯已关闭(`DAILY_CN_AI_NEWS=0`)。",
|
||||||
|
wecom_limit=wecom_limit,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def _format_news_section(
|
||||||
|
news: dict[str, Any],
|
||||||
|
*,
|
||||||
|
section_no: int,
|
||||||
|
title: str,
|
||||||
|
disabled_hint: str,
|
||||||
|
wecom_limit: int | None = None,
|
||||||
|
) -> list[str]:
|
||||||
if not news.get("enabled"):
|
if not news.get("enabled"):
|
||||||
return ["---", "", f"## {section_no}、国际 AI 时讯", "", "*AI 时讯已关闭(`DAILY_AI_NEWS=0`)。*", ""]
|
return ["---", "", f"## {section_no}、{title}", "", f"*{disabled_hint}*", ""]
|
||||||
|
|
||||||
categories = news.get("categories") or []
|
categories = news.get("categories") or []
|
||||||
hours = news.get("hours", 72)
|
hours = news.get("hours", 72)
|
||||||
lines = [
|
lines = [
|
||||||
"---",
|
"---",
|
||||||
"",
|
"",
|
||||||
f"## {section_no}、国际 AI 时讯",
|
f"## {section_no}、{title}",
|
||||||
"",
|
"",
|
||||||
f"> 近 **{hours}h** · {news.get('stats', {}).get('feeds_ok', 0)}/{news.get('stats', {}).get('feeds_total', 0)} 源可用",
|
f"> 近 **{hours}h** · {news.get('stats', {}).get('feeds_ok', 0)}/{news.get('stats', {}).get('feeds_total', 0)} 源可用",
|
||||||
"",
|
"",
|
||||||
@@ -460,3 +529,52 @@ def prepare_wecom_news_items(news: dict[str, Any]) -> list[dict[str, Any]]:
|
|||||||
}
|
}
|
||||||
)
|
)
|
||||||
return items
|
return items
|
||||||
|
|
||||||
|
|
||||||
|
def prepare_wecom_cn_news_items(news: dict[str, Any]) -> list[dict[str, Any]]:
|
||||||
|
if not news.get("enabled"):
|
||||||
|
return []
|
||||||
|
limit = _wecom_cn_limit()
|
||||||
|
flat = _dedupe_items(news.get("flat") or [])
|
||||||
|
flat.sort(key=_sort_key, reverse=True)
|
||||||
|
preferred = ("media", "tech", "dev")
|
||||||
|
picked: list[dict[str, Any]] = []
|
||||||
|
seen_links: set[str] = set()
|
||||||
|
seen_sources: set[str] = set()
|
||||||
|
|
||||||
|
for cat in preferred:
|
||||||
|
for item in flat:
|
||||||
|
link = _normalize_link(item.get("link", ""))
|
||||||
|
source = item.get("source_name", "?")
|
||||||
|
if item.get("category_id") != cat or not link or link in seen_links or source in seen_sources:
|
||||||
|
continue
|
||||||
|
picked.append(item)
|
||||||
|
seen_links.add(link)
|
||||||
|
seen_sources.add(source)
|
||||||
|
if len(picked) >= limit:
|
||||||
|
break
|
||||||
|
if len(picked) >= limit:
|
||||||
|
break
|
||||||
|
|
||||||
|
if len(picked) < limit:
|
||||||
|
for item in flat:
|
||||||
|
link = _normalize_link(item.get("link", ""))
|
||||||
|
if not link or link in seen_links:
|
||||||
|
continue
|
||||||
|
picked.append(item)
|
||||||
|
seen_links.add(link)
|
||||||
|
if len(picked) >= limit:
|
||||||
|
break
|
||||||
|
|
||||||
|
items: list[dict[str, Any]] = []
|
||||||
|
for item in picked[:limit]:
|
||||||
|
items.append(
|
||||||
|
{
|
||||||
|
"title": item.get("title", "?"),
|
||||||
|
"link": item.get("link", ""),
|
||||||
|
"source_name": item.get("source_name", "?"),
|
||||||
|
"published_fmt": item.get("published_fmt", ""),
|
||||||
|
"desc_short": _clean_text(item.get("summary", ""), 36),
|
||||||
|
}
|
||||||
|
)
|
||||||
|
return items
|
||||||
|
|||||||
@@ -8,7 +8,7 @@ from typing import Any
|
|||||||
|
|
||||||
from daily.config import OUTPUT_DIR, env_int
|
from daily.config import OUTPUT_DIR, env_int
|
||||||
from daily.delta import build_movement_baseline, build_movement_context, compare_depth
|
from daily.delta import build_movement_baseline, build_movement_context, compare_depth
|
||||||
from daily.news.fetch import prepare_wecom_news_items
|
from daily.news.fetch import prepare_wecom_cn_news_items, prepare_wecom_news_items
|
||||||
from daily.skills_group import group_skills_by_source
|
from daily.skills_group import group_skills_by_source
|
||||||
|
|
||||||
|
|
||||||
@@ -54,9 +54,14 @@ def _slim_github(item: dict[str, Any]) -> dict[str, Any]:
|
|||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
def _slim_news_items(ai_news: dict[str, Any], limit: int) -> list[dict[str, Any]]:
|
def _slim_news_items(
|
||||||
|
ai_news: dict[str, Any],
|
||||||
|
limit: int,
|
||||||
|
*,
|
||||||
|
prepare=prepare_wecom_news_items,
|
||||||
|
) -> list[dict[str, Any]]:
|
||||||
items: list[dict[str, Any]] = []
|
items: list[dict[str, Any]] = []
|
||||||
for item in prepare_wecom_news_items(ai_news):
|
for item in prepare(ai_news):
|
||||||
items.append(
|
items.append(
|
||||||
{
|
{
|
||||||
"link": item.get("link", ""),
|
"link": item.get("link", ""),
|
||||||
@@ -98,10 +103,12 @@ def build_llm_input(
|
|||||||
github_topic: list[dict[str, Any]],
|
github_topic: list[dict[str, Any]],
|
||||||
topic_name: str,
|
topic_name: str,
|
||||||
ai_news: dict[str, Any],
|
ai_news: dict[str, Any],
|
||||||
|
cn_ai_news: dict[str, Any],
|
||||||
wecom_limits: dict[str, int],
|
wecom_limits: dict[str, int],
|
||||||
) -> dict[str, Any]:
|
) -> dict[str, Any]:
|
||||||
"""供 Cursor 编辑的精简 JSON(不含完整 markdown)。"""
|
"""供 Cursor 编辑的精简 JSON(不含完整 markdown)。"""
|
||||||
news_limit = wecom_limits.get("ai_news", 10)
|
news_limit = wecom_limits.get("ai_news", 10)
|
||||||
|
cn_news_limit = wecom_limits.get("cn_ai_news", 8)
|
||||||
depth = compare_depth()
|
depth = compare_depth()
|
||||||
trend_cmp = trending[:depth]
|
trend_cmp = trending[:depth]
|
||||||
hot_cmp = hot[:depth]
|
hot_cmp = hot[:depth]
|
||||||
@@ -153,6 +160,11 @@ def build_llm_input(
|
|||||||
"repos": [_slim_github(x) for x in topic_slice],
|
"repos": [_slim_github(x) for x in topic_slice],
|
||||||
},
|
},
|
||||||
"ai_news": _slim_news_items(ai_news, news_limit) if ai_news.get("enabled") else [],
|
"ai_news": _slim_news_items(ai_news, news_limit) if ai_news.get("enabled") else [],
|
||||||
|
"cn_ai_news": _slim_news_items(
|
||||||
|
cn_ai_news, cn_news_limit, prepare=prepare_wecom_cn_news_items
|
||||||
|
)
|
||||||
|
if cn_ai_news.get("enabled")
|
||||||
|
else [],
|
||||||
"movement": movement,
|
"movement": movement,
|
||||||
"movement_baseline": movement_baseline,
|
"movement_baseline": movement_baseline,
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -29,6 +29,8 @@ Step 2 基于趋势 + 原始数据 → 写企微 Markdown 早报
|
|||||||
| `github_topic.topic` | Topic 名称,用于区块标题(如 `llm`) |
|
| `github_topic.topic` | Topic 名称,用于区块标题(如 `llm`) |
|
||||||
| `movement.*_moves` | 较昨日新增(**仅**用于 opening / signals,**不**用于列表区块) |
|
| `movement.*_moves` | 较昨日新增(**仅**用于 opening / signals,**不**用于列表区块) |
|
||||||
| `movement.*_summary` | 新增摘要(可选写入 signals) |
|
| `movement.*_summary` | 新增摘要(可选写入 signals) |
|
||||||
|
| `ai_news` | 国际 AI 时讯 Top N |
|
||||||
|
| `cn_ai_news` | 国内 AI 时讯 Top N |
|
||||||
|
|
||||||
**禁止**使用已合并的 `skills_moves` / `github_moves` 自行扩写;**禁止**排名变化、安装涨跌。
|
**禁止**使用已合并的 `skills_moves` / `github_moves` 自行扩写;**禁止**排名变化、安装涨跌。
|
||||||
|
|
||||||
@@ -98,6 +100,10 @@ Step 1 的 `signals` 与 `top_picks` **优先引用 Top 榜榜首/前列条目
|
|||||||
1. [{title_zh}]({link}) — {why 或摘要}
|
1. [{title_zh}]({link}) — {why 或摘要}
|
||||||
2. ...(**必须 10 条**,来自 `ai_news`,按重要性排序)
|
2. ...(**必须 10 条**,来自 `ai_news`,按重要性排序)
|
||||||
|
|
||||||
|
🇨🇳 **国内 AI · 精选 8**
|
||||||
|
1. [{title}]({link}) — {why 或摘要}
|
||||||
|
2. ...(**必须 8 条**,来自 `cn_ai_news`,按重要性排序;标题已是中文,可微调润色)
|
||||||
|
|
||||||
<!-- **不要写** Skills Trending / Skills Hot 区块,Python 会在推送前按 source 合并后自动插入 -->
|
<!-- **不要写** Skills Trending / Skills Hot 区块,Python 会在推送前按 source 合并后自动插入 -->
|
||||||
|
|
||||||
🐙 **GitHub Trending Top {N}**
|
🐙 **GitHub Trending Top {N}**
|
||||||
@@ -120,8 +126,9 @@ Step 1 的 `signals` 与 `top_picks` **优先引用 Top 榜榜首/前列条目
|
|||||||
3. **禁止**改用 `movement.*_moves` 作为列表来源;movement 仅用于 opening / signals 描述「今日新增」
|
3. **禁止**改用 `movement.*_moves` 作为列表来源;movement 仅用于 opening / signals 描述「今日新增」
|
||||||
4. **禁止**在条目后写 `(新入 … #n)` 类括号标注
|
4. **禁止**在条目后写 `(新入 … #n)` 类括号标注
|
||||||
5. **即使某榜较昨日无新增,仍须完整列出 Top 榜条目**
|
5. **即使某榜较昨日无新增,仍须完整列出 Top 榜条目**
|
||||||
6. **国际 AI 必须 10 条**
|
6. **国际 AI 必须 10 条**(来自 `ai_news`)
|
||||||
7. 禁止排名变化、安装涨跌、连霸描述
|
7. **国内 AI 必须 8 条**(来自 `cn_ai_news`;无数据时写「暂无可用条目」)
|
||||||
|
8. 禁止排名变化、安装涨跌、连霸描述
|
||||||
|
|
||||||
```markdown
|
```markdown
|
||||||
📈 **Skills Trending Top 10**
|
📈 **Skills Trending Top 10**
|
||||||
|
|||||||
Reference in New Issue
Block a user