From f6fac7b6422cee00555cc1456075492b45eb6882 Mon Sep 17 00:00:00 2001 From: yumao Date: Fri, 3 Jul 2026 16:38:48 +0800 Subject: [PATCH] fix: use browser User-Agent for RSS feeds blocked by InfoQ Co-authored-by: Cursor --- daily/news/fetch.py | 7 ++++--- tests/test_rss_parse.py | 6 ++++++ 2 files changed, 10 insertions(+), 3 deletions(-) diff --git a/daily/news/fetch.py b/daily/news/fetch.py index 4c4a4f6..b9adb35 100644 --- a/daily/news/fetch.py +++ b/daily/news/fetch.py @@ -23,12 +23,13 @@ from daily.news.feeds_cn import CN_AI_TITLE_KEYWORDS, CN_NEWS_CATEGORIES logger = logging.getLogger(__name__) -USER_AGENT = "Mozilla/5.0 (compatible; daily-robots/1.0; +https://skills.sh)" -BROWSER_USER_AGENT = ( +# 部分站点（如 InfoQ）会拦截含 bot 标识的 UA，RSS 抓取统一用浏览器 UA +RSS_USER_AGENT = ( "Mozilla/5.0 (Windows NT 10.0; Win64; x64) " "AppleWebKit/537.36 (KHTML, like Gecko) " "Chrome/131.0.0.0 Safari/537.36" ) +BROWSER_USER_AGENT = RSS_USER_AGENT STRIP_HTML = re.compile(r"<[^>]+>") WS = re.compile(r"\s+") @@ -338,7 +339,7 @@ def _fetch_news(categories: tuple[NewsCategory, ...]) -> dict[str, Any]: per_category = _per_category_limit() cutoff = _now_utc() - timedelta(hours=hours) - headers = {"User-Agent": USER_AGENT, "Accept": "application/rss+xml, application/atom+xml, application/xml, text/xml, */*"} + headers = {"User-Agent": RSS_USER_AGENT, "Accept": "application/rss+xml, application/atom+xml, application/xml, text/xml, */*"} tasks: list[tuple[NewsCategory, NewsFeed]] = [] for category in categories: for feed in category.feeds: diff --git a/tests/test_rss_parse.py b/tests/test_rss_parse.py index 269bf55..92ad7f1 100644 --- a/tests/test_rss_parse.py +++ b/tests/test_rss_parse.py @@ -23,6 +23,12 @@ def test_parse_sample_rss_fixture(): assert "minimal RSS item" in items[0]["summary"] +def test_rss_user_agent_avoids_bot_blocked_feeds(): + from daily.news.fetch import RSS_USER_AGENT + + assert "daily-robots" not in RSS_USER_AGENT + + def test_fetch_ai_news_offline(monkeypatch): from daily.news import fetch as news_fetch