ftb-diy/daily-robots
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fix: use browser User-Agent for RSS feeds blocked by InfoQ

Browse Source 
Co-authored-by: Cursor <cursoragent@cursor.com>
This commit is contained in:
王礼
2026-07-03 16:38:48 +08:00
parent 37931d2ef5
commit f6fac7b642
2 changed files with 10 additions and 3 deletions
Download Patch File Download Diff File Expand all files Collapse all files

7
daily/news/fetch.py
View File

@@ -23,12 +23,13 @@ from daily.news.feeds_cn import CN_AI_TITLE_KEYWORDS, CN_NEWS_CATEGORIES
logger = logging.getLogger(__name__) logger = logging.getLogger(__name__)
USER_AGENT = "Mozilla/5.0 (compatible; daily-robots/1.0; +https://skills.sh)" # 部分站点（如 InfoQ）会拦截含 bot 标识的 UA，RSS 抓取统一用浏览器 UA
BROWSER_USER_AGENT = ( RSS_USER_AGENT = (
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) " "Mozilla/5.0 (Windows NT 10.0; Win64; x64) "
"AppleWebKit/537.36 (KHTML, like Gecko) " "AppleWebKit/537.36 (KHTML, like Gecko) "
"Chrome/131.0.0.0 Safari/537.36" "Chrome/131.0.0.0 Safari/537.36"
) )
BROWSER_USER_AGENT = RSS_USER_AGENT
STRIP_HTML = re.compile(r"<[^>]+>") STRIP_HTML = re.compile(r"<[^>]+>")
WS = re.compile(r"\s+") WS = re.compile(r"\s+")
@@ -338,7 +339,7 @@ def _fetch_news(categories: tuple[NewsCategory, ...]) -> dict[str, Any]:
per_category = _per_category_limit() per_category = _per_category_limit()
cutoff = _now_utc() - timedelta(hours=hours) cutoff = _now_utc() - timedelta(hours=hours)
headers = {"User-Agent": USER_AGENT, "Accept": "application/rss+xml, application/atom+xml, application/xml, text/xml, */*"} headers = {"User-Agent": RSS_USER_AGENT, "Accept": "application/rss+xml, application/atom+xml, application/xml, text/xml, */*"}
tasks: list[tuple[NewsCategory, NewsFeed]] = [] tasks: list[tuple[NewsCategory, NewsFeed]] = []
for category in categories: for category in categories:
for feed in category.feeds: for feed in category.feeds:

6
tests/test_rss_parse.py
View File

@@ -23,6 +23,12 @@ def test_parse_sample_rss_fixture():
assert "minimal RSS item" in items[0]["summary"] assert "minimal RSS item" in items[0]["summary"]
def test_rss_user_agent_avoids_bot_blocked_feeds():
from daily.news.fetch import RSS_USER_AGENT
assert "daily-robots" not in RSS_USER_AGENT
def test_fetch_ai_news_offline(monkeypatch): def test_fetch_ai_news_offline(monkeypatch):
from daily.news import fetch as news_fetch from daily.news import fetch as news_fetch