ftb-diy/daily-robots
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fix: use browser User-Agent for RSS feeds blocked by InfoQ

Browse Source 
Co-authored-by: Cursor <cursoragent@cursor.com>
This commit is contained in:
王礼
2026-07-03 16:38:48 +08:00
parent 37931d2ef5
commit f6fac7b642
2 changed files with 10 additions and 3 deletions
Download Patch File Download Diff File Expand all files Collapse all files

7
daily/news/fetch.py
View File

@@ -23,12 +23,13 @@ from daily.news.feeds_cn import CN_AI_TITLE_KEYWORDS, CN_NEWS_CATEGORIES
logger = logging.getLogger(__name__)
USER_AGENT = "Mozilla/5.0 (compatible; daily-robots/1.0; +https://skills.sh)"
BROWSER_USER_AGENT = (
# 部分站点（如 InfoQ）会拦截含 bot 标识的 UA，RSS 抓取统一用浏览器 UA
RSS_USER_AGENT = (
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) "
"AppleWebKit/537.36 (KHTML, like Gecko) "
"Chrome/131.0.0.0 Safari/537.36"
)
BROWSER_USER_AGENT = RSS_USER_AGENT
STRIP_HTML = re.compile(r"<[^>]+>")
WS = re.compile(r"\s+")
@@ -338,7 +339,7 @@ def _fetch_news(categories: tuple[NewsCategory, ...]) -> dict[str, Any]:
per_category = _per_category_limit()
cutoff = _now_utc() - timedelta(hours=hours)
headers = {"User-Agent": USER_AGENT, "Accept": "application/rss+xml, application/atom+xml, application/xml, text/xml, */*"}
headers = {"User-Agent": RSS_USER_AGENT, "Accept": "application/rss+xml, application/atom+xml, application/xml, text/xml, */*"}
tasks: list[tuple[NewsCategory, NewsFeed]] = []
for category in categories:
for feed in category.feeds: