fix: retry transient RSS fetch failures with backoff

Co-authored-by: Cursor <cursoragent@cursor.com>
This commit is contained in:
2026-07-03 16:39:11 +08:00
parent f6fac7b642
commit 64179820d7
3 changed files with 52 additions and 9 deletions

View File

@@ -289,17 +289,22 @@ def _fetch_one(
category: NewsCategory,
feed: NewsFeed,
) -> tuple[list[dict[str, Any]], bool]:
max_attempts = max(1, env_int("DAILY_RSS_RETRY", 3))
last_exc: Exception | None = None
for url in _reddit_fetch_urls(feed.url):
try:
headers = _request_headers(dict(client.headers), url)
resp = client.get(url, headers=headers)
resp.raise_for_status()
entries = _parse_feed(resp.text, feed.name, category)
return _filter_ai_entries(entries, ai_filter=feed.ai_filter), True
except Exception as exc:
last_exc = exc
continue
for attempt in range(max_attempts):
try:
headers = _request_headers(dict(client.headers), url)
resp = client.get(url, headers=headers)
resp.raise_for_status()
entries = _parse_feed(resp.text, feed.name, category)
return _filter_ai_entries(entries, ai_filter=feed.ai_filter), True
except Exception as exc:
last_exc = exc
if attempt + 1 < max_attempts:
time.sleep(min(2.0 * (attempt + 1), 5.0))
continue
break
logger.warning("RSS fetch failed [%s] %s: %s", feed.name, feed.url, last_exc)
return [], False