fix: retry transient RSS fetch failures with backoff
Co-authored-by: Cursor <cursoragent@cursor.com>
This commit is contained in:
@@ -289,17 +289,22 @@ def _fetch_one(
|
||||
category: NewsCategory,
|
||||
feed: NewsFeed,
|
||||
) -> tuple[list[dict[str, Any]], bool]:
|
||||
max_attempts = max(1, env_int("DAILY_RSS_RETRY", 3))
|
||||
last_exc: Exception | None = None
|
||||
for url in _reddit_fetch_urls(feed.url):
|
||||
try:
|
||||
headers = _request_headers(dict(client.headers), url)
|
||||
resp = client.get(url, headers=headers)
|
||||
resp.raise_for_status()
|
||||
entries = _parse_feed(resp.text, feed.name, category)
|
||||
return _filter_ai_entries(entries, ai_filter=feed.ai_filter), True
|
||||
except Exception as exc:
|
||||
last_exc = exc
|
||||
continue
|
||||
for attempt in range(max_attempts):
|
||||
try:
|
||||
headers = _request_headers(dict(client.headers), url)
|
||||
resp = client.get(url, headers=headers)
|
||||
resp.raise_for_status()
|
||||
entries = _parse_feed(resp.text, feed.name, category)
|
||||
return _filter_ai_entries(entries, ai_filter=feed.ai_filter), True
|
||||
except Exception as exc:
|
||||
last_exc = exc
|
||||
if attempt + 1 < max_attempts:
|
||||
time.sleep(min(2.0 * (attempt + 1), 5.0))
|
||||
continue
|
||||
break
|
||||
logger.warning("RSS fetch failed [%s] %s: %s", feed.name, feed.url, last_exc)
|
||||
return [], False
|
||||
|
||||
|
||||
Reference in New Issue
Block a user