diff --git a/.env.example b/.env.example index 8ddb77c..8076c20 100644 --- a/.env.example +++ b/.env.example @@ -86,6 +86,8 @@ DAILY_AI_NEWS_PER_CATEGORY=5 # DAILY_AGENT_CN_NEWS_POOL=30 # RSS 源:编辑 config/feeds.yaml(intl / cn);缺失时回退内置默认 +# 单源失败重试次数(默认 3,含首次请求) +# DAILY_RSS_RETRY=3 # 内容过滤:config/sensitive_words.yaml + DAILY_CONTENT_FILTER=1 # DAILY_CONTENT_FILTER=0 diff --git a/daily/news/fetch.py b/daily/news/fetch.py index b9adb35..9e38b4e 100644 --- a/daily/news/fetch.py +++ b/daily/news/fetch.py @@ -289,17 +289,22 @@ def _fetch_one( category: NewsCategory, feed: NewsFeed, ) -> tuple[list[dict[str, Any]], bool]: + max_attempts = max(1, env_int("DAILY_RSS_RETRY", 3)) last_exc: Exception | None = None for url in _reddit_fetch_urls(feed.url): - try: - headers = _request_headers(dict(client.headers), url) - resp = client.get(url, headers=headers) - resp.raise_for_status() - entries = _parse_feed(resp.text, feed.name, category) - return _filter_ai_entries(entries, ai_filter=feed.ai_filter), True - except Exception as exc: - last_exc = exc - continue + for attempt in range(max_attempts): + try: + headers = _request_headers(dict(client.headers), url) + resp = client.get(url, headers=headers) + resp.raise_for_status() + entries = _parse_feed(resp.text, feed.name, category) + return _filter_ai_entries(entries, ai_filter=feed.ai_filter), True + except Exception as exc: + last_exc = exc + if attempt + 1 < max_attempts: + time.sleep(min(2.0 * (attempt + 1), 5.0)) + continue + break logger.warning("RSS fetch failed [%s] %s: %s", feed.name, feed.url, last_exc) return [], False diff --git a/tests/test_rss_parse.py b/tests/test_rss_parse.py index 92ad7f1..8c14292 100644 --- a/tests/test_rss_parse.py +++ b/tests/test_rss_parse.py @@ -29,6 +29,42 @@ def test_rss_user_agent_avoids_bot_blocked_feeds(): assert "daily-robots" not in RSS_USER_AGENT +def test_fetch_one_retries_transient_errors(monkeypatch): + import httpx + + from daily.news.feeds import NEWS_CATEGORIES + from daily.news import fetch as news_fetch + + monkeypatch.setenv("DAILY_RSS_RETRY", "2") + monkeypatch.setattr(news_fetch.time, "sleep", lambda _: None) + + category = NEWS_CATEGORIES[0] + feed = category.feeds[0] + calls = {"n": 0} + + class FakeResponse: + def raise_for_status(self): + return None + + @property + def text(self): + return (FIXTURES / "sample-rss.xml").read_text(encoding="utf-8") + + class FakeClient: + headers = {"User-Agent": "test"} + + def get(self, url, headers=None): + calls["n"] += 1 + if calls["n"] < 2: + raise httpx.RemoteProtocolError("Server disconnected") + return FakeResponse() + + entries, ok = news_fetch._fetch_one(FakeClient(), category, feed) + assert ok is True + assert len(entries) == 1 + assert calls["n"] == 2 + + def test_fetch_ai_news_offline(monkeypatch): from daily.news import fetch as news_fetch