yumao
391f887d73
标题相似度合并、信源/时效/昨日重复加权排序，Agent 模式扩大 LLM 新闻候选池并记录抓取失败源。 Co-authored-by: Cursor <cursoragent@cursor.com>
93 lines
2.9 KiB
Python
93 lines
2.9 KiB
Python
from __future__ import annotations
|
|
|
|
from datetime import datetime, timezone
|
|
|
|
from daily.news.rank import (
|
|
apply_news_ranking,
|
|
fuzzy_dedupe_by_title,
|
|
jaccard_similarity,
|
|
score_news_item,
|
|
_title_tokens,
|
|
)
|
|
|
|
|
|
def test_jaccard_similarity_detects_duplicate_headlines():
|
|
left = _title_tokens("OpenAI launches GPT-5 with new reasoning")
|
|
right = _title_tokens("OpenAI launches GPT 5 with reasoning upgrade")
|
|
assert jaccard_similarity(left, right) >= 0.55
|
|
|
|
|
|
def test_fuzzy_dedupe_keeps_higher_score():
|
|
items = [
|
|
{"title": "OpenAI launches GPT-5", "link": "https://a.example/1", "score": 20},
|
|
{"title": "OpenAI launches GPT 5 today", "link": "https://b.example/2", "score": 45},
|
|
]
|
|
deduped = fuzzy_dedupe_by_title(items, threshold=0.55)
|
|
assert len(deduped) == 1
|
|
assert deduped[0]["link"] == "https://b.example/2"
|
|
|
|
|
|
def test_score_news_item_prefers_fresh_official_and_penalizes_yesterday():
|
|
now = datetime(2026, 7, 3, 12, 0, tzinfo=timezone.utc)
|
|
fresh = {
|
|
"title": "Claude update",
|
|
"link": "https://anthropic.com/news/claude",
|
|
"category_id": "official",
|
|
"published": "Thu, 03 Jul 2026 10:00:00 GMT",
|
|
}
|
|
stale_seen = {
|
|
"title": "Old story",
|
|
"link": "https://example.com/old",
|
|
"category_id": "media",
|
|
"published": "Mon, 30 Jun 2026 10:00:00 GMT",
|
|
}
|
|
fresh_score = score_news_item(fresh, yesterday_links=set(), now=now)
|
|
stale_score = score_news_item(
|
|
stale_seen,
|
|
yesterday_links={"https://example.com/old"},
|
|
now=now,
|
|
)
|
|
assert fresh_score > stale_score
|
|
|
|
|
|
def test_apply_news_ranking_sets_flat_scores(monkeypatch, tmp_path):
|
|
import daily.config as config
|
|
import daily.news.rank as rank
|
|
|
|
out = tmp_path / "output"
|
|
out.mkdir()
|
|
monkeypatch.setattr(config, "OUTPUT_DIR", out)
|
|
monkeypatch.setattr(rank, "load_yesterday_news_links", lambda _date: set())
|
|
|
|
news = {
|
|
"enabled": True,
|
|
"categories": [
|
|
{
|
|
"id": "official",
|
|
"name": "厂商官方",
|
|
"icon": "🏢",
|
|
"items": [
|
|
{
|
|
"title": "OpenAI ships new model",
|
|
"link": "https://openai.com/a",
|
|
"category_id": "official",
|
|
"published": "Thu, 03 Jul 2026 08:00:00 GMT",
|
|
},
|
|
{
|
|
"title": "OpenAI ships a new model today",
|
|
"link": "https://techcrunch.com/a",
|
|
"category_id": "media",
|
|
"published": "Thu, 03 Jul 2026 07:00:00 GMT",
|
|
},
|
|
],
|
|
}
|
|
],
|
|
"flat": [],
|
|
"stats": {},
|
|
}
|
|
|
|
ranked = apply_news_ranking(news, date_str="2026-07-03")
|
|
assert ranked["flat"]
|
|
assert all("score" in item for item in ranked["flat"])
|
|
assert len(ranked["flat"]) == 1
|