diff --git a/.env.example b/.env.example index 5b63963..06ad1a2 100644 --- a/.env.example +++ b/.env.example @@ -32,7 +32,11 @@ DAILY_WECOM_GITHUB_TRENDING=10 DAILY_WECOM_GITHUB_EMERGING=10 DAILY_WECOM_GITHUB_TOPIC=10 DAILY_WECOM_AI_NEWS=10 -DAILY_WECOM_CN_AI_NEWS=8 +# research 模式额外技术类时讯条数（叠加在 AI 时讯精选之上） +DAILY_WECOM_AI_NEWS_TECH=5 +DAILY_WECOM_CN_AI_NEWS=10 +# 企微新闻摘要字数（句读/词边界截断，不加省略号） +# DAILY_WECOM_NEWS_DESC_LIMIT=72 # 企微 Skills 合并前扫描池大小（同 source 合并后仍凑满 Top N） # DAILY_WECOM_SKILL_POOL=200 @@ -44,6 +48,11 @@ DAILY_WECOM_CHUNK_BYTES=4096 # 企微列表模式：delta=仅展示新入榜 | full=全量 Top 榜（回退） DAILY_WECOM_MODE=delta +# Delta 模式下新入榜优先，不足时用当日 Top 榜补满各区块条数（0=仅展示变化） +# 补榜时会排除近 N 天 baseline 已出现过的条目，避免周内重复（默认 7 天） +DAILY_WECOM_DELTA_PAD=1 +# 补榜时从更大候选池选取（默认展示条数×5，至少 50） +# DAILY_WECOM_PAD_POOL=50 # 无历史 data.json 时：full=首日全量一次 | empty=列表为空 DAILY_DELTA_BASELINE_FALLBACK=full # 推送闸门不满足时跳过 webhook（仍写 output） @@ -52,6 +61,11 @@ DAILY_SKIP_PUSH_WHEN_SILENT=1 # 已推送新闻 link 去重天数 DAILY_NEWS_DEDUP_DAYS=7 +# 常驻调度（python -m daily schedule） +DAILY_SCHEDULE_TZ=Asia/Shanghai +DAILY_SCHEDULE_GENERATE_AT=08:50 +DAILY_SCHEDULE_PUSH_AT=09:00 + # 编辑指定今日首推（可选）：关键词，或 关键词|URL # Python Step 0 检索 → featured.json；Agent / classic 企微「今日首推」优先使用 # DAILY_FEATURED_PICK=gstack @@ -59,8 +73,10 @@ DAILY_NEWS_DEDUP_DAYS=7 # 国际 AI 时讯（RSS，见 daily/news/feeds.py） DAILY_AI_NEWS=1 -# 国内 AI 时讯（RSS，见 daily/news/feeds_cn.py） +# 国内 AI 时讯（RSS，见 daily/news/feeds_cn.py；research 模式下忽略） DAILY_CN_AI_NEWS=1 +# AI 时讯来源：rss=RSS 抓取 | research=Cursor SDK + deep-research（WebSearch） +# DAILY_AI_NEWS_MODE=research # 英文描述 → 简短中文（DAILY_CURSOR_EDITOR=0 时生效） # DAILY_ZH_DESC=1 # DAILY_ZH_DESC_BATCH=20 @@ -88,6 +104,17 @@ DAILY_AI_NEWS_HOURS=24 DAILY_AI_NEWS_PER_FEED=3 DAILY_AI_NEWS_PER_CATEGORY=5 +# 多样性 / 去重（见 docs/superpowers/specs/2026-07-14-wecom-diversity-dedup-design.md） +# DAILY_BOARD_DEDUP_DAYS=7 +# DAILY_BOARD_POOL_SIZE=50 +# DAILY_FEATURED_DEDUP_DAYS=30 +# DAILY_THEME_BAN_DAYS=7 +# DAILY_NARRATIVE_AXIS_DAYS=3 +DAILY_NEWS_BACKFILL=0 +# 国际时讯：在 24h 滚动窗口基础上，不早于今日 0 点（DAILY_AI_NEWS_TZ） +DAILY_AI_NEWS_FLOOR_TODAY=1 +# DAILY_AI_NEWS_TZ=Asia/Shanghai + # Reddit RSS（403/429 时在 Reddit 偏好设置 → RSS feeds 复制 user / feed 参数） # REDDIT_RSS_USER=your_username # REDDIT_RSS_FEED=your_feed_token diff --git a/daily/board_history.py b/daily/board_history.py new file mode 100644 index 0000000..f0d6e2f --- /dev/null +++ b/daily/board_history.py @@ -0,0 +1,83 @@ +"""企微展示历史：读写 data.wecom_shown_keys，与 movement_baseline 严格分离。""" + +from __future__ import annotations + +import json +import logging +from datetime import datetime, timedelta +from pathlib import Path +from typing import Any + +from daily.config import OUTPUT_DIR, board_dedup_days +from daily.delta import RECENT_BOARD_KEYS, skill_id + +logger = logging.getLogger(__name__) + +BOARD_KEYS = RECENT_BOARD_KEYS + + +def extract_shown_keys(board: str, items: list[dict[str, Any]]) -> list[str]: + """从最终展示 items 抽取稳定 identity key。""" + keys: list[str] = [] + seen: set[str] = set() + for item in items: + if board.startswith("skills_"): + key = skill_id(item) + else: + key = str(item.get("repo") or "") + if not key or key in seen: + continue + seen.add(key) + keys.append(key) + return keys + + +def load_recent_shown_keys( + date_str: str, + *, + lookback_days: int | None = None, +) -> dict[str, set[str]]: + """近 N 日 data.wecom_shown_keys 并集（不含当日）。缺省或读失败视为空集。""" + empty = {board: set() for board in BOARD_KEYS} + try: + dt = datetime.strptime(date_str, "%Y-%m-%d") + except ValueError: + return empty + + days = lookback_days if lookback_days is not None else board_dedup_days() + out: dict[str, set[str]] = {board: set() for board in BOARD_KEYS} + + for day_offset in range(1, days + 1): + prev_date = (dt - timedelta(days=day_offset)).strftime("%Y-%m-%d") + path = OUTPUT_DIR / f"{prev_date}.data.json" + if not path.exists(): + continue + try: + payload = json.loads(path.read_text(encoding="utf-8")) + except (OSError, ValueError) as exc: + logger.warning("读取 wecom_shown_keys %s 失败：%s", path, exc) + continue + data = payload.get("data") + if not isinstance(data, dict): + continue + shown = data.get("wecom_shown_keys") + if not isinstance(shown, dict): + continue + for board in BOARD_KEYS: + keys = shown.get(board) or [] + if not isinstance(keys, list): + continue + out[board].update(str(k) for k in keys if k) + return out + + +def merge_wecom_shown_into_data( + data: dict[str, Any], + shown: dict[str, list[str]], +) -> dict[str, Any]: + """写入 wecom_shown_keys，不修改 movement_baseline。""" + merged = dict(data) + merged["wecom_shown_keys"] = { + board: list(keys) for board, keys in shown.items() + } + return merged diff --git a/daily/config.py b/daily/config.py index 0e686f4..09de410 100644 --- a/daily/config.py +++ b/daily/config.py @@ -30,6 +30,24 @@ def wecom_skill_desc_limit() -> int: return env_int("DAILY_WECOM_SKILL_DESC_LIMIT", 56) +def wecom_news_desc_limit() -> int: + """企微新闻摘要建议字数；在句读/词边界截断，不加省略号。""" + return max(24, env_int("DAILY_WECOM_NEWS_DESC_LIMIT", 72)) + + +def wecom_ai_news_tech_limit() -> int: + """research 模式下技术类时讯条数（叠加在 DAILY_WECOM_AI_NEWS 之上）。""" + return max(0, env_int("DAILY_WECOM_AI_NEWS_TECH", 5)) + + +def wecom_pad_pool_size(display_limit: int) -> int: + """Delta 补榜候选池大小（展示条数之上多取，避免去重后凑不满）。""" + explicit = env_int("DAILY_WECOM_PAD_POOL", -1) + if explicit > 0: + return explicit + return max(display_limit * 5, 50) + + def full_desc_limit() -> int: """完整版早报摘要长度；0 表示不截断。""" return env_int("DAILY_FULL_DESC_LIMIT", 0) @@ -99,5 +117,53 @@ def delta_baseline_fallback() -> str: return raw if raw in {"full", "empty"} else "full" +def wecom_delta_pad() -> bool: + """Delta 模式下新入榜优先，不足时用当日 Top 榜补满；补榜排除近 N 天 baseline 已出现条目。""" + return env_bool("DAILY_WECOM_DELTA_PAD", True) + + +def delta_pad_lookback_days() -> int: + """补榜时排除近 N 天 baseline 已出现过的条目（默认与异动对比窗口一致）。""" + fallback = env_int("DAILY_DELTA_LOOKBACK_DAYS", 7) + return max(1, env_int("DAILY_DELTA_PAD_LOOKBACK_DAYS", fallback)) + + def force_push() -> bool: return env_bool("DAILY_FORCE_PUSH", False) + + +def schedule_timezone_name() -> str: + return (env("DAILY_SCHEDULE_TZ") or "Asia/Shanghai").strip() or "Asia/Shanghai" + + +def schedule_generate_at() -> str: + return (env("DAILY_SCHEDULE_GENERATE_AT") or "08:50").strip() or "08:50" + + +def schedule_push_at() -> str: + return (env("DAILY_SCHEDULE_PUSH_AT") or "09:00").strip() or "09:00" + + +def board_dedup_days() -> int: + return max(1, env_int("DAILY_BOARD_DEDUP_DAYS", 7)) + + +def board_pool_size() -> int: + fallback = env_int("DAILY_WECOM_SKILL_POOL", 50) + return max(1, env_int("DAILY_BOARD_POOL_SIZE", max(50, fallback))) + + +def featured_dedup_days() -> int: + return max(1, env_int("DAILY_FEATURED_DEDUP_DAYS", 30)) + + +def theme_ban_days() -> int: + return max(1, env_int("DAILY_THEME_BAN_DAYS", 7)) + + +def narrative_axis_days() -> int: + return max(1, env_int("DAILY_NARRATIVE_AXIS_DAYS", 3)) + + +def news_backfill_enabled() -> bool: + return env_bool("DAILY_NEWS_BACKFILL", False) diff --git a/daily/delta.py b/daily/delta.py index 339ecdc..5d08383 100644 --- a/daily/delta.py +++ b/daily/delta.py @@ -15,6 +15,15 @@ logger = logging.getLogger(__name__) KeyFn = Callable[[dict[str, Any]], str] +RECENT_BOARD_KEYS = ( + "skills_trending", + "skills_hot", + "github_trending", + "github_emerging", + "github_topic", +) + + def compare_depth() -> int: return env_int("DAILY_DELTA_COMPARE_DEPTH", 15) @@ -39,6 +48,23 @@ def _key_set(items: list[dict[str, Any]], key_fn: KeyFn, *, depth: int) -> set[s return {key_fn(item) for item in items[:depth] if key_fn(item)} +def load_recent_board_keys( + date_str: str, + *, + lookback_days: int | None = None, +) -> dict[str, set[str]]: + """近 N 天各榜已展示过的 skill id / repo（不含当日，供补榜去重）。 + + 委托 board_history.load_recent_shown_keys，只读 wecom_shown_keys， + 不读 movement_baseline。 + """ + from daily.board_history import load_recent_shown_keys + from daily.config import board_dedup_days + + days = lookback_days if lookback_days is not None else board_dedup_days() + return load_recent_shown_keys(date_str, lookback_days=days) + + def find_previous_data(date_str: str) -> tuple[str, dict[str, Any]] | None: """查找最近一份早于 date_str 的 data.json。""" try: diff --git a/tests/test_board_history.py b/tests/test_board_history.py new file mode 100644 index 0000000..61d8097 --- /dev/null +++ b/tests/test_board_history.py @@ -0,0 +1,65 @@ +# tests/test_board_history.py +from __future__ import annotations + +import json +import os +import tempfile +import unittest +from pathlib import Path +from unittest.mock import patch + +from daily.board_history import extract_shown_keys, load_recent_shown_keys, merge_wecom_shown_into_data +from daily.config import board_dedup_days, news_backfill_enabled + + +class ConfigDiversityTests(unittest.TestCase): + def test_board_dedup_days_default(self): + with patch.dict(os.environ, {}, clear=True): + self.assertEqual(board_dedup_days(), 7) + + def test_news_backfill_default_off(self): + with patch.dict(os.environ, {}, clear=True): + self.assertFalse(news_backfill_enabled()) + + +class ShownKeysTests(unittest.TestCase): + def test_extract_github_repo_keys(self): + items = [{"repo": "a/b"}, {"repo": "c/d"}] + self.assertEqual(extract_shown_keys("github_trending", items), ["a/b", "c/d"]) + + def test_load_recent_reads_wecom_shown_not_baseline(self): + with tempfile.TemporaryDirectory() as tmp: + out = Path(tmp) + # 前日：shown 只有 x/y；baseline raw 含 a/b —— 周去重只能看到 x/y + payload = { + "data": { + "date": "2026-07-13", + "movement_baseline": { + "github_trending": [{"repo": "a/b"}, {"repo": "x/y"}], + }, + "wecom_shown_keys": {"github_trending": ["x/y"]}, + } + } + (out / "2026-07-13.data.json").write_text( + json.dumps(payload, ensure_ascii=False), encoding="utf-8" + ) + with patch("daily.board_history.OUTPUT_DIR", out): + keys = load_recent_shown_keys("2026-07-14", lookback_days=7) + self.assertEqual(keys["github_trending"], {"x/y"}) + self.assertNotIn("a/b", keys["github_trending"]) + + def test_merge_shown_does_not_touch_baseline(self): + data = { + "movement_baseline": {"github_trending": [{"repo": "raw/one"}]}, + } + merged = merge_wecom_shown_into_data( + data, {"github_trending": ["shown/one"]} + ) + self.assertEqual( + merged["movement_baseline"]["github_trending"][0]["repo"], "raw/one" + ) + self.assertEqual(merged["wecom_shown_keys"]["github_trending"], ["shown/one"]) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_wecom_delta.py b/tests/test_wecom_delta.py index fb7dbb3..d67a83f 100644 --- a/tests/test_wecom_delta.py +++ b/tests/test_wecom_delta.py @@ -44,6 +44,12 @@ class ConfigHelpersTests(unittest.TestCase): with patch.dict(os.environ, {}, clear=True): self.assertEqual(delta_baseline_fallback(), "full") + def test_wecom_delta_pad_default_true(self): + from daily.config import wecom_delta_pad + + with patch.dict(os.environ, {}, clear=True): + self.assertTrue(wecom_delta_pad()) + class NewsPushedLinksTests(unittest.TestCase): def test_filter_and_record_roundtrip(self): @@ -121,7 +127,7 @@ class DeltaFormatTests(unittest.TestCase): ] text = build_skills_delta_sections(moves, []) self.assertIn("Skills Trending 变化", text) - self.assertIn("[新入 #3]", text) + self.assertNotIn("[新入 #", text) self.assertNotIn("Skills Hot 变化", text) def test_github_delta_omits_stable_board(self): @@ -129,14 +135,268 @@ class DeltaFormatTests(unittest.TestCase): movement = { "github_trending_moves": [ - {"repo": "a/b", "url": "https://github.com/a/b", "rank": 1, "language": "Go"} + { + "repo": "a/b", + "url": "https://github.com/a/b", + "rank": 1, + "language": "Go", + "description": "open-source codebase and curriculum", + } ], "github_emerging_moves": [], "github_topic_moves": [], } - text = build_github_delta_sections(movement, topic_name="llm") + with patch( + "daily.format_wecom.localize_brief_descriptions", + return_value={"github:a/b": "开源代码库与课程体系"}, + ): + text = build_github_delta_sections(movement, topic_name="llm") + self.assertIn("开源代码库", text) self.assertIn("GitHub Trending 变化", text) self.assertNotIn("新兴", text) + self.assertNotIn("[新入 #", text) + self.assertNotIn("\n > ", text) + + def test_delta_pad_groups_same_source_moves(self): + from daily.format_wecom import build_skills_delta_sections + + moves = [ + { + "id": f"lllllllama/rigorpilot-skills/s{i}", + "title": f"s{i}", + "source": "lllllllama/rigorpilot-skills", + "installs": 250 - i, + "installs_fmt": str(250 - i), + "link": f"https://www.skills.sh/lllllllama/rigorpilot-skills/s{i}", + "description": f"skill {i}", + } + for i in range(1, 11) + ] + with patch("daily.format_wecom.localize_brief_descriptions", return_value={}): + with patch("daily.format_wecom.needs_chinese", return_value=False): + text = build_skills_delta_sections(moves, [], trending_limit=10, pad=True) + self.assertIn("10 skills", text) + self.assertNotIn("[**s2**]", text) + + def test_delta_pad_fills_skills_to_limit(self): + from daily.format_wecom import build_skills_delta_sections + + moves = [ + { + "id": "a/b/new", + "rank": 3, + "title": "new", + "source": "a/b", + "installs": 99, + "installs_fmt": "99", + "link": "https://skills.sh/a/b/new", + "description": "new skill", + } + ] + full = [ + { + "id": "a/b/new", + "title": "new", + "source": "a/b", + "installs": 99, + "installs_fmt": "99", + "link": "https://skills.sh/a/b/new", + "description": "new skill", + }, + *[ + { + "id": f"src{i}/skill", + "title": "skill", + "source": f"src{i}/pkg", + "installs": 100 - i, + "installs_fmt": str(100 - i), + "link": f"https://skills.sh/src{i}/pkg/skill", + "description": f"skill from src{i}", + } + for i in range(1, 12) + ], + ] + with patch("daily.format_wecom.localize_brief_descriptions", return_value={}): + with patch("daily.format_wecom.needs_chinese", return_value=False): + text = build_skills_delta_sections( + moves, + [], + trending_full=full, + hot_full=[], + trending_limit=10, + pad=True, + ) + self.assertIn("Skills Trending Top 10", text) + self.assertNotIn("Skills Trending 变化", text) + + def test_delta_pad_uses_large_pool_when_recent_excludes_top(self): + from daily.format_wecom import build_skills_delta_sections + + full_small = [ + { + "id": f"seen/src/s{i}", + "title": f"s{i}", + "source": "seen/src", + "installs": 100 - i, + "installs_fmt": str(100 - i), + "link": f"https://skills.sh/seen/src/s{i}", + "description": f"seen {i}", + } + for i in range(1, 11) + ] + full_large = [ + { + "id": f"fresh/src{n}/skill", + "title": "skill", + "source": f"fresh/src{n}", + "installs": 50 - n, + "installs_fmt": str(50 - n), + "link": f"https://skills.sh/fresh/src{n}/skill", + "description": f"fresh {n}", + } + for n in range(1, 11) + ] + recent = {f"seen/src/s{i}" for i in range(1, 11)} + with patch("daily.format_wecom.localize_brief_descriptions", return_value={}): + with patch("daily.format_wecom.needs_chinese", return_value=False): + small = build_skills_delta_sections( + [], + [], + trending_full=full_small, + trending_limit=10, + pad=True, + recent_trending=recent, + ) + large = build_skills_delta_sections( + [], + [], + trending_full=full_large, + trending_limit=10, + pad=True, + recent_trending=recent, + ) + self.assertNotIn("Skills Trending Top 10", small) + self.assertIn("Skills Trending Top 10", large) + self.assertIn("fresh/src1", large) + + def test_delta_pad_fills_github_to_limit(self): + from daily.format_wecom import build_github_delta_sections + + movement = {"github_trending_moves": [], "github_emerging_moves": [], "github_topic_moves": []} + full = [ + { + "repo": f"org/r{i}", + "url": f"https://github.com/org/r{i}", + "language": "Go", + "stars_today_fmt": "100", + "total_stars_fmt": "1K", + "description": f"repo {i}", + "desc_short": f"repo {i}", + } + for i in range(1, 12) + ] + with patch("daily.format_wecom.localize_brief_descriptions", return_value={}): + text = build_github_delta_sections( + movement, + topic_name="llm", + github_trending=full, + trending_limit=10, + pad=True, + ) + self.assertIn("GitHub Trending Top 10", text) + self.assertNotIn("GitHub Trending 变化", text) + + def test_delta_pad_skips_recent_skills(self): + from daily.format_wecom import build_skills_delta_sections + + full = [ + { + "id": f"x/y/s{i}", + "title": f"s{i}", + "source": "x/y", + "installs": 100 - i, + "installs_fmt": str(100 - i), + "link": f"https://skills.sh/x/y/s{i}", + "description": f"skill {i}", + } + for i in range(4, 6) + ] + with patch("daily.format_wecom.localize_brief_descriptions", return_value={}): + with patch("daily.format_wecom.needs_chinese", return_value=False): + text = build_skills_delta_sections( + [], + [], + trending_full=full, + hot_full=[], + trending_limit=10, + pad=True, + recent_trending={f"x/y/s{i}" for i in range(1, 4)}, + ) + self.assertIn("Skills Trending Top 1", text) + self.assertIn("2 skills", text) + + def test_delta_pad_skips_recent_github(self): + from daily.format_wecom import build_github_delta_sections + + movement = {"github_trending_moves": [], "github_emerging_moves": [], "github_topic_moves": []} + full = [ + { + "repo": f"org/r{i}", + "url": f"https://github.com/org/r{i}", + "language": "Go", + "total_stars_fmt": "1K", + "description": f"repo {i}", + "desc_short": f"repo {i}", + } + for i in range(1, 6) + ] + with patch("daily.format_wecom.localize_brief_descriptions", return_value={}): + text = build_github_delta_sections( + movement, + topic_name="llm", + github_trending=full, + trending_limit=10, + pad=True, + recent_board_keys={"github_trending": {f"org/r{i}" for i in range(1, 4)}}, + ) + self.assertIn("GitHub Trending Top 2", text) + self.assertIn("org/r4", text) + self.assertNotIn("org/r1", text) + + def test_load_recent_board_keys_from_data_json(self): + import json + import tempfile + from pathlib import Path + + from daily.delta import load_recent_board_keys + + with tempfile.TemporaryDirectory() as tmp: + out = Path(tmp) + payload = { + "data": { + "date": "2026-07-09", + "movement_baseline": { + "skills_trending": [ + {"id": "a/b/raw", "title": "raw", "source": "a/b"}, + ], + "skills_hot": [], + "github_trending": [{"repo": "org/raw"}], + "github_emerging": [], + "github_topic": [], + }, + "wecom_shown_keys": { + "skills_trending": ["a/b/foo"], + "github_trending": ["org/bar"], + }, + } + } + (out / "2026-07-09.data.json").write_text(json.dumps(payload), encoding="utf-8") + with patch("daily.board_history.OUTPUT_DIR", out): + recent = load_recent_board_keys("2026-07-10", lookback_days=7) + self.assertIn("a/b/foo", recent["skills_trending"]) + self.assertIn("org/bar", recent["github_trending"]) + self.assertNotIn("a/b/raw", recent["skills_trending"]) + self.assertNotIn("org/raw", recent["github_trending"]) class PushGateTests(unittest.TestCase): @@ -243,6 +503,25 @@ class E2ESmokeTests(unittest.TestCase): trending=[], hot=[], topic_name="llm", + pad=False, ) self.assertIn("Skills Trending 变化", out) self.assertNotIn("Skills Trending Top", out) + + +class SyncMovementGithubTests(unittest.TestCase): + def test_sync_copies_localized_description(self): + from daily.generate import _sync_movement_github_descriptions + + movement = { + "github_trending_moves": [{"repo": "a/b", "description": "english"}], + "github_emerging_moves": [], + "github_topic_moves": [], + } + _sync_movement_github_descriptions( + movement, + github_trending=[{"repo": "a/b", "description": "中文描述"}], + github_emerging=[], + github_topic=[], + ) + self.assertEqual(movement["github_trending_moves"][0]["description"], "中文描述")