ftb-diy/daily-robots
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: 拆分 wecom_shown_keys 与 movement_baseline 历史层

Browse Source 
Co-authored-by: Cursor <cursoragent@cursor.com>
This commit is contained in:
王礼
2026-07-14 10:31:49 +08:00
parent 02e97e057f
commit f563239e0b
6 changed files with 551 additions and 5 deletions
Download Patch File Download Diff File Expand all files Collapse all files

31
.env.example
View File

@@ -32,7 +32,11 @@ DAILY_WECOM_GITHUB_TRENDING=10
DAILY_WECOM_GITHUB_EMERGING=10
DAILY_WECOM_GITHUB_TOPIC=10
DAILY_WECOM_AI_NEWS=10
DAILY_WECOM_CN_AI_NEWS=8
# research 模式额外技术类时讯条数（叠加在 AI 时讯精选之上）
DAILY_WECOM_AI_NEWS_TECH=5
DAILY_WECOM_CN_AI_NEWS=10
# 企微新闻摘要字数（句读/词边界截断，不加省略号）
# DAILY_WECOM_NEWS_DESC_LIMIT=72
# 企微 Skills 合并前扫描池大小（同 source 合并后仍凑满 Top N）
# DAILY_WECOM_SKILL_POOL=200
@@ -44,6 +48,11 @@ DAILY_WECOM_CHUNK_BYTES=4096
# 企微列表模式：delta=仅展示新入榜 | full=全量 Top 榜（回退）
DAILY_WECOM_MODE=delta
# Delta 模式下新入榜优先，不足时用当日 Top 榜补满各区块条数（0=仅展示变化）
# 补榜时会排除近 N 天 baseline 已出现过的条目，避免周内重复（默认 7 天）
DAILY_WECOM_DELTA_PAD=1
# 补榜时从更大候选池选取（默认展示条数×5，至少 50）
# DAILY_WECOM_PAD_POOL=50
# 无历史 data.json 时：full=首日全量一次 | empty=列表为空
DAILY_DELTA_BASELINE_FALLBACK=full
# 推送闸门不满足时跳过 webhook（仍写 output）
@@ -52,6 +61,11 @@ DAILY_SKIP_PUSH_WHEN_SILENT=1
# 已推送新闻 link 去重天数
DAILY_NEWS_DEDUP_DAYS=7
# 常驻调度（python -m daily schedule）
DAILY_SCHEDULE_TZ=Asia/Shanghai
DAILY_SCHEDULE_GENERATE_AT=08:50
DAILY_SCHEDULE_PUSH_AT=09:00
# 编辑指定今日首推（可选）：关键词，或 关键词|URL
# Python Step 0 检索 → featured.json；Agent / classic 企微「今日首推」优先使用
# DAILY_FEATURED_PICK=gstack
@@ -59,8 +73,10 @@ DAILY_NEWS_DEDUP_DAYS=7
# 国际 AI 时讯（RSS，见 daily/news/feeds.py）
DAILY_AI_NEWS=1
# 国内 AI 时讯（RSS，见 daily/news/feeds_cn.py）
# 国内 AI 时讯（RSS，见 daily/news/feeds_cn.py；research 模式下忽略）
DAILY_CN_AI_NEWS=1
# AI 时讯来源：rss=RSS 抓取 | research=Cursor SDK + deep-research（WebSearch）
# DAILY_AI_NEWS_MODE=research
# 英文描述 → 简短中文（DAILY_CURSOR_EDITOR=0 时生效）
# DAILY_ZH_DESC=1
# DAILY_ZH_DESC_BATCH=20
@@ -88,6 +104,17 @@ DAILY_AI_NEWS_HOURS=24
DAILY_AI_NEWS_PER_FEED=3
DAILY_AI_NEWS_PER_CATEGORY=5
# 多样性 / 去重（见 docs/superpowers/specs/2026-07-14-wecom-diversity-dedup-design.md）
# DAILY_BOARD_DEDUP_DAYS=7
# DAILY_BOARD_POOL_SIZE=50
# DAILY_FEATURED_DEDUP_DAYS=30
# DAILY_THEME_BAN_DAYS=7
# DAILY_NARRATIVE_AXIS_DAYS=3
DAILY_NEWS_BACKFILL=0
# 国际时讯：在 24h 滚动窗口基础上，不早于今日 0 点（DAILY_AI_NEWS_TZ）
DAILY_AI_NEWS_FLOOR_TODAY=1
# DAILY_AI_NEWS_TZ=Asia/Shanghai
# Reddit RSS（403/429 时在 Reddit 偏好设置 → RSS feeds 复制 user / feed 参数）
# REDDIT_RSS_USER=your_username
# REDDIT_RSS_FEED=your_feed_token

83
daily/board_history.py Normal file
View File

@@ -0,0 +1,83 @@
"""企微展示历史：读写 data.wecom_shown_keys，与 movement_baseline 严格分离。"""
from __future__ import annotations
import json
import logging
from datetime import datetime, timedelta
from pathlib import Path
from typing import Any
from daily.config import OUTPUT_DIR, board_dedup_days
from daily.delta import RECENT_BOARD_KEYS, skill_id
logger = logging.getLogger(__name__)
BOARD_KEYS = RECENT_BOARD_KEYS
def extract_shown_keys(board: str, items: list[dict[str, Any]]) -> list[str]:
"""从最终展示 items 抽取稳定 identity key。"""
keys: list[str] = []
seen: set[str] = set()
for item in items:
if board.startswith("skills_"):
key = skill_id(item)
else:
key = str(item.get("repo") or "")
if not key or key in seen:
continue
seen.add(key)
keys.append(key)
return keys
def load_recent_shown_keys(
date_str: str,
*,
lookback_days: int | None = None,
) -> dict[str, set[str]]:
"""近 N 日 data.wecom_shown_keys 并集（不含当日）。缺省或读失败视为空集。"""
empty = {board: set() for board in BOARD_KEYS}
try:
dt = datetime.strptime(date_str, "%Y-%m-%d")
except ValueError:
return empty
days = lookback_days if lookback_days is not None else board_dedup_days()
out: dict[str, set[str]] = {board: set() for board in BOARD_KEYS}
for day_offset in range(1, days + 1):
prev_date = (dt - timedelta(days=day_offset)).strftime("%Y-%m-%d")
path = OUTPUT_DIR / f"{prev_date}.data.json"
if not path.exists():
continue
try:
payload = json.loads(path.read_text(encoding="utf-8"))
except (OSError, ValueError) as exc:
logger.warning("读取 wecom_shown_keys %s 失败：%s", path, exc)
continue
data = payload.get("data")
if not isinstance(data, dict):
continue
shown = data.get("wecom_shown_keys")
if not isinstance(shown, dict):
continue
for board in BOARD_KEYS:
keys = shown.get(board) or []
if not isinstance(keys, list):
continue
out[board].update(str(k) for k in keys if k)
return out
def merge_wecom_shown_into_data(
data: dict[str, Any],
shown: dict[str, list[str]],
) -> dict[str, Any]:
"""写入 wecom_shown_keys，不修改 movement_baseline。"""
merged = dict(data)
merged["wecom_shown_keys"] = {
board: list(keys) for board, keys in shown.items()
}
return merged

66
daily/config.py
View File

@@ -30,6 +30,24 @@ def wecom_skill_desc_limit() -> int:
return env_int("DAILY_WECOM_SKILL_DESC_LIMIT", 56)
def wecom_news_desc_limit() -> int:
"""企微新闻摘要建议字数；在句读/词边界截断，不加省略号。"""
return max(24, env_int("DAILY_WECOM_NEWS_DESC_LIMIT", 72))
def wecom_ai_news_tech_limit() -> int:
"""research 模式下技术类时讯条数（叠加在 DAILY_WECOM_AI_NEWS 之上）。"""
return max(0, env_int("DAILY_WECOM_AI_NEWS_TECH", 5))
def wecom_pad_pool_size(display_limit: int) -> int:
"""Delta 补榜候选池大小（展示条数之上多取，避免去重后凑不满）。"""
explicit = env_int("DAILY_WECOM_PAD_POOL", -1)
if explicit > 0:
return explicit
return max(display_limit * 5, 50)
def full_desc_limit() -> int:
"""完整版早报摘要长度；0 表示不截断。"""
return env_int("DAILY_FULL_DESC_LIMIT", 0)
@@ -99,5 +117,53 @@ def delta_baseline_fallback() -> str:
return raw if raw in {"full", "empty"} else "full"
def wecom_delta_pad() -> bool:
"""Delta 模式下新入榜优先，不足时用当日 Top 榜补满；补榜排除近 N 天 baseline 已出现条目。"""
return env_bool("DAILY_WECOM_DELTA_PAD", True)
def delta_pad_lookback_days() -> int:
"""补榜时排除近 N 天 baseline 已出现过的条目（默认与异动对比窗口一致）。"""
fallback = env_int("DAILY_DELTA_LOOKBACK_DAYS", 7)
return max(1, env_int("DAILY_DELTA_PAD_LOOKBACK_DAYS", fallback))
def force_push() -> bool:
return env_bool("DAILY_FORCE_PUSH", False)
def schedule_timezone_name() -> str:
return (env("DAILY_SCHEDULE_TZ") or "Asia/Shanghai").strip() or "Asia/Shanghai"
def schedule_generate_at() -> str:
return (env("DAILY_SCHEDULE_GENERATE_AT") or "08:50").strip() or "08:50"
def schedule_push_at() -> str:
return (env("DAILY_SCHEDULE_PUSH_AT") or "09:00").strip() or "09:00"
def board_dedup_days() -> int:
return max(1, env_int("DAILY_BOARD_DEDUP_DAYS", 7))
def board_pool_size() -> int:
fallback = env_int("DAILY_WECOM_SKILL_POOL", 50)
return max(1, env_int("DAILY_BOARD_POOL_SIZE", max(50, fallback)))
def featured_dedup_days() -> int:
return max(1, env_int("DAILY_FEATURED_DEDUP_DAYS", 30))
def theme_ban_days() -> int:
return max(1, env_int("DAILY_THEME_BAN_DAYS", 7))
def narrative_axis_days() -> int:
return max(1, env_int("DAILY_NARRATIVE_AXIS_DAYS", 3))
def news_backfill_enabled() -> bool:
return env_bool("DAILY_NEWS_BACKFILL", False)

26
daily/delta.py
View File

@@ -15,6 +15,15 @@ logger = logging.getLogger(__name__)
KeyFn = Callable[[dict[str, Any]], str]
RECENT_BOARD_KEYS = (
"skills_trending",
"skills_hot",
"github_trending",
"github_emerging",
"github_topic",
)
def compare_depth() -> int:
return env_int("DAILY_DELTA_COMPARE_DEPTH", 15)
@@ -39,6 +48,23 @@ def _key_set(items: list[dict[str, Any]], key_fn: KeyFn, *, depth: int) -> set[s
return {key_fn(item) for item in items[:depth] if key_fn(item)}
def load_recent_board_keys(
date_str: str,
*,
lookback_days: int | None = None,
) -> dict[str, set[str]]:
"""近 N 天各榜已展示过的 skill id / repo（不含当日，供补榜去重）。
委托 board_history.load_recent_shown_keys，只读 wecom_shown_keys，
不读 movement_baseline。
"""
from daily.board_history import load_recent_shown_keys
from daily.config import board_dedup_days
days = lookback_days if lookback_days is not None else board_dedup_days()
return load_recent_shown_keys(date_str, lookback_days=days)
def find_previous_data(date_str: str) -> tuple[str, dict[str, Any]] | None:
"""查找最近一份早于 date_str 的 data.json。"""
try:

65
tests/test_board_history.py Normal file
View File

@@ -0,0 +1,65 @@
# tests/test_board_history.py
from __future__ import annotations
import json
import os
import tempfile
import unittest
from pathlib import Path
from unittest.mock import patch
from daily.board_history import extract_shown_keys, load_recent_shown_keys, merge_wecom_shown_into_data
from daily.config import board_dedup_days, news_backfill_enabled
class ConfigDiversityTests(unittest.TestCase):
def test_board_dedup_days_default(self):
with patch.dict(os.environ, {}, clear=True):
self.assertEqual(board_dedup_days(), 7)
def test_news_backfill_default_off(self):
with patch.dict(os.environ, {}, clear=True):
self.assertFalse(news_backfill_enabled())
class ShownKeysTests(unittest.TestCase):
def test_extract_github_repo_keys(self):
items = [{"repo": "a/b"}, {"repo": "c/d"}]
self.assertEqual(extract_shown_keys("github_trending", items), ["a/b", "c/d"])
def test_load_recent_reads_wecom_shown_not_baseline(self):
with tempfile.TemporaryDirectory() as tmp:
out = Path(tmp)
# 前日：shown 只有 x/y；baseline raw 含 a/b —— 周去重只能看到 x/y
payload = {
"data": {
"date": "2026-07-13",
"movement_baseline": {
"github_trending": [{"repo": "a/b"}, {"repo": "x/y"}],
},
"wecom_shown_keys": {"github_trending": ["x/y"]},
}
}
(out / "2026-07-13.data.json").write_text(
json.dumps(payload, ensure_ascii=False), encoding="utf-8"
)
with patch("daily.board_history.OUTPUT_DIR", out):
keys = load_recent_shown_keys("2026-07-14", lookback_days=7)
self.assertEqual(keys["github_trending"], {"x/y"})
self.assertNotIn("a/b", keys["github_trending"])
def test_merge_shown_does_not_touch_baseline(self):
data = {
"movement_baseline": {"github_trending": [{"repo": "raw/one"}]},
}
merged = merge_wecom_shown_into_data(
data, {"github_trending": ["shown/one"]}
)
self.assertEqual(
merged["movement_baseline"]["github_trending"][0]["repo"], "raw/one"
)
self.assertEqual(merged["wecom_shown_keys"]["github_trending"], ["shown/one"])
if __name__ == "__main__":
unittest.main()

283
tests/test_wecom_delta.py
View File

@@ -44,6 +44,12 @@ class ConfigHelpersTests(unittest.TestCase):
with patch.dict(os.environ, {}, clear=True):
self.assertEqual(delta_baseline_fallback(), "full")
def test_wecom_delta_pad_default_true(self):
from daily.config import wecom_delta_pad
with patch.dict(os.environ, {}, clear=True):
self.assertTrue(wecom_delta_pad())
class NewsPushedLinksTests(unittest.TestCase):
def test_filter_and_record_roundtrip(self):
@@ -121,7 +127,7 @@ class DeltaFormatTests(unittest.TestCase):
]
text = build_skills_delta_sections(moves, [])
self.assertIn("Skills Trending 变化", text)
self.assertIn("[新入 #3]", text)
self.assertNotIn("[新入 #", text)
self.assertNotIn("Skills Hot 变化", text)
def test_github_delta_omits_stable_board(self):
@@ -129,14 +135,268 @@ class DeltaFormatTests(unittest.TestCase):
movement = {
"github_trending_moves": [
{"repo": "a/b", "url": "https://github.com/a/b", "rank": 1, "language": "Go"}
{
"repo": "a/b",
"url": "https://github.com/a/b",
"rank": 1,
"language": "Go",
"description": "open-source codebase and curriculum",
}
],
"github_emerging_moves": [],
"github_topic_moves": [],
}
with patch(
"daily.format_wecom.localize_brief_descriptions",
return_value={"github:a/b": "开源代码库与课程体系"},
):
text = build_github_delta_sections(movement, topic_name="llm")
self.assertIn("开源代码库", text)
self.assertIn("GitHub Trending 变化", text)
self.assertNotIn("新兴", text)
self.assertNotIn("[新入 #", text)
self.assertNotIn("\n > ", text)
def test_delta_pad_groups_same_source_moves(self):
from daily.format_wecom import build_skills_delta_sections
moves = [
{
"id": f"lllllllama/rigorpilot-skills/s{i}",
"title": f"s{i}",
"source": "lllllllama/rigorpilot-skills",
"installs": 250 - i,
"installs_fmt": str(250 - i),
"link": f"https://www.skills.sh/lllllllama/rigorpilot-skills/s{i}",
"description": f"skill {i}",
}
for i in range(1, 11)
]
with patch("daily.format_wecom.localize_brief_descriptions", return_value={}):
with patch("daily.format_wecom.needs_chinese", return_value=False):
text = build_skills_delta_sections(moves, [], trending_limit=10, pad=True)
self.assertIn("10 skills", text)
self.assertNotIn("[**s2**]", text)
def test_delta_pad_fills_skills_to_limit(self):
from daily.format_wecom import build_skills_delta_sections
moves = [
{
"id": "a/b/new",
"rank": 3,
"title": "new",
"source": "a/b",
"installs": 99,
"installs_fmt": "99",
"link": "https://skills.sh/a/b/new",
"description": "new skill",
}
]
full = [
{
"id": "a/b/new",
"title": "new",
"source": "a/b",
"installs": 99,
"installs_fmt": "99",
"link": "https://skills.sh/a/b/new",
"description": "new skill",
},
*[
{
"id": f"src{i}/skill",
"title": "skill",
"source": f"src{i}/pkg",
"installs": 100 - i,
"installs_fmt": str(100 - i),
"link": f"https://skills.sh/src{i}/pkg/skill",
"description": f"skill from src{i}",
}
for i in range(1, 12)
],
]
with patch("daily.format_wecom.localize_brief_descriptions", return_value={}):
with patch("daily.format_wecom.needs_chinese", return_value=False):
text = build_skills_delta_sections(
moves,
[],
trending_full=full,
hot_full=[],
trending_limit=10,
pad=True,
)
self.assertIn("Skills Trending Top 10", text)
self.assertNotIn("Skills Trending 变化", text)
def test_delta_pad_uses_large_pool_when_recent_excludes_top(self):
from daily.format_wecom import build_skills_delta_sections
full_small = [
{
"id": f"seen/src/s{i}",
"title": f"s{i}",
"source": "seen/src",
"installs": 100 - i,
"installs_fmt": str(100 - i),
"link": f"https://skills.sh/seen/src/s{i}",
"description": f"seen {i}",
}
for i in range(1, 11)
]
full_large = [
{
"id": f"fresh/src{n}/skill",
"title": "skill",
"source": f"fresh/src{n}",
"installs": 50 - n,
"installs_fmt": str(50 - n),
"link": f"https://skills.sh/fresh/src{n}/skill",
"description": f"fresh {n}",
}
for n in range(1, 11)
]
recent = {f"seen/src/s{i}" for i in range(1, 11)}
with patch("daily.format_wecom.localize_brief_descriptions", return_value={}):
with patch("daily.format_wecom.needs_chinese", return_value=False):
small = build_skills_delta_sections(
[],
[],
trending_full=full_small,
trending_limit=10,
pad=True,
recent_trending=recent,
)
large = build_skills_delta_sections(
[],
[],
trending_full=full_large,
trending_limit=10,
pad=True,
recent_trending=recent,
)
self.assertNotIn("Skills Trending Top 10", small)
self.assertIn("Skills Trending Top 10", large)
self.assertIn("fresh/src1", large)
def test_delta_pad_fills_github_to_limit(self):
from daily.format_wecom import build_github_delta_sections
movement = {"github_trending_moves": [], "github_emerging_moves": [], "github_topic_moves": []}
full = [
{
"repo": f"org/r{i}",
"url": f"https://github.com/org/r{i}",
"language": "Go",
"stars_today_fmt": "100",
"total_stars_fmt": "1K",
"description": f"repo {i}",
"desc_short": f"repo {i}",
}
for i in range(1, 12)
]
with patch("daily.format_wecom.localize_brief_descriptions", return_value={}):
text = build_github_delta_sections(
movement,
topic_name="llm",
github_trending=full,
trending_limit=10,
pad=True,
)
self.assertIn("GitHub Trending Top 10", text)
self.assertNotIn("GitHub Trending 变化", text)
def test_delta_pad_skips_recent_skills(self):
from daily.format_wecom import build_skills_delta_sections
full = [
{
"id": f"x/y/s{i}",
"title": f"s{i}",
"source": "x/y",
"installs": 100 - i,
"installs_fmt": str(100 - i),
"link": f"https://skills.sh/x/y/s{i}",
"description": f"skill {i}",
}
for i in range(4, 6)
]
with patch("daily.format_wecom.localize_brief_descriptions", return_value={}):
with patch("daily.format_wecom.needs_chinese", return_value=False):
text = build_skills_delta_sections(
[],
[],
trending_full=full,
hot_full=[],
trending_limit=10,
pad=True,
recent_trending={f"x/y/s{i}" for i in range(1, 4)},
)
self.assertIn("Skills Trending Top 1", text)
self.assertIn("2 skills", text)
def test_delta_pad_skips_recent_github(self):
from daily.format_wecom import build_github_delta_sections
movement = {"github_trending_moves": [], "github_emerging_moves": [], "github_topic_moves": []}
full = [
{
"repo": f"org/r{i}",
"url": f"https://github.com/org/r{i}",
"language": "Go",
"total_stars_fmt": "1K",
"description": f"repo {i}",
"desc_short": f"repo {i}",
}
for i in range(1, 6)
]
with patch("daily.format_wecom.localize_brief_descriptions", return_value={}):
text = build_github_delta_sections(
movement,
topic_name="llm",
github_trending=full,
trending_limit=10,
pad=True,
recent_board_keys={"github_trending": {f"org/r{i}" for i in range(1, 4)}},
)
self.assertIn("GitHub Trending Top 2", text)
self.assertIn("org/r4", text)
self.assertNotIn("org/r1", text)
def test_load_recent_board_keys_from_data_json(self):
import json
import tempfile
from pathlib import Path
from daily.delta import load_recent_board_keys
with tempfile.TemporaryDirectory() as tmp:
out = Path(tmp)
payload = {
"data": {
"date": "2026-07-09",
"movement_baseline": {
"skills_trending": [
{"id": "a/b/raw", "title": "raw", "source": "a/b"},
],
"skills_hot": [],
"github_trending": [{"repo": "org/raw"}],
"github_emerging": [],
"github_topic": [],
},
"wecom_shown_keys": {
"skills_trending": ["a/b/foo"],
"github_trending": ["org/bar"],
},
}
}
(out / "2026-07-09.data.json").write_text(json.dumps(payload), encoding="utf-8")
with patch("daily.board_history.OUTPUT_DIR", out):
recent = load_recent_board_keys("2026-07-10", lookback_days=7)
self.assertIn("a/b/foo", recent["skills_trending"])
self.assertIn("org/bar", recent["github_trending"])
self.assertNotIn("a/b/raw", recent["skills_trending"])
self.assertNotIn("org/raw", recent["github_trending"])
class PushGateTests(unittest.TestCase):
@@ -243,6 +503,25 @@ class E2ESmokeTests(unittest.TestCase):
trending=[],
hot=[],
topic_name="llm",
pad=False,
)
self.assertIn("Skills Trending 变化", out)
self.assertNotIn("Skills Trending Top", out)
class SyncMovementGithubTests(unittest.TestCase):
def test_sync_copies_localized_description(self):
from daily.generate import _sync_movement_github_descriptions
movement = {
"github_trending_moves": [{"repo": "a/b", "description": "english"}],
"github_emerging_moves": [],
"github_topic_moves": [],
}
_sync_movement_github_descriptions(
movement,
github_trending=[{"repo": "a/b", "description": "中文描述"}],
github_emerging=[],
github_topic=[],
)
self.assertEqual(movement["github_trending_moves"][0]["description"], "中文描述")