feat: 早报系统重构与功能增强
- 新增常驻调度器 daily/scheduler.py + run-scheduler.ps1（定时生成/推送） - 新增 daily/bridge_manager.py：Windows 兼容的 Cursor SDK 桥接 - 新增 skills/daily-featured-pick 首推 Skill 与叙事轴/去重逻辑 - 新闻抓取窗口、GitHub 搜索、企微 delta 模式等多项改进 - 补充设计文档与 superpowers 计划/规范 - 新增对应测试（scheduler、featured_pick、github_search、news_fetch_window 等） Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
This commit is contained in:
@@ -5,6 +5,7 @@ from __future__ import annotations
|
||||
import sys
|
||||
|
||||
from daily.generate import main as generate_main
|
||||
from daily.scheduler import main as schedule_main
|
||||
from daily.webhook import main as push_main
|
||||
|
||||
|
||||
@@ -14,7 +15,14 @@ def main() -> int:
|
||||
return generate_main()
|
||||
if cmd in {"push", "send", "webhook"}:
|
||||
return push_main(sys.argv[2:])
|
||||
print(f"未知命令: {cmd}\n用法: python -m daily [generate|push] [report_path]", file=sys.stderr)
|
||||
if cmd in {"schedule", "scheduler", "daemon"}:
|
||||
return schedule_main()
|
||||
print(
|
||||
f"未知命令: {cmd}\n"
|
||||
"用法: python -m daily [generate|push|schedule] [report_path]\n"
|
||||
" python -m daily schedule [--once] [--dry-run]",
|
||||
file=sys.stderr,
|
||||
)
|
||||
return 1
|
||||
|
||||
|
||||
|
||||
@@ -4,6 +4,7 @@ from __future__ import annotations
|
||||
|
||||
import json
|
||||
import logging
|
||||
import re
|
||||
from datetime import datetime, timedelta
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
@@ -15,29 +16,119 @@ logger = logging.getLogger(__name__)
|
||||
|
||||
BOARD_KEYS = RECENT_BOARD_KEYS
|
||||
|
||||
_GITHUB_REPO_RE = re.compile(r"github\.com/([\w.-]+/[\w.-]+)", re.I)
|
||||
_SKILL_SH_RE = re.compile(r"skills\.sh/([\w.-]+/[\w.-]+(?:/[\w.-]+)?)", re.I)
|
||||
|
||||
# 与企微正文榜单标题对齐；顺序用于切分相邻 section
|
||||
_WECOM_SECTION_PATTERNS: tuple[tuple[str, re.Pattern[str]], ...] = (
|
||||
("skills_trending", re.compile(r"Skills\s+Trending", re.I)),
|
||||
("skills_hot", re.compile(r"Skills\s+Hot", re.I)),
|
||||
("github_trending", re.compile(r"GitHub\s+Trending", re.I)),
|
||||
("github_emerging", re.compile(r"GitHub\s+新兴", re.I)),
|
||||
("github_topic", re.compile(r"Topic\s+", re.I)),
|
||||
)
|
||||
|
||||
|
||||
def extract_shown_keys(board: str, items: list[dict[str, Any]]) -> list[str]:
|
||||
"""从最终展示 items 抽取稳定 identity key。"""
|
||||
"""从最终展示 items 抽取稳定 identity key。
|
||||
|
||||
Skills 榜同时写入 skill id 与 source，便于周去重按仓屏蔽。
|
||||
"""
|
||||
keys: list[str] = []
|
||||
seen: set[str] = set()
|
||||
for item in items:
|
||||
if board.startswith("skills_"):
|
||||
key = skill_id(item)
|
||||
candidates = [skill_id(item), str(item.get("source") or "").strip()]
|
||||
else:
|
||||
key = str(item.get("repo") or "")
|
||||
if not key or key in seen:
|
||||
continue
|
||||
seen.add(key)
|
||||
keys.append(key)
|
||||
candidates = [str(item.get("repo") or "")]
|
||||
for key in candidates:
|
||||
if not key or key in seen:
|
||||
continue
|
||||
seen.add(key)
|
||||
keys.append(key)
|
||||
return keys
|
||||
|
||||
|
||||
def _keys_from_board_items(board: str, data: dict[str, Any]) -> set[str]:
|
||||
if board == "github_topic":
|
||||
topic = data.get("github_topic") or {}
|
||||
items = topic.get("repos") if isinstance(topic, dict) else []
|
||||
else:
|
||||
items = data.get(board) or []
|
||||
if not isinstance(items, list):
|
||||
return set()
|
||||
return set(extract_shown_keys(board, items))
|
||||
|
||||
|
||||
def parse_wecom_shown_keys(md: str) -> dict[str, set[str]]:
|
||||
"""从企微 Markdown 按榜单 section 解析已展示 keys（冷启动兼容）。"""
|
||||
out: dict[str, set[str]] = {board: set() for board in BOARD_KEYS}
|
||||
if not (md or "").strip():
|
||||
return out
|
||||
|
||||
hits: list[tuple[int, str]] = []
|
||||
for board, pattern in _WECOM_SECTION_PATTERNS:
|
||||
for match in pattern.finditer(md):
|
||||
hits.append((match.start(), board))
|
||||
if not hits:
|
||||
return out
|
||||
hits.sort(key=lambda x: x[0])
|
||||
|
||||
for idx, (start, board) in enumerate(hits):
|
||||
end = hits[idx + 1][0] if idx + 1 < len(hits) else len(md)
|
||||
chunk = md[start:end]
|
||||
if board.startswith("skills_"):
|
||||
out[board].update(_SKILL_SH_RE.findall(chunk))
|
||||
else:
|
||||
out[board].update(_GITHUB_REPO_RE.findall(chunk))
|
||||
return out
|
||||
|
||||
|
||||
def _load_shown_keys_for_day(path: Path, date_str: str) -> dict[str, set[str]] | None:
|
||||
"""读一日历史：优先 wecom_shown_keys；缺省则回退 wecom.md，再回退 data 榜字段。"""
|
||||
empty = {board: set() for board in BOARD_KEYS}
|
||||
try:
|
||||
payload = json.loads(path.read_text(encoding="utf-8"))
|
||||
except (OSError, ValueError) as exc:
|
||||
logger.warning("读取 wecom_shown_keys %s 失败：%s", path, exc)
|
||||
return None
|
||||
|
||||
data = payload.get("data") if isinstance(payload, dict) else None
|
||||
if not isinstance(data, dict):
|
||||
return empty
|
||||
|
||||
out: dict[str, set[str]] = {board: set() for board in BOARD_KEYS}
|
||||
shown = data.get("wecom_shown_keys")
|
||||
if isinstance(shown, dict):
|
||||
for board in BOARD_KEYS:
|
||||
keys = shown.get(board) or []
|
||||
if isinstance(keys, list):
|
||||
out[board].update(str(k) for k in keys if k)
|
||||
if any(out.values()):
|
||||
return out
|
||||
|
||||
wecom_path = OUTPUT_DIR / f"{date_str}.wecom.md"
|
||||
if wecom_path.exists():
|
||||
try:
|
||||
md = wecom_path.read_text(encoding="utf-8")
|
||||
except OSError as exc:
|
||||
logger.warning("读取 wecom.md 回退 %s 失败：%s", wecom_path, exc)
|
||||
else:
|
||||
parsed = parse_wecom_shown_keys(md)
|
||||
if any(parsed.values()):
|
||||
return parsed
|
||||
|
||||
for board in BOARD_KEYS:
|
||||
out[board].update(_keys_from_board_items(board, data))
|
||||
return out
|
||||
|
||||
|
||||
def load_recent_shown_keys(
|
||||
date_str: str,
|
||||
*,
|
||||
lookback_days: int | None = None,
|
||||
) -> dict[str, set[str]]:
|
||||
"""近 N 日 data.wecom_shown_keys 并集（不含当日）。缺省或读失败视为空集。"""
|
||||
"""近 N 日已展示 keys 并集（不含当日）。缺省或读失败视为空集。"""
|
||||
empty = {board: set() for board in BOARD_KEYS}
|
||||
try:
|
||||
dt = datetime.strptime(date_str, "%Y-%m-%d")
|
||||
@@ -52,22 +143,11 @@ def load_recent_shown_keys(
|
||||
path = OUTPUT_DIR / f"{prev_date}.data.json"
|
||||
if not path.exists():
|
||||
continue
|
||||
try:
|
||||
payload = json.loads(path.read_text(encoding="utf-8"))
|
||||
except (OSError, ValueError) as exc:
|
||||
logger.warning("读取 wecom_shown_keys %s 失败：%s", path, exc)
|
||||
continue
|
||||
data = payload.get("data")
|
||||
if not isinstance(data, dict):
|
||||
continue
|
||||
shown = data.get("wecom_shown_keys")
|
||||
if not isinstance(shown, dict):
|
||||
day_keys = _load_shown_keys_for_day(path, prev_date)
|
||||
if day_keys is None:
|
||||
continue
|
||||
for board in BOARD_KEYS:
|
||||
keys = shown.get(board) or []
|
||||
if not isinstance(keys, list):
|
||||
continue
|
||||
out[board].update(str(k) for k in keys if k)
|
||||
out[board].update(day_keys.get(board) or set())
|
||||
return out
|
||||
|
||||
|
||||
|
||||
@@ -24,20 +24,22 @@ def board_select(
|
||||
from daily.skills_group import group_skills_by_source
|
||||
|
||||
pool = group_skills_by_source(items, limit=pool_size, pool_size=pool_size)
|
||||
|
||||
def key_fn(x: dict[str, Any]) -> str:
|
||||
return skill_id(x)
|
||||
else:
|
||||
pool = items[: max(pool_size, limit)]
|
||||
|
||||
def key_fn(x: dict[str, Any]) -> str:
|
||||
return str(x.get("repo") or "")
|
||||
|
||||
out: list[dict[str, Any]] = []
|
||||
for item in pool:
|
||||
k = key_fn(item)
|
||||
if not k or k in recent_keys:
|
||||
continue
|
||||
if kind == "skill":
|
||||
key = skill_id(item)
|
||||
source = str(item.get("source") or "").strip()
|
||||
if (key and key in recent_keys) or (source and source in recent_keys):
|
||||
continue
|
||||
if not key and not source:
|
||||
continue
|
||||
else:
|
||||
key = str(item.get("repo") or "")
|
||||
if not key or key in recent_keys:
|
||||
continue
|
||||
out.append(item)
|
||||
if len(out) >= limit:
|
||||
break
|
||||
|
||||
156
daily/bridge_manager.py
Normal file
156
daily/bridge_manager.py
Normal file
@@ -0,0 +1,156 @@
|
||||
"""Windows 兼容的 Cursor SDK bridge 管理。"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import codecs
|
||||
import json
|
||||
import logging
|
||||
import os
|
||||
import subprocess
|
||||
import threading
|
||||
import time
|
||||
from pathlib import Path
|
||||
from typing import Any, Mapping
|
||||
|
||||
from daily.config import ROOT, env
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
READY_LINE_PREFIX = "cursor-sdk-bridge ready "
|
||||
_bridge_lock = threading.Lock()
|
||||
_bridge_process: subprocess.Popen[bytes] | None = None
|
||||
|
||||
|
||||
def _cursor_cwd() -> str:
|
||||
return env("DAILY_CURSOR_CWD") or env("CURSOR_CWD") or str(ROOT)
|
||||
|
||||
|
||||
def _parse_discovery_line(line: str) -> Mapping[str, Any] | None:
|
||||
if not line.startswith(READY_LINE_PREFIX):
|
||||
return None
|
||||
payload = line[len(READY_LINE_PREFIX) :].strip()
|
||||
loaded = json.loads(payload)
|
||||
if not isinstance(loaded, dict):
|
||||
raise RuntimeError("Bridge discovery payload must be an object")
|
||||
return loaded
|
||||
|
||||
|
||||
def _read_discovery_polling(process: subprocess.Popen[bytes], timeout: float = 60) -> Mapping[str, Any]:
|
||||
"""不用 selectors，避免 Windows 上 WinError 10038。"""
|
||||
if process.stderr is None:
|
||||
raise RuntimeError("Bridge stderr unavailable")
|
||||
|
||||
fd = process.stderr.fileno()
|
||||
was_blocking = os.get_blocking(fd)
|
||||
os.set_blocking(fd, False)
|
||||
decoder = codecs.getincrementaldecoder("utf-8")(errors="replace")
|
||||
pending = ""
|
||||
stderr_lines: list[str] = []
|
||||
deadline = time.monotonic() + timeout
|
||||
|
||||
try:
|
||||
while time.monotonic() < deadline:
|
||||
try:
|
||||
chunk = os.read(fd, 8192)
|
||||
except BlockingIOError:
|
||||
chunk = b""
|
||||
|
||||
if chunk:
|
||||
pending += decoder.decode(chunk)
|
||||
while "\n" in pending:
|
||||
line, pending = pending.split("\n", 1)
|
||||
stderr_lines.append(line)
|
||||
discovery = _parse_discovery_line(line)
|
||||
if discovery is not None:
|
||||
return discovery
|
||||
else:
|
||||
code = process.poll()
|
||||
if code is not None:
|
||||
pending += decoder.decode(b"", final=True)
|
||||
if pending.strip():
|
||||
stderr_lines.append(pending.strip())
|
||||
joined = "\n".join(stderr_lines)[-2000:]
|
||||
raise RuntimeError(
|
||||
f"Bridge 启动失败 exit={code}: {joined or '无 stderr 输出'}"
|
||||
)
|
||||
time.sleep(0.05)
|
||||
finally:
|
||||
os.set_blocking(fd, was_blocking)
|
||||
|
||||
raise RuntimeError("等待 Cursor bridge 就绪超时")
|
||||
|
||||
|
||||
def _auth_token_from_discovery(discovery: Mapping[str, Any]) -> str:
|
||||
token = str(discovery.get("authToken") or "").strip()
|
||||
if token:
|
||||
return token
|
||||
token_file = discovery.get("authTokenFile")
|
||||
if token_file:
|
||||
return Path(str(token_file)).read_text(encoding="utf-8").strip()
|
||||
raise RuntimeError("Bridge discovery 缺少 auth token")
|
||||
|
||||
|
||||
def warm_cursor_bridge(force: bool = False) -> None:
|
||||
"""启动 cursor-sdk-bridge 并写入 CURSOR_SDK_BRIDGE_* 环境变量。"""
|
||||
global _bridge_process
|
||||
|
||||
with _bridge_lock:
|
||||
if (
|
||||
not force
|
||||
and _bridge_process is not None
|
||||
and _bridge_process.poll() is None
|
||||
and os.environ.get("CURSOR_SDK_BRIDGE_URL")
|
||||
and os.environ.get("CURSOR_SDK_BRIDGE_TOKEN")
|
||||
):
|
||||
return
|
||||
|
||||
if _bridge_process is not None and _bridge_process.poll() is None:
|
||||
_bridge_process.terminate()
|
||||
try:
|
||||
_bridge_process.wait(timeout=5)
|
||||
except subprocess.TimeoutExpired:
|
||||
_bridge_process.kill()
|
||||
|
||||
from cursor_sdk._vendor import resolve_bridge_path
|
||||
|
||||
cwd = _cursor_cwd()
|
||||
argv = [resolve_bridge_path(), "--workspace", cwd]
|
||||
logger.info("启动 Cursor bridge workspace=%s", cwd)
|
||||
|
||||
process = subprocess.Popen(
|
||||
argv,
|
||||
stdout=subprocess.DEVNULL,
|
||||
stderr=subprocess.PIPE,
|
||||
)
|
||||
try:
|
||||
discovery = _read_discovery_polling(process)
|
||||
except Exception:
|
||||
process.kill()
|
||||
process.wait(timeout=5)
|
||||
raise
|
||||
|
||||
url = str(discovery.get("url") or "").strip()
|
||||
if not url:
|
||||
host = str(discovery.get("host") or "127.0.0.1")
|
||||
port = discovery.get("port")
|
||||
url = f"http://{host}:{port}"
|
||||
|
||||
token = _auth_token_from_discovery(discovery)
|
||||
os.environ["CURSOR_SDK_BRIDGE_URL"] = url
|
||||
os.environ["CURSOR_SDK_BRIDGE_TOKEN"] = token
|
||||
_bridge_process = process
|
||||
logger.info("Cursor bridge 就绪: %s", url)
|
||||
|
||||
|
||||
def shutdown_cursor_bridge() -> None:
|
||||
global _bridge_process
|
||||
with _bridge_lock:
|
||||
if _bridge_process is None:
|
||||
return
|
||||
if _bridge_process.poll() is None:
|
||||
_bridge_process.terminate()
|
||||
try:
|
||||
_bridge_process.wait(timeout=5)
|
||||
except subprocess.TimeoutExpired:
|
||||
_bridge_process.kill()
|
||||
_bridge_process = None
|
||||
@@ -149,8 +149,8 @@ def board_dedup_days() -> int:
|
||||
|
||||
|
||||
def board_pool_size() -> int:
|
||||
fallback = env_int("DAILY_WECOM_SKILL_POOL", 50)
|
||||
return max(1, env_int("DAILY_BOARD_POOL_SIZE", max(50, fallback)))
|
||||
fallback = env_int("DAILY_WECOM_SKILL_POOL", 400)
|
||||
return max(1, env_int("DAILY_BOARD_POOL_SIZE", max(200, fallback)))
|
||||
|
||||
|
||||
def featured_dedup_days() -> int:
|
||||
|
||||
@@ -62,6 +62,10 @@ def load_recent_featured_keys(date_str: str, days: int | None = None) -> set[str
|
||||
if not isinstance(data, dict):
|
||||
continue
|
||||
key = str(data.get("featured_pick_key") or "").strip()
|
||||
if not key:
|
||||
featured = data.get("featured_pick")
|
||||
if isinstance(featured, dict):
|
||||
key = featured_identity_key(featured)
|
||||
if key:
|
||||
out.add(key)
|
||||
return out
|
||||
@@ -84,7 +88,12 @@ def load_yesterday_featured_key(date_str: str) -> str | None:
|
||||
if not isinstance(data, dict):
|
||||
return None
|
||||
key = str(data.get("featured_pick_key") or "").strip()
|
||||
return key or None
|
||||
if key:
|
||||
return key
|
||||
featured = data.get("featured_pick")
|
||||
if isinstance(featured, dict):
|
||||
return featured_identity_key(featured) or None
|
||||
return None
|
||||
|
||||
|
||||
def _featured_rng(date_str: str) -> random.Random:
|
||||
|
||||
@@ -7,7 +7,7 @@ from typing import Any
|
||||
|
||||
from daily.config import wecom_skill_desc_limit
|
||||
from daily.localize import LocalizeJob, localize_brief_descriptions, needs_chinese
|
||||
from daily.skills_group import group_skills_by_source, skill_id as board_skill_id
|
||||
from daily.skills_group import group_skills_by_source
|
||||
from daily.text_utils import trim_brief
|
||||
|
||||
ICONS = {
|
||||
@@ -469,6 +469,32 @@ def _prepare_grouped_wecom_skills(
|
||||
return wecom_items[:limit], keys
|
||||
|
||||
|
||||
def _normalize_skill_source_groups(items: list[dict[str, Any]]) -> list[dict[str, Any]]:
|
||||
"""将条目规范为按 source 合并的榜单项（展示始终为合并态）。"""
|
||||
flat: list[dict[str, Any]] = []
|
||||
for item in items:
|
||||
flat.extend(_flatten_skill_board_item(item))
|
||||
if not flat:
|
||||
return []
|
||||
return group_skills_by_source(flat, limit=len(flat), pool_size=len(flat))
|
||||
|
||||
|
||||
def _skill_primary_id(item: dict[str, Any]) -> str:
|
||||
return str(item.get("id") or f"{item.get('source')}/{item.get('title')}" or "").strip()
|
||||
|
||||
|
||||
def _source_from_skill_key(key: str) -> str:
|
||||
from daily.skills_group import source_from_skill_key
|
||||
|
||||
return source_from_skill_key(key)
|
||||
|
||||
|
||||
def expand_skill_recent_keys(keys: set[str] | None) -> set[str]:
|
||||
from daily.skills_group import expand_skill_recent_keys as _expand
|
||||
|
||||
return _expand(keys)
|
||||
|
||||
|
||||
def _merge_skill_board_items(
|
||||
moves: list[dict[str, Any]],
|
||||
full_items: list[dict[str, Any]],
|
||||
@@ -477,29 +503,80 @@ def _merge_skill_board_items(
|
||||
exclude_keys: set[str] | None = None,
|
||||
recent_keys: set[str] | None = None,
|
||||
) -> tuple[list[dict[str, Any]], set[str]]:
|
||||
"""异动优先，不足时用深池补满；按 source 合并态取条。
|
||||
|
||||
周去重按 source（含从 skill id 展开）；同日避开其它榜时也按 source。
|
||||
"""
|
||||
from daily.delta import skill_id as move_skill_id
|
||||
|
||||
exclude = exclude_keys or set()
|
||||
recent = recent_keys or set()
|
||||
seen: set[str] = set()
|
||||
flat: list[dict[str, Any]] = []
|
||||
recent = expand_skill_recent_keys(recent_keys)
|
||||
exclude_sources = {_source_from_skill_key(k) for k in exclude if k}
|
||||
exclude_sources.update(k for k in exclude if k)
|
||||
|
||||
def _blocked(item: dict[str, Any]) -> bool:
|
||||
primary = _skill_primary_id(item)
|
||||
source = str(item.get("source") or "").strip()
|
||||
if primary and primary in recent:
|
||||
return True
|
||||
if source and source in recent:
|
||||
return True
|
||||
if primary and primary in exclude:
|
||||
return True
|
||||
if source and (source in exclude_sources or source in exclude):
|
||||
return True
|
||||
return False
|
||||
|
||||
groups: list[dict[str, Any]] = []
|
||||
seen_sources: set[str] = set()
|
||||
|
||||
move_rows: list[dict[str, Any]] = []
|
||||
seen_move_ids: set[str] = set()
|
||||
for move in moves:
|
||||
key = move_skill_id(move)
|
||||
if not key or key in seen or key in exclude:
|
||||
if not key or key in seen_move_ids:
|
||||
continue
|
||||
seen.add(key)
|
||||
flat.append(_move_to_skill_row(move))
|
||||
pad_pool = max(limit * 5, len(flat), 50)
|
||||
for item in full_items:
|
||||
if len(flat) >= pad_pool:
|
||||
move_source = str(move.get("source") or "").strip()
|
||||
if key in exclude or (move_source and move_source in exclude_sources):
|
||||
continue
|
||||
if key in recent or (move_source and move_source in recent):
|
||||
continue
|
||||
seen_move_ids.add(key)
|
||||
move_rows.append(_move_to_skill_row(move))
|
||||
|
||||
for group in _normalize_skill_source_groups(move_rows):
|
||||
source = str(group.get("source") or "?")
|
||||
if source in seen_sources or _blocked(group):
|
||||
continue
|
||||
seen_sources.add(source)
|
||||
groups.append(group)
|
||||
if len(groups) >= limit:
|
||||
break
|
||||
for row in _flatten_skill_board_item(item):
|
||||
key = board_skill_id(row)
|
||||
if not key or key in seen or key in exclude or key in recent:
|
||||
|
||||
if len(groups) < limit:
|
||||
for group in _normalize_skill_source_groups(full_items):
|
||||
if len(groups) >= limit:
|
||||
break
|
||||
source = str(group.get("source") or "?")
|
||||
if source in seen_sources or _blocked(group):
|
||||
continue
|
||||
seen.add(key)
|
||||
flat.append(row)
|
||||
return _prepare_grouped_wecom_skills(flat, limit=limit)
|
||||
seen_sources.add(source)
|
||||
groups.append(group)
|
||||
|
||||
if not groups:
|
||||
return [], set()
|
||||
prepared = finalize_wecom_skill_groups(groups[:limit])
|
||||
wecom_items = [_grouped_skill_to_wecom_item(x) for x in prepared]
|
||||
# 供同日 Hot 排除：主键 + source
|
||||
keys: set[str] = set()
|
||||
for item in groups[:limit]:
|
||||
primary = _skill_primary_id(item)
|
||||
if primary:
|
||||
keys.add(primary)
|
||||
source = str(item.get("source") or "").strip()
|
||||
if source:
|
||||
keys.add(source)
|
||||
return wecom_items[:limit], keys
|
||||
|
||||
|
||||
def build_skills_delta_sections(
|
||||
@@ -508,8 +585,8 @@ def build_skills_delta_sections(
|
||||
*,
|
||||
trending_full: list[dict[str, Any]] | None = None,
|
||||
hot_full: list[dict[str, Any]] | None = None,
|
||||
trending_limit: int = 10,
|
||||
hot_limit: int = 10,
|
||||
trending_limit: int = 5,
|
||||
hot_limit: int = 5,
|
||||
pad: bool = False,
|
||||
recent_trending: set[str] | None = None,
|
||||
recent_hot: set[str] | None = None,
|
||||
@@ -517,11 +594,14 @@ def build_skills_delta_sections(
|
||||
sections: list[str] = []
|
||||
trending_keys: set[str] = set()
|
||||
if pad:
|
||||
skill_recent = expand_skill_recent_keys(
|
||||
(recent_trending or set()) | (recent_hot or set())
|
||||
)
|
||||
t_items, trending_keys = _merge_skill_board_items(
|
||||
trending_moves,
|
||||
trending_full or [],
|
||||
trending_limit,
|
||||
recent_keys=recent_trending,
|
||||
recent_keys=skill_recent,
|
||||
)
|
||||
if t_items:
|
||||
lines = [f"{ICONS['trending']} **Skills Trending Top {len(t_items)}**"]
|
||||
@@ -533,7 +613,7 @@ def build_skills_delta_sections(
|
||||
hot_full or [],
|
||||
hot_limit,
|
||||
exclude_keys=trending_keys,
|
||||
recent_keys=recent_hot,
|
||||
recent_keys=skill_recent,
|
||||
)
|
||||
if h_items:
|
||||
lines = [f"{ICONS['hot']} **Skills Hot Top {len(h_items)}**"]
|
||||
@@ -585,7 +665,7 @@ def _merge_github_board_items(
|
||||
for move in moves:
|
||||
repo = _github_move_to_repo(move)
|
||||
key = str(repo.get("repo") or "")
|
||||
if not key or key in seen:
|
||||
if not key or key in seen or key in recent:
|
||||
continue
|
||||
seen.add(key)
|
||||
merged.append(repo)
|
||||
@@ -607,15 +687,20 @@ def build_github_delta_sections(
|
||||
github_trending: list[dict[str, Any]] | None = None,
|
||||
github_emerging: list[dict[str, Any]] | None = None,
|
||||
github_topic: list[dict[str, Any]] | None = None,
|
||||
trending_limit: int = 10,
|
||||
emerging_limit: int = 10,
|
||||
topic_limit: int = 10,
|
||||
trending_limit: int = 5,
|
||||
emerging_limit: int = 5,
|
||||
topic_limit: int = 5,
|
||||
pad: bool = False,
|
||||
recent_board_keys: dict[str, set[str]] | None = None,
|
||||
) -> str:
|
||||
sections: list[str] = []
|
||||
recent = recent_board_keys or {}
|
||||
if pad:
|
||||
github_recent = (
|
||||
(recent.get("github_trending") or set())
|
||||
| (recent.get("github_emerging") or set())
|
||||
| (recent.get("github_topic") or set())
|
||||
)
|
||||
mapping = [
|
||||
("github_trending_moves", "github_trending", github_trending or [], trending_limit, "github", "GitHub Trending", False),
|
||||
("github_emerging_moves", "github_emerging", github_emerging or [], emerging_limit, "emerging", "GitHub 新兴", True),
|
||||
@@ -626,8 +711,9 @@ def build_github_delta_sections(
|
||||
movement.get(move_key) or [],
|
||||
full_repos,
|
||||
limit,
|
||||
recent_repos=recent.get(board_key),
|
||||
recent_repos=github_recent,
|
||||
)
|
||||
github_recent |= {str(r.get("repo") or "") for r in repos if r.get("repo")}
|
||||
if not repos:
|
||||
continue
|
||||
lines = [f"{ICONS[icon_key]} **{label} Top {len(repos)}**"]
|
||||
@@ -678,11 +764,11 @@ def resolve_wecom_board_items(
|
||||
github_trending: list[dict[str, Any]] | None = None,
|
||||
github_emerging: list[dict[str, Any]] | None = None,
|
||||
github_topic: list[dict[str, Any]] | None = None,
|
||||
wecom_trending: int = 10,
|
||||
wecom_hot: int = 10,
|
||||
wecom_github: int = 10,
|
||||
wecom_emerging: int = 10,
|
||||
wecom_topic: int = 10,
|
||||
wecom_trending: int = 5,
|
||||
wecom_hot: int = 5,
|
||||
wecom_github: int = 5,
|
||||
wecom_emerging: int = 5,
|
||||
wecom_topic: int = 5,
|
||||
pad: bool = False,
|
||||
date_str: str | None = None,
|
||||
trending_pad: list[dict[str, Any]] | None = None,
|
||||
@@ -704,8 +790,21 @@ def resolve_wecom_board_items(
|
||||
}
|
||||
|
||||
recent_board_keys: dict[str, set[str]] = {}
|
||||
skill_recent: set[str] = set()
|
||||
github_recent: set[str] = set()
|
||||
if pad and date_str:
|
||||
recent_board_keys = load_recent_board_keys(date_str)
|
||||
# Trending / Hot 共用周去重：任一类出现过的 source 两边都不再展示
|
||||
skill_recent = expand_skill_recent_keys(
|
||||
(recent_board_keys.get("skills_trending") or set())
|
||||
| (recent_board_keys.get("skills_hot") or set())
|
||||
)
|
||||
# GitHub 三榜共用周去重：任一类出现过的 repo 各榜都不再展示
|
||||
github_recent = (
|
||||
(recent_board_keys.get("github_trending") or set())
|
||||
| (recent_board_keys.get("github_emerging") or set())
|
||||
| (recent_board_keys.get("github_topic") or set())
|
||||
)
|
||||
|
||||
t_moves, h_moves = partition_skill_moves_for_wecom(
|
||||
movement.get("skills_trending_moves") or [],
|
||||
@@ -716,36 +815,41 @@ def resolve_wecom_board_items(
|
||||
t_moves,
|
||||
trending_pad if trending_pad else trending,
|
||||
wecom_trending,
|
||||
recent_keys=recent_board_keys.get("skills_trending"),
|
||||
recent_keys=skill_recent,
|
||||
)
|
||||
h_items, _ = _merge_skill_board_items(
|
||||
h_moves,
|
||||
hot_pad if hot_pad else hot,
|
||||
wecom_hot,
|
||||
exclude_keys=trending_keys,
|
||||
recent_keys=recent_board_keys.get("skills_hot"),
|
||||
recent_keys=skill_recent,
|
||||
)
|
||||
gt_items = _merge_github_board_items(
|
||||
movement.get("github_trending_moves") or [],
|
||||
github_trending_pad if github_trending_pad else (github_trending or []),
|
||||
wecom_github,
|
||||
recent_repos=github_recent,
|
||||
)
|
||||
github_recent |= {str(r.get("repo") or "") for r in gt_items if r.get("repo")}
|
||||
ge_items = _merge_github_board_items(
|
||||
movement.get("github_emerging_moves") or [],
|
||||
github_emerging_pad if github_emerging_pad else (github_emerging or []),
|
||||
wecom_emerging,
|
||||
recent_repos=github_recent,
|
||||
)
|
||||
github_recent |= {str(r.get("repo") or "") for r in ge_items if r.get("repo")}
|
||||
gtopic_items = _merge_github_board_items(
|
||||
movement.get("github_topic_moves") or [],
|
||||
github_topic_pad if github_topic_pad else (github_topic or []),
|
||||
wecom_topic,
|
||||
recent_repos=github_recent,
|
||||
)
|
||||
return {
|
||||
"skills_trending": t_items,
|
||||
"skills_hot": h_items,
|
||||
"github_trending": _merge_github_board_items(
|
||||
movement.get("github_trending_moves") or [],
|
||||
github_trending_pad if github_trending_pad else (github_trending or []),
|
||||
wecom_github,
|
||||
recent_repos=recent_board_keys.get("github_trending"),
|
||||
),
|
||||
"github_emerging": _merge_github_board_items(
|
||||
movement.get("github_emerging_moves") or [],
|
||||
github_emerging_pad if github_emerging_pad else (github_emerging or []),
|
||||
wecom_emerging,
|
||||
recent_repos=recent_board_keys.get("github_emerging"),
|
||||
),
|
||||
"github_topic": _merge_github_board_items(
|
||||
movement.get("github_topic_moves") or [],
|
||||
github_topic_pad if github_topic_pad else (github_topic or []),
|
||||
wecom_topic,
|
||||
recent_repos=recent_board_keys.get("github_topic"),
|
||||
),
|
||||
"github_trending": gt_items,
|
||||
"github_emerging": ge_items,
|
||||
"github_topic": gtopic_items,
|
||||
}
|
||||
|
||||
t_flat = [_move_to_skill_row(m) for m in t_moves]
|
||||
@@ -778,11 +882,11 @@ def replace_wecom_board_sections(
|
||||
github_trending: list[dict[str, Any]] | None = None,
|
||||
github_emerging: list[dict[str, Any]] | None = None,
|
||||
github_topic: list[dict[str, Any]] | None = None,
|
||||
wecom_trending: int = 10,
|
||||
wecom_hot: int = 10,
|
||||
wecom_github: int = 10,
|
||||
wecom_emerging: int = 10,
|
||||
wecom_topic: int = 10,
|
||||
wecom_trending: int = 5,
|
||||
wecom_hot: int = 5,
|
||||
wecom_github: int = 5,
|
||||
wecom_emerging: int = 5,
|
||||
wecom_topic: int = 5,
|
||||
pad: bool = False,
|
||||
date_str: str | None = None,
|
||||
trending_pad: list[dict[str, Any]] | None = None,
|
||||
@@ -854,11 +958,11 @@ def replace_wecom_skill_sections(
|
||||
github_trending: list[dict[str, Any]] | None = None,
|
||||
github_emerging: list[dict[str, Any]] | None = None,
|
||||
github_topic: list[dict[str, Any]] | None = None,
|
||||
wecom_trending: int = 10,
|
||||
wecom_hot: int = 10,
|
||||
wecom_github: int = 10,
|
||||
wecom_emerging: int = 10,
|
||||
wecom_topic: int = 10,
|
||||
wecom_trending: int = 5,
|
||||
wecom_hot: int = 5,
|
||||
wecom_github: int = 5,
|
||||
wecom_emerging: int = 5,
|
||||
wecom_topic: int = 5,
|
||||
pad: bool = False,
|
||||
date_str: str | None = None,
|
||||
trending_pad: list[dict[str, Any]] | None = None,
|
||||
|
||||
@@ -510,23 +510,26 @@ def _format_skill_section(items: list[dict[str, Any]], *, hot: bool = False) ->
|
||||
|
||||
|
||||
def generate_report() -> tuple[str, str, Path, Path]:
|
||||
trending_n = env_int("DAILY_TRENDING_LIMIT", 150)
|
||||
hot_n = max(env_int("DAILY_HOT_LIMIT", 150), compare_depth())
|
||||
# Hot/Trending 前排同 source 极密，需更深抓取才能凑够展示用的唯一 source
|
||||
trending_n = env_int("DAILY_TRENDING_LIMIT", 400)
|
||||
hot_n = max(env_int("DAILY_HOT_LIMIT", 400), compare_depth())
|
||||
compare_n = compare_depth()
|
||||
skill_pool = max(10, env_int("DAILY_WECOM_SKILL_POOL", 200))
|
||||
wecom_trending = env_int("DAILY_WECOM_TRENDING", 10)
|
||||
wecom_hot = env_int("DAILY_WECOM_HOT", 10)
|
||||
pad_pool = wecom_pad_pool_size(max(wecom_trending, wecom_hot, 10))
|
||||
skill_pool = max(10, env_int("DAILY_WECOM_SKILL_POOL", 400))
|
||||
wecom_trending = env_int("DAILY_WECOM_TRENDING", 5)
|
||||
wecom_hot = env_int("DAILY_WECOM_HOT", 5)
|
||||
pad_pool = wecom_pad_pool_size(max(wecom_trending, wecom_hot, 5))
|
||||
skill_pool = max(skill_pool, pad_pool)
|
||||
github_limit = env_int("DAILY_GITHUB_TRENDING_LIMIT", 10)
|
||||
wecom_github = env_int("DAILY_WECOM_GITHUB_TRENDING", env_int("DAILY_WECOM_REPOS", 10))
|
||||
github_fetch_n = max(github_limit, compare_n, wecom_github, pad_pool)
|
||||
wecom_github = env_int("DAILY_WECOM_GITHUB_TRENDING", env_int("DAILY_WECOM_REPOS", 5))
|
||||
# 周去重后顶刊 sticky，HTML/~30 条不够补满；深池默认 100（Search 已分页）
|
||||
github_pool = max(pad_pool, env_int("DAILY_GITHUB_POOL", 100))
|
||||
github_fetch_n = max(github_limit, compare_n, wecom_github, github_pool)
|
||||
emerging_limit = env_int("DAILY_GITHUB_EMERGING_LIMIT", 10)
|
||||
wecom_emerging = env_int("DAILY_WECOM_GITHUB_EMERGING", 10)
|
||||
emerging_fetch_n = max(emerging_limit, compare_n, wecom_emerging, pad_pool)
|
||||
wecom_emerging = env_int("DAILY_WECOM_GITHUB_EMERGING", 5)
|
||||
emerging_fetch_n = max(emerging_limit, compare_n, wecom_emerging, github_pool)
|
||||
topic_limit = env_int("DAILY_GITHUB_TOPIC_LIMIT", 10)
|
||||
wecom_topic = env_int("DAILY_WECOM_GITHUB_TOPIC", 10)
|
||||
topic_fetch_n = max(topic_limit, compare_n, wecom_topic, pad_pool)
|
||||
wecom_topic = env_int("DAILY_WECOM_GITHUB_TOPIC", 5)
|
||||
topic_fetch_n = max(topic_limit, compare_n, wecom_topic, github_pool)
|
||||
|
||||
feed = load_feed(force=True)
|
||||
prev_ids = _load_snapshot()
|
||||
@@ -576,10 +579,21 @@ def generate_report() -> tuple[str, str, Path, Path]:
|
||||
}
|
||||
pool = max(board_pool_size(), skill_pool, pad_pool)
|
||||
recent_shown = load_recent_shown_keys(date_str)
|
||||
from daily.skills_group import expand_skill_recent_keys
|
||||
|
||||
skill_recent = expand_skill_recent_keys(
|
||||
recent_shown["skills_trending"] | recent_shown["skills_hot"]
|
||||
)
|
||||
# GitHub 三榜共用周去重：任一类出现过的 repo 各榜都不再展示
|
||||
github_recent = (
|
||||
recent_shown["github_trending"]
|
||||
| recent_shown["github_emerging"]
|
||||
| recent_shown["github_topic"]
|
||||
)
|
||||
selected_trending = board_select(
|
||||
board="skills_trending",
|
||||
items=trending,
|
||||
recent_keys=recent_shown["skills_trending"],
|
||||
recent_keys=skill_recent,
|
||||
limit=wecom_trending,
|
||||
pool_size=pool,
|
||||
kind="skill",
|
||||
@@ -587,7 +601,7 @@ def generate_report() -> tuple[str, str, Path, Path]:
|
||||
selected_hot = board_select(
|
||||
board="skills_hot",
|
||||
items=hot,
|
||||
recent_keys=recent_shown["skills_hot"],
|
||||
recent_keys=skill_recent,
|
||||
limit=wecom_hot,
|
||||
pool_size=pool,
|
||||
kind="skill",
|
||||
@@ -595,23 +609,25 @@ def generate_report() -> tuple[str, str, Path, Path]:
|
||||
selected_github = board_select(
|
||||
board="github_trending",
|
||||
items=github_trending,
|
||||
recent_keys=recent_shown["github_trending"],
|
||||
recent_keys=github_recent,
|
||||
limit=wecom_github,
|
||||
pool_size=pool,
|
||||
kind="github",
|
||||
)
|
||||
github_recent |= {str(r.get("repo") or "") for r in selected_github if r.get("repo")}
|
||||
selected_emerging = board_select(
|
||||
board="github_emerging",
|
||||
items=github_emerging,
|
||||
recent_keys=recent_shown["github_emerging"],
|
||||
recent_keys=github_recent,
|
||||
limit=wecom_emerging,
|
||||
pool_size=pool,
|
||||
kind="github",
|
||||
)
|
||||
github_recent |= {str(r.get("repo") or "") for r in selected_emerging if r.get("repo")}
|
||||
selected_topic = board_select(
|
||||
board="github_topic",
|
||||
items=github_topic,
|
||||
recent_keys=recent_shown["github_topic"],
|
||||
recent_keys=github_recent,
|
||||
limit=wecom_topic,
|
||||
pool_size=pool,
|
||||
kind="github",
|
||||
@@ -842,7 +858,7 @@ def generate_report() -> tuple[str, str, Path, Path]:
|
||||
gt_pad = board_select(
|
||||
board="skills_trending",
|
||||
items=trending,
|
||||
recent_keys=recent_shown["skills_trending"],
|
||||
recent_keys=skill_recent,
|
||||
limit=pad_pool,
|
||||
pool_size=pool,
|
||||
kind="skill",
|
||||
@@ -850,7 +866,7 @@ def generate_report() -> tuple[str, str, Path, Path]:
|
||||
gh_pad = board_select(
|
||||
board="skills_hot",
|
||||
items=hot,
|
||||
recent_keys=recent_shown["skills_hot"],
|
||||
recent_keys=skill_recent,
|
||||
limit=pad_pool,
|
||||
pool_size=pool,
|
||||
kind="skill",
|
||||
@@ -858,12 +874,17 @@ def generate_report() -> tuple[str, str, Path, Path]:
|
||||
wecom_github_items = [_prepare_github_item(item) for item in selected_github]
|
||||
wecom_emerging_items = [_prepare_github_item(item) for item in selected_emerging]
|
||||
wecom_topic_items = [_prepare_github_item(item) for item in selected_topic]
|
||||
github_pad_recent = (
|
||||
recent_shown["github_trending"]
|
||||
| recent_shown["github_emerging"]
|
||||
| recent_shown["github_topic"]
|
||||
)
|
||||
wecom_github_pad = [
|
||||
_prepare_github_item(item)
|
||||
for item in board_select(
|
||||
board="github_trending",
|
||||
items=github_trending,
|
||||
recent_keys=recent_shown["github_trending"],
|
||||
recent_keys=github_pad_recent,
|
||||
limit=pad_pool,
|
||||
pool_size=pool,
|
||||
kind="github",
|
||||
@@ -874,7 +895,7 @@ def generate_report() -> tuple[str, str, Path, Path]:
|
||||
for item in board_select(
|
||||
board="github_emerging",
|
||||
items=github_emerging,
|
||||
recent_keys=recent_shown["github_emerging"],
|
||||
recent_keys=github_pad_recent,
|
||||
limit=pad_pool,
|
||||
pool_size=pool,
|
||||
kind="github",
|
||||
@@ -885,7 +906,7 @@ def generate_report() -> tuple[str, str, Path, Path]:
|
||||
for item in board_select(
|
||||
board="github_topic",
|
||||
items=github_topic,
|
||||
recent_keys=recent_shown["github_topic"],
|
||||
recent_keys=github_pad_recent,
|
||||
limit=pad_pool,
|
||||
pool_size=pool,
|
||||
kind="github",
|
||||
|
||||
@@ -43,38 +43,59 @@ def search_github_repos(
|
||||
logger.warning("GitHub Search 需要 GITHUB_TOKEN: %s", query[:80])
|
||||
return []
|
||||
|
||||
target = max(1, min(int(limit), 1000))
|
||||
per_page = min(100, target)
|
||||
repos: list[dict[str, Any]] = []
|
||||
seen: set[str] = set()
|
||||
page = 1
|
||||
|
||||
try:
|
||||
with httpx.Client(
|
||||
timeout=20.0,
|
||||
verify=certifi.where(),
|
||||
headers=github_api_headers(),
|
||||
) as client:
|
||||
resp = client.get(
|
||||
"https://api.github.com/search/repositories",
|
||||
params={
|
||||
"q": query,
|
||||
"sort": sort,
|
||||
"order": "desc",
|
||||
"per_page": min(max(limit, 1), 30),
|
||||
},
|
||||
)
|
||||
if resp.status_code != 200:
|
||||
logger.warning("GitHub Search 失败 (%s): %s", resp.status_code, query[:80])
|
||||
return []
|
||||
items = resp.json().get("items") or []
|
||||
while len(repos) < target:
|
||||
resp = client.get(
|
||||
"https://api.github.com/search/repositories",
|
||||
params={
|
||||
"q": query,
|
||||
"sort": sort,
|
||||
"order": "desc",
|
||||
"per_page": per_page,
|
||||
"page": page,
|
||||
},
|
||||
)
|
||||
if resp.status_code != 200:
|
||||
logger.warning(
|
||||
"GitHub Search 失败 (%s page=%s): %s",
|
||||
resp.status_code,
|
||||
page,
|
||||
query[:80],
|
||||
)
|
||||
break
|
||||
items = resp.json().get("items") or []
|
||||
if not items:
|
||||
break
|
||||
for item in items:
|
||||
full_name = item.get("full_name") or ""
|
||||
if not full_name or full_name in seen:
|
||||
continue
|
||||
seen.add(full_name)
|
||||
repos.append(_repo_from_api_item(item, source="api-search"))
|
||||
if len(repos) >= target:
|
||||
break
|
||||
if len(items) < per_page:
|
||||
break
|
||||
page += 1
|
||||
# Search API 最多约 1000 条 / 10 页
|
||||
if page > 10:
|
||||
break
|
||||
except Exception as exc:
|
||||
logger.warning("GitHub Search 异常: %s", exc)
|
||||
return []
|
||||
return repos[:target]
|
||||
|
||||
repos: list[dict[str, Any]] = []
|
||||
for item in items:
|
||||
full_name = item.get("full_name") or ""
|
||||
if not full_name:
|
||||
continue
|
||||
repos.append(_repo_from_api_item(item, source="api-search"))
|
||||
if len(repos) >= limit:
|
||||
break
|
||||
return repos
|
||||
return repos[:target]
|
||||
|
||||
|
||||
def _date_days_ago(days: int) -> str:
|
||||
|
||||
@@ -76,16 +76,9 @@ def _cursor_chat(system: str, user: str) -> str:
|
||||
return ""
|
||||
from cursor_sdk import Agent, AgentOptions, CursorAgentError, LocalAgentOptions
|
||||
|
||||
from daily.config import ensure_bot_on_path
|
||||
|
||||
ensure_bot_on_path()
|
||||
try:
|
||||
from bridge_manager import warm_cursor_bridge
|
||||
except ImportError:
|
||||
warm_cursor_bridge = lambda: None # noqa: E731
|
||||
from daily.bridge_manager import warm_cursor_bridge
|
||||
|
||||
cwd = env("DAILY_CURSOR_CWD") or str(ROOT)
|
||||
# bridge_manager 读 bot env_config 的 CURSOR_CWD，早报侧须先对齐工作目录
|
||||
os.environ["CURSOR_CWD"] = cwd
|
||||
warm_cursor_bridge()
|
||||
model = env("CURSOR_MODEL") or "composer-2.5"
|
||||
@@ -115,5 +108,16 @@ def llm_chat(system: str, user: str) -> str:
|
||||
return ""
|
||||
|
||||
|
||||
def has_cursor_configured() -> bool:
|
||||
return bool((env("CURSOR_API_KEY") or "").strip())
|
||||
|
||||
|
||||
def cursor_agent_prompt(system: str, user: str) -> str:
|
||||
"""仅 Cursor SDK Agent（可用 WebSearch 等工具），不走 OpenAI 兼容 API。"""
|
||||
if not has_cursor_configured():
|
||||
return ""
|
||||
return _cursor_chat(system, user)
|
||||
|
||||
|
||||
def has_llm_configured() -> bool:
|
||||
return bool(env("DAILY_LLM_API_KEY") or env("OPENAI_API_KEY") or env("CURSOR_API_KEY"))
|
||||
|
||||
@@ -1,13 +1,26 @@
|
||||
from daily.news.fetch import (
|
||||
|
||||
fetch_ai_news,
|
||||
|
||||
fetch_cn_ai_news,
|
||||
|
||||
format_cn_news_section,
|
||||
|
||||
format_news_section,
|
||||
|
||||
)
|
||||
|
||||
|
||||
|
||||
__all__ = [
|
||||
|
||||
"fetch_ai_news",
|
||||
|
||||
"fetch_cn_ai_news",
|
||||
|
||||
"format_news_section",
|
||||
|
||||
"format_cn_news_section",
|
||||
|
||||
]
|
||||
|
||||
|
||||
@@ -27,7 +27,7 @@ NEWS_CATEGORIES: tuple[NewsCategory, ...] = (
|
||||
name="厂商官方",
|
||||
icon="🏢",
|
||||
feeds=(
|
||||
NewsFeed("Anthropic Claude 更新", "https://docs.anthropic.com/en/release-notes/feed"),
|
||||
NewsFeed("Anthropic Claude 更新", "https://platform.claude.com/docs/en/release-notes/overview"),
|
||||
NewsFeed("OpenAI", "https://openai.com/news/rss.xml"),
|
||||
NewsFeed("Google AI", "https://blog.google/technology/ai/rss/"),
|
||||
NewsFeed("DeepMind", "https://deepmind.google/blog/rss.xml"),
|
||||
|
||||
@@ -44,7 +44,6 @@ CN_NEWS_CATEGORIES: tuple[NewsCategory, ...] = (
|
||||
icon="📰",
|
||||
feeds=(
|
||||
NewsFeed("量子位", "https://www.qbitai.com/feed"),
|
||||
NewsFeed("InfoQ 中文", "https://www.infoq.cn/feed/AI"),
|
||||
),
|
||||
),
|
||||
NewsCategory(
|
||||
@@ -60,12 +59,4 @@ CN_NEWS_CATEGORIES: tuple[NewsCategory, ...] = (
|
||||
),
|
||||
),
|
||||
),
|
||||
NewsCategory(
|
||||
id="dev",
|
||||
name="开发者社区",
|
||||
icon="💻",
|
||||
feeds=(
|
||||
NewsFeed("掘金", "https://juejin.cn/rss", ai_filter=True),
|
||||
),
|
||||
),
|
||||
)
|
||||
|
||||
@@ -94,7 +94,7 @@ def _slim_news_items(
|
||||
|
||||
|
||||
def _wecom_skill_pool() -> int:
|
||||
return max(10, env_int("DAILY_WECOM_SKILL_POOL", 200))
|
||||
return max(10, env_int("DAILY_WECOM_SKILL_POOL", 400))
|
||||
|
||||
|
||||
def build_llm_input(
|
||||
|
||||
284
daily/scheduler.py
Normal file
284
daily/scheduler.py
Normal file
@@ -0,0 +1,284 @@
|
||||
"""常驻调度：按配置时刻生成早报并推送企微。"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import logging
|
||||
import subprocess
|
||||
import sys
|
||||
import time
|
||||
from dataclasses import dataclass
|
||||
from datetime import date, datetime, time as dt_time, timedelta
|
||||
from pathlib import Path
|
||||
from zoneinfo import ZoneInfo
|
||||
|
||||
from daily.config import (
|
||||
CACHE_DIR,
|
||||
LOG_DIR,
|
||||
OUTPUT_DIR,
|
||||
ROOT,
|
||||
schedule_generate_at,
|
||||
schedule_push_at,
|
||||
schedule_timezone_name,
|
||||
)
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
_STATE_FILE = CACHE_DIR / "scheduler-state.json"
|
||||
_POLL_SECONDS = 15
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class ClockTime:
|
||||
hour: int
|
||||
minute: int
|
||||
|
||||
|
||||
@dataclass
|
||||
class SchedulerState:
|
||||
last_generate_date: str | None = None
|
||||
last_push_date: str | None = None
|
||||
|
||||
@classmethod
|
||||
def load(cls) -> SchedulerState:
|
||||
if not _STATE_FILE.exists():
|
||||
return cls()
|
||||
try:
|
||||
raw = json.loads(_STATE_FILE.read_text(encoding="utf-8"))
|
||||
except (OSError, ValueError):
|
||||
return cls()
|
||||
if not isinstance(raw, dict):
|
||||
return cls()
|
||||
return cls(
|
||||
last_generate_date=raw.get("last_generate_date"),
|
||||
last_push_date=raw.get("last_push_date"),
|
||||
)
|
||||
|
||||
def save(self) -> None:
|
||||
_STATE_FILE.parent.mkdir(parents=True, exist_ok=True)
|
||||
payload = {
|
||||
"last_generate_date": self.last_generate_date,
|
||||
"last_push_date": self.last_push_date,
|
||||
}
|
||||
_STATE_FILE.write_text(json.dumps(payload, ensure_ascii=False, indent=2), encoding="utf-8")
|
||||
|
||||
|
||||
def parse_hhmm(value: str) -> ClockTime:
|
||||
raw = (value or "").strip()
|
||||
parts = raw.split(":", 1)
|
||||
if len(parts) != 2:
|
||||
raise ValueError(f"无效时间格式: {value!r}，应为 HH:MM")
|
||||
hour = int(parts[0])
|
||||
minute = int(parts[1])
|
||||
if not (0 <= hour <= 23 and 0 <= minute <= 59):
|
||||
raise ValueError(f"无效时间: {value!r}")
|
||||
return ClockTime(hour=hour, minute=minute)
|
||||
|
||||
|
||||
def load_timezone() -> ZoneInfo:
|
||||
name = schedule_timezone_name()
|
||||
try:
|
||||
return ZoneInfo(name)
|
||||
except Exception as exc:
|
||||
raise RuntimeError(f"无效时区 DAILY_SCHEDULE_TZ={name!r}") from exc
|
||||
|
||||
|
||||
def _localize(day: date, clock: ClockTime, tz: ZoneInfo) -> datetime:
|
||||
return datetime.combine(day, dt_time(clock.hour, clock.minute), tz)
|
||||
|
||||
|
||||
def next_occurrence_after(clock: ClockTime, tz: ZoneInfo, after: datetime) -> datetime:
|
||||
local = after.astimezone(tz)
|
||||
candidate = local.replace(hour=clock.hour, minute=clock.minute, second=0, microsecond=0)
|
||||
if candidate <= local:
|
||||
candidate += timedelta(days=1)
|
||||
return candidate
|
||||
|
||||
|
||||
def _today_slot(day: date, clock: ClockTime, tz: ZoneInfo) -> datetime:
|
||||
return _localize(day, clock, tz)
|
||||
|
||||
|
||||
def _run_daily_subcommand(subcmd: str, *extra: str) -> int:
|
||||
cmd = [sys.executable, "-m", "daily", subcmd, *extra]
|
||||
logger.info("执行: %s", " ".join(cmd))
|
||||
proc = subprocess.run(cmd, cwd=str(ROOT), check=False)
|
||||
return int(proc.returncode)
|
||||
|
||||
|
||||
def run_generate() -> int:
|
||||
return _run_daily_subcommand("generate")
|
||||
|
||||
|
||||
def run_push_for_date(date_str: str) -> int:
|
||||
report = OUTPUT_DIR / f"{date_str}.wecom.md"
|
||||
if not report.exists():
|
||||
logger.error("推送失败：报告不存在 %s", report)
|
||||
return 1
|
||||
return _run_daily_subcommand("push", str(report))
|
||||
|
||||
|
||||
def _setup_logging() -> Path:
|
||||
LOG_DIR.mkdir(parents=True, exist_ok=True)
|
||||
log_path = LOG_DIR / "scheduler.log"
|
||||
logging.basicConfig(
|
||||
level=logging.INFO,
|
||||
format="%(asctime)s %(levelname)s %(message)s",
|
||||
handlers=[
|
||||
logging.FileHandler(log_path, encoding="utf-8"),
|
||||
logging.StreamHandler(sys.stdout),
|
||||
],
|
||||
)
|
||||
return log_path
|
||||
|
||||
|
||||
def _sleep_until(target: datetime, tz: ZoneInfo) -> None:
|
||||
while True:
|
||||
now = datetime.now(tz)
|
||||
seconds = (target - now).total_seconds()
|
||||
if seconds <= 0:
|
||||
return
|
||||
time.sleep(min(seconds, _POLL_SECONDS))
|
||||
|
||||
|
||||
def plan_next_action(
|
||||
*,
|
||||
now: datetime,
|
||||
tz: ZoneInfo,
|
||||
state: SchedulerState,
|
||||
generate_at: ClockTime,
|
||||
push_at: ClockTime,
|
||||
) -> tuple[datetime, str] | None:
|
||||
"""返回下一次应执行的动作；若今日已全部完成则返回明日 generate。"""
|
||||
today = now.astimezone(tz).date()
|
||||
today_str = today.isoformat()
|
||||
gen_done = state.last_generate_date == today_str
|
||||
push_done = state.last_push_date == today_str
|
||||
gen_slot = _today_slot(today, generate_at, tz)
|
||||
push_slot = _today_slot(today, push_at, tz)
|
||||
|
||||
# 推送窗口内：generate 未做则立即补跑（须先于 push）
|
||||
if not gen_done and gen_slot <= now <= push_slot:
|
||||
return now, "generate"
|
||||
# 已过推送时刻：仅当 generate 已完成时补跑 push
|
||||
if not push_done and gen_done and now >= push_slot:
|
||||
return now, "push"
|
||||
|
||||
candidates: list[tuple[datetime, str]] = []
|
||||
if not gen_done and gen_slot > now:
|
||||
candidates.append((gen_slot, "generate"))
|
||||
if not push_done and push_slot > now:
|
||||
candidates.append((push_slot, "push"))
|
||||
if candidates:
|
||||
return min(candidates, key=lambda item: item[0])
|
||||
|
||||
tomorrow_gen = next_occurrence_after(generate_at, tz, now)
|
||||
return tomorrow_gen, "generate"
|
||||
|
||||
|
||||
def run_scheduled_action(action: str, *, today_str: str) -> int:
|
||||
if action == "generate":
|
||||
return run_generate()
|
||||
if action == "push":
|
||||
return run_push_for_date(today_str)
|
||||
raise ValueError(f"未知动作: {action}")
|
||||
|
||||
|
||||
def tick_once(
|
||||
*,
|
||||
now: datetime | None = None,
|
||||
tz: ZoneInfo | None = None,
|
||||
state: SchedulerState | None = None,
|
||||
generate_at: ClockTime | None = None,
|
||||
push_at: ClockTime | None = None,
|
||||
dry_run: bool = False,
|
||||
) -> SchedulerState:
|
||||
tz = tz or load_timezone()
|
||||
now = now or datetime.now(tz)
|
||||
state = state or SchedulerState.load()
|
||||
generate_at = generate_at or parse_hhmm(schedule_generate_at())
|
||||
push_at = push_at or parse_hhmm(schedule_push_at())
|
||||
|
||||
today_str = now.astimezone(tz).date().isoformat()
|
||||
planned = plan_next_action(
|
||||
now=now,
|
||||
tz=tz,
|
||||
state=state,
|
||||
generate_at=generate_at,
|
||||
push_at=push_at,
|
||||
)
|
||||
if not planned:
|
||||
return state
|
||||
|
||||
run_at, action = planned
|
||||
if run_at > now:
|
||||
if not dry_run:
|
||||
logger.info("下次 %s @ %s (%s)", action, run_at.isoformat(), tz.key)
|
||||
_sleep_until(run_at, tz)
|
||||
elif not dry_run:
|
||||
slot = _today_slot(now.astimezone(tz).date(), generate_at if action == "generate" else push_at, tz)
|
||||
logger.info(
|
||||
"补跑 %s（计划 %02d:%02d，当前 %s）",
|
||||
action,
|
||||
slot.hour,
|
||||
slot.minute,
|
||||
now.astimezone(tz).strftime("%H:%M"),
|
||||
)
|
||||
|
||||
if dry_run:
|
||||
logger.info("[dry-run] 将执行 %s @ %s", action, run_at.isoformat())
|
||||
return state
|
||||
|
||||
logger.info("开始 %s（%s）", action, today_str)
|
||||
code = run_scheduled_action(action, today_str=today_str)
|
||||
if code != 0:
|
||||
logger.error("%s 失败，exit=%s", action, code)
|
||||
else:
|
||||
if action == "generate":
|
||||
state.last_generate_date = today_str
|
||||
elif action == "push":
|
||||
state.last_push_date = today_str
|
||||
state.save()
|
||||
logger.info("%s 完成", action)
|
||||
return state
|
||||
|
||||
|
||||
def main() -> int:
|
||||
dry_run = "--dry-run" in sys.argv[1:]
|
||||
once = "--once" in sys.argv[1:]
|
||||
|
||||
log_path = _setup_logging()
|
||||
tz = load_timezone()
|
||||
generate_at = parse_hhmm(schedule_generate_at())
|
||||
push_at = parse_hhmm(schedule_push_at())
|
||||
|
||||
logger.info(
|
||||
"调度器启动 tz=%s generate=%02d:%02d push=%02d:%02d log=%s",
|
||||
tz.key,
|
||||
generate_at.hour,
|
||||
generate_at.minute,
|
||||
push_at.hour,
|
||||
push_at.minute,
|
||||
log_path,
|
||||
)
|
||||
|
||||
state = SchedulerState.load()
|
||||
try:
|
||||
while True:
|
||||
state = tick_once(
|
||||
tz=tz,
|
||||
state=state,
|
||||
generate_at=generate_at,
|
||||
push_at=push_at,
|
||||
dry_run=dry_run,
|
||||
)
|
||||
if once or dry_run:
|
||||
break
|
||||
except KeyboardInterrupt:
|
||||
logger.info("调度器已停止（KeyboardInterrupt）")
|
||||
return 0
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
@@ -2,14 +2,16 @@
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import logging
|
||||
import re
|
||||
import time
|
||||
from typing import Any, Literal
|
||||
|
||||
import certifi
|
||||
import httpx
|
||||
|
||||
from daily.config import env
|
||||
from daily.config import CACHE_DIR, env
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
@@ -17,6 +19,81 @@ Board = Literal["trending", "hot"]
|
||||
SKILLS_SITE = "https://www.skills.sh"
|
||||
USER_AGENT = "Mozilla/5.0 (compatible; skills-hot-daily/1.0; +https://skills.sh)"
|
||||
|
||||
FEED_URLS = [
|
||||
"https://cdn.jsdelivr.net/gh/NeverSight/skills.sh_feed@main/data/feed.json",
|
||||
"https://raw.githubusercontent.com/NeverSight/skills.sh_feed/main/data/feed.json",
|
||||
]
|
||||
FEED_CACHE_TTL = 600
|
||||
FEED_CACHE_FILE = CACHE_DIR / "feed.json"
|
||||
|
||||
_feed_cache: dict[str, Any] = {"data": None, "fetched_at": 0.0}
|
||||
|
||||
|
||||
def format_installs(n: int | float) -> str:
|
||||
if n >= 1_000_000:
|
||||
return f"{n / 1_000_000:.1f}M"
|
||||
if n >= 1_000:
|
||||
return f"{n / 1_000:.1f}K"
|
||||
return str(int(n))
|
||||
|
||||
|
||||
def _fetch_feed_json(url: str) -> dict[str, Any]:
|
||||
headers = {"User-Agent": USER_AGENT, "Accept": "application/json"}
|
||||
with httpx.Client(
|
||||
timeout=httpx.Timeout(20.0, connect=10.0),
|
||||
verify=certifi.where(),
|
||||
follow_redirects=True,
|
||||
) as client:
|
||||
resp = client.get(url, headers=headers)
|
||||
resp.raise_for_status()
|
||||
return resp.json()
|
||||
|
||||
|
||||
def _load_feed_disk_cache() -> dict[str, Any] | None:
|
||||
if not FEED_CACHE_FILE.exists():
|
||||
return None
|
||||
try:
|
||||
return json.loads(FEED_CACHE_FILE.read_text(encoding="utf-8"))
|
||||
except (OSError, json.JSONDecodeError) as exc:
|
||||
logger.warning("读取 feed 本地缓存失败: %s", exc)
|
||||
return None
|
||||
|
||||
|
||||
def _save_feed_disk_cache(data: dict[str, Any]) -> None:
|
||||
FEED_CACHE_FILE.parent.mkdir(parents=True, exist_ok=True)
|
||||
FEED_CACHE_FILE.write_text(json.dumps(data, ensure_ascii=False), encoding="utf-8")
|
||||
|
||||
|
||||
def load_feed(force: bool = False) -> dict[str, Any]:
|
||||
now = time.time()
|
||||
if not force and _feed_cache["data"] and now - _feed_cache["fetched_at"] < FEED_CACHE_TTL:
|
||||
return _feed_cache["data"]
|
||||
|
||||
errors: list[str] = []
|
||||
for url in FEED_URLS:
|
||||
for attempt in range(3):
|
||||
try:
|
||||
data = _fetch_feed_json(url)
|
||||
_feed_cache["data"] = data
|
||||
_feed_cache["fetched_at"] = now
|
||||
_save_feed_disk_cache(data)
|
||||
logger.info("skills feed 已更新: %s", url)
|
||||
return data
|
||||
except Exception as exc:
|
||||
msg = f"{url} (#{attempt + 1}): {exc}"
|
||||
errors.append(msg)
|
||||
logger.debug("拉取失败 %s", msg)
|
||||
time.sleep(0.5 * (attempt + 1))
|
||||
|
||||
stale = _load_feed_disk_cache()
|
||||
if stale:
|
||||
logger.warning("网络不可用，回退到 feed 本地缓存")
|
||||
_feed_cache["data"] = stale
|
||||
_feed_cache["fetched_at"] = now
|
||||
return stale
|
||||
|
||||
raise RuntimeError(f"无法获取 skills 数据。最近错误: {errors[-1] if errors else 'unknown'}")
|
||||
|
||||
_SKILL_RE = re.compile(
|
||||
r'\{"source":"(?P<source>[^"]+)","skillId":"(?P<skill_id>[^"]+)",'
|
||||
r'"name":"(?P<name>[^"]+)","installs":(?P<installs>\d+)'
|
||||
|
||||
@@ -9,6 +9,28 @@ def skill_id(item: dict[str, Any]) -> str:
|
||||
return str(item.get("id") or f"{item.get('source')}/{item.get('title')}")
|
||||
|
||||
|
||||
def source_from_skill_key(key: str) -> str:
|
||||
"""从 skill id（source/title…）还原 source：去掉最后一段 title。"""
|
||||
parts = [p for p in str(key or "").split("/") if p]
|
||||
if len(parts) >= 2:
|
||||
return "/".join(parts[:-1])
|
||||
return str(key or "").strip()
|
||||
|
||||
|
||||
def expand_skill_recent_keys(keys: set[str] | None) -> set[str]:
|
||||
"""周去重 blocklist：保留原始 key，并展开为 source。"""
|
||||
out: set[str] = set()
|
||||
for key in keys or set():
|
||||
k = str(key or "").strip()
|
||||
if not k:
|
||||
continue
|
||||
out.add(k)
|
||||
src = source_from_skill_key(k)
|
||||
if src:
|
||||
out.add(src)
|
||||
return out
|
||||
|
||||
|
||||
def format_installs(n: int | float) -> str:
|
||||
if n >= 1_000_000:
|
||||
return f"{n / 1_000_000:.1f}M"
|
||||
|
||||
@@ -15,7 +15,7 @@ def clip_text(text: str, limit: int) -> str:
|
||||
|
||||
|
||||
def trim_brief(text: str, limit: int) -> str:
|
||||
"""企微简要：控制在 limit 内，优先在句读处截断，不加省略号。"""
|
||||
"""企微简要：控制在 limit 内，优先在句读/词边界截断，不加省略号。"""
|
||||
text = _WS.sub(" ", (text or "").strip())
|
||||
if not text or limit <= 0 or len(text) <= limit:
|
||||
return text
|
||||
@@ -27,4 +27,12 @@ def trim_brief(text: str, limit: int) -> str:
|
||||
pos = text.find(sep)
|
||||
if pos != -1 and pos + 1 <= limit:
|
||||
return text[: pos + 1]
|
||||
return text[:limit].rstrip("，、；： ")
|
||||
for sep in (". ", "! ", "? ", "; "):
|
||||
pos = text.rfind(sep, 0, limit + 1)
|
||||
if pos != -1 and pos + 1 >= min(limit // 2, 20):
|
||||
return text[: pos + 1].rstrip()
|
||||
if len(text) > limit:
|
||||
space = text.rfind(" ", 0, limit + 1)
|
||||
if space >= min(limit // 2, 20):
|
||||
return text[:space].rstrip("，、；： ,.;")
|
||||
return text[:limit].rstrip("，、；： ,.;")
|
||||
|
||||
Reference in New Issue
Block a user