From f533e31adb71fac3d56ec861467209e840b8ab0b Mon Sep 17 00:00:00 2001 From: yumao Date: Fri, 3 Jul 2026 14:45:39 +0800 Subject: [PATCH] =?UTF-8?q?refactor:=20Phase=201=20=E6=8A=BD=E5=87=BA=20sh?= =?UTF-8?q?ared/skills=5Fdata=20=E8=A7=A3=E8=80=A6=20daily=20=E4=B8=8E=20b?= =?UTF-8?q?ot?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit daily 不再通过 sys.path 导入 bot/skills_service，skills feed 数据层上移到 shared/。 Co-authored-by: Cursor --- bot/env_config.py | 5 ++ bot/skills_service.py | 113 ++++-------------------------------------- daily/config.py | 8 --- daily/generate.py | 13 ++--- daily/llm_client.py | 9 ++-- shared/__init__.py | 10 ++++ shared/skills_data.py | 109 ++++++++++++++++++++++++++++++++++++++++ 7 files changed, 146 insertions(+), 121 deletions(-) create mode 100644 shared/__init__.py create mode 100644 shared/skills_data.py diff --git a/bot/env_config.py b/bot/env_config.py index 703c286..0e70704 100644 --- a/bot/env_config.py +++ b/bot/env_config.py @@ -3,11 +3,16 @@ from __future__ import annotations import os +import sys from pathlib import Path from dotenv import load_dotenv _BOT_DIR = Path(__file__).resolve().parent +_REPO_ROOT = _BOT_DIR.parent +if str(_REPO_ROOT) not in sys.path: + sys.path.insert(0, str(_REPO_ROOT)) + load_dotenv(_BOT_DIR / ".env") load_dotenv(_BOT_DIR / ".env.local", override=True) diff --git a/bot/skills_service.py b/bot/skills_service.py index dc7e821..4bc78c2 100644 --- a/bot/skills_service.py +++ b/bot/skills_service.py @@ -1,32 +1,20 @@ -"""skills.sh 数据查询与命令解析。""" +"""skills.sh 快查命令解析与企微回复格式化。""" from __future__ import annotations -import json -import logging import re -import time +import sys from dataclasses import dataclass from pathlib import Path from typing import Any, Literal -import certifi -import httpx +_REPO_ROOT = Path(__file__).resolve().parent.parent +if str(_REPO_ROOT) not in sys.path: + sys.path.insert(0, str(_REPO_ROOT)) -logger = logging.getLogger(__name__) +from shared.skills_data import Board, board_items, format_installs, load_feed, warm_feed_cache -FEED_URLS = [ - # jsDelivr 在国内通常比 raw.githubusercontent.com 更稳定 - "https://cdn.jsdelivr.net/gh/NeverSight/skills.sh_feed@main/data/feed.json", - "https://raw.githubusercontent.com/NeverSight/skills.sh_feed/main/data/feed.json", -] -CACHE_TTL_SECONDS = 600 -CACHE_DIR = Path(__file__).resolve().parent / ".cache" -CACHE_FILE = CACHE_DIR / "feed.json" - -_cache: dict[str, Any] = {"data": None, "fetched_at": 0.0} - -Board = Literal["trending", "hot", "all"] +__all__ = ["Command", "handle_command", "parse_command", "warm_feed_cache"] @dataclass @@ -37,72 +25,6 @@ class Command: query: str = "" -def _fetch_json(url: str) -> dict[str, Any]: - headers = { - "User-Agent": "skills-hot-bot/1.0", - "Accept": "application/json", - } - with httpx.Client( - timeout=httpx.Timeout(20.0, connect=10.0), - verify=certifi.where(), - follow_redirects=True, - ) as client: - resp = client.get(url, headers=headers) - resp.raise_for_status() - return resp.json() - - -def _load_disk_cache() -> dict[str, Any] | None: - if not CACHE_FILE.exists(): - return None - try: - return json.loads(CACHE_FILE.read_text(encoding="utf-8")) - except (OSError, json.JSONDecodeError) as exc: - logger.warning("读取本地缓存失败: %s", exc) - return None - - -def _save_disk_cache(data: dict[str, Any]) -> None: - CACHE_DIR.mkdir(parents=True, exist_ok=True) - CACHE_FILE.write_text(json.dumps(data, ensure_ascii=False), encoding="utf-8") - - -def load_feed(force: bool = False) -> dict[str, Any]: - now = time.time() - if not force and _cache["data"] and now - _cache["fetched_at"] < CACHE_TTL_SECONDS: - return _cache["data"] - - errors: list[str] = [] - for url in FEED_URLS: - for attempt in range(3): - try: - data = _fetch_json(url) - _cache["data"] = data - _cache["fetched_at"] = now - _save_disk_cache(data) - logger.info("skills 数据已更新: %s", url) - return data - except Exception as exc: - msg = f"{url} (#{attempt + 1}): {exc}" - errors.append(msg) - logger.debug("拉取失败 %s", msg) - time.sleep(0.5 * (attempt + 1)) - - stale = _load_disk_cache() - if stale: - logger.warning("网络不可用，回退到本地缓存") - _cache["data"] = stale - _cache["fetched_at"] = now - return stale - - raise RuntimeError(f"无法获取 skills 数据。最近错误: {errors[-1]}") - - -def warm_feed_cache() -> None: - """启动时预加载，避免首条消息才触发网络请求。""" - load_feed(force=True) - - def _normalize_text(text: str) -> str: text = re.sub(r"@\S+\s*", "", text) return text.strip().lower() @@ -157,19 +79,6 @@ def parse_command(text: str) -> Command: return Command(kind="search", query=raw, limit=5) -def _format_installs(n: int | float) -> str: - if n >= 1_000_000: - return f"{n / 1_000_000:.1f}M" - if n >= 1_000: - return f"{n / 1_000:.1f}K" - return str(int(n)) - - -def _board_items(feed: dict[str, Any], board: Board) -> list[dict[str, Any]]: - key = {"trending": "topTrending", "hot": "topHot", "all": "topAllTime"}[board] - return feed.get(key, []) - - def _board_title(board: Board) -> str: return { "trending": "Trending（近期增长）", @@ -180,7 +89,7 @@ def _board_title(board: Board) -> str: def format_list(board: Board, limit: int) -> str: feed = load_feed() - items = _board_items(feed, board)[:limit] + items = board_items(feed, board)[:limit] updated = feed.get("updatedAt", "未知")[:10] lines = [ @@ -192,7 +101,7 @@ def format_list(board: Board, limit: int) -> str: for i, item in enumerate(items, 1): title = item.get("title", "?") source = item.get("source", "?") - installs = _format_installs(item.get("installs", 0)) + installs = format_installs(item.get("installs", 0)) desc = item.get("description", "") if len(desc) > 80: desc = desc[:77] + "..." @@ -241,7 +150,7 @@ def format_search(query: str, limit: int) -> str: for i, item in enumerate(matches, 1): title = item.get("title", "?") source = item.get("source", "?") - installs = _format_installs(item.get("installs", 0)) + installs = format_installs(item.get("installs", 0)) link = item.get("link", "") lines.append(f"{i}. **{title}** · {installs} · `{source}`") if link: @@ -270,7 +179,7 @@ def format_detail(name: str) -> str: [ f"**{best.get('title', '?')}**", f"`{best.get('source', '?')}`", - f"安装量：**{_format_installs(best.get('installs', 0))}**", + f"安装量：**{format_installs(best.get('installs', 0))}**", "", desc, "", diff --git a/daily/config.py b/daily/config.py index 8308aac..a940642 100644 --- a/daily/config.py +++ b/daily/config.py @@ -3,13 +3,11 @@ from __future__ import annotations import os -import sys from pathlib import Path from dotenv import load_dotenv ROOT = Path(__file__).resolve().parent.parent -BOT_DIR = ROOT / "bot" OUTPUT_DIR = ROOT / "output" LOG_DIR = ROOT / "logs" CACHE_DIR = ROOT / ".cache" @@ -49,12 +47,6 @@ load_dotenv(ROOT / ".env") load_dotenv(ROOT / ".env.local", override=True) -def ensure_bot_on_path() -> None: - bot = str(BOT_DIR) - if bot not in sys.path: - sys.path.insert(0, bot) - - def _clean_env_value(raw: str | None) -> str | None: if raw is None: return None diff --git a/daily/generate.py b/daily/generate.py index dfca5ba..df0a8c9 100644 --- a/daily/generate.py +++ b/daily/generate.py @@ -22,7 +22,6 @@ from daily.config import ( LOG_DIR, OUTPUT_DIR, SNAPSHOT_FILE, - ensure_bot_on_path, env, env_int, full_desc_limit, @@ -58,9 +57,7 @@ from daily.report_data import ( ) from daily.skills_board import load_boards from daily.skills_group import group_skills_by_source - -ensure_bot_on_path() -from skills_service import _format_installs, load_feed # noqa: E402 +from shared.skills_data import format_installs, load_feed THEME_RULES: list[tuple[str, str, list[str]]] = [ ("🎬", "AI 多媒体 / 视频", ["runcomfy", "remotion", "video", "seedance", "inpaint", "lipsync"]), @@ -232,7 +229,7 @@ def _prepare_skill_item(item: dict[str, Any], prev_ids: set[str], rank: int) -> badge = "🆕" elif rank == 1: badge = "👑" - installs_fmt = item.get("installs_fmt") or _format_installs(item.get("installs", 0)) + installs_fmt = item.get("installs_fmt") or format_installs(item.get("installs", 0)) title = item.get("source", "?") if item.get("cluster") else item.get("title", "?") desc = item.get("wecom_desc") or item.get("description") or item.get("cluster_titles") or "" limit = wecom_skill_desc_limit() @@ -303,7 +300,7 @@ def _build_highlights( ) if trending: t0 = trending[0] - points.append(f"📈 Skills 榜首 **{t0.get('title')}**（{_format_installs(t0.get('installs', 0))}）") + points.append(f"📈 Skills 榜首 **{t0.get('title')}**（{format_installs(t0.get('installs', 0))}）") if github_trending: g0 = github_trending[0] stars = g0.get("stars_today_fmt", "") @@ -315,7 +312,7 @@ def _build_highlights( points.append(f"🌱 新兴 [{e0['repo']}]({e0['url']})（⭐ {e0.get('total_stars_fmt', '?')}）") elif hot: h0 = hot[0] - points.append(f"🔥 Skills Hot 榜首 **{h0.get('title')}**（1H {_format_installs(h0.get('installs', 0))}）") + points.append(f"🔥 Skills Hot 榜首 **{h0.get('title')}**（1H {format_installs(h0.get('installs', 0))}）") while len(points) < 3 and len(trending) > len(points): item = trending[len(points)] points.append(f"✨ **{item.get('title')}** · `{item.get('source')}`") @@ -397,7 +394,7 @@ def _format_skill_section(items: list[dict[str, Any]], *, hot: bool = False) -> for i, item in enumerate(items, 1): skill_id = item.get("id") or f"{item.get('source', '?')}/{item.get('title', '?')}" link = item.get("link", "") - installs = _format_installs(item.get("installs", 0)) + installs = format_installs(item.get("installs", 0)) meta = f"1H {installs}" if hot else f"总安装 {installs}" if link: lines.append(f"{i}. **[{skill_id}]({link})** · {meta}") diff --git a/daily/llm_client.py b/daily/llm_client.py index b6712e4..05caef0 100644 --- a/daily/llm_client.py +++ b/daily/llm_client.py @@ -74,11 +74,14 @@ def _cursor_chat(system: str, user: str) -> str: api_key = (env("CURSOR_API_KEY") or "").strip() if not api_key: return "" + import sys + + from daily.config import ROOT from cursor_sdk import Agent, AgentOptions, CursorAgentError, LocalAgentOptions - from daily.config import ensure_bot_on_path - - ensure_bot_on_path() + _bot = str(ROOT / "bot") + if _bot not in sys.path: + sys.path.insert(0, _bot) try: from bridge_manager import warm_cursor_bridge except ImportError: diff --git a/shared/__init__.py b/shared/__init__.py new file mode 100644 index 0000000..c112c41 --- /dev/null +++ b/shared/__init__.py @@ -0,0 +1,10 @@ +"""Repo-root shared modules (skills feed data, etc.).""" + +from shared.skills_data import ( + Board, + format_installs, + load_feed, + warm_feed_cache, +) + +__all__ = ["Board", "format_installs", "load_feed", "warm_feed_cache"] diff --git a/shared/skills_data.py b/shared/skills_data.py new file mode 100644 index 0000000..bd72295 --- /dev/null +++ b/shared/skills_data.py @@ -0,0 +1,109 @@ +"""skills.sh feed.json 拉取、缓存与榜单数据。""" + +from __future__ import annotations + +import json +import logging +import time +from pathlib import Path +from typing import Any, Literal + +import certifi +import httpx + +logger = logging.getLogger(__name__) + +ROOT = Path(__file__).resolve().parent.parent +FEED_URLS = [ + "https://cdn.jsdelivr.net/gh/NeverSight/skills.sh_feed@main/data/feed.json", + "https://raw.githubusercontent.com/NeverSight/skills.sh_feed/main/data/feed.json", +] +CACHE_TTL_SECONDS = 600 +CACHE_DIR = ROOT / ".cache" +CACHE_FILE = CACHE_DIR / "skills-feed.json" +USER_AGENT = "Mozilla/5.0 (compatible; daily-robots/1.0; +https://skills.sh)" + +_cache: dict[str, Any] = {"data": None, "fetched_at": 0.0} + +Board = Literal["trending", "hot", "all"] + +BOARD_FEED_KEYS = { + "trending": "topTrending", + "hot": "topHot", + "all": "topAllTime", +} + + +def _fetch_json(url: str) -> dict[str, Any]: + headers = {"User-Agent": USER_AGENT, "Accept": "application/json"} + with httpx.Client( + timeout=httpx.Timeout(20.0, connect=10.0), + verify=certifi.where(), + follow_redirects=True, + ) as client: + resp = client.get(url, headers=headers) + resp.raise_for_status() + return resp.json() + + +def _load_disk_cache() -> dict[str, Any] | None: + if not CACHE_FILE.exists(): + return None + try: + return json.loads(CACHE_FILE.read_text(encoding="utf-8")) + except (OSError, json.JSONDecodeError) as exc: + logger.warning("读取 skills feed 本地缓存失败: %s", exc) + return None + + +def _save_disk_cache(data: dict[str, Any]) -> None: + CACHE_DIR.mkdir(parents=True, exist_ok=True) + CACHE_FILE.write_text(json.dumps(data, ensure_ascii=False), encoding="utf-8") + + +def load_feed(force: bool = False) -> dict[str, Any]: + now = time.time() + if not force and _cache["data"] and now - _cache["fetched_at"] < CACHE_TTL_SECONDS: + return _cache["data"] + + errors: list[str] = [] + for url in FEED_URLS: + for attempt in range(3): + try: + data = _fetch_json(url) + _cache["data"] = data + _cache["fetched_at"] = now + _save_disk_cache(data) + logger.info("skills 数据已更新: %s", url) + return data + except Exception as exc: + msg = f"{url} (#{attempt + 1}): {exc}" + errors.append(msg) + logger.debug("拉取失败 %s", msg) + time.sleep(0.5 * (attempt + 1)) + + stale = _load_disk_cache() + if stale: + logger.warning("网络不可用，回退到本地 skills feed 缓存") + _cache["data"] = stale + _cache["fetched_at"] = now + return stale + + raise RuntimeError(f"无法获取 skills 数据。最近错误: {errors[-1]}") + + +def warm_feed_cache() -> None: + """启动时预加载，避免首条消息才触发网络请求。""" + load_feed(force=True) + + +def format_installs(n: int | float) -> str: + if n >= 1_000_000: + return f"{n / 1_000_000:.1f}M" + if n >= 1_000: + return f"{n / 1_000:.1f}K" + return str(int(n)) + + +def board_items(feed: dict[str, Any], board: Board) -> list[dict[str, Any]]: + return feed.get(BOARD_FEED_KEYS[board], [])