daily-robots/daily/skills_board.py

"""从 skills.sh 官网抓取 Trending / Hot 完整榜单（突破 feed.json 50 条限制）。"""

from __future__ import annotations

import json
import logging
import re
import time
from typing import Any, Literal

import certifi
import httpx

from daily.config import CACHE_DIR, env

logger = logging.getLogger(__name__)

Board = Literal["trending", "hot"]
SKILLS_SITE = "https://www.skills.sh"
USER_AGENT = "Mozilla/5.0 (compatible; skills-hot-daily/1.0; +https://skills.sh)"

FEED_URLS = [
    "https://cdn.jsdelivr.net/gh/NeverSight/skills.sh_feed@main/data/feed.json",
    "https://raw.githubusercontent.com/NeverSight/skills.sh_feed/main/data/feed.json",
]
FEED_CACHE_TTL = 600
FEED_CACHE_FILE = CACHE_DIR / "feed.json"

_feed_cache: dict[str, Any] = {"data": None, "fetched_at": 0.0}


def format_installs(n: int | float) -> str:
    if n >= 1_000_000:
        return f"{n / 1_000_000:.1f}M"
    if n >= 1_000:
        return f"{n / 1_000:.1f}K"
    return str(int(n))


def _fetch_feed_json(url: str) -> dict[str, Any]:
    headers = {"User-Agent": USER_AGENT, "Accept": "application/json"}
    with httpx.Client(
        timeout=httpx.Timeout(20.0, connect=10.0),
        verify=certifi.where(),
        follow_redirects=True,
    ) as client:
        resp = client.get(url, headers=headers)
        resp.raise_for_status()
        return resp.json()


def _load_feed_disk_cache() -> dict[str, Any] | None:
    if not FEED_CACHE_FILE.exists():
        return None
    try:
        return json.loads(FEED_CACHE_FILE.read_text(encoding="utf-8"))
    except (OSError, json.JSONDecodeError) as exc:
        logger.warning("读取 feed 本地缓存失败: %s", exc)
        return None


def _save_feed_disk_cache(data: dict[str, Any]) -> None:
    FEED_CACHE_FILE.parent.mkdir(parents=True, exist_ok=True)
    FEED_CACHE_FILE.write_text(json.dumps(data, ensure_ascii=False), encoding="utf-8")


def load_feed(force: bool = False) -> dict[str, Any]:
    now = time.time()
    if not force and _feed_cache["data"] and now - _feed_cache["fetched_at"] < FEED_CACHE_TTL:
        return _feed_cache["data"]

    errors: list[str] = []
    for url in FEED_URLS:
        for attempt in range(3):
            try:
                data = _fetch_feed_json(url)
                _feed_cache["data"] = data
                _feed_cache["fetched_at"] = now
                _save_feed_disk_cache(data)
                logger.info("skills feed 已更新: %s", url)
                return data
            except Exception as exc:
                msg = f"{url} (#{attempt + 1}): {exc}"
                errors.append(msg)
                logger.debug("拉取失败 %s", msg)
                time.sleep(0.5 * (attempt + 1))

    stale = _load_feed_disk_cache()
    if stale:
        logger.warning("网络不可用，回退到 feed 本地缓存")
        _feed_cache["data"] = stale
        _feed_cache["fetched_at"] = now
        return stale

    raise RuntimeError(f"无法获取 skills 数据。最近错误: {errors[-1] if errors else 'unknown'}")

_SKILL_RE = re.compile(
    r'\{"source":"(?P<source>[^"]+)","skillId":"(?P<skill_id>[^"]+)",'
    r'"name":"(?P<name>[^"]+)","installs":(?P<installs>\d+)'
)
_RSC_CHUNK_RE = re.compile(r"self\.__next_f\.push\(\[1,\"(.*?)\"\]\)", re.DOTALL)


def board_source() -> str:
    return (env("SKILLS_BOARD_SOURCE") or "website").strip().lower()


def _fetch_html(path: str) -> str:
    url = f"{SKILLS_SITE}{path}"
    headers = {"User-Agent": USER_AGENT, "Accept": "text/html"}
    with httpx.Client(timeout=30.0, verify=certifi.where(), follow_redirects=True) as client:
        resp = client.get(url, headers=headers)
        resp.raise_for_status()
        return resp.text


def _rsc_blob(html: str) -> str:
    chunks = _RSC_CHUNK_RE.findall(html)
    blob = "\n".join(chunks)
    return blob.encode("utf-8").decode("unicode_escape", errors="ignore")


def _parse_initial_skills(blob: str, *, limit: int) -> list[dict[str, Any]]:
    items: list[dict[str, Any]] = []
    seen: set[str] = set()
    for match in _SKILL_RE.finditer(blob):
        source = match.group("source")
        skill_id = match.group("skill_id")
        uid = f"{source}/{skill_id}"
        if uid in seen:
            continue
        seen.add(uid)
        items.append(
            {
                "id": uid,
                "title": skill_id,
                "source": source,
                "installs": int(match.group("installs")),
                "link": f"{SKILLS_SITE}/{source}/{skill_id}",
                "description": "",
            }
        )
        if len(items) >= limit:
            break
    return items


def fetch_board(board: Board, *, limit: int) -> list[dict[str, Any]]:
    path = "/trending" if board == "trending" else "/hot"
    try:
        html = _fetch_html(path)
        items = _parse_initial_skills(_rsc_blob(html), limit=limit)
        if items:
            logger.info("skills.sh %s: %d items (limit=%d)", board, len(items), limit)
            return items
    except Exception as exc:
        logger.warning("skills.sh %s fetch failed, fallback to feed.json: %s", board, exc)
    return []


def enrich_from_feed(items: list[dict[str, Any]], feed: dict[str, Any]) -> None:
    desc_by_id: dict[str, str] = {}
    for key in ("topTrending", "topHot", "topAllTime"):
        for row in feed.get(key, []):
            uid = str(row.get("id") or f"{row.get('source')}/{row.get('title')}")
            desc = (row.get("description") or "").strip()
            if desc:
                desc_by_id[uid] = desc
    for item in items:
        if not item.get("description"):
            item["description"] = desc_by_id.get(item["id"], "")


def load_boards(
    feed: dict[str, Any],
    *,
    trending_limit: int,
    hot_limit: int,
) -> tuple[list[dict[str, Any]], list[dict[str, Any]]]:
    """优先 skills.sh 官网；失败时回退 feed.json。"""
    if board_source() == "feed":
        return (
            list(feed.get("topTrending", [])[:trending_limit]),
            list(feed.get("topHot", [])[:hot_limit]),
        )

    trending = fetch_board("trending", limit=trending_limit)
    hot = fetch_board("hot", limit=hot_limit)
    if not trending:
        trending = list(feed.get("topTrending", [])[:trending_limit])
    else:
        enrich_from_feed(trending, feed)
    if not hot:
        hot = list(feed.get("topHot", [])[:hot_limit])
    else:
        enrich_from_feed(hot, feed)
    return trending, hot