项目初始化
This commit is contained in:
9
daily/github/__init__.py
Normal file
9
daily/github/__init__.py
Normal file
@@ -0,0 +1,9 @@
|
||||
from daily.github.search import fetch_emerging_repos, fetch_topic_hot_repos
|
||||
from daily.github.trending import fetch_github_trending, trending_data_source_note
|
||||
|
||||
__all__ = [
|
||||
"fetch_emerging_repos",
|
||||
"fetch_topic_hot_repos",
|
||||
"fetch_github_trending",
|
||||
"trending_data_source_note",
|
||||
]
|
||||
81
daily/github/auth.py
Normal file
81
daily/github/auth.py
Normal file
@@ -0,0 +1,81 @@
|
||||
"""GitHub 请求共用：GITHUB_TOKEN、请求头、仓库 API。"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
from typing import Any
|
||||
|
||||
import certifi
|
||||
import httpx
|
||||
|
||||
from daily.config import env
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
DEFAULT_USER_AGENT = (
|
||||
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 "
|
||||
"(KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36"
|
||||
)
|
||||
|
||||
|
||||
def github_token() -> str | None:
|
||||
raw = (env("GITHUB_TOKEN") or "").strip()
|
||||
return raw or None
|
||||
|
||||
|
||||
def github_api_headers() -> dict[str, str]:
|
||||
headers = {
|
||||
"User-Agent": env("GITHUB_TRENDING_USER_AGENT", DEFAULT_USER_AGENT),
|
||||
"Accept": "application/vnd.github+json",
|
||||
"X-GitHub-Api-Version": "2022-11-28",
|
||||
}
|
||||
token = github_token()
|
||||
if token:
|
||||
headers["Authorization"] = f"Bearer {token}"
|
||||
return headers
|
||||
|
||||
|
||||
def github_html_headers() -> dict[str, str]:
|
||||
headers = {
|
||||
"User-Agent": env("GITHUB_TRENDING_USER_AGENT", DEFAULT_USER_AGENT),
|
||||
"Accept": "text/html,application/xhtml+xml",
|
||||
"Accept-Language": "en-US,en;q=0.9",
|
||||
}
|
||||
token = github_token()
|
||||
if token:
|
||||
headers["Authorization"] = f"Bearer {token}"
|
||||
return headers
|
||||
|
||||
|
||||
def fetch_repo_api(repo: str) -> dict[str, Any] | None:
|
||||
url = f"https://api.github.com/repos/{repo}"
|
||||
try:
|
||||
with httpx.Client(
|
||||
timeout=12.0,
|
||||
verify=certifi.where(),
|
||||
headers=github_api_headers(),
|
||||
) as client:
|
||||
resp = client.get(url)
|
||||
if resp.status_code != 200:
|
||||
return None
|
||||
data = resp.json()
|
||||
return {
|
||||
"description": data.get("description") or "",
|
||||
"language": data.get("language") or "",
|
||||
"stars": data.get("stargazers_count", 0),
|
||||
}
|
||||
except Exception as exc:
|
||||
logger.debug("GitHub API repo %s 失败: %s", repo, exc)
|
||||
return None
|
||||
|
||||
|
||||
def format_star_count(value: int | float | str) -> str:
|
||||
try:
|
||||
n = int(value)
|
||||
except (TypeError, ValueError):
|
||||
return str(value)
|
||||
if n >= 1_000_000:
|
||||
return f"{n / 1_000_000:.1f}M".replace(".0M", "M")
|
||||
if n >= 1_000:
|
||||
return f"{n / 1_000:.1f}K".replace(".0K", "K")
|
||||
return f"{n:,}"
|
||||
121
daily/github/search.py
Normal file
121
daily/github/search.py
Normal file
@@ -0,0 +1,121 @@
|
||||
"""GitHub Search API：新兴项目、Topic 热点。"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
from datetime import datetime, timedelta, timezone
|
||||
from typing import Any
|
||||
|
||||
import certifi
|
||||
import httpx
|
||||
|
||||
from daily.config import env, env_int
|
||||
from daily.github.auth import format_star_count, github_api_headers, github_token
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def _repo_from_api_item(item: dict[str, Any], *, source: str) -> dict[str, Any]:
|
||||
full_name = item.get("full_name") or ""
|
||||
stars = item.get("stargazers_count", 0)
|
||||
created = (item.get("created_at") or "")[:10]
|
||||
return {
|
||||
"repo": full_name,
|
||||
"url": item.get("html_url") or f"https://github.com/{full_name}",
|
||||
"description": item.get("description") or "",
|
||||
"language": item.get("language") or "",
|
||||
"stars_today": None,
|
||||
"stars_today_fmt": "",
|
||||
"total_stars_fmt": format_star_count(stars),
|
||||
"created_at": created,
|
||||
"source": source,
|
||||
}
|
||||
|
||||
|
||||
def search_github_repos(
|
||||
query: str,
|
||||
limit: int,
|
||||
*,
|
||||
sort: str = "stars",
|
||||
require_token: bool = True,
|
||||
) -> list[dict[str, Any]]:
|
||||
if require_token and not github_token():
|
||||
logger.warning("GitHub Search 需要 GITHUB_TOKEN: %s", query[:80])
|
||||
return []
|
||||
|
||||
try:
|
||||
with httpx.Client(
|
||||
timeout=20.0,
|
||||
verify=certifi.where(),
|
||||
headers=github_api_headers(),
|
||||
) as client:
|
||||
resp = client.get(
|
||||
"https://api.github.com/search/repositories",
|
||||
params={
|
||||
"q": query,
|
||||
"sort": sort,
|
||||
"order": "desc",
|
||||
"per_page": min(max(limit, 1), 30),
|
||||
},
|
||||
)
|
||||
if resp.status_code != 200:
|
||||
logger.warning("GitHub Search 失败 (%s): %s", resp.status_code, query[:80])
|
||||
return []
|
||||
items = resp.json().get("items") or []
|
||||
except Exception as exc:
|
||||
logger.warning("GitHub Search 异常: %s", exc)
|
||||
return []
|
||||
|
||||
repos: list[dict[str, Any]] = []
|
||||
for item in items:
|
||||
full_name = item.get("full_name") or ""
|
||||
if not full_name:
|
||||
continue
|
||||
repos.append(_repo_from_api_item(item, source="api-search"))
|
||||
if len(repos) >= limit:
|
||||
break
|
||||
return repos
|
||||
|
||||
|
||||
def _date_days_ago(days: int) -> str:
|
||||
return (datetime.now(timezone.utc) - timedelta(days=days)).strftime("%Y-%m-%d")
|
||||
|
||||
|
||||
def fetch_emerging_repos(
|
||||
limit: int = 3,
|
||||
*,
|
||||
days: int | None = None,
|
||||
min_stars: int | None = None,
|
||||
exclude: set[str] | None = None,
|
||||
) -> list[dict[str, Any]]:
|
||||
days = days if days is not None else env_int("GITHUB_EMERGING_DAYS", 14)
|
||||
min_stars = min_stars if min_stars is not None else env_int("GITHUB_EMERGING_MIN_STARS", 200)
|
||||
created_after = _date_days_ago(days)
|
||||
query = f"created:>{created_after} stars:>{min_stars} fork:false"
|
||||
repos = search_github_repos(query, limit + len(exclude or set()))
|
||||
if exclude:
|
||||
repos = [r for r in repos if r["repo"] not in exclude]
|
||||
for item in repos:
|
||||
item["source"] = "api-emerging"
|
||||
return repos[:limit]
|
||||
|
||||
|
||||
def fetch_topic_hot_repos(
|
||||
limit: int = 3,
|
||||
*,
|
||||
topic: str | None = None,
|
||||
pushed_days: int | None = None,
|
||||
min_stars: int | None = None,
|
||||
exclude: set[str] | None = None,
|
||||
) -> tuple[str, list[dict[str, Any]]]:
|
||||
topic = (topic or env("GITHUB_TOPIC") or "llm").strip()
|
||||
pushed_days = pushed_days if pushed_days is not None else env_int("GITHUB_TOPIC_PUSHED_DAYS", 7)
|
||||
min_stars = min_stars if min_stars is not None else env_int("GITHUB_TOPIC_MIN_STARS", 50)
|
||||
pushed_after = _date_days_ago(pushed_days)
|
||||
query = f"topic:{topic} pushed:>{pushed_after} stars:>{min_stars} fork:false"
|
||||
repos = search_github_repos(query, limit + len(exclude or set()))
|
||||
if exclude:
|
||||
repos = [r for r in repos if r["repo"] not in exclude]
|
||||
for item in repos:
|
||||
item["source"] = "api-topic"
|
||||
return topic, repos[:limit]
|
||||
206
daily/github/trending.py
Normal file
206
daily/github/trending.py
Normal file
@@ -0,0 +1,206 @@
|
||||
"""GitHub Trending：页面爬取或 Search API。"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
import re
|
||||
from datetime import datetime, timedelta, timezone
|
||||
from html import unescape
|
||||
from typing import Any, Literal
|
||||
from urllib.parse import urlencode
|
||||
|
||||
import certifi
|
||||
import httpx
|
||||
|
||||
from daily.config import env
|
||||
from daily.github.auth import (
|
||||
fetch_repo_api,
|
||||
format_star_count,
|
||||
github_html_headers,
|
||||
github_token,
|
||||
)
|
||||
from daily.github.search import search_github_repos
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
TrendingSince = Literal["daily", "weekly", "monthly"]
|
||||
TrendingMode = Literal["scrape", "api"]
|
||||
|
||||
DEFAULT_LIMIT = 5
|
||||
DEFAULT_SINCE: TrendingSince = "daily"
|
||||
DEFAULT_MODE: TrendingMode = "scrape"
|
||||
TRENDING_URL = "https://github.com/trending"
|
||||
|
||||
_ARTICLE_RE = re.compile(r'<article class="Box-row">.*?</article>', re.S)
|
||||
_REPO_HREF_RE = re.compile(r'h2[^>]*>\s*<a[^>]+href="([^"]+)"')
|
||||
_DESC_RE = re.compile(r'<p class="col-9[^"]*"[^>]*>([^<]*)</p>')
|
||||
_STARS_TODAY_RE = re.compile(r"([\d,]+)\s+stars?\s+today", re.I)
|
||||
_LANG_RE = re.compile(r'itemprop="programmingLanguage"[^>]*>([^<]+)<')
|
||||
_TOTAL_STARS_RE = re.compile(
|
||||
r'href="/[^/]+/[^/]+/stargazers"[^>]*>\s*<svg[^>]*octicon-star[^>]*>.*?</svg>\s*([\d.,kKmM]+)',
|
||||
re.S,
|
||||
)
|
||||
|
||||
|
||||
def _strip_html(text: str) -> str:
|
||||
return unescape(re.sub(r"\s+", " ", text or "")).strip()
|
||||
|
||||
|
||||
def trending_mode() -> TrendingMode:
|
||||
raw = (env("GITHUB_TRENDING_MODE") or DEFAULT_MODE).strip().lower()
|
||||
if raw in {"api", "token", "search"}:
|
||||
return "api"
|
||||
return "scrape"
|
||||
|
||||
|
||||
def trending_data_source_note() -> str:
|
||||
if trending_mode() == "api":
|
||||
return "> 数据来源：GitHub Search API（`GITHUB_TRENDING_MODE=api`，需 `GITHUB_TOKEN`）"
|
||||
return "> 数据来源：[github.com/trending](https://github.com/trending?since=daily)（页面爬取，失败时 API 降级）"
|
||||
|
||||
|
||||
def _parse_article(article_html: str) -> dict[str, Any] | None:
|
||||
href_match = _REPO_HREF_RE.search(article_html)
|
||||
if not href_match:
|
||||
return None
|
||||
href = href_match.group(1).strip("/")
|
||||
if href.count("/") != 1:
|
||||
return None
|
||||
owner, name = href.split("/", 1)
|
||||
repo = f"{owner}/{name}"
|
||||
desc_match = _DESC_RE.search(article_html)
|
||||
stars_today_match = _STARS_TODAY_RE.search(article_html)
|
||||
lang_match = _LANG_RE.search(article_html)
|
||||
total_stars_match = _TOTAL_STARS_RE.search(article_html)
|
||||
stars_today_raw = stars_today_match.group(1).replace(",", "") if stars_today_match else ""
|
||||
stars_today = int(stars_today_raw) if stars_today_raw.isdigit() else None
|
||||
return {
|
||||
"repo": repo,
|
||||
"url": f"https://github.com/{repo}",
|
||||
"description": _strip_html(desc_match.group(1)) if desc_match else "",
|
||||
"language": _strip_html(lang_match.group(1)) if lang_match else "",
|
||||
"stars_today": stars_today,
|
||||
"stars_today_fmt": stars_today_match.group(1) if stars_today_match else "",
|
||||
"total_stars_fmt": _strip_html(total_stars_match.group(1)) if total_stars_match else "",
|
||||
"source": "scrape",
|
||||
}
|
||||
|
||||
|
||||
def _build_trending_url(*, since: TrendingSince = DEFAULT_SINCE, language: str = "") -> str:
|
||||
if language:
|
||||
return f"{TRENDING_URL}/{language}?{urlencode({'since': since})}"
|
||||
return f"{TRENDING_URL}?{urlencode({'since': since})}"
|
||||
|
||||
|
||||
def _since_push_date(since: TrendingSince) -> str:
|
||||
now = datetime.now(timezone.utc)
|
||||
if since == "weekly":
|
||||
delta = timedelta(days=7)
|
||||
elif since == "monthly":
|
||||
delta = timedelta(days=30)
|
||||
else:
|
||||
delta = timedelta(days=1)
|
||||
return (now - delta).strftime("%Y-%m-%d")
|
||||
|
||||
|
||||
def _enrich_repo_from_api(item: dict[str, Any]) -> dict[str, Any]:
|
||||
if not github_token() and env("GITHUB_API_ENRICH", "1") != "1":
|
||||
return item
|
||||
meta = fetch_repo_api(item["repo"])
|
||||
if not meta:
|
||||
return item
|
||||
enriched = dict(item)
|
||||
if not enriched.get("description"):
|
||||
enriched["description"] = meta["description"]
|
||||
if not enriched.get("language"):
|
||||
enriched["language"] = meta["language"]
|
||||
if not enriched.get("total_stars_fmt") and meta["stars"]:
|
||||
enriched["total_stars_fmt"] = format_star_count(meta["stars"])
|
||||
enriched["source"] = enriched.get("source", "scrape") + "+api"
|
||||
return enriched
|
||||
|
||||
|
||||
def _fetch_trending_html(url: str) -> str | None:
|
||||
try:
|
||||
with httpx.Client(
|
||||
timeout=20.0,
|
||||
verify=certifi.where(),
|
||||
follow_redirects=True,
|
||||
headers=github_html_headers(),
|
||||
) as client:
|
||||
resp = client.get(url)
|
||||
if resp.status_code in {403, 429} and not github_token():
|
||||
logger.warning("GitHub Trending %s（匿名可能被限），可配置 GITHUB_TOKEN", resp.status_code)
|
||||
resp.raise_for_status()
|
||||
return resp.text
|
||||
except Exception as exc:
|
||||
logger.warning("GitHub Trending 页面抓取失败: %s", exc)
|
||||
return None
|
||||
|
||||
|
||||
def _parse_trending_html(html: str, limit: int) -> list[dict[str, Any]]:
|
||||
repos: list[dict[str, Any]] = []
|
||||
for article_html in _ARTICLE_RE.findall(html):
|
||||
item = _parse_article(article_html)
|
||||
if item:
|
||||
repos.append(_enrich_repo_from_api(item))
|
||||
if len(repos) >= limit:
|
||||
break
|
||||
return repos
|
||||
|
||||
|
||||
def _fetch_trending_via_api(limit: int, since: TrendingSince, language: str) -> list[dict[str, Any]]:
|
||||
pushed_after = _since_push_date(since)
|
||||
parts = [f"pushed:>{pushed_after}", "stars:>50", "fork:false"]
|
||||
if language:
|
||||
parts.append(f"language:{language}")
|
||||
query = " ".join(parts)
|
||||
repos = search_github_repos(query, limit, require_token=True)
|
||||
for item in repos:
|
||||
item["source"] = "api-search"
|
||||
return repos
|
||||
|
||||
|
||||
def fetch_github_trending(
|
||||
limit: int = DEFAULT_LIMIT,
|
||||
*,
|
||||
since: TrendingSince | None = None,
|
||||
language: str = "",
|
||||
) -> list[dict[str, Any]]:
|
||||
since = since or env("GITHUB_TRENDING_SINCE", DEFAULT_SINCE) # type: ignore[assignment]
|
||||
if since not in ("daily", "weekly", "monthly"):
|
||||
since = DEFAULT_SINCE
|
||||
lang = (language or env("GITHUB_TRENDING_LANGUAGE") or "").strip()
|
||||
mode = trending_mode()
|
||||
|
||||
if mode == "api":
|
||||
repos = _fetch_trending_via_api(limit, since, lang)
|
||||
if not repos and not github_token():
|
||||
logger.warning("GITHUB_TRENDING_MODE=api 需要配置 GITHUB_TOKEN")
|
||||
elif repos:
|
||||
logger.info("GitHub Trending 使用 API 模式，共 %d 条", len(repos))
|
||||
return repos[:limit]
|
||||
|
||||
url = _build_trending_url(since=since, language=lang)
|
||||
html = _fetch_trending_html(url)
|
||||
repos: list[dict[str, Any]] = []
|
||||
if html:
|
||||
repos = _parse_trending_html(html, limit)
|
||||
if not repos:
|
||||
logger.warning("GitHub Trending 页面解析为空: %s", url)
|
||||
|
||||
if len(repos) < limit:
|
||||
before = len(repos)
|
||||
fallback = _fetch_trending_via_api(limit, since, lang)
|
||||
seen = {r["repo"] for r in repos}
|
||||
for item in fallback:
|
||||
if item["repo"] in seen:
|
||||
continue
|
||||
repos.append(item)
|
||||
seen.add(item["repo"])
|
||||
if len(repos) >= limit:
|
||||
break
|
||||
if len(repos) > before:
|
||||
logger.info("已用 GitHub API 补充 %d 条 Trending 数据", len(repos) - before)
|
||||
|
||||
return repos[:limit]
|
||||
Reference in New Issue
Block a user