yumao
ce04d5a342
提取 collect/formatters/themes 等 pipeline 模块，新增 wecom 分条、RSS、delta、Agent 工作流测试。 Co-authored-by: Cursor <cursoragent@cursor.com>
117 lines
4.2 KiB
Python
117 lines
4.2 KiB
Python
"""归档内容中文化。"""
|
|
|
|
from __future__ import annotations
|
|
|
|
from typing import Any
|
|
|
|
from daily.config import full_desc_limit, news_summary_limit
|
|
from daily.localize import LocalizeJob, localize_descriptions, needs_chinese
|
|
|
|
from daily.pipeline.snapshot import skill_id
|
|
|
|
|
|
def localize_descriptions_in_place(
|
|
trending: list[dict[str, Any]],
|
|
hot: list[dict[str, Any]],
|
|
github_trending: list[dict[str, Any]],
|
|
github_emerging: list[dict[str, Any]],
|
|
github_topic: list[dict[str, Any]],
|
|
ai_news: dict[str, Any],
|
|
cn_ai_news: dict[str, Any] | None = None,
|
|
) -> None:
|
|
full_limit = full_desc_limit()
|
|
news_limit = news_summary_limit()
|
|
jobs: list[LocalizeJob] = []
|
|
seen_skill: set[str] = set()
|
|
for item in trending + hot:
|
|
sid = skill_id(item)
|
|
if sid in seen_skill:
|
|
continue
|
|
seen_skill.add(sid)
|
|
desc = (item.get("description") or "").strip()
|
|
if desc:
|
|
jobs.append(LocalizeJob(f"skill:{sid}", desc, full_limit))
|
|
|
|
seen_repo: set[str] = set()
|
|
for repo_list in (github_trending, github_emerging, github_topic):
|
|
for item in repo_list:
|
|
repo = item.get("repo", "")
|
|
if not repo or repo in seen_repo:
|
|
continue
|
|
seen_repo.add(repo)
|
|
desc = (item.get("description") or "").strip()
|
|
if desc:
|
|
jobs.append(LocalizeJob(f"github:{repo}", desc, full_limit))
|
|
|
|
if ai_news.get("enabled"):
|
|
seen_news: set[str] = set()
|
|
for item in ai_news.get("flat") or []:
|
|
link = item.get("link", "")
|
|
if not link or link in seen_news:
|
|
continue
|
|
seen_news.add(link)
|
|
summary = (item.get("summary") or "").strip()
|
|
if summary:
|
|
jobs.append(LocalizeJob(f"news:{link}", summary, news_limit))
|
|
|
|
zh_map = localize_descriptions(jobs, archive=True)
|
|
if not zh_map and not jobs:
|
|
return
|
|
|
|
def _apply_zh(mapping: dict[str, str]) -> None:
|
|
for item in trending + hot:
|
|
key = f"skill:{skill_id(item)}"
|
|
if key in mapping:
|
|
item["description"] = mapping[key]
|
|
for repo_list in (github_trending, github_emerging, github_topic):
|
|
for item in repo_list:
|
|
key = f"github:{item.get('repo', '')}"
|
|
if key in mapping:
|
|
item["description"] = mapping[key]
|
|
if ai_news.get("enabled"):
|
|
for cat in ai_news.get("categories") or []:
|
|
for item in cat.get("items") or []:
|
|
key = f"news:{item.get('link', '')}"
|
|
if key in mapping:
|
|
item["summary"] = mapping[key]
|
|
for item in ai_news.get("flat") or []:
|
|
key = f"news:{item.get('link', '')}"
|
|
if key in mapping:
|
|
item["summary"] = mapping[key]
|
|
|
|
_apply_zh(zh_map)
|
|
|
|
retry_jobs: list[LocalizeJob] = []
|
|
seen_skill.clear()
|
|
for item in trending + hot:
|
|
sid = skill_id(item)
|
|
if sid in seen_skill:
|
|
continue
|
|
seen_skill.add(sid)
|
|
desc = (item.get("description") or "").strip()
|
|
if needs_chinese(desc):
|
|
retry_jobs.append(LocalizeJob(f"skill:{sid}", desc, full_limit))
|
|
seen_repo.clear()
|
|
for repo_list in (github_trending, github_emerging, github_topic):
|
|
for item in repo_list:
|
|
repo = item.get("repo", "")
|
|
if not repo or repo in seen_repo:
|
|
continue
|
|
seen_repo.add(repo)
|
|
desc = (item.get("description") or "").strip()
|
|
if needs_chinese(desc):
|
|
retry_jobs.append(LocalizeJob(f"github:{repo}", desc, full_limit))
|
|
if ai_news.get("enabled"):
|
|
seen_news.clear()
|
|
for item in ai_news.get("flat") or []:
|
|
link = item.get("link", "")
|
|
if not link or link in seen_news:
|
|
continue
|
|
seen_news.add(link)
|
|
summary = (item.get("summary") or "").strip()
|
|
if needs_chinese(summary):
|
|
retry_jobs.append(LocalizeJob(f"news:{link}", summary, news_limit))
|
|
|
|
if retry_jobs:
|
|
_apply_zh(localize_descriptions(retry_jobs, archive=True))
|