daily-robots/daily/pipeline/collect.py

"""抓取与结构化输入组装。"""

from __future__ import annotations

from dataclasses import dataclass
from datetime import datetime
from typing import Any

from daily.agent_workflow import is_agent_mode
from daily.config import env_int
from daily.delta import compare_depth
from daily.github.search import fetch_emerging_repos, fetch_topic_hot_repos
from daily.github.trending import fetch_github_trending
from daily.news.fetch import fetch_ai_news, fetch_cn_ai_news
from daily.news.rank import apply_news_ranking
from daily.report_data import build_llm_input
from daily.skills_board import load_boards
from shared.skills_data import load_feed


@dataclass
class ReportLimits:
    trending_n: int
    hot_n: int
    skill_pool: int
    wecom_trending: int
    wecom_hot: int
    github_limit: int
    wecom_github: int
    emerging_limit: int
    wecom_emerging: int
    topic_limit: int
    wecom_topic: int


@dataclass
class ReportContext:
    feed: dict[str, Any]
    date_str: str
    time_str: str
    updated: str
    trending: list[dict[str, Any]]
    hot: list[dict[str, Any]]
    github_trending: list[dict[str, Any]]
    github_emerging: list[dict[str, Any]]
    github_topic: list[dict[str, Any]]
    topic_name: str
    ai_news: dict[str, Any]
    cn_ai_news: dict[str, Any]
    limits: ReportLimits
    wecom_limits: dict[str, int]
    llm_input: dict[str, Any]


def resolve_limits() -> ReportLimits:
    compare_n = compare_depth()
    trending_n = env_int("DAILY_TRENDING_LIMIT", 150)
    hot_n = max(env_int("DAILY_HOT_LIMIT", 150), compare_n)
    skill_pool = max(10, env_int("DAILY_WECOM_SKILL_POOL", 200))
    wecom_trending = env_int("DAILY_WECOM_TRENDING", 10)
    wecom_hot = env_int("DAILY_WECOM_HOT", 10)
    github_limit = env_int("DAILY_GITHUB_TRENDING_LIMIT", 10)
    wecom_github = env_int("DAILY_WECOM_GITHUB_TRENDING", env_int("DAILY_WECOM_REPOS", 10))
    emerging_limit = env_int("DAILY_GITHUB_EMERGING_LIMIT", 10)
    wecom_emerging = env_int("DAILY_WECOM_GITHUB_EMERGING", 10)
    topic_limit = env_int("DAILY_GITHUB_TOPIC_LIMIT", 10)
    wecom_topic = env_int("DAILY_WECOM_GITHUB_TOPIC", 10)
    return ReportLimits(
        trending_n=trending_n,
        hot_n=hot_n,
        skill_pool=skill_pool,
        wecom_trending=wecom_trending,
        wecom_hot=wecom_hot,
        github_limit=github_limit,
        wecom_github=wecom_github,
        emerging_limit=emerging_limit,
        wecom_emerging=wecom_emerging,
        topic_limit=topic_limit,
        wecom_topic=wecom_topic,
    )


def collect_report_context(now: datetime) -> ReportContext:
    limits = resolve_limits()
    compare_n = compare_depth()
    github_fetch_n = max(limits.github_limit, compare_n, limits.wecom_github)
    emerging_fetch_n = max(limits.emerging_limit, compare_n, limits.wecom_emerging)
    topic_fetch_n = max(limits.topic_limit, compare_n, limits.wecom_topic)

    feed = load_feed(force=True)
    date_str = now.strftime("%Y-%m-%d")
    time_str = now.strftime("%H:%M") + " (UTC+8)"
    updated = (feed.get("updatedAt") or "")[:10]

    trending, hot = load_boards(feed, trending_limit=limits.trending_n, hot_limit=limits.hot_n)
    github_trending = fetch_github_trending(github_fetch_n)
    seen_repos = {r["repo"] for r in github_trending}
    github_emerging = fetch_emerging_repos(emerging_fetch_n, exclude=seen_repos)
    seen_repos.update(r["repo"] for r in github_emerging)
    topic_name, github_topic = fetch_topic_hot_repos(topic_fetch_n, exclude=seen_repos)
    ai_news = apply_news_ranking(fetch_ai_news(), date_str=date_str)
    cn_ai_news = apply_news_ranking(fetch_cn_ai_news(), date_str=date_str)

    wecom_limits = {
        "trending": limits.wecom_trending,
        "hot": limits.wecom_hot,
        "trending_pool": limits.skill_pool,
        "hot_pool": limits.skill_pool,
        "github": limits.wecom_github,
        "emerging": limits.wecom_emerging,
        "topic": limits.wecom_topic,
        "ai_news": env_int("DAILY_WECOM_AI_NEWS", 10),
        "cn_ai_news": env_int("DAILY_WECOM_CN_AI_NEWS", 8),
    }
    llm_input = build_llm_input(
        date_str=date_str,
        updated=updated,
        trending=trending,
        hot=hot,
        github_trending=github_trending,
        github_emerging=github_emerging,
        github_topic=github_topic,
        topic_name=topic_name,
        ai_news=ai_news,
        cn_ai_news=cn_ai_news,
        wecom_limits=wecom_limits,
        agent_mode=is_agent_mode(),
    )
    return ReportContext(
        feed=feed,
        date_str=date_str,
        time_str=time_str,
        updated=updated,
        trending=trending,
        hot=hot,
        github_trending=github_trending,
        github_emerging=github_emerging,
        github_topic=github_topic,
        topic_name=topic_name,
        ai_news=ai_news,
        cn_ai_news=cn_ai_news,
        limits=limits,
        wecom_limits=wecom_limits,
        llm_input=llm_input,
    )