Skip to content

chenyuxiaojin/xiaochen-skills / plugins/cyxj-youtube-topics/skills/cyxj-youtube-topics

cyxj-youtube-topics

Verified

YouTube 选题发现 + 判断。搜索 "Claude Code" 相关最近 48 小时新视频, 去重、按话题聚类、做硬信号 + 字幕内容分析,输出带 verdict(值得做/观望/跟风/跳过)+ 理由 + 差异化切口建议的选题报告。写入 Obsidian 选题库。 触发方式:「YouTube 选题」「找 YouTube 选题」「YouTube 最近有什么」「帮我找找最近的新选题」「跑一下选题发现」「有什么新视频」 边界:只做 YouTube 新视频的选题发现与判断;泛选题、短视频 IP 人群选题、文案式出选题不归本 skill。

FileKindSize
SKILL.mdskill md12.0 KB
backfill_channel_ids.pyscript5.1 KB
chinese_reference.pyscript9.2 KB
paths.pyscript6.9 KB
references/setup.mdmarkdown3.3 KB
requirements.txtmanifest9 B
subtitle_fetcher.pyscript5.8 KB
topic_judge.pyscript8.4 KB
write_topics.pyscript39.5 KB
youtube_search.pyscript29.3 KB

youtube_search.py

View on GitHub
#!/usr/bin/env python3
"""YouTube 选题发现 — 三段式:召回 → 硬过滤 → 排序输出"""

import json
import os
import re
import sys
import time
from datetime import datetime, timedelta, timezone
from functools import lru_cache
from pathlib import Path

import requests

from paths import get_state_dir, get_topic_dir, load_youtube_api_keys

# ── 常量 ──────────────────────────────────────────────

# 第一段:召回关键词(大词 + 受众词 + 功能词 + 内容类型词)
KEYWORDS = [
    "Claude Code",              # 兜底大词
    "Claude Code tutorial",     # 教程类
    "Claude Code beginner",     # 新手向
    "Claude Code plan",         # 功能更新
    "Claude Code skills",       # 功能特性
    "Claude Code agent",        # 热点话题
    "Claude Code MCP",          # 生态热点
    "Claude Code build",        # 实战展示
    "Claude Code update",       # 资讯更新
    "Anthropic Academy",        # A 社官方课程/教育
    "Claude Desktop",           # 桌面版动态
    # Claude 同族(非 Code)—— 补「用 Claude 做X / 课程」这类此前漏网的视频
    "Claude course",
    "build with Claude",
    "Claude for business",
]

# 爆款层:按播放量召回的宽词(少而宽,专抓被 order=date 埋掉的高播放视频)
KEYWORDS_VIEWCOUNT = [
    "Claude Code",
    "Claude",
    "Anthropic Claude",
]
HOURS_WINDOW = max(1, min(168, int(os.environ.get("CYXJ_LOOKBACK_HOURS", "48"))))
MAX_RESULTS_PER_KEYWORD = 50  # 拉满上限,search.list 无论取多少条都扣 100 点
API_BASE = "https://www.googleapis.com/youtube/v3"

# 第二段:硬过滤阈值
MIN_VIEW_COUNT = 100
MIN_DURATION_SECONDS = 180  # 3 分钟(2026-06-02:含信任频道,统一砍 shorts)

# 信任频道:召回阶段单独查询(绕开关键词竞争),并豁免关键词相关性复查;但仍受时长门槛约束(2026-06-02:不再豁免时长,shorts 一律砍)。
# 种子列表保证 Day-1 即生效;之外的创作者由 load_promoted_channels() 从创作者索引自动晋升。
SEED_TRUSTED_CHANNELS = [
    ("UCoy6cTJ7Tg0dqS-DI-_REsA", "Chase AI"),
    ("UC2ojq-nuP8ceeHqiroeKhBA", "Nate Herk"),
    ("UCiZotp9tZ4uXgXEjHDUYzBQ", "John Kim"),
    ("UC_x36zCEGilGpB1m-V4gmjg", "IndyDevDan"),
    ("UCSxPE9PHHxQUEt6ajGmQyMA", "Brian Casel"),
    ("UCxVxcTULO9cFU6SB9qVaisQ", "Jack Roberts"),
    ("UCZRp6-Xvzo_dBFvt9L7y3Qw", "Build Great Products"),
    ("UC4x3CR25WSlvMJUtSPPzwwg", "Chris Raroque"),
    # 2026-06-08 用户手动收编:连续 6 天领头羊榜、未在白名单
    ("UClXAalunTPaX1YV185DWUeg", "Vaibhav Sisinty"),
    ("UCbo-KbSjJDG6JWQ_MTZ_rNA", "Nick Saraev"),
]

# 自动晋升阈值
PROMOTION_MIN_VIDEOS = 1     # 至少召回过 1 条
PROMOTION_MAX_CHANNELS = 50  # 上限防失控

# 信任频道直查每个频道拉视频上限
TRUSTED_MAX_RESULTS_PER_CHANNEL = 20
APIFY_SEARCH_ACTOR = "streamers~youtube-scraper"

# 噪音标题关键词(大小写不敏感)
NOISE_TITLE_WORDS = re.compile(
    r"\b(shorts?|clip|highlights?|teaser|trailer|livestream|live\s*stream|"
    r"live\s*coding\s*stream|replay|stream\s*archive)\b",
    re.IGNORECASE,
)

# 非英文字符集
NON_ENGLISH_PATTERN = re.compile(
    r"[\u4e00-\u9fff\u3040-\u30ff\uac00-\ud7af\u0400-\u04ff"
    r"\u0600-\u06ff\u0e00-\u0e7f\u0900-\u097f]"
)

STATE_DIR = get_state_dir()           # 本地非同步:状态文件(与 write_topics.py 同源)
SEEN_IDS_PATH = STATE_DIR / ".seen_video_ids.json"


@lru_cache(maxsize=1)
def _topic_dir() -> Path:
    """iCloud / Obsidian:.md 总览(去重兜底扫这里)。
    惰性求值:未设 CYXJ_TOPIC_DIR 时 get_topic_dir() 会 sys.exit,
    推迟到真正用到才求值,保证 `import youtube_search`(如 chinese_reference.py)不会因此退出。"""
    return get_topic_dir()

# 匹配 YouTube URL 中的 11 位 Video ID
VIDEO_ID_PATTERN = re.compile(
    r"(?:youtube\.com/watch\?v=|youtu\.be/|youtube\.com/shorts/|youtube\.com/embed/)"
    r"([0-9A-Za-z_-]{11})"
)


# ── 多 key 轮询 ───────────────────────────────────────

QUOTA_REASONS = {"quotaExceeded", "dailyLimitExceeded", "rateLimitExceeded", "userRateLimitExceeded"}


class KeyRotator:
    """YouTube API key 轮询器。current 返回当前 key,advance() 切下一个,耗尽返回 False。"""

    def __init__(self, keys: list[str]):
        if not keys:
            raise ValueError("KeyRotator 至少需要 1 个 key")
        self.keys = keys
        self.idx = 0

    @property
    def current(self) -> str:
        return self.keys[self.idx]

    def advance(self) -> bool:
        if self.idx + 1 >= len(self.keys):
            return False
        self.idx += 1
        return True

    def __bool__(self) -> bool:
        return True

    def __len__(self) -> int:
        return len(self.keys)


def _is_quota_error(resp: requests.Response) -> bool:
    # 429 = 短期 QPS 限流;不同 GCP 项目的 key 配额独立,切 key 能绕开
    if resp.status_code == 429:
        return True
    if resp.status_code != 403:
        return False
    try:
        errors = resp.json().get("error", {}).get("errors", [])
        return any(e.get("reason") in QUOTA_REASONS for e in errors)
    except Exception:
        return False


# 网络层退避重试的间隔(秒)。2026-07-04 教训:代理未就绪时 SSL 握手被掐,
# 14 个关键词 40 秒内连败——之前一次异常就判死该词,一次退避都没给。
NETWORK_RETRY_DELAYS = [5, 20]


def _mask_keys(text: str) -> str:
    """把错误信息里的 API key 打码。requests 异常自带完整 URL(含 key=...),
    不打码会明文进当天日志和 failures/ 失败现场(2026-07-04 实锤泄漏)。"""
    return re.sub(r"key=[A-Za-z0-9_-]{10,}", "key=***", text)


def youtube_get(rotator: KeyRotator, endpoint: str, params: dict, **kwargs) -> requests.Response:
    """带配额轮询 + 网络退避重试的 GET。

    - 网络层异常(连接/SSL/超时)按 NETWORK_RETRY_DELAYS 退避重试,仍失败抛 key 已打码的 RuntimeError;
    - 403 quotaExceeded / 429 时切下一个 key 重试,所有 key 耗尽则抛(HTTP 错误信息同样打码)。"""
    while True:
        params["key"] = rotator.current
        resp = None
        for attempt in range(len(NETWORK_RETRY_DELAYS) + 1):
            try:
                resp = requests.get(f"{API_BASE}{endpoint}", params=params, **kwargs)
                break
            except (requests.ConnectionError, requests.Timeout) as e:
                if attempt >= len(NETWORK_RETRY_DELAYS):
                    # from None:不链原始异常,避免未打码的 URL 随 traceback 泄漏
                    raise RuntimeError(_mask_keys(f"{type(e).__name__}: {e}")) from None
                delay = NETWORK_RETRY_DELAYS[attempt]
                print(
                    f"警告:{endpoint} 网络异常({_mask_keys(str(e))[:120]}),{delay}s 后重试"
                    f"(第 {attempt + 2}/{len(NETWORK_RETRY_DELAYS) + 1} 次)",
                    file=sys.stderr,
                )
                time.sleep(delay)
        if _is_quota_error(resp):
            print(
                f"警告:key #{rotator.idx + 1} 配额耗尽,尝试切换下一个 key", file=sys.stderr
            )
            if rotator.advance():
                continue
            print(
                f"错误:所有 {len(rotator)} 个 YouTube API key 都已耗尽配额", file=sys.stderr
            )
        try:
            resp.raise_for_status()
        except requests.HTTPError as e:
            raise RuntimeError(_mask_keys(str(e))) from None
        return resp


# ── 工具函数 ──────────────────────────────────────────

def parse_duration(duration_str: str) -> int:
    match = re.match(r"PT(?:(\d+)H)?(?:(\d+)M)?(?:(\d+)S)?", duration_str)
    if not match:
        return 0
    return int(match.group(1) or 0) * 3600 + int(match.group(2) or 0) * 60 + int(match.group(3) or 0)


def format_relative_time(published_at: str) -> str:
    pub_time = datetime.fromisoformat(published_at.replace("Z", "+00:00"))
    diff = datetime.now(timezone.utc) - pub_time
    hours = int(diff.total_seconds() / 3600)
    if hours < 1:
        return "刚刚"
    if hours < 24:
        return f"{hours}小时前"
    return f"{hours // 24}天前"


def format_view_count(count: int) -> str:
    if count >= 10000:
        return f"{count / 10000:.1f}万"
    if count >= 1000:
        return f"{count / 1000:.1f}千"
    return str(count)


# ── 第一段:召回 ──────────────────────────────────────

def recall(
    rotator: KeyRotator,
    published_after: str,
    keywords: list[str],
    order: str = "date",
    pages: int = 1,
    critical: bool = True,
) -> tuple[list[dict], int]:
    """多关键词搜索,尽量多拿候选,按 Video ID 去重(本次调用内)。
    order 控制排序(date / viewCount);pages>1 时按 nextPageToken 翻页。
    critical=True 时「全部关键词都首页失败」会 sys.exit(主召回用,判 API 死);
    补充层(deep/viewCount)应传 critical=False,单点抖动不拖垮整条流水线。
    返回 (候选列表, 本函数发起的 search.list 调用次数)。"""
    all_videos = []
    seen_ids = set()
    failed_keywords = []
    search_call_count = 0

    for keyword in keywords:
        page_token = None
        for _ in range(max(1, pages)):
            params = {
                "q": keyword,
                "part": "snippet",
                "type": "video",
                "order": order,
                "publishedAfter": published_after,
                "relevanceLanguage": "en",
                "maxResults": MAX_RESULTS_PER_KEYWORD,
            }
            if page_token:
                params["pageToken"] = page_token
            try:
                resp = youtube_get(rotator, "/search", params, timeout=30)
                search_call_count += 1
            except Exception as e:
                print(f"警告:关键词 '{keyword}' 搜索失败 ({e})", file=sys.stderr)
                # 仅首页就失败才算该词「颗粒无收」;翻页中途失败(page_token 已有)不算整词失败
                if page_token is None:
                    failed_keywords.append(keyword)
                break

            data = resp.json()
            for item in data.get("items", []):
                video_id = item["id"]["videoId"]
                if video_id in seen_ids:
                    continue
                seen_ids.add(video_id)
                snippet = item["snippet"]
                all_videos.append({
                    "video_id": video_id,
                    "title": snippet["title"],
                    "channel": snippet["channelTitle"],
                    "channel_id": snippet.get("channelId", ""),
                    "description": snippet.get("description", ""),
                    "published_at": snippet["publishedAt"],
                    "url": f"https://www.youtube.com/watch?v={video_id}",
                })

            page_token = data.get("nextPageToken")
            if not page_token:
                break

    if critical and keywords and len(failed_keywords) == len(keywords):
        print(f"错误:所有 {len(keywords)} 个关键词搜索全部失败,疑似 API key 失效或网络问题", file=sys.stderr)
        sys.exit(1)

    print(f"召回(order={order},{len(keywords)} 词,{search_call_count} 次调用):{len(all_videos)} 个候选", file=sys.stderr)
    return all_videos, search_call_count


# ── 第二段:硬过滤 ──────────────────────────────────────

def enrich_and_filter(rotator: KeyRotator | None, videos: list[dict]) -> list[dict]:
    """获取详情 + 硬过滤:语言、时长、播放量、噪音标题。
    已带 duration_seconds + view_count 的视频(如 Apify backend 召回的)跳过 videos.list。"""
    if not videos:
        return []

    # 只对缺少 stats/duration 的视频调 videos.list(节省 quota)
    need_enrich = [
        v for v in videos
        if "duration_seconds" not in v or "view_count" not in v
    ]
    stats_map = {}
    duration_map = {}
    lang_map = {}

    if need_enrich and rotator:
        video_ids = [v["video_id"] for v in need_enrich]
        for i in range(0, len(video_ids), 50):
            batch = video_ids[i:i + 50]
            resp = youtube_get(rotator, "/videos", {
                "part": "statistics,contentDetails,snippet",
                "id": ",".join(batch),
            }, timeout=30)
            for item in resp.json().get("items", []):
                vid = item["id"]
                stats_map[vid] = int(item["statistics"].get("viewCount", 0))
                duration_map[vid] = parse_duration(item["contentDetails"].get("duration", "PT0S"))
                snippet = item.get("snippet", {})
                lang = snippet.get("defaultAudioLanguage") or snippet.get("defaultLanguage") or ""
                lang_map[vid] = lang.lower()

    for video in videos:
        # 已带字段就保留,否则用 videos.list 拿到的值;都没有就 0/空
        video["view_count"] = video.get("view_count", stats_map.get(video["video_id"], 0))
        video["duration_seconds"] = video.get("duration_seconds", duration_map.get(video["video_id"], 0))
        video["language"] = video.get("language", lang_map.get(video["video_id"], ""))

    # 硬过滤
    filtered = []
    for v in videos:
        title = v["title"]

        # 相关性过滤:标题或描述必须包含相关关键词(不区分大小写)。
        # 信任频道豁免——已白名单的创作者不再做关键词复查(与下方时长豁免同理):
        # search.list 返回的描述是截断的,开头若是推广链接会把关键词挤出截断段,
        # 导致标题不含关键词的相关视频(如 Nate Herk 的发布解读)被误砍。真跑偏的
        # 视频仍由下游 LLM 聚类/判断阶段拦截。
        text = (title + " " + v.get("description", "")).lower()
        if v.get("source") != "trusted_channel" and not any(
            kw in text for kw in ("claude", "anthropic")
        ):
            continue

        # 语言过滤:只保留英语(en, en-US, en-GB 等)或未标注语言的视频
        lang = v.get("language", "")
        if lang and not lang.startswith("en"):
            continue

        # 非英文字符集(兜底:拦截 CJK、俄文等未标注语言的非英文视频)
        if NON_ENGLISH_PATTERN.search(title):
            continue

        # 噪音标题
        if NOISE_TITLE_WORDS.search(title):
            continue

        # 时长 < 3 分钟:一律砍(含信任频道;shorts 对选题无价值——2026-06-02 移除豁免)
        if v["duration_seconds"] < MIN_DURATION_SECONDS:
            continue

        # 播放量 < 100
        if v["view_count"] < MIN_VIEW_COUNT:
            continue

        filtered.append(v)

    print(f"硬过滤后:{len(filtered)} 个", file=sys.stderr)
    return filtered


# ── 去重 ──────────────────────────────────────────────

def load_seen_ids() -> set[str]:
    """从独立索引文件加载已见过的 Video ID,md 扫描作为兜底"""
    seen = set()

    # 主索引:seen_video_ids.json
    if SEEN_IDS_PATH.exists():
        try:
            seen.update(json.loads(SEEN_IDS_PATH.read_text(encoding="utf-8")))
        except Exception:
            pass

    # 兜底:扫描选题库 markdown
    topic_dir = _topic_dir()
    if topic_dir.exists():
        for md_file in topic_dir.glob("**/*.md"):
            try:
                content = md_file.read_text(encoding="utf-8")
                seen.update(VIDEO_ID_PATTERN.findall(content))
            except Exception as e:
                print(f"警告:无法读取 {md_file.name}({e})", file=sys.stderr)

    return seen


# ── 第三段:排序 + 输出 ──────────────────────────────────

def sort_and_output(videos: list[dict]) -> list[dict]:
    """按播放量降序排列,添加格式化字段"""
    videos.sort(key=lambda v: v["view_count"], reverse=True)

    for v in videos:
        v["relative_time"] = format_relative_time(v["published_at"])
        v["view_count_formatted"] = format_view_count(v["view_count"])
        mins, secs = divmod(v["duration_seconds"], 60)
        v["duration_formatted"] = f"{mins}分{secs}秒" if secs else f"{mins}分钟"
        pub = datetime.fromisoformat(v["published_at"].replace("Z", "+00:00"))
        hours = max((datetime.now(timezone.utc) - pub).total_seconds() / 3600, 1)
        v["view_velocity"] = round(v["view_count"] / hours, 1)

    return videos


# ── 信任频道直查 ──────────────────────────────────────

def _load_creator_index() -> dict:
    """从本地状态目录读创作者索引,找不到返回空字典。"""
    p = STATE_DIR / "创作者索引.json"
    if not p.exists():
        return {}
    try:
        data = json.loads(p.read_text(encoding="utf-8"))
        return data.get("creators", {})
    except Exception as e:
        print(f"警告:读创作者索引失败({e}),跳过自动晋升", file=sys.stderr)
        return {}


def load_promoted_channels() -> list[tuple[str, str]]:
    """自动晋升:从创作者索引筛 is_quality=True AND total_videos≥N AND channel_id 非空,
    按 avg_views 降序取前 PROMOTION_MAX_CHANNELS 个。"""
    creators = _load_creator_index()
    candidates = []
    for name, c in creators.items():
        if not c.get("is_quality"):
            continue
        if c.get("total_videos", 0) < PROMOTION_MIN_VIDEOS:
            continue
        cid = c.get("channel_id", "")
        if not cid:
            continue
        candidates.append((cid, name, c.get("avg_views", 0)))
    candidates.sort(key=lambda x: -x[2])
    return [(cid, name) for cid, name, _ in candidates[:PROMOTION_MAX_CHANNELS]]


def get_trusted_channels() -> list[tuple[str, str]]:
    """合并种子 + 自动晋升,按 channel_id 去重(种子优先保留名字)。"""
    seen = set()
    out = []
    for cid, name in SEED_TRUSTED_CHANNELS:
        if cid in seen:
            continue
        seen.add(cid)
        out.append((cid, name))
    for cid, name in load_promoted_channels():
        if cid in seen:
            continue
        seen.add(cid)
        out.append((cid, name))
    return out


def _parse_apify_duration(value) -> int:
    """Apify 的 duration 可能是 'mm:ss' / 'hh:mm:ss' 字符串或 lengthSeconds 数字。"""
    if isinstance(value, (int, float)):
        return int(value)
    if isinstance(value, str):
        parts = value.strip().split(":")
        try:
            nums = [int(p) for p in parts]
        except ValueError:
            return 0
        if len(nums) == 3:
            return nums[0] * 3600 + nums[1] * 60 + nums[2]
        if len(nums) == 2:
            return nums[0] * 60 + nums[1]
        if len(nums) == 1:
            return nums[0]
    return 0


def _trusted_recall_youtube_api(
    rotator: KeyRotator, published_after: str, channels: list[tuple[str, str]]
) -> list[dict]:
    """生产 backend:每个频道单独调一次 search.list?channelId=X。每个频道 100 配额。"""
    out = []
    failed = 0
    for cid, name in channels:
        try:
            resp = youtube_get(rotator, "/search", {
                "channelId": cid,
                "part": "snippet",
                "type": "video",
                "order": "date",
                "publishedAfter": published_after,
                "maxResults": TRUSTED_MAX_RESULTS_PER_CHANNEL,
            }, timeout=30)
        except Exception as e:
            print(f"警告:信任频道 '{name}' ({cid}) 拉取失败 ({e})", file=sys.stderr)
            failed += 1
            continue
        for item in resp.json().get("items", []):
            try:
                vid = item["id"]["videoId"]
            except (KeyError, TypeError):
                continue
            snippet = item["snippet"]
            out.append({
                "video_id": vid,
                "title": snippet["title"],
                "channel": snippet.get("channelTitle") or name,
                "channel_id": snippet.get("channelId") or cid,
                "description": snippet.get("description", ""),
                "published_at": snippet["publishedAt"],
                "url": f"https://www.youtube.com/watch?v={vid}",
                "source": "trusted_channel",
            })
    print(
        f"信任频道直查(YouTube API):{len(channels)} 频道,{failed} 失败,召回 {len(out)} 条",
        file=sys.stderr,
    )
    return out


APIFY_BATCH_SIZE = 4  # Apify sync 单次 5 分钟超时,4 频道 × ~60s = 安全


def _apify_recall_batch(
    token: str, batch: list[tuple[str, str]], published_after: str,
    cid_to_name: dict[str, str],
) -> list[dict]:
    """单批 Apify 调用:≤APIFY_BATCH_SIZE 个频道 URL 一次性提交。"""
    apify_url = f"https://api.apify.com/v2/acts/{APIFY_SEARCH_ACTOR}/run-sync-get-dataset-items"
    payload = {
        "startUrls": [
            {"url": f"https://www.youtube.com/channel/{cid}/videos"}
            for cid, _ in batch
        ],
        "maxResults": TRUSTED_MAX_RESULTS_PER_CHANNEL,
        "maxResultsShorts": 0,
        "maxResultStreams": 0,
        "dateFilter": "week",
        "videoType": "video",
        "downloadSubtitles": False,
    }
    try:
        resp = requests.post(
            apify_url,
            params={"token": token, "timeout": "300"},
            json=payload,
            timeout=360,
        )
        resp.raise_for_status()
        items = resp.json()
        if not isinstance(items, list):
            raise RuntimeError(f"Apify 返回非列表: {type(items)}")
    except Exception as e:
        print(f"警告:Apify 批 {[n for _, n in batch]} 失败 ({e})", file=sys.stderr)
        return []

    out = []
    for item in items:
        video_url = item.get("url") or item.get("videoUrl") or ""
        m = VIDEO_ID_PATTERN.search(video_url)
        vid = m.group(1) if m else (item.get("id") or "")
        if not vid or len(vid) != 11:
            continue
        channel_url = item.get("channelUrl") or ""
        cid_match = re.search(r"/channel/(UC[A-Za-z0-9_-]{22})", channel_url)
        cid = cid_match.group(1) if cid_match else (item.get("channelId") or "")
        if cid not in cid_to_name:
            continue  # 跳过非本批信任频道的视频
        duration_s = _parse_apify_duration(item.get("lengthSeconds") or item.get("duration"))
        view_count = int(item.get("viewCount") or 0)
        pub_raw = item.get("date") or item.get("publishedAt") or ""
        if pub_raw:
            # 用 datetime 解析后再比较——Apify 的 date 格式不保证是 ISO,
            # 字符串直接比会静默误判。解析失败按「保留」处理,交给下游过滤。
            try:
                pub_dt = datetime.fromisoformat(pub_raw.replace("Z", "+00:00"))
                if pub_dt.tzinfo is None:
                    pub_dt = pub_dt.replace(tzinfo=timezone.utc)
                cutoff_dt = datetime.fromisoformat(published_after.replace("Z", "+00:00"))
                if pub_dt < cutoff_dt:
                    continue
            except (ValueError, TypeError):
                print(
                    f"警告:无法解析发布时间 '{pub_raw}'({vid}),按保留处理",
                    file=sys.stderr,
                )
        out.append({
            "video_id": vid,
            "title": item.get("title", ""),
            "channel": item.get("channelName") or cid_to_name.get(cid, ""),
            "channel_id": cid,
            "description": item.get("text") or item.get("description") or "",
            "published_at": pub_raw,
            "url": video_url or f"https://www.youtube.com/watch?v={vid}",
            "duration_seconds": duration_s,
            "view_count": view_count,
            "language": "",
            "source": "trusted_channel",
        })
    return out


def _trusted_recall_apify(
    published_after: str, channels: list[tuple[str, str]]
) -> list[dict]:
    """测试 backend:不烧 YouTube quota,用 Apify streamers/youtube-scraper。
    分批调用(每批 APIFY_BATCH_SIZE 个频道)避免 Apify sync 5 分钟超时。
    返回 dict 已包含 duration/views,后续 enrich_and_filter 会跳过 videos.list。"""
    from paths import load_apify_token
    token = load_apify_token()
    cid_to_name = {cid: name for cid, name in channels}

    out = []
    failed_batches = 0
    for i in range(0, len(channels), APIFY_BATCH_SIZE):
        batch = channels[i:i + APIFY_BATCH_SIZE]
        print(f"  Apify 批 {i // APIFY_BATCH_SIZE + 1}:{[n for _, n in batch]}", file=sys.stderr)
        batch_out = _apify_recall_batch(token, batch, published_after, cid_to_name)
        if not batch_out:
            failed_batches += 1
        out.extend(batch_out)

    print(
        f"信任频道直查(Apify 分批):{len(channels)} 频道分 "
        f"{(len(channels) + APIFY_BATCH_SIZE - 1) // APIFY_BATCH_SIZE} 批,"
        f"{failed_batches} 批失败,召回 {len(out)} 条",
        file=sys.stderr,
    )
    return out


def recall_from_trusted_channels(rotator: KeyRotator | None, published_after: str) -> list[dict]:
    """白名单(种子)频道直查。env CYXJ_TRUSTED_BACKEND 切换:youtube_api(默认)/ apify(测试)。

    2026-06-02:直查范围从「种子 + 自动晋升」收窄为【仅种子白名单】。自动晋升频道不再获抓取
    特权(直查 + 关键词豁免)——它只作为「领头羊榜」的排名展示(见 write_topics.py),不再翻它
    的频道库存。这样只有种子视频带 source=trusted_channel,关键词豁免也只对种子生效。
    (load_promoted_channels / get_trusted_channels 已退出抓取链路,保留仅备查。)"""
    channels = list(SEED_TRUSTED_CHANNELS)
    if not channels:
        return []
    backend = os.environ.get("CYXJ_TRUSTED_BACKEND", "youtube_api")
    print(
        f"白名单直查:{len(channels)} 个种子频道,backend={backend}",
        file=sys.stderr,
    )
    if backend == "apify":
        return _trusted_recall_apify(published_after, channels)
    return _trusted_recall_youtube_api(rotator, published_after, channels)


# ── 入口 ──────────────────────────────────────────────

def main():
    api_key_required = os.environ.get("CYXJ_TRUSTED_BACKEND", "youtube_api") != "apify"
    rotator: KeyRotator | None = None
    if api_key_required:
        keys = load_youtube_api_keys()
        rotator = KeyRotator(keys)
        print(f"YouTube API key 池:共 {len(keys)} 个,按序号轮询", file=sys.stderr)
    published_after = (
        datetime.now(timezone.utc) - timedelta(hours=HOURS_WINDOW)
    ).strftime("%Y-%m-%dT%H:%M:%SZ")

    # 1a. 关键词召回(Apify 模式下不跑——避免烧 YouTube quota)
    if rotator:
        date_videos, n_date = recall(rotator, published_after, KEYWORDS, order="date")
        deep_videos, n_deep = recall(rotator, published_after, ["Claude Code"], order="date", pages=2, critical=False)
        view_videos, n_view = recall(rotator, published_after, KEYWORDS_VIEWCOUNT, order="viewCount", critical=False)
        merged: dict[str, dict] = {}
        for v in date_videos + deep_videos + view_videos:
            merged.setdefault(v["video_id"], v)
        keyword_candidates = list(merged.values())
        for v in keyword_candidates:
            v["source"] = "keyword"
        print(f"召回搜索调用次数:date={n_date} + deep={n_deep} + viewCount={n_view} = {n_date + n_deep + n_view}", file=sys.stderr)
    else:
        print("信号:Apify 模式跳过关键词召回(避免烧 YouTube quota)", file=sys.stderr)
        keyword_candidates = []

    # 1b. 信任频道直查
    trusted_candidates = recall_from_trusted_channels(rotator, published_after)

    # 1c. 合并:video_id 去重,trusted 覆盖 keyword(保留 trusted source)
    by_id: dict[str, dict] = {}
    for v in keyword_candidates + trusted_candidates:
        existing = by_id.get(v["video_id"])
        if existing is None or v.get("source") == "trusted_channel":
            by_id[v["video_id"]] = v
    candidates = list(by_id.values())
    print(f"召回合并:{len(candidates)} 个候选(去重后)", file=sys.stderr)

    # 2. 硬过滤
    clean = enrich_and_filter(rotator, candidates)

    # 3. 去重
    seen_ids = load_seen_ids()
    new_videos = [v for v in clean if v["video_id"] not in seen_ids]
    print(f"去重后:{len(new_videos)} 个新视频", file=sys.stderr)

    # 4. 排序 + 输出
    #    注意:.seen_video_ids.json 不在这里更新——必须等 write_topics.py 成功写入
    #    总览文件后才能标记"已处理",否则中途失败会丢视频。
    result = sort_and_output(new_videos)

    print(json.dumps(result, ensure_ascii=False, indent=2))


if __name__ == "__main__":
    main()
reverse-engineer-anythingmorluto/reaReverse engineer native, managed, Electron/JavaScript, packaged, firmware, and browser targets with REA, and build agent environments (AgentEnv bundles) of web applications from a live instance or a capture, and of Linux desktop apps with a task from a request. Use shipped-artifact or requested runtime evidence to explain features, compare versions, decompile code, or guide a reconstruction. Skip REA for ordinary source-repository architecture analysis.Stars: 472kUpdated Claude CodeCodexVerifiedInstall
diagnosing-superpowersobra/superpowersUse when a superpowers session went wrong and your human partner wants to know why — repeated work, ignored plans, stumbles, poor results, a skill that didn't fire, "it took too long", "why is it so expensive", "what is it doing" — or wants to build a bug report for the superpowers maintainers, for the current session or a past one identified by id or path, on any harness.Stars: 323kUpdated Claude CodeCodexVerifiedInstall
dispatching-parallel-agentsobra/superpowersUse when facing 2+ independent tasks that can be worked on without shared state or sequential dependenciesStars: 323kUpdated Claude CodeCodexVerifiedInstall
executing-plansobra/superpowersUse when executing an implementation plan in the current session as the implementer yourself — your human partner chose inline execution, or no subagent tool is availableStars: 323kUpdated Claude CodeCodexVerifiedInstall