chenyuxiaojin/xiaochen-skills / plugins/cyxj-youtube-topics/skills/cyxj-youtube-topics
cyxj-youtube-topics
VerifiedYouTube 选题发现 + 判断。搜索 "Claude Code" 相关最近 48 小时新视频, 去重、按话题聚类、做硬信号 + 字幕内容分析,输出带 verdict(值得做/观望/跟风/跳过)+ 理由 + 差异化切口建议的选题报告。写入 Obsidian 选题库。 触发方式:「YouTube 选题」「找 YouTube 选题」「YouTube 最近有什么」「帮我找找最近的新选题」「跑一下选题发现」「有什么新视频」 边界:只做 YouTube 新视频的选题发现与判断;泛选题、短视频 IP 人群选题、文案式出选题不归本 skill。
| File | Kind | Size |
|---|---|---|
| SKILL.md | skill md | 12.0 KB |
| backfill_channel_ids.py | script | 5.1 KB |
| chinese_reference.py | script | 9.2 KB |
| paths.py | script | 6.9 KB |
| references/setup.md | markdown | 3.3 KB |
| requirements.txt | manifest | 9 B |
| subtitle_fetcher.py | script | 5.8 KB |
| topic_judge.py | script | 8.4 KB |
| write_topics.py | script | 39.5 KB |
| youtube_search.py | script | 29.3 KB |
youtube_search.py
View on GitHub#!/usr/bin/env python3
"""YouTube 选题发现 — 三段式:召回 → 硬过滤 → 排序输出"""
import json
import os
import re
import sys
import time
from datetime import datetime, timedelta, timezone
from functools import lru_cache
from pathlib import Path
import requests
from paths import get_state_dir, get_topic_dir, load_youtube_api_keys
# ── 常量 ──────────────────────────────────────────────
# 第一段:召回关键词(大词 + 受众词 + 功能词 + 内容类型词)
KEYWORDS = [
"Claude Code", # 兜底大词
"Claude Code tutorial", # 教程类
"Claude Code beginner", # 新手向
"Claude Code plan", # 功能更新
"Claude Code skills", # 功能特性
"Claude Code agent", # 热点话题
"Claude Code MCP", # 生态热点
"Claude Code build", # 实战展示
"Claude Code update", # 资讯更新
"Anthropic Academy", # A 社官方课程/教育
"Claude Desktop", # 桌面版动态
# Claude 同族(非 Code)—— 补「用 Claude 做X / 课程」这类此前漏网的视频
"Claude course",
"build with Claude",
"Claude for business",
]
# 爆款层:按播放量召回的宽词(少而宽,专抓被 order=date 埋掉的高播放视频)
KEYWORDS_VIEWCOUNT = [
"Claude Code",
"Claude",
"Anthropic Claude",
]
HOURS_WINDOW = max(1, min(168, int(os.environ.get("CYXJ_LOOKBACK_HOURS", "48"))))
MAX_RESULTS_PER_KEYWORD = 50 # 拉满上限,search.list 无论取多少条都扣 100 点
API_BASE = "https://www.googleapis.com/youtube/v3"
# 第二段:硬过滤阈值
MIN_VIEW_COUNT = 100
MIN_DURATION_SECONDS = 180 # 3 分钟(2026-06-02:含信任频道,统一砍 shorts)
# 信任频道:召回阶段单独查询(绕开关键词竞争),并豁免关键词相关性复查;但仍受时长门槛约束(2026-06-02:不再豁免时长,shorts 一律砍)。
# 种子列表保证 Day-1 即生效;之外的创作者由 load_promoted_channels() 从创作者索引自动晋升。
SEED_TRUSTED_CHANNELS = [
("UCoy6cTJ7Tg0dqS-DI-_REsA", "Chase AI"),
("UC2ojq-nuP8ceeHqiroeKhBA", "Nate Herk"),
("UCiZotp9tZ4uXgXEjHDUYzBQ", "John Kim"),
("UC_x36zCEGilGpB1m-V4gmjg", "IndyDevDan"),
("UCSxPE9PHHxQUEt6ajGmQyMA", "Brian Casel"),
("UCxVxcTULO9cFU6SB9qVaisQ", "Jack Roberts"),
("UCZRp6-Xvzo_dBFvt9L7y3Qw", "Build Great Products"),
("UC4x3CR25WSlvMJUtSPPzwwg", "Chris Raroque"),
# 2026-06-08 用户手动收编:连续 6 天领头羊榜、未在白名单
("UClXAalunTPaX1YV185DWUeg", "Vaibhav Sisinty"),
("UCbo-KbSjJDG6JWQ_MTZ_rNA", "Nick Saraev"),
]
# 自动晋升阈值
PROMOTION_MIN_VIDEOS = 1 # 至少召回过 1 条
PROMOTION_MAX_CHANNELS = 50 # 上限防失控
# 信任频道直查每个频道拉视频上限
TRUSTED_MAX_RESULTS_PER_CHANNEL = 20
APIFY_SEARCH_ACTOR = "streamers~youtube-scraper"
# 噪音标题关键词(大小写不敏感)
NOISE_TITLE_WORDS = re.compile(
r"\b(shorts?|clip|highlights?|teaser|trailer|livestream|live\s*stream|"
r"live\s*coding\s*stream|replay|stream\s*archive)\b",
re.IGNORECASE,
)
# 非英文字符集
NON_ENGLISH_PATTERN = re.compile(
r"[\u4e00-\u9fff\u3040-\u30ff\uac00-\ud7af\u0400-\u04ff"
r"\u0600-\u06ff\u0e00-\u0e7f\u0900-\u097f]"
)
STATE_DIR = get_state_dir() # 本地非同步:状态文件(与 write_topics.py 同源)
SEEN_IDS_PATH = STATE_DIR / ".seen_video_ids.json"
@lru_cache(maxsize=1)
def _topic_dir() -> Path:
"""iCloud / Obsidian:.md 总览(去重兜底扫这里)。
惰性求值:未设 CYXJ_TOPIC_DIR 时 get_topic_dir() 会 sys.exit,
推迟到真正用到才求值,保证 `import youtube_search`(如 chinese_reference.py)不会因此退出。"""
return get_topic_dir()
# 匹配 YouTube URL 中的 11 位 Video ID
VIDEO_ID_PATTERN = re.compile(
r"(?:youtube\.com/watch\?v=|youtu\.be/|youtube\.com/shorts/|youtube\.com/embed/)"
r"([0-9A-Za-z_-]{11})"
)
# ── 多 key 轮询 ───────────────────────────────────────
QUOTA_REASONS = {"quotaExceeded", "dailyLimitExceeded", "rateLimitExceeded", "userRateLimitExceeded"}
class KeyRotator:
"""YouTube API key 轮询器。current 返回当前 key,advance() 切下一个,耗尽返回 False。"""
def __init__(self, keys: list[str]):
if not keys:
raise ValueError("KeyRotator 至少需要 1 个 key")
self.keys = keys
self.idx = 0
@property
def current(self) -> str:
return self.keys[self.idx]
def advance(self) -> bool:
if self.idx + 1 >= len(self.keys):
return False
self.idx += 1
return True
def __bool__(self) -> bool:
return True
def __len__(self) -> int:
return len(self.keys)
def _is_quota_error(resp: requests.Response) -> bool:
# 429 = 短期 QPS 限流;不同 GCP 项目的 key 配额独立,切 key 能绕开
if resp.status_code == 429:
return True
if resp.status_code != 403:
return False
try:
errors = resp.json().get("error", {}).get("errors", [])
return any(e.get("reason") in QUOTA_REASONS for e in errors)
except Exception:
return False
# 网络层退避重试的间隔(秒)。2026-07-04 教训:代理未就绪时 SSL 握手被掐,
# 14 个关键词 40 秒内连败——之前一次异常就判死该词,一次退避都没给。
NETWORK_RETRY_DELAYS = [5, 20]
def _mask_keys(text: str) -> str:
"""把错误信息里的 API key 打码。requests 异常自带完整 URL(含 key=...),
不打码会明文进当天日志和 failures/ 失败现场(2026-07-04 实锤泄漏)。"""
return re.sub(r"key=[A-Za-z0-9_-]{10,}", "key=***", text)
def youtube_get(rotator: KeyRotator, endpoint: str, params: dict, **kwargs) -> requests.Response:
"""带配额轮询 + 网络退避重试的 GET。
- 网络层异常(连接/SSL/超时)按 NETWORK_RETRY_DELAYS 退避重试,仍失败抛 key 已打码的 RuntimeError;
- 403 quotaExceeded / 429 时切下一个 key 重试,所有 key 耗尽则抛(HTTP 错误信息同样打码)。"""
while True:
params["key"] = rotator.current
resp = None
for attempt in range(len(NETWORK_RETRY_DELAYS) + 1):
try:
resp = requests.get(f"{API_BASE}{endpoint}", params=params, **kwargs)
break
except (requests.ConnectionError, requests.Timeout) as e:
if attempt >= len(NETWORK_RETRY_DELAYS):
# from None:不链原始异常,避免未打码的 URL 随 traceback 泄漏
raise RuntimeError(_mask_keys(f"{type(e).__name__}: {e}")) from None
delay = NETWORK_RETRY_DELAYS[attempt]
print(
f"警告:{endpoint} 网络异常({_mask_keys(str(e))[:120]}),{delay}s 后重试"
f"(第 {attempt + 2}/{len(NETWORK_RETRY_DELAYS) + 1} 次)",
file=sys.stderr,
)
time.sleep(delay)
if _is_quota_error(resp):
print(
f"警告:key #{rotator.idx + 1} 配额耗尽,尝试切换下一个 key", file=sys.stderr
)
if rotator.advance():
continue
print(
f"错误:所有 {len(rotator)} 个 YouTube API key 都已耗尽配额", file=sys.stderr
)
try:
resp.raise_for_status()
except requests.HTTPError as e:
raise RuntimeError(_mask_keys(str(e))) from None
return resp
# ── 工具函数 ──────────────────────────────────────────
def parse_duration(duration_str: str) -> int:
match = re.match(r"PT(?:(\d+)H)?(?:(\d+)M)?(?:(\d+)S)?", duration_str)
if not match:
return 0
return int(match.group(1) or 0) * 3600 + int(match.group(2) or 0) * 60 + int(match.group(3) or 0)
def format_relative_time(published_at: str) -> str:
pub_time = datetime.fromisoformat(published_at.replace("Z", "+00:00"))
diff = datetime.now(timezone.utc) - pub_time
hours = int(diff.total_seconds() / 3600)
if hours < 1:
return "刚刚"
if hours < 24:
return f"{hours}小时前"
return f"{hours // 24}天前"
def format_view_count(count: int) -> str:
if count >= 10000:
return f"{count / 10000:.1f}万"
if count >= 1000:
return f"{count / 1000:.1f}千"
return str(count)
# ── 第一段:召回 ──────────────────────────────────────
def recall(
rotator: KeyRotator,
published_after: str,
keywords: list[str],
order: str = "date",
pages: int = 1,
critical: bool = True,
) -> tuple[list[dict], int]:
"""多关键词搜索,尽量多拿候选,按 Video ID 去重(本次调用内)。
order 控制排序(date / viewCount);pages>1 时按 nextPageToken 翻页。
critical=True 时「全部关键词都首页失败」会 sys.exit(主召回用,判 API 死);
补充层(deep/viewCount)应传 critical=False,单点抖动不拖垮整条流水线。
返回 (候选列表, 本函数发起的 search.list 调用次数)。"""
all_videos = []
seen_ids = set()
failed_keywords = []
search_call_count = 0
for keyword in keywords:
page_token = None
for _ in range(max(1, pages)):
params = {
"q": keyword,
"part": "snippet",
"type": "video",
"order": order,
"publishedAfter": published_after,
"relevanceLanguage": "en",
"maxResults": MAX_RESULTS_PER_KEYWORD,
}
if page_token:
params["pageToken"] = page_token
try:
resp = youtube_get(rotator, "/search", params, timeout=30)
search_call_count += 1
except Exception as e:
print(f"警告:关键词 '{keyword}' 搜索失败 ({e})", file=sys.stderr)
# 仅首页就失败才算该词「颗粒无收」;翻页中途失败(page_token 已有)不算整词失败
if page_token is None:
failed_keywords.append(keyword)
break
data = resp.json()
for item in data.get("items", []):
video_id = item["id"]["videoId"]
if video_id in seen_ids:
continue
seen_ids.add(video_id)
snippet = item["snippet"]
all_videos.append({
"video_id": video_id,
"title": snippet["title"],
"channel": snippet["channelTitle"],
"channel_id": snippet.get("channelId", ""),
"description": snippet.get("description", ""),
"published_at": snippet["publishedAt"],
"url": f"https://www.youtube.com/watch?v={video_id}",
})
page_token = data.get("nextPageToken")
if not page_token:
break
if critical and keywords and len(failed_keywords) == len(keywords):
print(f"错误:所有 {len(keywords)} 个关键词搜索全部失败,疑似 API key 失效或网络问题", file=sys.stderr)
sys.exit(1)
print(f"召回(order={order},{len(keywords)} 词,{search_call_count} 次调用):{len(all_videos)} 个候选", file=sys.stderr)
return all_videos, search_call_count
# ── 第二段:硬过滤 ──────────────────────────────────────
def enrich_and_filter(rotator: KeyRotator | None, videos: list[dict]) -> list[dict]:
"""获取详情 + 硬过滤:语言、时长、播放量、噪音标题。
已带 duration_seconds + view_count 的视频(如 Apify backend 召回的)跳过 videos.list。"""
if not videos:
return []
# 只对缺少 stats/duration 的视频调 videos.list(节省 quota)
need_enrich = [
v for v in videos
if "duration_seconds" not in v or "view_count" not in v
]
stats_map = {}
duration_map = {}
lang_map = {}
if need_enrich and rotator:
video_ids = [v["video_id"] for v in need_enrich]
for i in range(0, len(video_ids), 50):
batch = video_ids[i:i + 50]
resp = youtube_get(rotator, "/videos", {
"part": "statistics,contentDetails,snippet",
"id": ",".join(batch),
}, timeout=30)
for item in resp.json().get("items", []):
vid = item["id"]
stats_map[vid] = int(item["statistics"].get("viewCount", 0))
duration_map[vid] = parse_duration(item["contentDetails"].get("duration", "PT0S"))
snippet = item.get("snippet", {})
lang = snippet.get("defaultAudioLanguage") or snippet.get("defaultLanguage") or ""
lang_map[vid] = lang.lower()
for video in videos:
# 已带字段就保留,否则用 videos.list 拿到的值;都没有就 0/空
video["view_count"] = video.get("view_count", stats_map.get(video["video_id"], 0))
video["duration_seconds"] = video.get("duration_seconds", duration_map.get(video["video_id"], 0))
video["language"] = video.get("language", lang_map.get(video["video_id"], ""))
# 硬过滤
filtered = []
for v in videos:
title = v["title"]
# 相关性过滤:标题或描述必须包含相关关键词(不区分大小写)。
# 信任频道豁免——已白名单的创作者不再做关键词复查(与下方时长豁免同理):
# search.list 返回的描述是截断的,开头若是推广链接会把关键词挤出截断段,
# 导致标题不含关键词的相关视频(如 Nate Herk 的发布解读)被误砍。真跑偏的
# 视频仍由下游 LLM 聚类/判断阶段拦截。
text = (title + " " + v.get("description", "")).lower()
if v.get("source") != "trusted_channel" and not any(
kw in text for kw in ("claude", "anthropic")
):
continue
# 语言过滤:只保留英语(en, en-US, en-GB 等)或未标注语言的视频
lang = v.get("language", "")
if lang and not lang.startswith("en"):
continue
# 非英文字符集(兜底:拦截 CJK、俄文等未标注语言的非英文视频)
if NON_ENGLISH_PATTERN.search(title):
continue
# 噪音标题
if NOISE_TITLE_WORDS.search(title):
continue
# 时长 < 3 分钟:一律砍(含信任频道;shorts 对选题无价值——2026-06-02 移除豁免)
if v["duration_seconds"] < MIN_DURATION_SECONDS:
continue
# 播放量 < 100
if v["view_count"] < MIN_VIEW_COUNT:
continue
filtered.append(v)
print(f"硬过滤后:{len(filtered)} 个", file=sys.stderr)
return filtered
# ── 去重 ──────────────────────────────────────────────
def load_seen_ids() -> set[str]:
"""从独立索引文件加载已见过的 Video ID,md 扫描作为兜底"""
seen = set()
# 主索引:seen_video_ids.json
if SEEN_IDS_PATH.exists():
try:
seen.update(json.loads(SEEN_IDS_PATH.read_text(encoding="utf-8")))
except Exception:
pass
# 兜底:扫描选题库 markdown
topic_dir = _topic_dir()
if topic_dir.exists():
for md_file in topic_dir.glob("**/*.md"):
try:
content = md_file.read_text(encoding="utf-8")
seen.update(VIDEO_ID_PATTERN.findall(content))
except Exception as e:
print(f"警告:无法读取 {md_file.name}({e})", file=sys.stderr)
return seen
# ── 第三段:排序 + 输出 ──────────────────────────────────
def sort_and_output(videos: list[dict]) -> list[dict]:
"""按播放量降序排列,添加格式化字段"""
videos.sort(key=lambda v: v["view_count"], reverse=True)
for v in videos:
v["relative_time"] = format_relative_time(v["published_at"])
v["view_count_formatted"] = format_view_count(v["view_count"])
mins, secs = divmod(v["duration_seconds"], 60)
v["duration_formatted"] = f"{mins}分{secs}秒" if secs else f"{mins}分钟"
pub = datetime.fromisoformat(v["published_at"].replace("Z", "+00:00"))
hours = max((datetime.now(timezone.utc) - pub).total_seconds() / 3600, 1)
v["view_velocity"] = round(v["view_count"] / hours, 1)
return videos
# ── 信任频道直查 ──────────────────────────────────────
def _load_creator_index() -> dict:
"""从本地状态目录读创作者索引,找不到返回空字典。"""
p = STATE_DIR / "创作者索引.json"
if not p.exists():
return {}
try:
data = json.loads(p.read_text(encoding="utf-8"))
return data.get("creators", {})
except Exception as e:
print(f"警告:读创作者索引失败({e}),跳过自动晋升", file=sys.stderr)
return {}
def load_promoted_channels() -> list[tuple[str, str]]:
"""自动晋升:从创作者索引筛 is_quality=True AND total_videos≥N AND channel_id 非空,
按 avg_views 降序取前 PROMOTION_MAX_CHANNELS 个。"""
creators = _load_creator_index()
candidates = []
for name, c in creators.items():
if not c.get("is_quality"):
continue
if c.get("total_videos", 0) < PROMOTION_MIN_VIDEOS:
continue
cid = c.get("channel_id", "")
if not cid:
continue
candidates.append((cid, name, c.get("avg_views", 0)))
candidates.sort(key=lambda x: -x[2])
return [(cid, name) for cid, name, _ in candidates[:PROMOTION_MAX_CHANNELS]]
def get_trusted_channels() -> list[tuple[str, str]]:
"""合并种子 + 自动晋升,按 channel_id 去重(种子优先保留名字)。"""
seen = set()
out = []
for cid, name in SEED_TRUSTED_CHANNELS:
if cid in seen:
continue
seen.add(cid)
out.append((cid, name))
for cid, name in load_promoted_channels():
if cid in seen:
continue
seen.add(cid)
out.append((cid, name))
return out
def _parse_apify_duration(value) -> int:
"""Apify 的 duration 可能是 'mm:ss' / 'hh:mm:ss' 字符串或 lengthSeconds 数字。"""
if isinstance(value, (int, float)):
return int(value)
if isinstance(value, str):
parts = value.strip().split(":")
try:
nums = [int(p) for p in parts]
except ValueError:
return 0
if len(nums) == 3:
return nums[0] * 3600 + nums[1] * 60 + nums[2]
if len(nums) == 2:
return nums[0] * 60 + nums[1]
if len(nums) == 1:
return nums[0]
return 0
def _trusted_recall_youtube_api(
rotator: KeyRotator, published_after: str, channels: list[tuple[str, str]]
) -> list[dict]:
"""生产 backend:每个频道单独调一次 search.list?channelId=X。每个频道 100 配额。"""
out = []
failed = 0
for cid, name in channels:
try:
resp = youtube_get(rotator, "/search", {
"channelId": cid,
"part": "snippet",
"type": "video",
"order": "date",
"publishedAfter": published_after,
"maxResults": TRUSTED_MAX_RESULTS_PER_CHANNEL,
}, timeout=30)
except Exception as e:
print(f"警告:信任频道 '{name}' ({cid}) 拉取失败 ({e})", file=sys.stderr)
failed += 1
continue
for item in resp.json().get("items", []):
try:
vid = item["id"]["videoId"]
except (KeyError, TypeError):
continue
snippet = item["snippet"]
out.append({
"video_id": vid,
"title": snippet["title"],
"channel": snippet.get("channelTitle") or name,
"channel_id": snippet.get("channelId") or cid,
"description": snippet.get("description", ""),
"published_at": snippet["publishedAt"],
"url": f"https://www.youtube.com/watch?v={vid}",
"source": "trusted_channel",
})
print(
f"信任频道直查(YouTube API):{len(channels)} 频道,{failed} 失败,召回 {len(out)} 条",
file=sys.stderr,
)
return out
APIFY_BATCH_SIZE = 4 # Apify sync 单次 5 分钟超时,4 频道 × ~60s = 安全
def _apify_recall_batch(
token: str, batch: list[tuple[str, str]], published_after: str,
cid_to_name: dict[str, str],
) -> list[dict]:
"""单批 Apify 调用:≤APIFY_BATCH_SIZE 个频道 URL 一次性提交。"""
apify_url = f"https://api.apify.com/v2/acts/{APIFY_SEARCH_ACTOR}/run-sync-get-dataset-items"
payload = {
"startUrls": [
{"url": f"https://www.youtube.com/channel/{cid}/videos"}
for cid, _ in batch
],
"maxResults": TRUSTED_MAX_RESULTS_PER_CHANNEL,
"maxResultsShorts": 0,
"maxResultStreams": 0,
"dateFilter": "week",
"videoType": "video",
"downloadSubtitles": False,
}
try:
resp = requests.post(
apify_url,
params={"token": token, "timeout": "300"},
json=payload,
timeout=360,
)
resp.raise_for_status()
items = resp.json()
if not isinstance(items, list):
raise RuntimeError(f"Apify 返回非列表: {type(items)}")
except Exception as e:
print(f"警告:Apify 批 {[n for _, n in batch]} 失败 ({e})", file=sys.stderr)
return []
out = []
for item in items:
video_url = item.get("url") or item.get("videoUrl") or ""
m = VIDEO_ID_PATTERN.search(video_url)
vid = m.group(1) if m else (item.get("id") or "")
if not vid or len(vid) != 11:
continue
channel_url = item.get("channelUrl") or ""
cid_match = re.search(r"/channel/(UC[A-Za-z0-9_-]{22})", channel_url)
cid = cid_match.group(1) if cid_match else (item.get("channelId") or "")
if cid not in cid_to_name:
continue # 跳过非本批信任频道的视频
duration_s = _parse_apify_duration(item.get("lengthSeconds") or item.get("duration"))
view_count = int(item.get("viewCount") or 0)
pub_raw = item.get("date") or item.get("publishedAt") or ""
if pub_raw:
# 用 datetime 解析后再比较——Apify 的 date 格式不保证是 ISO,
# 字符串直接比会静默误判。解析失败按「保留」处理,交给下游过滤。
try:
pub_dt = datetime.fromisoformat(pub_raw.replace("Z", "+00:00"))
if pub_dt.tzinfo is None:
pub_dt = pub_dt.replace(tzinfo=timezone.utc)
cutoff_dt = datetime.fromisoformat(published_after.replace("Z", "+00:00"))
if pub_dt < cutoff_dt:
continue
except (ValueError, TypeError):
print(
f"警告:无法解析发布时间 '{pub_raw}'({vid}),按保留处理",
file=sys.stderr,
)
out.append({
"video_id": vid,
"title": item.get("title", ""),
"channel": item.get("channelName") or cid_to_name.get(cid, ""),
"channel_id": cid,
"description": item.get("text") or item.get("description") or "",
"published_at": pub_raw,
"url": video_url or f"https://www.youtube.com/watch?v={vid}",
"duration_seconds": duration_s,
"view_count": view_count,
"language": "",
"source": "trusted_channel",
})
return out
def _trusted_recall_apify(
published_after: str, channels: list[tuple[str, str]]
) -> list[dict]:
"""测试 backend:不烧 YouTube quota,用 Apify streamers/youtube-scraper。
分批调用(每批 APIFY_BATCH_SIZE 个频道)避免 Apify sync 5 分钟超时。
返回 dict 已包含 duration/views,后续 enrich_and_filter 会跳过 videos.list。"""
from paths import load_apify_token
token = load_apify_token()
cid_to_name = {cid: name for cid, name in channels}
out = []
failed_batches = 0
for i in range(0, len(channels), APIFY_BATCH_SIZE):
batch = channels[i:i + APIFY_BATCH_SIZE]
print(f" Apify 批 {i // APIFY_BATCH_SIZE + 1}:{[n for _, n in batch]}", file=sys.stderr)
batch_out = _apify_recall_batch(token, batch, published_after, cid_to_name)
if not batch_out:
failed_batches += 1
out.extend(batch_out)
print(
f"信任频道直查(Apify 分批):{len(channels)} 频道分 "
f"{(len(channels) + APIFY_BATCH_SIZE - 1) // APIFY_BATCH_SIZE} 批,"
f"{failed_batches} 批失败,召回 {len(out)} 条",
file=sys.stderr,
)
return out
def recall_from_trusted_channels(rotator: KeyRotator | None, published_after: str) -> list[dict]:
"""白名单(种子)频道直查。env CYXJ_TRUSTED_BACKEND 切换:youtube_api(默认)/ apify(测试)。
2026-06-02:直查范围从「种子 + 自动晋升」收窄为【仅种子白名单】。自动晋升频道不再获抓取
特权(直查 + 关键词豁免)——它只作为「领头羊榜」的排名展示(见 write_topics.py),不再翻它
的频道库存。这样只有种子视频带 source=trusted_channel,关键词豁免也只对种子生效。
(load_promoted_channels / get_trusted_channels 已退出抓取链路,保留仅备查。)"""
channels = list(SEED_TRUSTED_CHANNELS)
if not channels:
return []
backend = os.environ.get("CYXJ_TRUSTED_BACKEND", "youtube_api")
print(
f"白名单直查:{len(channels)} 个种子频道,backend={backend}",
file=sys.stderr,
)
if backend == "apify":
return _trusted_recall_apify(published_after, channels)
return _trusted_recall_youtube_api(rotator, published_after, channels)
# ── 入口 ──────────────────────────────────────────────
def main():
api_key_required = os.environ.get("CYXJ_TRUSTED_BACKEND", "youtube_api") != "apify"
rotator: KeyRotator | None = None
if api_key_required:
keys = load_youtube_api_keys()
rotator = KeyRotator(keys)
print(f"YouTube API key 池:共 {len(keys)} 个,按序号轮询", file=sys.stderr)
published_after = (
datetime.now(timezone.utc) - timedelta(hours=HOURS_WINDOW)
).strftime("%Y-%m-%dT%H:%M:%SZ")
# 1a. 关键词召回(Apify 模式下不跑——避免烧 YouTube quota)
if rotator:
date_videos, n_date = recall(rotator, published_after, KEYWORDS, order="date")
deep_videos, n_deep = recall(rotator, published_after, ["Claude Code"], order="date", pages=2, critical=False)
view_videos, n_view = recall(rotator, published_after, KEYWORDS_VIEWCOUNT, order="viewCount", critical=False)
merged: dict[str, dict] = {}
for v in date_videos + deep_videos + view_videos:
merged.setdefault(v["video_id"], v)
keyword_candidates = list(merged.values())
for v in keyword_candidates:
v["source"] = "keyword"
print(f"召回搜索调用次数:date={n_date} + deep={n_deep} + viewCount={n_view} = {n_date + n_deep + n_view}", file=sys.stderr)
else:
print("信号:Apify 模式跳过关键词召回(避免烧 YouTube quota)", file=sys.stderr)
keyword_candidates = []
# 1b. 信任频道直查
trusted_candidates = recall_from_trusted_channels(rotator, published_after)
# 1c. 合并:video_id 去重,trusted 覆盖 keyword(保留 trusted source)
by_id: dict[str, dict] = {}
for v in keyword_candidates + trusted_candidates:
existing = by_id.get(v["video_id"])
if existing is None or v.get("source") == "trusted_channel":
by_id[v["video_id"]] = v
candidates = list(by_id.values())
print(f"召回合并:{len(candidates)} 个候选(去重后)", file=sys.stderr)
# 2. 硬过滤
clean = enrich_and_filter(rotator, candidates)
# 3. 去重
seen_ids = load_seen_ids()
new_videos = [v for v in clean if v["video_id"] not in seen_ids]
print(f"去重后:{len(new_videos)} 个新视频", file=sys.stderr)
# 4. 排序 + 输出
# 注意:.seen_video_ids.json 不在这里更新——必须等 write_topics.py 成功写入
# 总览文件后才能标记"已处理",否则中途失败会丢视频。
result = sort_and_output(new_videos)
print(json.dumps(result, ensure_ascii=False, indent=2))
if __name__ == "__main__":
main()
Related skills
| Skill | Stars | Updated | Works with | Status | Install |
|---|---|---|---|---|---|
| reverse-engineer-anythingmorluto/reaReverse engineer native, managed, Electron/JavaScript, packaged, firmware, and browser targets with REA, and build agent environments (AgentEnv bundles) of web applications from a live instance or a capture, and of Linux desktop apps with a task from a request. Use shipped-artifact or requested runtime evidence to explain features, compare versions, decompile code, or guide a reconstruction. Skip REA for ordinary source-repository architecture analysis. | Stars: 472k | Updated | Claude CodeCodex | Verified | Install |
| diagnosing-superpowersobra/superpowersUse when a superpowers session went wrong and your human partner wants to know why — repeated work, ignored plans, stumbles, poor results, a skill that didn't fire, "it took too long", "why is it so expensive", "what is it doing" — or wants to build a bug report for the superpowers maintainers, for the current session or a past one identified by id or path, on any harness. | Stars: 323k | Updated | Claude CodeCodex | Verified | Install |
| dispatching-parallel-agentsobra/superpowersUse when facing 2+ independent tasks that can be worked on without shared state or sequential dependencies | Stars: 323k | Updated | Claude CodeCodex | Verified | Install |
| executing-plansobra/superpowersUse when executing an implementation plan in the current session as the implementer yourself — your human partner chose inline execution, or no subagent tool is available | Stars: 323k | Updated | Claude CodeCodex | Verified | Install |