zenstory-ai/video-recap-skills / skills/video-recap
video-recap
Verified把一段或多集视频做成带画框字幕的中文解说成片或纯原声高光短片。用户提供 .mp4 / .mov / .mkv / .webm,要求做影视解说、短剧二创、加旁白配音、跨集混剪或导出剪映草稿时使用;串联完成看片识台词、定主线与留原声、按台词停顿切镜头、写词检查、TTS 配音与混音加字幕。触发词:视频解说、影视解说、视频旁白、生成解说、视频 recap、video recap、voiceover、narration、recap。
scripts/detect.py
View on GitHubimport json
import os
import re
import subprocess
from concurrent.futures import ThreadPoolExecutor
from pathlib import Path
from lib import CONFIG
from lib import log, run_cmd, get_video_duration, file_identity
# ── Step 2: 场景检测 ──────────────────────────────────────────────────
def detect_scenes(video_path, work_dir, threshold=None):
"""使用 ffmpeg scdet 滤镜检测场景切换"""
threshold = CONFIG["scene_threshold"] if threshold is None else threshold
scdet_threshold = int(threshold * 100)
cmd = ["ffmpeg", "-i", str(video_path),
"-vf", f"scdet=threshold={scdet_threshold}",
"-f", "null", "-"]
result = run_cmd(cmd)
if result.returncode != 0:
raise RuntimeError(f"场景检测失败: {result.stderr}")
# 解析 lavfi.scd.time 和 lavfi.scd.score
times = []
for line in result.stderr.split("\n"):
match = re.search(r"lavfi\.scd\.time[:=]\s*(\S+)", line)
if match:
times.append(float(match.group(1)))
if not times:
# 没有检测到场景切换,整个视频作为一个场景
duration = get_video_duration(video_path)
scenes = [{"start": 0.0, "end": duration}]
log(f"未检测到场景切换,整个视频作为一个场景 ({duration:.1f}s)")
else:
scenes = []
prev = 0.0
for t in times:
scenes.append({"start": round(prev, 2), "end": round(t, 2)})
prev = t
# 最后一个场景到视频结束
duration = get_video_duration(video_path)
scenes.append({"start": round(prev, 2), "end": round(duration, 2)})
log(f"检测到 {len(scenes)} 个场景")
if CONFIG["scene_junk_filter"]:
scenes = _filter_junk_scenes(scenes, video_path)
# 保存
scenes_file = work_dir / "scenes.json"
scenes_file.write_text(json.dumps(scenes, ensure_ascii=False, indent=2), encoding="utf-8")
for i, s in enumerate(scenes):
log(f" 场景 {i+1}: {s['start']:.1f}s - {s['end']:.1f}s ({s['end']-s['start']:.1f}s)")
return scenes
def _sample_frame_luma(video_path, timestamp, sample_size=64):
"""Extract one frame via ffmpeg and return luma values without extra deps."""
cmd = [
"ffmpeg",
"-hide_banner",
"-loglevel",
"error",
"-ss",
f"{max(0.0, float(timestamp)):.3f}",
"-i",
str(video_path),
"-frames:v",
"1",
"-vf",
f"scale={sample_size}:{sample_size},format=rgb24",
"-f",
"rawvideo",
"-",
]
result = subprocess.run(cmd, capture_output=True)
if result.returncode != 0 or not result.stdout:
raise RuntimeError(result.stderr.decode("utf-8", errors="replace")[:300])
data = result.stdout
return [
0.2126 * data[i] + 0.7152 * data[i + 1] + 0.0722 * data[i + 2]
for i in range(0, len(data) - 2, 3)
]
def _is_junk_scene(video_path, timestamp, threshold_dark=None, threshold_bright=None):
"""Return True for near-black or near-white scene-start frames."""
threshold_dark = CONFIG["scene_junk_dark_luma"] if threshold_dark is None else threshold_dark
threshold_bright = CONFIG["scene_junk_bright_luma"] if threshold_bright is None else threshold_bright
pixel_ratio = min(1.0, float(CONFIG["scene_junk_pixel_ratio"]))
try:
lumas = _sample_frame_luma(video_path, timestamp)
except Exception as exc:
log(f"场景亮度采样失败,保留场景 {timestamp:.1f}s: {exc}")
return False
if not lumas:
return False
avg_luma = sum(lumas) / len(lumas)
dark_ratio = sum(1 for value in lumas if value <= threshold_dark) / len(lumas)
bright_ratio = sum(1 for value in lumas if value >= threshold_bright) / len(lumas)
return (
avg_luma <= threshold_dark and dark_ratio >= pixel_ratio
) or (
avg_luma >= threshold_bright and bright_ratio >= pixel_ratio
)
def _scene_is_junk(scene, video_path):
"""Whether every probe point of one scene is a black/white transition frame."""
start = float(scene["start"])
end = float(scene["end"])
# 多点采样:起始、中点、结尾各探一帧;只有全部为垃圾帧才删除,
# 含任意非垃圾帧的场景必须保留(避免短黑场并入长真实场景后被整段误删)
probe_times = [
min(end, start + 0.1),
(start + end) / 2.0,
max(start, end - 0.1),
]
# `all` 的短路很重要:绝大多数场景第一探就否掉,只花一次 ffmpeg。
return all(_is_junk_scene(video_path, t) for t in probe_times)
def _filter_junk_scenes(scenes, video_path):
"""Filter black/white transition scenes while never deleting the whole video."""
if len(scenes) <= 1:
return scenes
# 每个探点都是一次 ffmpeg 进程(seek + 解一帧),一部 30 分钟片有几百个场景,串行下来
# 光进程启动就要几十秒。按场景并行:每个 worker 内部仍然短路,所以既保留了「多数场景只
# 探一次」,又把进程启动开销摊平。判定逻辑与探点完全不变。
workers = int(CONFIG["scene_junk_workers"])
with ThreadPoolExecutor(max_workers=min(workers, len(scenes))) as executor:
verdicts = list(executor.map(lambda s: _scene_is_junk(s, video_path), scenes))
filtered = []
removed = []
for scene, is_junk in zip(scenes, verdicts):
if is_junk:
removed.append(scene)
else:
filtered.append(scene)
if not filtered:
log("场景黑/白帧过滤会删除全部场景,已放弃过滤")
return scenes
if removed:
log(f"过滤黑/白帧过渡场景: {len(scenes)} → {len(filtered)}")
return filtered
# ── Step 3.5: 静音检测 ─────────────────────────────────────────────
# The silence-detection audio is always extracted below as 16 kHz mono signed-16 PCM.
_SILENCE_AUDIO_BYTES_PER_SECOND = 16000 * 1 * 2
def _wav_seconds_estimate(audio_path):
"""Approximate duration from file size — no ffprobe subprocess on this path."""
try:
return max(0.0, Path(audio_path).stat().st_size / _SILENCE_AUDIO_BYTES_PER_SECOND)
except OSError:
return 0.0
def _compact_ffmpeg_error(stderr, limit=400):
"""Keep the actionable tail without dumping ffmpeg build/configuration banners."""
text = " ".join(str(stderr or "").split())
if len(text) <= limit:
return text
return "…" + text[-limit:]
def annotate_quiet_windows_with_asr(periods, asr_result=None, *, video_duration=None, configured_segment_seconds=None):
"""Pure helper: annotate quiet windows with ASR-overlap confidence.
Coarse grid ASR (large synthetic windows from chunking) must not turn every quiet
window into speech. Returns the annotated periods; inputs are copied.
"""
out = [dict(p) for p in (periods or [])]
for qp in out:
qp.setdefault("speech_overlap_ratio", 0.0)
qp.setdefault("asr_overlap_seconds", 0.0)
qp.setdefault("asr_granularity", "none")
qp.setdefault("has_speech", False)
qp.setdefault("has_speech_reason", "no_asr_overlap")
if not asr_result:
return out
valid_segments = []
for seg in asr_result:
ss, se = float(seg["start"]), float(seg["end"])
if se > ss:
valid_segments.append((ss, se))
asr_coverage = sum(se - ss for ss, se in valid_segments)
avg_seg_dur = asr_coverage / len(valid_segments) if valid_segments else 0
if video_duration is None:
ends = [float(p["end"]) for p in out] + [se for _, se in valid_segments]
video_duration = max(ends or [0.0])
configured = float(configured_segment_seconds if configured_segment_seconds is not None else CONFIG["asr_segment_seconds"])
coarse_asr = (
(len(valid_segments) <= 5 and asr_coverage > float(video_duration) * 0.8) or
asr_coverage > float(video_duration) * 1.5 or
avg_seg_dur > max(45.0, configured * 1.5) or
(avg_seg_dur >= configured * 0.9 and asr_coverage > float(video_duration) * 0.7)
)
granularity = "coarse_grid" if coarse_asr else "segment"
for qp in out:
overlap_seconds = 0.0
for ss, se in valid_segments:
overlap_seconds += max(0.0, min(float(qp["end"]), se) - max(float(qp["start"]), ss))
ratio = overlap_seconds / max(0.001, float(qp["duration"]))
qp["asr_overlap_seconds"] = round(overlap_seconds, 3)
qp["speech_overlap_ratio"] = round(ratio, 4)
qp["asr_granularity"] = granularity
if coarse_asr:
qp["has_speech"] = False
qp["has_speech_reason"] = "coarse_asr_overlap_ignored" if overlap_seconds > 0 else "coarse_asr_no_overlap"
continue
if ratio >= 0.3:
qp["has_speech"] = True
qp["has_speech_reason"] = "asr_overlap_high_confidence"
elif overlap_seconds > 0:
qp["has_speech"] = False
qp["has_speech_reason"] = "asr_overlap_low_confidence_quiet"
else:
qp["has_speech"] = False
qp["has_speech_reason"] = "no_asr_overlap"
return out
def detect_silence_periods(video_path, work_dir, asr_result=None):
"""用 ffmpeg silencedetect 检测安静时段,作为解说插入的候选窗口"""
audio_path = work_dir / "audio.wav"
if not _audio_cache_matches(audio_path, video_path):
if audio_path.exists():
audio_path.unlink()
# 提取到临时文件,成功后原子移动到位,避免被中断的 -y 运行留下半截 audio.wav
tmp_path = work_dir / "audio.wav.tmp"
extract = run_cmd([
"ffmpeg", "-y", "-i", str(video_path),
"-vn", "-ar", "16000", "-ac", "1",
"-f", "wav", str(tmp_path) # .tmp extension hides the format from ffmpeg; state it
])
if extract.returncode != 0 or not tmp_path.exists():
log(
"音频提取失败,无法检测静音窗口(视频可能无音轨): "
f"{_compact_ffmpeg_error(extract.stderr)}"
)
if tmp_path.exists():
tmp_path.unlink()
detect_speech_boundary_anchors(work_dir, asr_result or [])
return []
os.replace(str(tmp_path), str(audio_path))
_write_audio_meta(work_dir, video_path)
# Sentence-entry anchors use short acoustic pauses aligned to terminal ASR punctuation.
# They are deliberately separate from silence_periods.json: a 200ms sentence pause is a
# safe place to ENTER narration, but not a multi-second quiet window that can own a block.
detect_speech_boundary_anchors(work_dir, asr_result or [])
noise = CONFIG["silence_noise_threshold"]
min_dur = CONFIG["silence_min_duration"]
cmd = ["ffmpeg", "-i", str(audio_path),
"-af", f"silencedetect=noise={noise}:d={min_dur}",
"-f", "null", "-"]
# Everything else in this function degrades to `return []` (narration then falls back to
# non-quiet placement). A timeout must degrade the same way instead of raising
# TimeoutExpired through the whole understanding stage. Scale the budget with the audio
# so a long feature is not cut off by a constant tuned for short clips — estimated from
# the file size (this is the 16 kHz mono s16 WAV we just wrote), not from another probe.
timeout = max(120.0, _wav_seconds_estimate(audio_path) * 0.5)
try:
result = run_cmd(cmd, timeout=timeout)
except subprocess.TimeoutExpired:
log(f"静音检测超时(>{timeout:.0f}s),跳过安静窗口检测")
return []
if result.returncode != 0:
log(f"静音检测失败: {_compact_ffmpeg_error(result.stderr)}")
return []
output = result.stderr
# 解析 silence_start / silence_end(片头静音可能出现负起点如 -0.0013,统一钳制到 0.0)
starts = [max(0.0, float(m)) for m in re.findall(r'silence_start:\s*(-?[\d.]+)', output)]
ends = [max(0.0, float(m)) for m in re.findall(r'silence_end:\s*(-?[\d.]+)', output)]
# 配对所有静音段(不过滤时长,后面合并后再过滤)
raw_periods = []
for s, e in zip(starts, ends):
raw_periods.append({"start": round(s, 2), "end": round(e, 2),
"duration": round(e - s, 2)})
# 末尾静音
if len(starts) > len(ends):
dur_start = starts[len(ends)]
total_dur = get_video_duration(str(audio_path))
raw_periods.append({"start": round(dur_start, 2), "end": round(total_dur, 2),
"duration": round(total_dur - dur_start, 2)})
# 合并相邻静音段(间隔 < merge_gap 的合并为一个大窗口)
merge_gap = CONFIG["silence_merge_gap"]
merged = []
for rp in sorted(raw_periods, key=lambda x: x["start"]):
if merged and rp["start"] - merged[-1]["end"] < merge_gap:
merged[-1]["end"] = rp["end"]
merged[-1]["duration"] = round(merged[-1]["end"] - merged[-1]["start"], 2)
else:
merged.append({"start": rp["start"], "end": rp["end"],
"duration": rp["duration"], "has_speech": False})
# 过滤最短时长
quiet_min = CONFIG["quiet_window_min"]
periods = [p for p in merged if p["duration"] >= quiet_min]
# 与 ASR 交叉验证:标记有语音的窗口,并记录可观测 confidence/reason。
periods = annotate_quiet_windows_with_asr(
periods,
asr_result,
video_duration=get_video_duration(str(audio_path)) if asr_result else None,
configured_segment_seconds=CONFIG["asr_segment_seconds"],
)
# 保存
(work_dir / "silence_periods.json").write_text(
json.dumps(periods, ensure_ascii=False, indent=2), encoding="utf-8")
log(f"检测到 {len(periods)} 个安静窗口 (≥{quiet_min}s)")
for qp in periods:
flag = " [有语音]" if qp["has_speech"] else ""
log(f" {qp['start']:.1f}s-{qp['end']:.1f}s ({qp['duration']:.1f}s){flag}")
return periods
SPEECH_BOUNDARY_ANCHORS_SCHEMA_VERSION = 2
def _anchor_confidence(seconds):
return "high" if seconds <= 0.6 else ("medium" if seconds <= 1.2 else "low")
def _classify_anchor(timing_bound, error):
"""(confidence, boundary_use) from the ASR-window bound and the pause-snap distance."""
confidence = _anchor_confidence(max(timing_bound, error))
if confidence in {"high", "medium"}:
return confidence, "verified"
return confidence, ("unverified" if error <= 1.2 else "none")
def detect_speech_boundary_anchors(work_dir, asr_result):
"""Write sentence-end entry anchors by aligning ASR punctuation to short pauses.
MiMo ASR timestamps are window-level, not word-level. We therefore estimate each terminal
punctuation time from its character position inside the ASR window, then snap it to the
closest short acoustic pause. The output is guidance + a deterministic pre-TTS gate; it
never rewrites narration timing on its own.
The character-position estimate is a guess, so `confidence` is not taken from the snap
distance alone: the sentence end can be anywhere inside the ASR window, which bounds the
real error by `timing_bound_seconds` (the farther window edge from the pause). Only a
narrow window can yield `high`/`medium` (`boundary_use: verified`); a wide window yields
`low`, and an anchor that still snapped within the alignment limit is `unverified`
(usable by gates, labelled as an estimate) rather than dropped.
"""
work_dir = Path(work_dir)
audio_path = work_dir / "audio.wav"
out_path = work_dir / "speech_boundary_anchors.json"
segments = list(asr_result or [])
report = {
"schema_version": SPEECH_BOUNDARY_ANCHORS_SCHEMA_VERSION,
"artifact": "speech_boundary_anchors.json",
"status": "completed",
"detector": {
"noise_threshold": CONFIG["source_boundary_noise_threshold"],
"min_pause_seconds": float(CONFIG["source_boundary_min_pause"]),
"alignment": "terminal_punctuation_to_nearest_acoustic_pause",
},
"sentence_anchors": [],
"acoustic_pauses": [],
}
if not audio_path.exists() or not segments:
report["status"] = "unavailable"
report["reason"] = "missing_audio_or_asr"
out_path.write_text(json.dumps(report, ensure_ascii=False, indent=2), encoding="utf-8")
return report
audio_dur = max(
_wav_seconds_estimate(audio_path),
max((float(s["end"]) for s in segments), default=0.0),
)
timeout = max(120, int(audio_dur * 0.5) + 60)
try:
result = run_cmd([
"ffmpeg", "-hide_banner", "-nostats", "-i", str(audio_path),
"-af", (
"silencedetect="
f"noise={CONFIG['source_boundary_noise_threshold']}:"
f"d={float(CONFIG['source_boundary_min_pause'])}"
),
"-f", "null", "-",
], timeout=timeout)
except subprocess.TimeoutExpired:
report["status"] = "failed"
report["reason"] = "silencedetect_timeout"
out_path.write_text(json.dumps(report, ensure_ascii=False, indent=2), encoding="utf-8")
return report
if result.returncode != 0:
report["status"] = "failed"
report["reason"] = "silencedetect_failed"
out_path.write_text(json.dumps(report, ensure_ascii=False, indent=2), encoding="utf-8")
return report
starts = [max(0.0, float(value)) for value in re.findall(r"silence_start:\s*(-?[\d.]+)", result.stderr or "")]
ends = [max(0.0, float(value)) for value in re.findall(r"silence_end:\s*(-?[\d.]+)", result.stderr or "")]
pauses = []
for index, (start, end) in enumerate(zip(starts, ends)):
if end <= start:
continue
pauses.append({
"index": index,
"start": round(start, 3),
"end": round(end, 3),
"midpoint": round((start + end) / 2.0, 3),
"duration": round(end - start, 3),
})
report["acoustic_pauses"] = pauses
max_error = float(CONFIG["source_boundary_max_alignment_error"])
used = set()
anchors = []
for asr_index, segment in enumerate(segments):
seg_start = float(segment["start"])
seg_end = float(segment["end"])
text = segment["text"].strip()
if seg_end <= seg_start or not text:
continue
last_midpoint = seg_start - 1e-6
for match in re.finditer(r"(?:[。!?!?;;]|(?<![\d.])\.(?![\d.]))", text):
if match.group(0) == "." and _is_non_terminal_english_dot(
text, match.start(), match.end()
):
continue
expected = seg_start + (seg_end - seg_start) * (match.end() / max(1, len(text)))
candidates = [
pause for pause in pauses
if pause["index"] not in used
and seg_start - 0.3 <= pause["midpoint"] <= seg_end + 0.3
and pause["midpoint"] > last_midpoint
and abs(pause["midpoint"] - expected) <= max_error
]
if not candidates:
continue
pause = min(candidates, key=lambda item: abs(item["midpoint"] - expected))
used.add(pause["index"])
last_midpoint = pause["midpoint"]
error = abs(pause["midpoint"] - expected)
timing_bound = max(pause["start"] - seg_start, seg_end - pause["start"], 0.0)
confidence, boundary_use = _classify_anchor(timing_bound, error)
anchors.append({
"time": pause["end"],
"pause_start": pause["start"],
"pause_end": pause["end"],
"expected_time": round(expected, 3),
"alignment_error": round(error, 3),
"timing_bound_seconds": round(timing_bound, 3),
"timing_basis": "asr_window",
"confidence": confidence,
"boundary_use": boundary_use,
"punctuation": match.group(0),
"text_tail": text[max(0, match.end() - 32):match.end()],
"asr_segment_index": asr_index,
})
report["sentence_anchors"] = sorted(anchors, key=lambda item: item["time"])
out_path.write_text(json.dumps(report, ensure_ascii=False, indent=2), encoding="utf-8")
log(f"检测到 {len(anchors)} 个原声句末安全切入点")
return report
_EN_ABBREV_BEFORE_DOT_RE = re.compile(
r"(?:^|[^A-Za-z])(?:mr|mrs|ms|dr|prof|sr|jr|st|mt|vs|etc|e\.g|i\.e|a\.m|p\.m)$",
re.IGNORECASE,
)
_EN_INITIAL_BEFORE_DOT_RE = re.compile(r"(?:^|[\s(\[{\".]|(?<![A-Za-z])')[A-Za-z]$")
_EN_DOTTED_ABBREV_CONTINUATION_RE = re.compile(r"^[A-Za-z]\.")
def _is_non_terminal_english_dot(text, dot_start, dot_end):
"""True when `.` is mid-sentence after a common English abbreviation or initial."""
tail = text[dot_end:]
if not tail.strip():
return False
if _EN_DOTTED_ABBREV_CONTINUATION_RE.match(tail):
return True
prefix = text[:dot_start]
return bool(
_EN_ABBREV_BEFORE_DOT_RE.search(prefix)
or _EN_INITIAL_BEFORE_DOT_RE.search(prefix)
)
def _audio_meta_path(work_dir):
return Path(work_dir) / "audio.wav.meta.json"
def _audio_cache_matches(audio_path, video_path):
audio_path = Path(audio_path)
if not audio_path.exists():
return False
meta_path = _audio_meta_path(audio_path.parent)
if not meta_path.exists():
return False
try:
meta = json.loads(meta_path.read_text(encoding="utf-8"))
except (OSError, ValueError, TypeError):
return False
try:
expected = file_identity(video_path)
except OSError:
return False
return (
isinstance(meta, dict)
and meta.get("source_video") == str(Path(video_path).resolve())
and meta.get("source_video_identity") == expected
)
def _write_audio_meta(work_dir, video_path):
_audio_meta_path(work_dir).write_text(
json.dumps({
"schema_version": 1,
"source_video": str(Path(video_path).resolve()),
"source_video_identity": file_identity(video_path),
"audio": "audio.wav",
}, ensure_ascii=False, indent=2),
encoding="utf-8",
)
Related skills
| Skill | Stars | Updated | Works with | Status | Install |
|---|---|---|---|---|---|
| slack-gif-creatoranthropics/skillsKnowledge and utilities for creating animated GIFs optimized for Slack. Provides constraints, validation tools, and animation concepts. Use when users request animated GIFs for Slack like "make me a GIF of X doing Y for Slack." | Stars: 198k | Updated | Claude CodeCodex | Verified | Install |
| ponytailDietrichGebert/ponytailLazy senior dev mode for any coding task: the smallest change that fully solves it, and a reply a busy human understands in one read. | Stars: 196k | Updated | Claude CodeCodex | Verified | Install |
| ponytail-debtDietrichGebert/ponytailHarvest every shortcut: comment into one debt ledger, so deferrals get tracked instead of forgotten. One-shot report. | Stars: 196k | Updated | Claude CodeCodex | Verified | Install |
| ponytail-debtDietrichGebert/ponytailList every `shortcut:` comment (and older `ponytail:` ones) as a debt ledger. One-shot report, changes nothing. Use for "ponytail debt", "what did ponytail defer", "list the shortcuts", /ponytail-debt. | Stars: 196k | Updated | Claude CodeCodex | Verified | Install |