zenstory-ai/video-recap-skills / skills/video-recap
video-recap
Verified把一段或多集视频做成带画框字幕的中文解说成片或纯原声高光短片。用户提供 .mp4 / .mov / .mkv / .webm,要求做影视解说、短剧二创、加旁白配音、跨集混剪或导出剪映草稿时使用;串联完成看片识台词、定主线与留原声、按台词停顿切镜头、写词检查、TTS 配音与混音加字幕。触发词:视频解说、影视解说、视频旁白、生成解说、视频 recap、video recap、voiceover、narration、recap。
scripts/audio_mix.py
View on GitHub"""Source handoffs, ducking envelopes, and audio mix graphs."""
from pathlib import Path
from artifacts import _load_work_json
from audio_automation import (
coalesce_duck_windows,
ducking_expression,
release_ducking_expression,
)
from lib import CONFIG, log
from speech_ownership import _dialogue_speech_spans
def _seg_place_window(seg):
"""A segment's actual placed (start, end) on the output timeline; zero-width when unplaced."""
return seg["actual_place_start"], seg["actual_place_end"]
# After narration ends, original audio may stay ducked this long waiting for a sentence-end
# anchor. Beyond it the duck releases at the narration end: coarse-ASR anchors are estimates,
# and holding a duck for tens of seconds buries dialogue the viewer must hear.
SOURCE_HANDOFF_MAX_HOLD_SECONDS = 3.0
def _load_sentence_handoff_anchors(work_dir):
"""Load usable sentence anchors (`boundary_use` verified/unverified) and their pauses."""
work_dir = Path(work_dir)
cut_mode = (work_dir / "edited_source.mp4").exists() or (
work_dir / "clip_plan_validated.json"
).exists()
artifact = "speech_boundary_anchors_output.json" if cut_mode else "speech_boundary_anchors.json"
payload = _load_work_json(work_dir, artifact)
if payload is None:
return [], None, {"require_measured": cut_mode}
if cut_mode:
# Output-clock anchors are only trusted when they are at least as new as the cut plan.
plan_path = work_dir / "clip_plan_validated.json"
fresh = (
payload.get("schema_version") == 2
and payload.get("timeline") == "cut_output"
and plan_path.exists()
and (work_dir / artifact).stat().st_mtime_ns >= plan_path.stat().st_mtime_ns
)
if not fresh:
return [], None, {"require_measured": True}
payload = {**payload, "require_measured": True}
anchors = {}
for item in payload["sentence_anchors"]:
# Schema-1 anchors (no `boundary_use`) came from the old coarse estimator: high/medium
# labels there are usable but unverified.
use = item.get("boundary_use") or (
"unverified" if item["confidence"] in {"high", "medium"} else "none"
)
if use == "none":
continue
when = float(item["time"])
pause_start = float(item.get("pause_start", when - 0.12))
row = {
"time": round(when, 4),
"pause_start": round(max(0.0, min(pause_start, when)), 4),
"verified": use == "verified",
}
key = (row["time"], row["pause_start"])
row["verified"] = row["verified"] or anchors.get(key, {}).get("verified", False)
anchors[key] = row
return sorted(anchors.values(), key=lambda row: row["time"]), artifact, payload
def _prefer_verified(matches):
"""First verified anchor among time-ordered matches, else the first match, else None."""
return next((anchor for anchor in matches if anchor["verified"]), None) or (
matches[0] if matches else None
)
def _timed_rows(rows):
return [{"start": float(row["start"]), "end": float(row["end"])} for row in rows]
def _asr_segments(work_dir):
"""Raw asr_result.json segments; [] when absent."""
return _load_work_json(work_dir, "asr_result.json") or []
def _handoff_speech_evidence(work_dir, payload):
"""(speech, quiet, dialogue) rows; `dialogue` drops interjection-only windows and
decides only whether a narration entry interrupts source speech.
ASR fallback rows with empty text are windows where nothing was recognized, not
speech (cut.py and validate.py drop them the same way). Output-clock
`speech_spans` keep the raw ASR text; a row with no `text` counts as dialogue.
"""
rows = payload.get("speech_spans", [])
quiet = _timed_rows(payload.get("quiet_windows", []))
if not payload.get("require_measured"):
rows = rows or [row for row in _asr_segments(work_dir) if row["text"].strip()]
if not quiet:
asr_evidence = _load_work_json(work_dir, "asr_timing_evidence.json") or {}
if isinstance(asr_evidence, dict) and asr_evidence.get("status") == "NO_AUDIO_STREAM":
quiet = [{"start": 0.0, "end": 1e9}]
else:
silence = _load_work_json(work_dir, "silence_periods.json") or []
quiet = _timed_rows(row for row in silence if not row["has_speech"])
return _timed_rows(rows), quiet, _timed_rows(_dialogue_speech_spans(rows))
def _merged_handoff_intervals(start, end, rows):
intervals = sorted(
(max(start, row["start"]), min(end, row["end"]))
for row in rows
if row["end"] > start and row["start"] < end
)
merged = []
for left, right in intervals:
if right <= left:
continue
if merged and left <= merged[-1][1]:
merged[-1] = (merged[-1][0], max(merged[-1][1], right))
else:
merged.append((left, right))
return merged
def _speech_overlap_excluding_quiet(start, end, speech, quiet):
speech_intervals = _merged_handoff_intervals(start, end, speech)
quiet_intervals = _merged_handoff_intervals(start, end, quiet)
overlap = sum(right - left for left, right in speech_intervals)
for speech_left, speech_right in speech_intervals:
overlap -= sum(
max(0.0, min(speech_right, quiet_right) - max(speech_left, quiet_left))
for quiet_left, quiet_right in quiet_intervals
)
return max(0.0, overlap)
def _measured_speech_owned(
start, end, speech, quiet, anchors, authored, require_measured=False
):
duration = max(0.0, end - start)
quiet_min = max(0.3, duration * CONFIG["quiet_overlap_min_ratio"])
if speech:
return _speech_overlap_excluding_quiet(start, end, speech, quiet) > 0.05
quiet_overlap = sum(
right - left for left, right in _merged_handoff_intervals(start, end, quiet)
)
if quiet and quiet_overlap >= quiet_min:
return False
return True if anchors or require_measured else bool(authored)
def _entry_speech_owned(
start, speech, dialogue, quiet, anchors, authored, require_measured=False, tolerance=0.05
):
if any(row["start"] - tolerance <= start <= row["end"] + tolerance for row in quiet):
return False
if any(row["start"] - tolerance <= start < row["end"] - tolerance for row in dialogue):
return True
if speech:
return False
return True if anchors or require_measured else bool(authored)
def _unowned_entry_status(start, speech, quiet, tolerance=0.05):
"""Entry status when a narration entry does not interrupt source dialogue.
`non_dialogue_source` when the entry lands inside measured speech that holds only
interjections (a scream, "Hi."); `quiet_source` for measured quiet or no speech.
"""
if any(row["start"] - tolerance <= start <= row["end"] + tolerance for row in quiet):
return "quiet_source"
if any(row["start"] <= start < row["end"] for row in speech):
return "non_dialogue_source"
return "quiet_source"
def _work_has_source_speech(work_dir, speech_spans, require_measured):
if speech_spans or require_measured:
return True
return any(item["text"].strip() for item in _asr_segments(work_dir))
def _separated_by_measured_silence(run_end, anchor, quiet_windows, acoustic_pauses):
"""True when `anchor` belongs to a fresh utterance starting after silence at/after `run_end`."""
utterance_end = anchor["pause_start"]
for row in (*quiet_windows, *acoustic_pauses):
start, end = row["start"], row["end"]
if end >= utterance_end - 0.05:
continue
if start - 0.05 <= run_end <= end + 0.05 and end > run_end - 0.02:
return True
if start >= run_end - 0.05 and end - max(run_end, start) >= 0.35 and end < utterance_end - 0.10:
return True
return False
def _apply_source_sentence_handoffs(tts_segments, work_dir, video_duration):
"""Keep source audio ducked until a nearby sentence boundary after narration.
This does not move or trim narration. It only extends the ORIGINAL-audio duck
envelope so returning the source track does not reveal the middle of a sentence —
for at most SOURCE_HANDOFF_MAX_HOLD_SECONDS. With no anchor in that window the duck
releases at the narration end (`bounded_release`) instead of burying source dialogue
until a distant anchor.
"""
fade = CONFIG["duck_fade_seconds"]
bridge = CONFIG["duck_bridge_seconds"]
anchors, artifact, evidence_payload = _load_sentence_handoff_anchors(work_dir)
speech_spans, quiet_windows, dialogue_spans = _handoff_speech_evidence(
work_dir, evidence_payload
)
acoustic_pauses = _timed_rows(evidence_payload.get("acoustic_pauses", []))
require_measured = evidence_payload.get("require_measured", False)
placed = []
for seg in tts_segments:
start, end = _seg_place_window(seg)
if end > start:
placed.append((start, end, seg))
placed.sort(key=lambda item: (item[0], item[1]))
if not placed:
return []
runs = []
for start, end, seg in placed:
if runs and start - runs[-1]["end"] <= bridge + 1e-6:
runs[-1]["end"] = max(runs[-1]["end"], end)
runs[-1]["segments"].append(seg)
else:
runs.append({"start": start, "end": end, "segments": [seg]})
source_has_speech = _work_has_source_speech(work_dir, speech_spans, require_measured)
report = []
for run in runs:
ownership = []
for seg in run["segments"]:
start, end = _seg_place_window(seg)
measured = _measured_speech_owned(
start,
end,
speech_spans,
quiet_windows,
anchors,
seg["overlaps_speech"],
require_measured=require_measured,
)
seg["overlaps_speech"] = measured
ownership.append(measured)
first = run["segments"][0]
entry_owned = _entry_speech_owned(
run["start"],
speech_spans,
dialogue_spans,
quiet_windows,
anchors,
first["overlaps_speech"],
require_measured=require_measured,
)
speech_owned = entry_owned or any(ownership)
if not entry_owned:
first["source_entry_status"] = _unowned_entry_status(
run["start"], speech_spans, quiet_windows
)
if not speech_owned:
report.append({"start": run["start"], "end": run["end"], "status": "quiet_source"})
continue
last = run["segments"][-1]
entry_anchor = _prefer_verified([
anchor
for anchor in anchors
if anchor["pause_start"] - 0.05 <= run["start"] <= anchor["time"] + 0.08
])
start_safe = run["start"] <= 0.25 or entry_anchor is not None
if entry_owned and anchors and not start_safe:
first["source_handoff_blocking"] = True
first["source_entry_status"] = "unsafe_entry"
elif entry_owned and not anchors:
first["source_entry_status"] = "unverified"
elif entry_owned and entry_anchor is not None and not entry_anchor["verified"]:
first["source_entry_status"] = "sentence_boundary_unverified"
elif entry_owned:
first["source_entry_status"] = "sentence_boundary"
max_hold = SOURCE_HANDOFF_MAX_HOLD_SECONDS
restore_anchor = _prefer_verified([
anchor
for anchor in anchors
if run["end"] - 0.01 <= anchor["time"] <= run["end"] + max_hold
and not _separated_by_measured_silence(
run["end"], anchor, quiet_windows, acoustic_pauses
)
])
if restore_anchor is not None:
# Hold the source low through its last spoken sample, then fit the release
# entirely inside the measured pause. Never begin the ramp `fade` seconds
# before the anchor when that would expose the final source phoneme.
duck_end = max(run["end"], restore_anchor["pause_start"])
restore_at = max(duck_end, restore_anchor["time"])
status = (
"sentence_boundary"
if restore_anchor["verified"]
else "sentence_boundary_unverified"
)
elif anchors and float(video_duration) - run["end"] <= max_hold:
# No complete source sentence before a near tail: never expose a fragment there.
restore_at = float(video_duration)
duck_end = float(video_duration)
status = "held_to_timeline_end"
elif anchors:
# No anchor within the bound: release smoothly at the narration end instead of
# burying upcoming dialogue across coarse ASR windows or distant anchors.
restore_at = run["end"] + fade
duck_end = run["end"]
status = "bounded_release"
elif source_has_speech:
first["source_handoff_blocking"] = True
first["source_entry_status"] = "anchors_unavailable"
restore_at = run["end"] + fade
duck_end = run["end"]
status = "anchors_unavailable"
else:
restore_at = run["end"] + fade
duck_end = run["end"]
status = "no_source_speech"
last["source_duck_end"] = round(min(float(video_duration), duck_end), 4)
last["source_restore_at"] = round(min(float(video_duration), restore_at), 4)
last["source_handoff_status"] = status
report.append({
"start": round(run["start"], 4),
"end": round(run["end"], 4),
"restore_at": last["source_restore_at"],
"hold_seconds": round(max(0.0, last["source_restore_at"] - run["end"]), 4),
"status": status,
"anchor_artifact": artifact,
})
return report
def _amix_tail(narr_vol, bgm_chain=""):
"""Mix the prepared original track [orig] (+ optional BGM bed) with the boosted
narration [narr] into [aout]. bgm_chain, when given, defines [bgm] from input [2:a]."""
vocal_fx = (
",highpass=f=80,equalizer=f=175:t=q:w=1.1:g=1.4,"
"equalizer=f=320:t=q:w=1.4:g=-1.6,equalizer=f=3200:t=q:w=1.2:g=1.8,"
"equalizer=f=6800:t=q:w=2.0:g=-1.5,"
"acompressor=threshold=0.112:ratio=2.4:attack=12:release=140:makeup=1.19"
if CONFIG.get("studio_vocal_polish", False)
else ""
)
narr = f"[1:a]volume={narr_vol}{vocal_fx},aresample=48000[narr];"
if bgm_chain:
return bgm_chain + narr + "[orig][bgm][narr]amix=inputs=3:duration=first:dropout_transition=0:normalize=0[aout]"
return narr + "[orig][narr]amix=inputs=2:duration=first:dropout_transition=0:normalize=0[aout]"
def _duck_envelope(tts_segments, idle, speech_vol, quiet_vol, fade, bridge):
"""Per-beat ducking automation for the ORIGINAL track.
Uses the shared ducking contract: [start-fade,start] pre-roll ramp down,
[start,end] held at the selected duck level, and [end,end+fade] release.
Bridged spans use the most-ducked (lowest) level, matching timeline.json /
JianYing keyframes. Returns a volume= expression.
"""
windows = []
for seg in tts_segments:
start, narration_end = _seg_place_window(seg)
hold_end = max(narration_end, seg.get("source_duck_end", narration_end))
restore_at = max(hold_end, seg.get("source_restore_at", hold_end + fade))
level = speech_vol if seg["overlaps_speech"] else quiet_vol
windows.append((start, hold_end, level, restore_at))
return release_ducking_expression(windows, idle, fade, bridge=bridge)
def _bgm_envelope(tts_segments, base, duck, fade, bridge):
"""Per-beat ducking automation for the BGM track using the shared contract."""
windows = [
(start, end, duck)
for start, end in map(_seg_place_window, tts_segments)
if end > start
]
return ducking_expression(coalesce_duck_windows(windows, bridge), base, fade)
def _tail_fade_suffix(video_duration):
"""Smooth master tail fade-out on bed tracks so room tone/BGM never chop abruptly."""
fade_ms = float(CONFIG.get("ending_audio_fade_ms", 0.0))
if fade_ms <= 0.0 or video_duration is None or float(video_duration) <= 0.0:
return ""
dur = float(video_duration)
fade_sec = min(fade_ms / 1000.0, dur / 3.0)
if fade_sec < 0.02:
return ""
start = max(0.0, dur - fade_sec)
return f",afade=t=out:st={start:.3f}:d={fade_sec:.3f}"
def _build_audio_filter_complex(
tts_segments,
has_bgm=False,
*,
original_audio_label="0:a",
bgm_audio_label="2:a",
video_duration=None,
):
"""Compose the audio tracks into [aout], like a cut-software timeline.
Tracks:
- original (input [0:a], the video's own audio): ducked under each narration
window by a per-beat volume envelope, but held up at `idle_orig_volume` in
the gaps so the recap never drops to dead air between sentences.
- bgm (input [2:a], optional): a looped music bed, gently ducked under narration.
- narration (input [1:a]): the TTS, boosted and laid on top.
Placement comes from actual_place_start/end.
"""
narr_vol = CONFIG["ducking_narr_weight"]
fade = CONFIG["duck_fade_seconds"]
bridge = CONFIG["duck_bridge_seconds"]
original_in = f"[{original_audio_label}]"
bgm_in = f"[{bgm_audio_label}]"
tail_fade = _tail_fade_suffix(video_duration)
# BGM bed (input [2:a]): ducked under each narration window when present, with a subtle
# vocal-presence pocket (-2.2 dB @ 2.2 kHz) when studio_vocal_polish is enabled.
bgm_chain = ""
if has_bgm:
base = CONFIG["bgm_volume"]
bgm_expr = _bgm_envelope(tts_segments, base, CONFIG["bgm_ducking_volume"], fade, bridge)
bgm_eq = ",equalizer=f=2200:t=q:w=1.2:g=-2.2" if CONFIG.get("studio_vocal_polish", False) else ""
bgm_chain = f"{bgm_in}volume='{bgm_expr}':eval=frame{bgm_eq},aresample=48000{tail_fade}[bgm];"
# Gap-fill ducking envelope on the original track.
idle = CONFIG["idle_orig_volume"]
speech_vol = CONFIG["speech_ducking_volume"]
quiet_vol = CONFIG["zone_ducking_volume"]
expr = _duck_envelope(tts_segments, idle, speech_vol, quiet_vol, fade, bridge)
n_overlap = sum(1 for s in tts_segments if s["overlaps_speech"])
n_quiet = len(tts_segments) - n_overlap
log(f"gap-fill ducking: 间隙原声={idle}, 对白段={speech_vol}({n_overlap}), 安静段={quiet_vol}({n_quiet}), 桥接间隙<{bridge}s")
orig = f"{original_in}volume='{expr}':eval=frame,aresample=48000{tail_fade}[orig];"
return orig + _amix_tail(narr_vol, bgm_chain)
Related skills
| Skill | Stars | Updated | Works with | Status | Install |
|---|---|---|---|---|---|
| slack-gif-creatoranthropics/skillsKnowledge and utilities for creating animated GIFs optimized for Slack. Provides constraints, validation tools, and animation concepts. Use when users request animated GIFs for Slack like "make me a GIF of X doing Y for Slack." | Stars: 198k | Updated | Claude CodeCodex | Verified | Install |
| ponytailDietrichGebert/ponytailLazy senior dev mode for any coding task: the smallest change that fully solves it, and a reply a busy human understands in one read. | Stars: 196k | Updated | Claude CodeCodex | Verified | Install |
| ponytail-debtDietrichGebert/ponytailHarvest every shortcut: comment into one debt ledger, so deferrals get tracked instead of forgotten. One-shot report. | Stars: 196k | Updated | Claude CodeCodex | Verified | Install |
| ponytail-debtDietrichGebert/ponytailList every `shortcut:` comment (and older `ponytail:` ones) as a debt ledger. One-shot report, changes nothing. Use for "ponytail debt", "what did ponytail defer", "list the shortcuts", /ponytail-debt. | Stars: 196k | Updated | Claude CodeCodex | Verified | Install |