jianshuo/claude-skills / wjs-dubbing-video
wjs-dubbing-video
VerifiedUse when the user has a video + a target-language SRT and wants the video to actually speak that language — generates a time-aligned TTS voice dub. Routes by voice ID — Volcano (豆包) TTS for Chinese, edge-tts neural for any language. Defaults to one voice (single-speaker); opt-in multi-speaker via visual diarization. Outputs `*_<lang>_dub.mp4` with the dub audio in place of the original. Final mixing (audio bed + burn-in) is handed off to `/wjs-burning-subtitles`. Triggers — "配音", "中文配音", "Chinese dub", "voice over this", "dub the video", "TTS this SRT", "different voice for each speaker".
| File | Kind | Size |
|---|---|---|
| SKILL.md | skill md | 21.0 KB |
| scripts/dub.pyexecutable | script | 13.0 KB |
| scripts/visual_diarize.pyexecutable | script | 8.4 KB |
scripts/visual_diarize.py
View on GitHub#!/usr/bin/env python3
"""Visual speaker diarization via mouth-movement detection.
Reads a video + an SRT, detects up to N faces per frame using
MediaPipe Face Mesh, clusters them by horizontal screen position into
speakers (left = A, right = B), measures each face's mouth aperture
over time, and integrates frame-to-frame mouth movement within each
SRT cue's [start, end] window. The face with the most movement during
a cue wins that cue's speaker tag.
Outputs a tagged SRT with `[A]`/`[B]` prefixes and a JSON report of
per-cue scores so you can spot-check ambiguous calls.
Usage:
visual_diarize.py --video V.mp4 --srt CLEAN.srt --out TAGGED.srt
[--sample-fps 5] [--num-speakers 2]
[--report report.json]
"""
import argparse, json, re, sys
from pathlib import Path
import cv2
import numpy as np
import mediapipe as mp
from mediapipe.tasks import python as mp_python
from mediapipe.tasks.python import vision as mp_vision
# MediaPipe Tasks API needs a model file. Use a stable cache location.
MODEL_PATH = Path("/tmp/mp_models/face_landmarker.task")
MODEL_URL = ("https://storage.googleapis.com/mediapipe-models/"
"face_landmarker/face_landmarker/float16/1/face_landmarker.task")
def ensure_model():
if MODEL_PATH.exists():
return
import urllib.request
MODEL_PATH.parent.mkdir(parents=True, exist_ok=True)
print(f"downloading {MODEL_URL} → {MODEL_PATH}", file=sys.stderr)
urllib.request.urlretrieve(MODEL_URL, MODEL_PATH)
def ts_to_sec(ts):
h, m, rest = ts.split(":")
s, ms = rest.split(",")
return int(h)*3600 + int(m)*60 + int(s) + int(ms)/1000
def parse_srt(text):
out = []
for b in re.split(r"\n\s*\n", text.strip()):
lines = b.strip().splitlines()
if len(lines) < 3:
continue
idx = lines[0]
m = re.match(r"(\S+)\s+-->\s+(\S+)", lines[1])
out.append({
"idx": idx,
"time_line": lines[1],
"start": ts_to_sec(m.group(1)),
"end": ts_to_sec(m.group(2)),
# Strip any pre-existing [X] tag from cue text — we'll re-tag
"text": re.sub(r'^\[[A-Za-z][A-Za-z0-9_-]*\]\s*',
'', "\n".join(lines[2:])),
})
return out
def main():
ap = argparse.ArgumentParser()
ap.add_argument("--video", required=True)
ap.add_argument("--srt", required=True, help="clean SRT (no [X] tags)")
ap.add_argument("--out", required=True, help="output tagged SRT")
ap.add_argument("--sample-fps", type=float, default=5.0,
help="frames per second to sample (default 5)")
ap.add_argument("--num-speakers", type=int, default=2)
ap.add_argument("--labels", default="A,B,C,D,E",
help="comma-separated labels in left-to-right order")
ap.add_argument("--report", default=None,
help="write per-cue diarization JSON for spot-checking")
args = ap.parse_args()
cap = cv2.VideoCapture(args.video)
fps = cap.get(cv2.CAP_PROP_FPS)
n_frames = int(cap.get(cv2.CAP_PROP_FRAME_COUNT))
duration = n_frames / fps
print(f"video: {duration:.1f}s @ {fps:.2f}fps, {n_frames} frames", file=sys.stderr)
# Sample every Nth frame
step = max(1, int(round(fps / args.sample_fps)))
actual_sample_fps = fps / step
print(f"sampling every {step} frames ({actual_sample_fps:.2f}fps)", file=sys.stderr)
ensure_model()
base = mp_python.BaseOptions(model_asset_path=str(MODEL_PATH))
options = mp_vision.FaceLandmarkerOptions(
base_options=base,
running_mode=mp_vision.RunningMode.VIDEO,
num_faces=args.num_speakers,
min_face_detection_confidence=0.4,
min_face_presence_confidence=0.4,
min_tracking_confidence=0.4,
)
landmarker = mp_vision.FaceLandmarker.create_from_options(options)
# Per-frame: collect (face_center_x, mouth_aperture) for each detected face
timeline = [] # list of (timestamp, [(cx, aperture), ...])
fi = 0
while True:
ret, frame = cap.read()
if not ret:
break
if fi % step == 0:
ts = fi / fps
rgb = cv2.cvtColor(frame, cv2.COLOR_BGR2RGB)
mp_image = mp.Image(image_format=mp.ImageFormat.SRGB, data=rgb)
result = landmarker.detect_for_video(mp_image, int(ts * 1000))
faces = []
for lm_list in result.face_landmarks:
# Use nose-tip x (idx 1) as horizontal anchor
cx = lm_list[1].x
# Mouth aperture: inner upper (13) - inner lower (14)
upper = lm_list[13]
lower = lm_list[14]
aperture = abs(upper.y - lower.y)
faces.append((cx, aperture))
timeline.append((ts, faces))
fi += 1
cap.release()
landmarker.close()
print(f"sampled {len(timeline)} frames", file=sys.stderr)
# Cluster face positions: simple bucket along x-axis. For 2 speakers,
# split at the median of all detected face_center_x. For N>2, use
# equal-percentile bins.
all_cx = np.array([cx for _, faces in timeline for cx, _ in faces])
if all_cx.size == 0:
sys.exit("no faces detected anywhere — diarization not possible")
n = args.num_speakers
quantiles = np.linspace(0, 1, n + 1)
bin_edges = np.quantile(all_cx, quantiles)
bin_edges[0] -= 1e-6
bin_edges[-1] += 1e-6
labels = args.labels.split(",")[:n]
print(f"x-bin edges: {bin_edges.round(3).tolist()} → labels {labels}",
file=sys.stderr)
# Per-frame per-speaker aperture
apertures = {lab: [] for lab in labels}
timestamps = []
for ts, faces in timeline:
timestamps.append(ts)
per = {lab: np.nan for lab in labels}
for cx, ap in faces:
# Find which bin this face falls in
bin_idx = np.searchsorted(bin_edges, cx, side='right') - 1
bin_idx = max(0, min(n - 1, bin_idx))
lab = labels[bin_idx]
# If multiple faces hit the same bin (shouldn't happen for a
# well-posed scene), keep the larger aperture
cur = per[lab]
per[lab] = ap if np.isnan(cur) else max(cur, ap)
for lab in labels:
apertures[lab].append(per[lab])
timestamps = np.array(timestamps)
movement = {}
for lab in labels:
a = np.array(apertures[lab], dtype=float)
# Forward-fill NaNs so diff isn't blasted to NaN at gaps
# (a missing detection ≠ no movement; we just don't know)
diff = np.zeros_like(a)
for i in range(1, len(a)):
if not np.isnan(a[i]) and not np.isnan(a[i-1]):
diff[i] = abs(a[i] - a[i-1])
# else 0 (insufficient data)
movement[lab] = diff
# Per cue: integrate movement
cues = parse_srt(Path(args.srt).read_text())
report = []
for cue in cues:
mask = (timestamps >= cue["start"]) & (timestamps < cue["end"])
scores = {lab: float(np.nansum(movement[lab][mask])) for lab in labels}
max_lab = max(scores, key=scores.get)
# Confidence: ratio of winner to runner-up
sorted_v = sorted(scores.values(), reverse=True)
ratio = (sorted_v[0] / sorted_v[1]) if len(sorted_v) > 1 and sorted_v[1] > 0 else float('inf')
report.append({
"idx": cue["idx"],
"time": cue["time_line"],
"start": cue["start"],
"end": cue["end"],
"scores": scores,
"speaker": max_lab,
"confidence_ratio": ratio,
"frames_in_cue": int(mask.sum()),
})
# Write tagged SRT
with open(args.out, "w") as f:
for cue, r in zip(cues, report):
f.write(f"{cue['idx']}\n{cue['time_line']}\n[{r['speaker']}] {cue['text']}\n\n")
# Print summary
counts = {}
for r in report:
counts[r["speaker"]] = counts.get(r["speaker"], 0) + 1
print(f"\nspeaker assignments: {counts}", file=sys.stderr)
print(f"low-confidence cues (ratio < 1.5):", file=sys.stderr)
for r in report:
if r["confidence_ratio"] < 1.5:
print(f" cue {r['idx']} → [{r['speaker']}] "
f"scores={r['scores']} ratio={r['confidence_ratio']:.2f}",
file=sys.stderr)
if args.report:
Path(args.report).write_text(json.dumps(report, indent=2))
print(f"wrote per-cue report → {args.report}", file=sys.stderr)
if __name__ == "__main__":
main()
Related skills
| Skill | Stars | Updated | Works with | Status | Install |
|---|---|---|---|---|---|
| slack-gif-creatoranthropics/skillsKnowledge and utilities for creating animated GIFs optimized for Slack. Provides constraints, validation tools, and animation concepts. Use when users request animated GIFs for Slack like "make me a GIF of X doing Y for Slack." | Stars: 198k | Updated | Claude CodeCodex | Verified | Install |
| ponytailDietrichGebert/ponytailLazy senior dev mode for any coding task: the smallest change that fully solves it, and a reply a busy human understands in one read. | Stars: 194k | Updated | Claude CodeCodex | Verified | Install |
| ponytail-debtDietrichGebert/ponytailHarvest every shortcut: comment into one debt ledger, so deferrals get tracked instead of forgotten. One-shot report. | Stars: 194k | Updated | Claude CodeCodex | Verified | Install |
| ponytail-debtDietrichGebert/ponytailList every `shortcut:` comment (and older `ponytail:` ones) as a debt ledger. One-shot report, changes nothing. Use for "ponytail debt", "what did ponytail defer", "list the shortcuts", /ponytail-debt. | Stars: 194k | Updated | Claude CodeCodex | Verified | Install |