How to add dual-language captions with Claude

Add dual-language captions to input.mp4. Transcribe the speech with faster-whisper (model base.en, word timestamps), group the words four at a time, and translate each group into Spanish. Show each group from its first word's start to its last word's end and nothing after the last word: the English in white Arial Bold 6.4% of the frame width tall, centered horizontally with its center 80% of the way down the frame, and the Spanish in yellow (255, 214, 10) Arial Bold 5.4% of the frame width tall, centered 5.5% of the frame height below it. Put both on one black strip at 67% opacity (alpha 170) with rounded corners, reaching 3% of the frame width past the wider line on each side and 2.1% above the English and below the Spanish. Use Python with Pillow to draw the text (don't use ffmpeg's drawtext or subtitles filter) and ffmpeg to put it on the video, keep the audio, and save it as output.mp4.

#!/usr/bin/env python3
"""Dual-language captions: the spoken phrase in white with its translation in yellow underneath, on a dark strip.

Usage: python3 dual-language-captions.py input.mp4 output.mp4
Whisper transcribes the speech and splits it into phrases of CHUNK words. TRANSLATION holds one line per phrase
in the second language (ask Claude to translate them); if the count doesn't match, the script prints the phrases
and stops so you can fill it in.
Needs ffmpeg, Pillow and faster-whisper (pip install pillow faster-whisper).
"""
import json
import math
import subprocess
import sys
from pathlib import Path

from faster_whisper import WhisperModel
from PIL import Image, ImageDraw, ImageFont

TRANSLATION = ["Este sérum limpió mi",   # one line per spoken phrase, in order
               "piel en dos semanas"]
CHUNK = 4                  # words per phrase
SIZE = 0.064               # spoken line's text height as a share of the frame width
SECOND_SIZE = 0.054        # translation's text height as a share of the frame width
Y = 0.80                   # where the spoken line's center sits (0 = top, 1 = bottom)
GAP = 0.055                # distance down to the translation's center, as a share of the frame height
TEXT = "white"             # spoken line color
SECOND_TEXT = (255, 214, 10)  # translation color
STRIP = (0, 0, 0, 170)     # color and opacity (0-255) of the strip behind both lines
PAD = 0.03                 # space around the text inside the strip, as a share of the frame width
FONTS = ["/System/Library/Fonts/Supplemental/Arial Bold.ttf",     # macOS
         "/usr/share/fonts/truetype/dejavu/DejaVuSans-Bold.ttf",  # Linux
         "C:/Windows/Fonts/arialbd.ttf"]                          # Windows
FONT_FILE = next((f for f in FONTS if Path(f).exists()), None) or sys.exit("No bold font found: add one to FONTS")


def phrases(src):
    """[(start, end, text)] for every CHUNK spoken words, from Whisper's word timings."""
    model = WhisperModel("base.en", device="cpu", compute_type="int8")
    segments, _ = model.transcribe(src, word_timestamps=True)
    words = [word for segment in segments for word in segment.words]
    groups = [words[i:i + CHUNK] for i in range(0, len(words), CHUNK)]
    return [(g[0].start, g[-1].end, " ".join(word.word.strip() for word in g)) for g in groups]


def caption(spoken, translated):
    """Both lines drawn once, centered, on one rounded strip."""
    img = Image.new("RGBA", (w, h))
    draw = ImageDraw.Draw(img)
    top, bottom = (w / 2, h * Y), (w / 2, h * (Y + GAP))
    a = draw.textbbox(top, spoken, font=font, anchor="mm")
    b = draw.textbbox(bottom, translated, font=second_font, anchor="mm")
    pad = w * PAD
    draw.rounded_rectangle([min(a[0], b[0]) - pad, a[1] - pad * 0.7, max(a[2], b[2]) + pad, b[3] + pad * 0.7],
                           radius=pad * 0.6, fill=STRIP)
    draw.text(top, spoken, font=font, anchor="mm", fill=TEXT)
    draw.text(bottom, translated, font=second_font, anchor="mm", fill=SECOND_TEXT)
    return img


def overlay(t):
    for i, (start, end, _) in enumerate(cues):
        if start <= t < end:
            return drawn[i]
    return blank


def probe(path):
    info = json.loads(subprocess.check_output(["ffprobe", "-v", "error", "-select_streams", "v:0", "-show_entries",
                                               "stream=width,height,r_frame_rate:format=duration", "-of", "json", path]))
    s = info["streams"][0]
    num, den = s["r_frame_rate"].split("/")
    return s["width"], s["height"], s["r_frame_rate"], float(num) / float(den), float(info["format"]["duration"])


def render(src, out):
    """Draw overlay(t) on every frame of src and save to out, keeping the audio."""
    ff = subprocess.Popen(["ffmpeg", "-y", "-v", "error", "-i", src,
                           "-f", "rawvideo", "-pix_fmt", "rgba", "-s", f"{w}x{h}", "-r", rate, "-i", "-",
                           "-filter_complex", "[0:v][1:v]overlay=shortest=1,format=yuv420p[v]",
                           "-map", "[v]", "-map", "0:a?", "-c:v", "libx264", "-crf", "18", "-c:a", "copy", out],
                          stdin=subprocess.PIPE)
    for i in range(math.ceil(duration * fps)):
        ff.stdin.write(overlay(i / fps).tobytes())
    ff.stdin.close()
    if ff.wait():
        sys.exit("ffmpeg failed")


src, out = sys.argv[1:3]
cues = phrases(src)
if len(cues) != len(TRANSLATION):
    sys.exit("Set TRANSLATION to one line for each of these phrases:\n" + "\n".join(text for _, _, text in cues))
w, h, rate, fps, duration = probe(src)
font = ImageFont.truetype(FONT_FILE, round(w * SIZE))
second_font = ImageFont.truetype(FONT_FILE, round(w * SECOND_SIZE))
blank = Image.new("RGBA", (w, h))
drawn = [caption(text, translated) for (_, _, text), translated in zip(cues, TRANSLATION)]
render(src, out)