How to add translated captions with Claude

Add translated captions to input.mp4. Transcribe the speech with faster-whisper (model base.en, word timestamps), group the words four at a time, and translate each group into French ("Ce sérum a purifié", "ma peau en deux semaines"). Show only the French, each line from its group's first word's start to its last word's end and nothing after the last word: white Arial Bold 6% of the frame width tall, centered horizontally with its center 82% of the way down the frame, on a black strip at 67% opacity (alpha 170) with rounded corners reaching 3% of the frame width past the text on the left and right and 2.1% above and below. While a caption is on screen, show a yellow (255, 214, 10) rounded tag reading "FR" in black Arial Bold 4.5% of the frame width tall, centered horizontally 5.5% of the frame height above the caption's center, reaching 1.8% of the frame width past its letters on the sides and 1.05% above and below. Use Python with Pillow to draw the text (don't use ffmpeg's drawtext or subtitles filter) and ffmpeg to put it on the video, keep the audio, and save it as output.mp4.

#!/usr/bin/env python3
"""Translated captions: each spoken phrase shown only in another language, in white on a dark strip under a yellow language tag.

Usage: python3 translated-captions.py input.mp4 output.mp4
Whisper transcribes the speech and splits it into phrases of CHUNK words. TRANSLATION holds one line per phrase
in the new language (ask Claude to translate them); if the count doesn't match, the script prints the phrases
and stops so you can fill it in.
Needs ffmpeg, Pillow and faster-whisper (pip install pillow faster-whisper).
"""
import json
import math
import subprocess
import sys
from pathlib import Path

from faster_whisper import WhisperModel
from PIL import Image, ImageDraw, ImageFont

TRANSLATION = ["Ce sérum a purifié",        # one line per spoken phrase, in order
               "ma peau en deux semaines"]
LANGUAGE = "FR"            # what the tag above the captions says
CHUNK = 4                  # words per phrase
SIZE = 0.06                # caption text height as a share of the frame width
Y = 0.82                   # where the caption's center sits (0 = top, 1 = bottom)
TEXT = "white"             # caption color
STRIP = (0, 0, 0, 170)     # color and opacity (0-255) of the strip behind the caption
PAD = 0.03                 # space around the text inside the strip, as a share of the frame width
TAG_SIZE = 0.045           # tag text height as a share of the frame width
TAG_GAP = 0.055            # distance up from the caption's center to the tag's center, as a share of the frame height
TAG = (255, 214, 10)       # tag color (its letters are black)
FONTS = ["/System/Library/Fonts/Supplemental/Arial Bold.ttf",     # macOS
         "/usr/share/fonts/truetype/dejavu/DejaVuSans-Bold.ttf",  # Linux
         "C:/Windows/Fonts/arialbd.ttf"]                          # Windows
FONT_FILE = next((f for f in FONTS if Path(f).exists()), None) or sys.exit("No bold font found: add one to FONTS")


def phrases(src):
    """[(start, end, text)] for every CHUNK spoken words, from Whisper's word timings."""
    model = WhisperModel("base.en", device="cpu", compute_type="int8")
    segments, _ = model.transcribe(src, word_timestamps=True)
    words = [word for segment in segments for word in segment.words]
    groups = [words[i:i + CHUNK] for i in range(0, len(words), CHUNK)]
    return [(g[0].start, g[-1].end, " ".join(word.word.strip() for word in g)) for g in groups]


def caption(translated):
    """The translated line on a rounded strip, with the language tag centered above it."""
    img = Image.new("RGBA", (w, h))
    draw = ImageDraw.Draw(img)
    pad = w * PAD
    x0, y0, x1, y1 = draw.textbbox((w / 2, h * Y), translated, font=font, anchor="mm")
    draw.rounded_rectangle([x0 - pad, y0 - pad * 0.7, x1 + pad, y1 + pad * 0.7], radius=pad * 0.6, fill=STRIP)
    draw.text((w / 2, h * Y), translated, font=font, anchor="mm", fill=TEXT)
    tag_center = (w / 2, h * (Y - TAG_GAP))
    x0, y0, x1, y1 = draw.textbbox(tag_center, LANGUAGE, font=tag_font, anchor="mm")
    draw.rounded_rectangle([x0 - pad * 0.6, y0 - pad * 0.35, x1 + pad * 0.6, y1 + pad * 0.35],
                           radius=pad * 0.4, fill=TAG)
    draw.text(tag_center, LANGUAGE, font=tag_font, anchor="mm", fill="black")
    return img


def overlay(t):
    for i, (start, end, _) in enumerate(cues):
        if start <= t < end:
            return drawn[i]
    return blank


def probe(path):
    info = json.loads(subprocess.check_output(["ffprobe", "-v", "error", "-select_streams", "v:0", "-show_entries",
                                               "stream=width,height,r_frame_rate:format=duration", "-of", "json", path]))
    s = info["streams"][0]
    num, den = s["r_frame_rate"].split("/")
    return s["width"], s["height"], s["r_frame_rate"], float(num) / float(den), float(info["format"]["duration"])


def render(src, out):
    """Draw overlay(t) on every frame of src and save to out, keeping the audio."""
    ff = subprocess.Popen(["ffmpeg", "-y", "-v", "error", "-i", src,
                           "-f", "rawvideo", "-pix_fmt", "rgba", "-s", f"{w}x{h}", "-r", rate, "-i", "-",
                           "-filter_complex", "[0:v][1:v]overlay=shortest=1,format=yuv420p[v]",
                           "-map", "[v]", "-map", "0:a?", "-c:v", "libx264", "-crf", "18", "-c:a", "copy", out],
                          stdin=subprocess.PIPE)
    for i in range(math.ceil(duration * fps)):
        ff.stdin.write(overlay(i / fps).tobytes())
    ff.stdin.close()
    if ff.wait():
        sys.exit("ffmpeg failed")


src, out = sys.argv[1:3]
cues = phrases(src)
if len(cues) != len(TRANSLATION):
    sys.exit("Set TRANSLATION to one line for each of these phrases:\n" + "\n".join(text for _, _, text in cues))
w, h, rate, fps, duration = probe(src)
font = ImageFont.truetype(FONT_FILE, round(w * SIZE))
tag_font = ImageFont.truetype(FONT_FILE, round(w * TAG_SIZE))
blank = Image.new("RGBA", (w, h))
drawn = [caption(translated) for translated in TRANSLATION]
render(src, out)