How to add build-up captions with Claude

Add build-up captions to input.mp4 from its voiceover. Transcribe it with faster-whisper (model "base.en", device cpu, compute_type int8, word_timestamps=True) and split the words into sentences at words ending in . ? or !. Lay out each whole sentence first, in capitals with punctuation removed, as white bold text (Arial Bold) 8.5% of the frame width tall with a black outline 1.4% of the frame width thick, words separated by a space plus the outline thickness, wrapped into lines no wider than 86% of the frame width, each line centered horizontally, line spacing 1.25 times the text height, and the block of lines centered 70% of the way down the frame. Then show each word at its own place from the moment it starts being spoken, so words never move as later ones arrive, and keep every word of the sentence on screen until 0.6 seconds after its last word ends, then clear it. Nothing appears before the first word. Draw the text with Python and Pillow (not ffmpeg's drawtext filter) and put it on the video with ffmpeg, keep the audio, and save it as output.mp4.

#!/usr/bin/env python3
"""Build-up captions: the voiceover is transcribed and each spoken word joins the caption, which stays until the sentence ends.

Usage: python3 build-up-captions.py input.mp4 output.mp4
Needs ffmpeg, Pillow and faster-whisper (pip install pillow faster-whisper). The first run downloads Whisper's
"base.en" model (about 150 MB); after that it runs offline.
"""
import json
import math
import subprocess
import sys
from pathlib import Path

from faster_whisper import WhisperModel
from PIL import Image, ImageDraw, ImageFont

SIZE = 0.085       # text height as a share of the frame width
Y = 0.70           # where the caption block's center sits (0 = top, 1 = bottom)
WIDTH = 0.86       # widest a line can get, as a share of the frame width
LEADING = 1.25     # line spacing as a multiple of the text height
HOLD = 0.6         # seconds the full sentence stays after its last word
STROKE = 0.014     # black outline thickness as a share of the frame width
FONTS = ["/System/Library/Fonts/Supplemental/Arial Bold.ttf",     # macOS
         "/usr/share/fonts/truetype/dejavu/DejaVuSans-Bold.ttf",  # Linux
         "C:/Windows/Fonts/arialbd.ttf"]                          # Windows
FONT_FILE = next((f for f in FONTS if Path(f).exists()), None) or sys.exit("No bold font found: add one to FONTS")


def transcribe(path):
    """Whisper's words, split into sentences at . ? or !"""
    model = WhisperModel("base.en", device="cpu", compute_type="int8")
    segments, _ = model.transcribe(path, word_timestamps=True)
    sentences, current = [], []
    for segment in segments:
        for word in segment.words:
            current.append(word)
            if word.word.strip().endswith((".", "?", "!")):
                sentences.append(current)
                current = []
    return sentences + [current] if current else sentences


def layout(sentence):
    """Where each word of the finished sentence goes, wrapped into centered lines, so words never move as
    later ones arrive: [(text, x, y, start)]."""
    space = font.getlength(" ") + stroke  # the outline would otherwise eat the gap between words
    lines, line = [], []
    for word in sentence:
        text = word.word.strip(" .,!?").upper()
        width = sum(font.getlength(t) + space for t, _ in line) + font.getlength(text)
        if line and width > w * WIDTH:
            lines.append(line)
            line = []
        line.append((text, word.start))
    lines.append(line)
    step = w * SIZE * LEADING
    top = h * Y - step * (len(lines) - 1) / 2
    placed = []
    for row, line in enumerate(lines):
        x = w / 2 - (sum(font.getlength(t) for t, _ in line) + space * (len(line) - 1)) / 2
        for text, start in line:
            placed.append((text, x, top + row * step, start))
            x += font.getlength(text) + space
    return placed


def overlay(t):
    img = Image.new("RGBA", (w, h))
    draw = ImageDraw.Draw(img)
    for sentence, placed in zip(sentences, layouts):
        if sentence[0].start <= t < sentence[-1].end + HOLD:
            for text, x, y, start in placed:
                if start <= t:
                    draw.text((x, y), text, font=font, anchor="lm", fill="white",
                              stroke_width=stroke, stroke_fill="black")
    return img


def probe(path):
    info = json.loads(subprocess.check_output(["ffprobe", "-v", "error", "-select_streams", "v:0", "-show_entries",
                                               "stream=width,height,r_frame_rate:format=duration", "-of", "json", path]))
    s = info["streams"][0]
    num, den = s["r_frame_rate"].split("/")
    return s["width"], s["height"], s["r_frame_rate"], float(num) / float(den), float(info["format"]["duration"])


def render(src, out):
    """Draw overlay(t) on every frame of src and save to out, keeping the audio."""
    ff = subprocess.Popen(["ffmpeg", "-y", "-v", "error", "-i", src,
                           "-f", "rawvideo", "-pix_fmt", "rgba", "-s", f"{w}x{h}", "-r", rate, "-i", "-",
                           "-filter_complex", "[0:v][1:v]overlay=shortest=1,format=yuv420p[v]",
                           "-map", "[v]", "-map", "0:a?", "-c:v", "libx264", "-crf", "18", "-c:a", "copy", out],
                          stdin=subprocess.PIPE)
    for i in range(math.ceil(duration * fps)):
        ff.stdin.write(overlay(i / fps).tobytes())
    ff.stdin.close()
    if ff.wait():
        sys.exit("ffmpeg failed")


src, out = sys.argv[1:3]
w, h, rate, fps, duration = probe(src)
font = ImageFont.truetype(FONT_FILE, round(w * SIZE))
stroke = round(w * STROKE)
sentences = transcribe(src)
layouts = [layout(sentence) for sentence in sentences]
render(src, out)