How to add pop-in captions with Claude

Add pop-in captions to input.mp4. Transcribe its voiceover with faster-whisper (model "base.en", device cpu, compute_type int8, word_timestamps=True) and group the words two at a time. Show each pair in capitals with punctuation removed, as white bold text with a black outline 1.2% of the frame width thick, text about 11% of the frame width tall, centered horizontally 70% of the way down, from the first word's start to the second word's end. Each caption pops in around its own center: scale 0 at its start, 1.2x at 0.12 seconds, 0.95x at 0.21 seconds and settled at 1.0x by 0.3 seconds, easing between those points. Draw the text with Python and Pillow (not ffmpeg's drawtext filter) and put it on the video with ffmpeg, keep the audio, and save it as output.mp4.

#!/usr/bin/env python3
"""Pop-in captions: the voiceover is transcribed and each two-word phrase pops in, overshoots and settles with a bounce.

Usage: python3 pop-in-captions.py input.mp4 output.mp4
Needs ffmpeg, Pillow and faster-whisper (pip install pillow faster-whisper). The first run downloads Whisper's
"base.en" model (about 150 MB); after that it runs offline.
"""
import json
import math
import subprocess
import sys
from pathlib import Path

from faster_whisper import WhisperModel
from PIL import Image, ImageDraw, ImageFont

WORDS = 2          # words per caption
SIZE = 0.11        # text height as a share of the frame width
Y = 0.70           # where the caption's center sits (0 = top, 1 = bottom)
POP = 0.30         # seconds from nothing to settled
OVERSHOOT = 1.20   # how far past full size the caption grows before settling back
STROKE = 0.012     # black outline thickness as a share of the frame width
FONTS = ["/System/Library/Fonts/Supplemental/Arial Bold.ttf",     # macOS
         "/usr/share/fonts/truetype/dejavu/DejaVuSans-Bold.ttf",  # Linux
         "C:/Windows/Fonts/arialbd.ttf"]                          # Windows
FONT_FILE = next((f for f in FONTS if Path(f).exists()), None) or sys.exit("No bold font found: add one to FONTS")


def transcribe(path):
    """Word timings from Whisper, grouped into captions of WORDS words: (text, start, end)."""
    model = WhisperModel("base.en", device="cpu", compute_type="int8")
    segments, _ = model.transcribe(path, word_timestamps=True)
    words = [word for segment in segments for word in segment.words]
    groups = [words[i:i + WORDS] for i in range(0, len(words), WORDS)]
    return [(" ".join(word.word.strip(" .,!?").upper() for word in group), group[0].start, group[-1].end)
            for group in groups]


def caption_image(text):
    """White bold text with a black outline, drawn once at full size."""
    font = ImageFont.truetype(FONT_FILE, round(w * SIZE))
    stroke = round(w * STROKE)
    x0, y0, x1, y1 = ImageDraw.Draw(Image.new("RGBA", (1, 1))).textbbox((0, 0), text, font=font, stroke_width=stroke)
    img = Image.new("RGBA", (x1 - x0, y1 - y0))
    ImageDraw.Draw(img).text((-x0, -y0), text, font=font, fill="white", stroke_width=stroke, stroke_fill="black")
    return img


def pop_scale(p):
    """0 -> OVERSHOOT at 40% of the pop, a small dip below 1 at 70%, then 1."""
    dip = 1 - (OVERSHOOT - 1) / 4
    keys = [(0, 0), (0.4, OVERSHOOT), (0.7, dip), (1, 1)]
    if p >= 1:
        return 1
    for (p0, s0), (p1, s1) in zip(keys, keys[1:]):
        if p < p1:
            k = (1 - math.cos(math.pi * (p - p0) / (p1 - p0))) / 2  # ease in and out between keys
            return s0 + (s1 - s0) * k


def overlay(t):
    img = Image.new("RGBA", (w, h))
    for (text, start, end), full in zip(captions, images):
        if start <= t < end:
            scale = pop_scale((t - start) / POP)
            if scale > 0.02:
                c = full.resize((max(1, round(full.width * scale)), max(1, round(full.height * scale))), Image.LANCZOS)
                img.alpha_composite(c, (round(w / 2 - c.width / 2), round(h * Y - c.height / 2)))
    return img


def probe(path):
    info = json.loads(subprocess.check_output(["ffprobe", "-v", "error", "-select_streams", "v:0", "-show_entries",
                                               "stream=width,height,r_frame_rate:format=duration", "-of", "json", path]))
    s = info["streams"][0]
    num, den = s["r_frame_rate"].split("/")
    return s["width"], s["height"], s["r_frame_rate"], float(num) / float(den), float(info["format"]["duration"])


def render(src, out):
    """Draw overlay(t) on every frame of src and save to out, keeping the audio."""
    ff = subprocess.Popen(["ffmpeg", "-y", "-v", "error", "-i", src,
                           "-f", "rawvideo", "-pix_fmt", "rgba", "-s", f"{w}x{h}", "-r", rate, "-i", "-",
                           "-filter_complex", "[0:v][1:v]overlay=shortest=1,format=yuv420p[v]",
                           "-map", "[v]", "-map", "0:a?", "-c:v", "libx264", "-crf", "18", "-c:a", "copy", out],
                          stdin=subprocess.PIPE)
    for i in range(math.ceil(duration * fps)):
        ff.stdin.write(overlay(i / fps).tobytes())
    ff.stdin.close()
    if ff.wait():
        sys.exit("ffmpeg failed")


src, out = sys.argv[1:3]
w, h, rate, fps, duration = probe(src)
captions = transcribe(src)
images = [caption_image(text) for text, _, _ in captions]
render(src, out)