How to add auto captions with Claude

Add automatic captions to input.mp4 from its speech, with no script or subtitle file. Transcribe the audio with faster-whisper (model "base.en", device cpu, compute_type int8, word_timestamps=True). Group the words in order into lines of at most 4 words that fit in 86% of the frame width, and show each line from its first word's start until 0.4 seconds after its last word ends, or until the next line starts if that is sooner, with nothing on screen after the last line. Draw each line in white bold text (Arial Bold) 7.5% of the frame width tall, centered horizontally with its center 84% of the way down the frame, over a soft shadow: the same text in black at alpha 230 with a stroke 12% of the text height thick, shifted 6% of the text height right and down and blurred with a Gaussian blur of 12% of the text height. No box behind the text. Use Python with Pillow to draw the text (don't use ffmpeg's drawtext or subtitles filter) and ffmpeg to put it on the video, keep the audio, and save it as output.mp4.

#!/usr/bin/env python3
"""Auto captions: Whisper transcribes the speech, then each line is drawn in white with a soft shadow as it is spoken.

Usage: python3 auto-captions.py input.mp4 output.mp4
No subtitle file needed: faster-whisper listens to the audio on your computer and times every word.
Needs ffmpeg, faster-whisper and Pillow (pip install faster-whisper pillow). The first run downloads
the "base.en" speech model (about 140 MB); use "base" or "small" for speech that isn't English.
"""
import json
import math
import subprocess
import sys
from pathlib import Path

from faster_whisper import WhisperModel
from PIL import Image, ImageDraw, ImageFilter, ImageFont

MODEL = "base.en"     # Whisper model: bigger ("small.en", "medium.en") is more accurate but slower
SIZE = 0.075          # text height as a share of the frame width
Y = 0.84              # where the line's center sits (0 = top, 1 = bottom)
MAX_WORDS = 4         # most words on one line
MAX_WIDTH = 0.86      # longest line as a share of the frame width
HOLD = 0.4            # seconds a line stays up after its last word, unless the next line starts sooner
TEXT = "white"        # text color
SHADOW = (0, 0, 0, 230)  # color and opacity (0-255) of the soft shadow under the text
FONTS = ["/System/Library/Fonts/Supplemental/Arial Bold.ttf",     # macOS
         "/usr/share/fonts/truetype/dejavu/DejaVuSans-Bold.ttf",  # Linux
         "C:/Windows/Fonts/arialbd.ttf"]                          # Windows
FONT_FILE = next((f for f in FONTS if Path(f).exists()), None) or sys.exit("No bold font found: add one to FONTS")


def transcribe(path):
    """Every spoken word as (text, start, end), times in seconds."""
    model = WhisperModel(MODEL, device="cpu", compute_type="int8")
    segments, _ = model.transcribe(path, word_timestamps=True)
    return [(word.word.strip(), word.start, word.end) for segment in segments for word in segment.words]


def caption_lines(words):
    """Group the words into lines of up to MAX_WORDS that fit MAX_WIDTH: [(text, start, end)]."""
    groups = []
    for word in words:
        longer = " ".join(text for text, _, _ in (groups[-1] if groups else []) + [word])
        if groups and len(groups[-1]) < MAX_WORDS and font.getlength(longer) <= w * MAX_WIDTH:
            groups[-1].append(word)
        else:
            groups.append([word])
    lines = []
    for i, group in enumerate(groups):
        end = group[-1][2] + HOLD
        if i + 1 < len(groups):
            end = min(end, groups[i + 1][0][1])
        lines.append((" ".join(text for text, _, _ in group), group[0][1], end))
    return lines


def caption(text):
    """One line drawn once: white text over a blurred shadow."""
    shadow = Image.new("RGBA", (w, h))
    offset = font.size * 0.06
    ImageDraw.Draw(shadow).text((w / 2 + offset, h * Y + offset), text, font=font, anchor="mm", fill=SHADOW,
                                stroke_width=round(font.size * 0.12), stroke_fill=SHADOW)
    img = shadow.filter(ImageFilter.GaussianBlur(font.size * 0.12))
    ImageDraw.Draw(img).text((w / 2, h * Y), text, font=font, anchor="mm", fill=TEXT)
    return img


def overlay(t):
    for i, (text, start, end) in enumerate(lines):
        if start <= t < end:
            return drawn.get(i) or drawn.setdefault(i, caption(text))
    return blank


def probe(path):
    info = json.loads(subprocess.check_output(["ffprobe", "-v", "error", "-select_streams", "v:0", "-show_entries",
                                               "stream=width,height,r_frame_rate:format=duration", "-of", "json", path]))
    s = info["streams"][0]
    num, den = s["r_frame_rate"].split("/")
    return s["width"], s["height"], s["r_frame_rate"], float(num) / float(den), float(info["format"]["duration"])


def render(src, out):
    """Draw overlay(t) on every frame of src and save to out, keeping the audio."""
    ff = subprocess.Popen(["ffmpeg", "-y", "-v", "error", "-i", src,
                           "-f", "rawvideo", "-pix_fmt", "rgba", "-s", f"{w}x{h}", "-r", rate, "-i", "-",
                           "-filter_complex", "[0:v][1:v]overlay=shortest=1,format=yuv420p[v]",
                           "-map", "[v]", "-map", "0:a?", "-c:v", "libx264", "-crf", "18", "-c:a", "copy", out],
                          stdin=subprocess.PIPE)
    for i in range(math.ceil(duration * fps)):
        ff.stdin.write(overlay(i / fps).tobytes())
    ff.stdin.close()
    if ff.wait():
        sys.exit("ffmpeg failed")


src, out = sys.argv[1:3]
w, h, rate, fps, duration = probe(src)
font = ImageFont.truetype(FONT_FILE, round(w * SIZE))
lines = caption_lines(transcribe(src))
blank = Image.new("RGBA", (w, h))
drawn = {}
render(src, out)