How to add a lyric video with Claude

Make a lyric video from input.mp4. Transcribe its vocals with faster-whisper (model base.en, word timestamps) and group the words into lines of at most 5 words, starting a new line after a pause longer than 0.4 seconds or a word ending in punctuation. Show one line at a time in white capitals (Arial Black or another heavy bold font, 12% of the frame width tall, wrapped to fit 86% of the width) with a soft black shadow, centered horizontally with its middle 45% of the way down. Each line is on screen from its first word's start until the next line's first word starts (the last line stays 0.8 seconds after its last word). A line fades in over 0.3 seconds while rising into place from 5% of the frame height lower, and fades out over its last 0.3 seconds while drifting up by the same amount. Darken the whole footage with a 45% black layer from the first line until the last line has gone, fading that in and out over 0.3 seconds. Draw the text with Python and Pillow (don't use ffmpeg's drawtext filter), put it on the video with ffmpeg, keep the audio, and save it as output.mp4.

#!/usr/bin/env python3
"""Lyric video: the vocals are transcribed and each lyric line rises onto the middle of the screen in big type as it is sung.

Usage: python3 lyric-video.py input.mp4 output.mp4
Needs ffmpeg, Pillow and faster-whisper (pip install pillow faster-whisper); the "base.en" model downloads on first run.
For a song, separate the vocals first (for example with Demucs) and transcribe those for cleaner timing.
"""
import json
import math
import subprocess
import sys
from pathlib import Path

from faster_whisper import WhisperModel
from PIL import Image, ImageDraw, ImageFilter, ImageFont

LINE_WORDS = 5          # most words on one lyric line
GAP = 0.4               # a pause this long (seconds) also starts a new line
SIZE = 0.12             # text height as a share of the frame width (shrinks so each row fits)
Y = 0.45                # where the line's center sits (0 = top, 1 = bottom)
RISE = 0.05             # how far a line rises as it comes in, as a share of the frame height
FADE = 0.3              # seconds a line takes to fade in or out
HOLD = 0.8              # seconds the last line stays after its last word
DIM = 0.45              # how much the footage darkens while lyrics are up (0 = not at all)
FONTS = ["/System/Library/Fonts/Supplemental/Arial Black.ttf",    # macOS
         "/usr/share/fonts/truetype/dejavu/DejaVuSans-Bold.ttf",  # Linux
         "C:/Windows/Fonts/ariblk.ttf"]                           # Windows
FONT_FILE = next((f for f in FONTS if Path(f).exists()), None) or sys.exit("No bold font found: add one to FONTS")


def lyric_lines(src):
    """Word timings from Whisper, grouped into lines of at most LINE_WORDS, breaking early at pauses and punctuation."""
    model = WhisperModel("base.en", device="cpu", compute_type="int8")
    segments, _ = model.transcribe(src, word_timestamps=True)
    lines, line = [], []
    for word in (w for s in segments for w in s.words):
        if line and (len(line) == LINE_WORDS or word.start - line[-1].end > GAP or line[-1].word.strip()[-1] in ".,!?"):
            lines.append(line)
            line = []
        line.append(word)
    lines.append(line)
    timed = []
    for i, line in enumerate(lines):
        end = lines[i + 1][0].start if i + 1 < len(lines) else line[-1].end + HOLD
        timed.append((line[0].start, end, " ".join(w.word.strip(" .,!?") for w in line).upper()))
    return timed


def line_image(text):
    """The line in white, wrapped onto rows that fit 86% of the frame width, over a soft shadow."""
    size = round(w * SIZE)
    while True:
        font = ImageFont.truetype(FONT_FILE, size)
        rows, row = [], []
        for word in text.split():
            if row and font.getlength(" ".join(row + [word])) > w * 0.86:
                rows.append(" ".join(row))
                row = []
            row.append(word)
        rows.append(" ".join(row))
        if all(font.getlength(r) <= w * 0.86 for r in rows) or size <= 16:
            break
        size -= 2
    leading = round(size * 1.15)
    img = Image.new("RGBA", (w, leading * len(rows) + size))
    draw = ImageDraw.Draw(img)
    for i, r in enumerate(rows):
        draw.text((w / 2, size / 2 + leading * (i + 0.5)), r, font=font, anchor="mm", fill="white")
    shadow = Image.new("RGBA", img.size, (0, 0, 0, 0))
    shadow.putalpha(img.getchannel("A").filter(ImageFilter.GaussianBlur(size / 6)).point(lambda a: a * 0.9))
    return Image.alpha_composite(shadow, img)


def overlay(t):
    img = Image.new("RGBA", (w, h))
    if lines[0][0] <= t < lines[-1][1]:
        dim = DIM * min(1, (t - lines[0][0]) / FADE, (lines[-1][1] - t) / FADE)
        img.paste((0, 0, 0, round(255 * dim)), (0, 0, w, h))
    for start, end, art in lines:
        if not start <= t < end:
            continue
        come = min(1, (t - start) / FADE)    # rises in from below...
        go = min(1, (end - t) / FADE)        # ...and drifts up as it fades before the next line
        rise = ((1 - come) ** 2 - (1 - go) ** 2) * RISE * h
        layer = art.copy()
        layer.putalpha(art.getchannel("A").point(lambda a: a * come * go))
        img.alpha_composite(layer, (0, round(h * Y - art.height / 2 + rise)))
    return img


def probe(path):
    info = json.loads(subprocess.check_output(["ffprobe", "-v", "error", "-select_streams", "v:0", "-show_entries",
                                               "stream=width,height,r_frame_rate:format=duration", "-of", "json", path]))
    s = info["streams"][0]
    num, den = s["r_frame_rate"].split("/")
    return s["width"], s["height"], s["r_frame_rate"], float(num) / float(den), float(info["format"]["duration"])


def render(src, out):
    """Draw overlay(t) on every frame of src and save to out, keeping the audio."""
    ff = subprocess.Popen(["ffmpeg", "-y", "-v", "error", "-i", src,
                           "-f", "rawvideo", "-pix_fmt", "rgba", "-s", f"{w}x{h}", "-r", rate, "-i", "-",
                           "-filter_complex", "[0:v][1:v]overlay=shortest=1,format=yuv420p[v]",
                           "-map", "[v]", "-map", "0:a?", "-c:v", "libx264", "-crf", "18", "-c:a", "copy", out],
                          stdin=subprocess.PIPE)
    for i in range(math.ceil(duration * fps)):
        ff.stdin.write(overlay(i / fps).tobytes())
    ff.stdin.close()
    if ff.wait():
        sys.exit("ffmpeg failed")


src, out = sys.argv[1:3]
w, h, rate, fps, duration = probe(src)
lines = [(start, end, line_image(text)) for start, end, text in lyric_lines(src)]
render(src, out)