How to add an audiogram with Claude

Turn input.mp4 (540x960, 30fps, 6 seconds, with a voiceover) into a podcast audiogram: a still picture with a live waveform and captions, same size and length, keeping the audio. Take the first frame as the only picture. Background: that frame blurred with a Gaussian blur of radius 6% of the frame width (32 px) and darkened to 40% brightness. Cover: a 400x400 square cut from the first frame at x=70, y=182 (centered across, top edge 19% down) and pasted back at the same spot, unscaled, with 27 px rounded corners and a soft black shadow under it (170/255 opacity, offset 11 px down, blurred 16 px). Above it, centered across, draw "EPISODE 12" in light gray (200, 200, 200) bold text 17 px tall centered 10.5% down the frame and "THE GLOW PODCAST" in white bold text 32 px tall centered 14.5% down. Waveform: 36 amber (255, 190, 40) bars with fully rounded ends spread evenly from 10% to 90% across the frame (each bar 60% of its 12 px slot wide), centered 70% down the frame, each extending up and down from that line by its level times 6% of the frame height, never less than a round dot. For the levels, read the audio as mono 48 kHz, and for each frame take a Hann-windowed 2048-sample FFT centered on the frame's time; split 80 Hz to 6 kHz into 18 bands spaced evenly in pitch, take each band's mean magnitude, divide it by that band's largest value over the whole clip and take the square root; smooth each band frame by frame, closing 70% of the gap when it rises and 25% when it falls. Mirror the 18 bands so the lowest sits in the middle and the highest at both ends. Captions: transcribe the audio with faster-whisper (model "base.en", device cpu, compute_type int8, word_timestamps=True), group the words into lines of at most 4, and show each line from its first word's start until 0.4 seconds after its last word ends, or until the next line starts, with nothing after the last line. Draw each line in white bold text 37 px tall, centered across with its center 80% down the frame, with the word being spoken in the same amber. Along the bottom, centered 89% down, draw a 4 px gray (110, 110, 110) track from 10% to 90% across with a white fill that grows from the left edge to the right edge over the clip's length. Draw everything with Pillow (don't use ffmpeg's drawtext filter), encode H.264 yuv420p at 30fps, keep the input's audio and save it as output.mp4.

#!/usr/bin/env python3
"""Audiogram: a clip's audio turned into a podcast-style card, with a still cover, bouncing bars and live captions.

Usage: python3 audiogram.py input.mp4 output.mp4
The cover is the input's first frame, so the picture holds still while the sound plays. No subtitle file
needed: faster-whisper transcribes the audio on your computer and times every word.
Needs ffmpeg, faster-whisper, numpy and Pillow (pip install faster-whisper numpy pillow). The first run
downloads the "base.en" speech model (about 140 MB).
"""
import json
import math
import subprocess
import sys
from pathlib import Path

import numpy as np
from faster_whisper import WhisperModel
from PIL import Image, ImageDraw, ImageEnhance, ImageFilter, ImageFont

MODEL = "base.en"          # Whisper model: bigger ("small.en") is more accurate but slower
SHOW, EPISODE = "THE GLOW PODCAST", "EPISODE 12"   # title over the cover, and the small line above it
COVER = 0.74               # cover width (and height) as a share of the frame width
COVER_TOP = 0.19           # cover's top edge (0-1 down the frame); it's cut from the same spot of the first frame
BACKDROP = 0.4             # brightness of the blurred first frame behind everything (1 = unchanged)
ACCENT = (255, 190, 40)    # color of the bars and of the word being spoken
BARS = 36                  # bars in the waveform, mirrored so low notes sit in the middle
WAVE_Y, WAVE_HEIGHT = 0.70, 0.06   # waveform center line and tallest bar above and below it, as shares of the frame height
CAPTION_Y = 0.80           # caption line's center (0-1 down the frame)
SIZE = 0.068               # caption text height as a share of the frame width
MAX_WORDS = 4              # most words on one caption line
HOLD = 0.4                 # seconds a line stays up after its last word, unless the next line starts sooner
PROGRESS_Y = 0.89          # thin progress bar along the bottom, 0-1 down the frame
ATTACK, RELEASE = 0.7, 0.25   # share of the gap to each bar's new height closed per frame, rising and falling
RATE = 48000               # audio samples per second read from the input
FONTS = ["/System/Library/Fonts/Supplemental/Arial Bold.ttf",     # macOS
         "/usr/share/fonts/truetype/dejavu/DejaVuSans-Bold.ttf",  # Linux
         "C:/Windows/Fonts/arialbd.ttf"]                          # Windows
FONT_FILE = next((f for f in FONTS if Path(f).exists()), None) or sys.exit("No bold font found: add one to FONTS")


def transcribe(path):
    """Every spoken word as (text, start, end), times in seconds."""
    model = WhisperModel(MODEL, device="cpu", compute_type="int8")
    segments, _ = model.transcribe(path, word_timestamps=True)
    return [(word.word.strip(), word.start, word.end) for segment in segments for word in segment.words]


def caption_lines(words):
    """Group the words into lines of up to MAX_WORDS that fit the frame: [(words, start, end)]."""
    groups = []
    for word in words:
        longer = " ".join(text for text, _, _ in (groups[-1] if groups else []) + [word])
        if groups and len(groups[-1]) < MAX_WORDS and font.getlength(longer) <= w * 0.88:
            groups[-1].append(word)
        else:
            groups.append([word])
    starts = [group[0][1] for group in groups[1:]] + [math.inf]
    return [(group, group[0][1], min(group[-1][2] + HOLD, next_start)) for group, next_start in zip(groups, starts)]


def bar_levels(path):
    """Height (0-1) of each half of the waveform's bars for every frame, from the sound's spectrum around it:
    BARS // 2 frequency bands spaced evenly in pitch from 80 Hz to 6 kHz."""
    raw = subprocess.check_output(["ffmpeg", "-v", "error", "-i", path, "-f", "s16le", "-ac", "1", "-ar", str(RATE), "-"])
    sound = np.frombuffer(raw, np.int16).astype(np.float32) / 32768
    sound = np.pad(sound, (1024, 2048 + max(0, round(duration * RATE) - len(sound))))   # silence around it, so every window is full
    freqs = np.fft.rfftfreq(2048, 1 / RATE)
    edges = np.geomspace(80, 6000, BARS // 2 + 1)
    window = np.hanning(2048)
    power = []
    for i in range(frames):
        start = round(i / fps * RATE)
        spectrum = np.abs(np.fft.rfft(sound[start:start + 2048] * window))
        power.append([spectrum[(freqs >= lo) & (freqs < hi)].mean() for lo, hi in zip(edges, edges[1:])])
    power = np.array(power)
    power = np.sqrt(power / np.maximum(power.max(axis=0), 1e-6))   # each band's loudest moment reaches full height
    level, out = np.zeros(BARS // 2), []
    for p in power:
        level += (p - level) * np.where(p > level, ATTACK, RELEASE)
        out.append(level.copy())
    return out


def background(first):
    """Everything that stays still: the blurred, darkened first frame, the titles and the cover cut from it."""
    img = ImageEnhance.Brightness(first.filter(ImageFilter.GaussianBlur(w * 0.06))).enhance(BACKDROP).convert("RGBA")
    side, x, y = round(w * COVER), round((w - w * COVER) / 2), round(h * COVER_TOP)
    radius = round(w * 0.05)
    shadow = Image.new("RGBA", (w, h))
    ImageDraw.Draw(shadow).rounded_rectangle([x, y + w * 0.02, x + side, y + side + w * 0.02], radius, fill=(0, 0, 0, 170))
    img.alpha_composite(shadow.filter(ImageFilter.GaussianBlur(w * 0.03)))
    mask = Image.new("L", (side, side))
    ImageDraw.Draw(mask).rounded_rectangle([0, 0, side - 1, side - 1], radius, fill=255)
    img.paste(first.crop((x, y, x + side, y + side)), (x, y), mask)
    draw = ImageDraw.Draw(img)
    draw.text((w / 2, h * 0.105), EPISODE, font=small_font, anchor="mm", fill=(200, 200, 200))
    draw.text((w / 2, h * 0.145), SHOW, font=title_font, anchor="mm", fill="white")
    return img


def frame(i):
    """The still background with this frame's bars, caption line and progress bar."""
    t = i / fps
    img = still.copy()
    draw = ImageDraw.Draw(img)
    heights = np.concatenate([levels[i][::-1], levels[i]])    # low notes in the middle, high notes at both ends
    pitch = w * 0.8 / BARS
    bar = pitch * 0.6
    for k, level in enumerate(heights):
        x = w * 0.1 + k * pitch + (pitch - bar) / 2
        half = max(level * h * WAVE_HEIGHT, bar / 2)          # never thinner than a dot, so silence shows a row of dots
        draw.rounded_rectangle([x, h * WAVE_Y - half, x + bar, h * WAVE_Y + half], bar / 2, fill=ACCENT)
    for group, start, end in lines:
        if start <= t < end:
            active = max(k for k, (_, word_start, _) in enumerate(group) if word_start <= t)
            x = (w - font.getlength(" ".join(text for text, _, _ in group))) / 2
            for k, (text, _, _) in enumerate(group):
                draw.text((x, h * CAPTION_Y), text, font=font, anchor="lm", fill=ACCENT if k == active else "white")
                x += font.getlength(text + " ")
    y, thick = h * PROGRESS_Y, max(2, round(h * 0.004))
    draw.rounded_rectangle([w * 0.1, y, w * 0.9, y + thick], thick / 2, fill=(110, 110, 110))
    draw.rounded_rectangle([w * 0.1, y, w * 0.1 + w * 0.8 * min(t / duration, 1), y + thick], thick / 2, fill="white")
    return img.convert("RGB")


def probe(path):
    info = json.loads(subprocess.check_output(["ffprobe", "-v", "error", "-select_streams", "v:0", "-show_entries",
                                               "stream=width,height:format=duration", "-of", "json", path]))
    s = info["streams"][0]
    return s["width"], s["height"], float(info["format"]["duration"])


def first_frame(path):
    raw = subprocess.check_output(["ffmpeg", "-v", "error", "-i", path, "-frames:v", "1", "-f", "rawvideo", "-pix_fmt", "rgb24", "-"])
    return Image.frombytes("RGB", (w, h), raw)


def render(src, out):
    """Write every frame at 30fps and save to out with src's audio."""
    ff = subprocess.Popen(["ffmpeg", "-y", "-v", "error", "-f", "rawvideo", "-pix_fmt", "rgb24", "-s", f"{w}x{h}",
                           "-r", "30", "-i", "-", "-i", src, "-map", "0:v", "-map", "1:a?", "-c:v", "libx264",
                           "-crf", "18", "-pix_fmt", "yuv420p", "-c:a", "copy", "-shortest", out],
                          stdin=subprocess.PIPE)
    for i in range(frames):
        ff.stdin.write(frame(i).tobytes())
    ff.stdin.close()
    if ff.wait():
        sys.exit("ffmpeg failed")


src, out = sys.argv[1:3]
w, h, duration = probe(src)
fps = 30
frames = math.ceil(duration * fps)
font = ImageFont.truetype(FONT_FILE, round(w * SIZE))
title_font = ImageFont.truetype(FONT_FILE, round(w * 0.06))
small_font = ImageFont.truetype(FONT_FILE, round(w * 0.032))
still = background(first_frame(src))
levels = bar_levels(src)
lines = caption_lines(transcribe(src))
render(src, out)