How to add a volume-reactive avatar with Claude

Add a talking avatar to input.mp4 (it has a voiceover). Draw two versions of a round avatar badge with Pillow, 34% of the frame width across, centered 22% across and 83% down: a white circle with a violet (88, 64, 200) disc inside it inset by 4% of the width, a cartoon face on it (skin (246, 196, 158) oval, dark brown (58, 38, 32) hair and dot eyes, soft pink cheeks), one with a closed smile drawn as a brown arc and one with an open mouth: a red (170, 25, 45) oval 18% of the badge width across and 16% tall, centered 67% down the badge, with a pink tongue. Measure the voice loudness for each video frame (RMS of the audio over that frame, divided by the loudest frame's value). Open the mouth when the loudness reaches 0.30 and keep it open until it drops below 0.15, so it doesn't flicker. While the mouth is open, lift the avatar 3% of its width and show a yellow (255, 205, 40) ring 7% of the badge width thick just outside the white edge, with a soft halo; when the mouth closes, fade the ring out by multiplying its opacity by 0.75 every frame. Use Python with Pillow and numpy to draw the frames and ffmpeg to put them on the video (don't rely on ffmpeg's drawtext filter), keep the audio, and save it as output.mp4.

#!/usr/bin/env python3
"""Volume-reactive avatar: a cartoon face in the bottom-left corner opens its mouth and glows yellow whenever the voice gets loud.

Usage: python3 volume-reactive-avatar.py input.mp4 output.mp4  (the input needs a voice on its audio track)
Needs ffmpeg, numpy and Pillow (pip install numpy pillow).
"""
import json
import math
import subprocess
import sys

import numpy as np
from PIL import Image, ImageDraw, ImageFilter

X, Y = 0.22, 0.83           # avatar center (0-1 across and down the frame)
SIZE = 0.34                 # avatar width as a share of the frame width
OPEN_AT = 0.30              # loudness (0-1, 1 = loudest frame) that opens the mouth
CLOSE_AT = 0.15             # loudness that closes it again; lower than OPEN_AT so the mouth doesn't flicker
BOB = 0.03                  # how far the avatar hops up while talking, as a share of its width
BADGE = (88, 64, 200)       # badge color behind the face
SKIN = (246, 196, 158)      # face color
HAIR = (58, 38, 32)         # hair and eye color
MOUTH = (170, 25, 45)       # inside of the open mouth
GLOW = (255, 205, 40)       # ring that lights up while the mouth is open
RELEASE = 0.75              # how much of the glow carries over to the next frame after the mouth closes
RATE = 16000                # audio sample rate the loudness uses


def probe(path):
    info = json.loads(subprocess.check_output(["ffprobe", "-v", "error", "-select_streams", "v:0", "-show_entries",
                                               "stream=width,height,r_frame_rate:format=duration", "-of", "json", path]))
    s = info["streams"][0]
    num, den = s["r_frame_rate"].split("/")
    return s["width"], s["height"], s["r_frame_rate"], float(num) / float(den), float(info["format"]["duration"])


def mouth_open_per_frame(src, frames):
    """Whether the mouth is open at each video frame: opens above OPEN_AT, stays open until below CLOSE_AT."""
    pcm = subprocess.check_output(["ffmpeg", "-v", "error", "-i", src, "-ac", "1", "-ar", str(RATE), "-f", "f32le", "-"])
    audio = np.frombuffer(pcm, np.float32)
    step = RATE / fps
    rms = np.array([np.sqrt(np.mean(audio[round(i * step):round((i + 1) * step)] ** 2) + 1e-12) for i in range(frames)])
    level = rms / max(rms.max(), 1e-9)
    is_open, state = [], False
    for v in level:
        state = v >= CLOSE_AT if state else v >= OPEN_AT
        is_open.append(state)
    return is_open


def avatar(mouth_open):
    """The round avatar badge, with the mouth closed in a smile or open mid-word (drawn 4x size for smooth edges)."""
    d = round(w * SIZE) * 4
    img = Image.new("RGBA", (d, d))
    draw = ImageDraw.Draw(img)
    draw.ellipse([0, 0, d - 1, d - 1], fill="white")
    draw.ellipse([d * 0.04, d * 0.04, d * 0.96, d * 0.96], fill=BADGE)
    draw.ellipse([d * 0.22, d * 0.12, d * 0.78, d * 0.66], fill=HAIR)                     # hair behind the head
    draw.ellipse([d * 0.25, d * 0.20, d * 0.75, d * 0.80], fill=SKIN)                     # face
    draw.chord([d * 0.24, d * 0.15, d * 0.76, d * 0.52], 180, 360, fill=HAIR)             # fringe
    for ex in (0.40, 0.60):
        draw.ellipse([d * (ex - 0.03), d * 0.44, d * (ex + 0.03), d * 0.53], fill=HAIR)   # eyes
        draw.ellipse([d * (ex - 0.01), d * 0.455, d * (ex + 0.01), d * 0.475], fill="white")
    for cx in (0.34, 0.66):
        draw.ellipse([d * (cx - 0.04), d * 0.56, d * (cx + 0.04), d * 0.60], fill=(240, 160, 150))  # cheeks
    if mouth_open:
        draw.ellipse([d * 0.41, d * 0.59, d * 0.59, d * 0.75], fill=MOUTH)
        draw.chord([d * 0.44, d * 0.67, d * 0.56, d * 0.77], 180, 360, fill=(235, 110, 120))  # tongue
    else:
        draw.arc([d * 0.43, d * 0.57, d * 0.57, d * 0.66], 20, 160, fill=HAIR, width=max(2, d // 45))
    # mask everything to the round badge
    mask = Image.new("L", (d, d))
    ImageDraw.Draw(mask).ellipse([0, 0, d - 1, d - 1], fill=255)
    img.putalpha(Image.fromarray(np.minimum(np.asarray(img.getchannel("A")), np.asarray(mask))))
    return img.resize((d // 4, d // 4), Image.LANCZOS)


def glow_ring():
    """A thick yellow ring with a soft halo, a little wider than the avatar."""
    d = round(w * SIZE)
    pad = d // 5
    big = Image.new("RGBA", ((d + 2 * pad) * 4,) * 2)
    ring = [(pad - d * 0.05) * 4, (pad - d * 0.05) * 4, (pad + d * 1.05) * 4, (pad + d * 1.05) * 4]
    ImageDraw.Draw(big).ellipse(ring, outline=GLOW, width=round(d * 0.07 * 4))
    img = big.resize((d + 2 * pad,) * 2, Image.LANCZOS)
    halo = img.filter(ImageFilter.GaussianBlur(d / 18))
    halo.alpha_composite(img)
    return halo


def overlay(mouth_open, glow):
    img = Image.new("RGBA", (w, h))
    face = open_face if mouth_open else closed_face
    cx, cy = round(w * X), round(h * Y - (face.width * BOB if mouth_open else 0))
    if glow > 0.02:
        ring = ring_img.copy()
        ring.putalpha(Image.fromarray((np.asarray(ring.getchannel("A")) * glow).astype(np.uint8)))
        img.alpha_composite(ring, (cx - ring.width // 2, cy - ring.height // 2))
    img.alpha_composite(face, (cx - face.width // 2, cy - face.height // 2))
    return img


def render(src, out):
    """Draw the avatar on every frame of src and save to out, keeping the audio."""
    frames = math.ceil(duration * fps)
    mouth = mouth_open_per_frame(src, frames)
    ff = subprocess.Popen(["ffmpeg", "-y", "-v", "error", "-i", src,
                           "-f", "rawvideo", "-pix_fmt", "rgba", "-s", f"{w}x{h}", "-r", rate, "-i", "-",
                           "-filter_complex", "[0:v][1:v]overlay=shortest=1,format=yuv420p[v]",
                           "-map", "[v]", "-map", "0:a?", "-c:v", "libx264", "-crf", "18", "-c:a", "copy", out],
                          stdin=subprocess.PIPE)
    glow = 0.0
    for i in range(frames):
        glow = 1.0 if mouth[i] else glow * RELEASE
        ff.stdin.write(overlay(mouth[i], glow).tobytes())
    ff.stdin.close()
    if ff.wait():
        sys.exit("ffmpeg failed")


src, out = sys.argv[1:3]
w, h, rate, fps, duration = probe(src)
closed_face, open_face, ring_img = avatar(False), avatar(True), glow_ring()
render(src, out)