assets/generate_diagram.py

#!/usr/bin/env python3
"""Generate the how_it_works_architecture.png diagram for the README."""

import matplotlib
matplotlib.use("Agg")
import matplotlib.pyplot as plt
import matplotlib.patches as mpatches
from matplotlib.patches import FancyBboxPatch, FancyArrowPatch

fig, axes = plt.subplots(1, 3, figsize=(22, 14))
for ax in axes:
    ax.set_xlim(0, 10)
    ax.set_ylim(0, 16)
    ax.axis("off")

# ── Colors ──
BLUE_BG = "#EDECFB"
BLUE_BORDER = "#3B3BD3"
GREEN_BG = "#E8F5E9"
GREEN_BORDER = "#388E3C"
CYAN_BG = "#E0F7FA"
CYAN_BORDER = "#00838F"
RED_BG = "#FFEBEE"
RED_BORDER = "#C62828"
GRAY_BG = "#F5F5F5"
GRAY_BORDER = "#9E9E9E"
TEXT_COLOR = "#3F4547"
HOOK_COLOR = "#C62828"

def box(ax, x, y, w, h, label, bg=BLUE_BG, border=BLUE_BORDER, fontsize=9,
        fontstyle="normal", fontweight="normal", ha="center"):
    rect = FancyBboxPatch((x, y), w, h, boxstyle="round,pad=0.15",
                          facecolor=bg, edgecolor=border, linewidth=1.5)
    ax.add_patch(rect)
    ax.text(x + w/2, y + h/2, label, ha=ha if ha == "center" else "center",
            va="center", fontsize=fontsize, color=TEXT_COLOR,
            fontweight=fontweight, fontstyle=fontstyle, wrap=True)

def arrow(ax, x1, y1, x2, y2, color=BLUE_BORDER):
    ax.annotate("", xy=(x2, y2), xytext=(x1, y1),
                arrowprops=dict(arrowstyle="->,head_width=0.3,head_length=0.2",
                                color=color, lw=1.5))

def hook_label(ax, x, y, label, color=HOOK_COLOR):
    rect = FancyBboxPatch((x, y), 2.2, 0.6, boxstyle="round,pad=0.1",
                          facecolor=RED_BG, edgecolor=color, linewidth=1.2,
                          linestyle="--")
    ax.add_patch(rect)
    ax.text(x + 1.1, y + 0.3, label, ha="center", va="center",
            fontsize=7.5, color=color, fontweight="bold")

# ═══════════════════════════════════════════════════════════════════════
# COLUMN 1: Model Architecture & Where We Intercept
# ═══════════════════════════════════════════════════════════════════════
ax = axes[0]
ax.set_title("Model architecture & where we intercept", fontsize=13,
             fontweight="bold", color=BLUE_BORDER, pad=15)

# Camera image
box(ax, 2, 14.5, 6, 0.8, "Camera image (H x W x 3)", bg=GRAY_BG, border=GRAY_BORDER)
arrow(ax, 5, 14.5, 5, 14.0)

# SigLIP vision encoder block
encoder_rect = FancyBboxPatch((1.5, 11.0), 7, 2.8, boxstyle="round,pad=0.2",
                               facecolor="#F3F2FF", edgecolor=BLUE_BORDER,
                               linewidth=2, linestyle="-")
ax.add_patch(encoder_rect)
ax.text(5, 13.6, "SigLIP Vision Encoder", ha="center", va="center",
        fontsize=10, color=BLUE_BORDER, fontweight="bold")
ax.text(5, 13.1, "(12 layers, 12 heads, no CLS token)", ha="center", va="center",
        fontsize=8, color="#7F8385")

# Layers inside encoder
box(ax, 2.5, 12.2, 5, 0.5, "Patch embedding  (512px / 16px = 32x32 = 1024 patches)",
    fontsize=7.5)
arrow(ax, 5, 12.2, 5, 11.9)
box(ax, 2.5, 11.2, 5, 0.6, "Self-attention layers 1 ... 12",
    fontsize=8, fontweight="bold")

# Hook on encoder
hook_label(ax, 6.8, 11.3, "HOOK: fwd hook\non attn layers")

arrow(ax, 5, 11.0, 5, 10.5)

# Connector
box(ax, 2, 9.8, 6, 0.7, "Connector (pixel shuffle)\n1024 patches  ->  64 vision tokens",
    fontsize=8)
arrow(ax, 5, 9.8, 5, 9.3)

# VLM / SmolLM2
vlm_rect = FancyBboxPatch((1.5, 7.0), 7, 2.1, boxstyle="round,pad=0.2",
                            facecolor="#F3F2FF", edgecolor=BLUE_BORDER,
                            linewidth=2, linestyle="-")
ax.add_patch(vlm_rect)
ax.text(5, 8.8, "SmolLM2 (VLM)", ha="center", va="center",
        fontsize=10, color=BLUE_BORDER, fontweight="bold")

# Prefix tokens inside VLM
box(ax, 2.2, 7.3, 2, 0.7, "64 vision\ntokens", fontsize=7.5, bg="#E8EAF6", border="#5C6BC0")
box(ax, 4.3, 7.3, 2, 0.7, "language\ntokens", fontsize=7.5, bg="#E8EAF6", border="#5C6BC0")
box(ax, 6.5, 7.3, 1.7, 0.7, "state\ntokens", fontsize=7.5, bg="#E8EAF6", border="#5C6BC0")

ax.text(5, 8.2, "Self-attention over prefix  ->  builds KV cache",
        ha="center", va="center", fontsize=8, color="#7F8385")

# Arrow from VLM down: KV cache
arrow(ax, 5, 7.0, 5, 6.5)
ax.text(5.1, 6.7, "KV cache", ha="left", va="center", fontsize=8,
        color=GREEN_BORDER, fontweight="bold")

# Action expert
expert_rect = FancyBboxPatch((1.5, 3.8), 7, 2.5, boxstyle="round,pad=0.2",
                              facecolor="#E8F5E9", edgecolor=GREEN_BORDER,
                              linewidth=2, linestyle="-")
ax.add_patch(expert_rect)
ax.text(5, 6.0, "Action Expert", ha="center", va="center",
        fontsize=10, color=GREEN_BORDER, fontweight="bold")
ax.text(5, 5.5, "Q = expert action tokens", ha="center", va="center",
        fontsize=8, color=TEXT_COLOR)
ax.text(5, 5.0, "K, V = VLM prefix KV cache", ha="center", va="center",
        fontsize=8, color=TEXT_COLOR)
ax.text(5, 4.4, "Cross-attn: Q_expert attends to K_prefix\n(detected when Q_len != K_len)",
        ha="center", va="center", fontsize=7.5, color="#7F8385")

# Monkey-patch label
hook_label(ax, 6.8, 4.0, "MONKEY-PATCH:\neager_attn_fwd")

arrow(ax, 5, 3.8, 5, 3.3)

# Action output
box(ax, 2, 2.5, 6, 0.7, "Action chunk  ->  Robot actions",
    bg=GREEN_BG, border=GREEN_BORDER, fontsize=9, fontweight="bold")

# ═══════════════════════════════════════════════════════════════════════
# COLUMN 2: Self-Attention -> Heatmap Pipeline
# ═══════════════════════════════════════════════════════════════════════
ax = axes[1]
ax.set_title("Self-attention  ->  heatmap", fontsize=13,
             fontweight="bold", color=BLUE_BORDER, pad=15)

box(ax, 2, 14.2, 6, 0.8, "Attention weights\n(heads, patches, patches)",
    fontsize=9)
arrow(ax, 5, 14.2, 5, 13.6)

box(ax, 2, 12.8, 6, 0.8, "Aggregation method:", fontsize=9, fontweight="bold")
ax.text(5, 12.5, "last-layer: use layer 12 only\n"
        "rollout: multiply across all 12 layers\n"
        "all-layers: keep each layer separate",
        ha="center", va="top", fontsize=7.5, color="#7F8385")
arrow(ax, 5, 11.6, 5, 11.2)

box(ax, 2, 10.4, 6, 0.8, "Average across 12 heads\n-> per-patch importance (1024,)",
    fontsize=8.5)
arrow(ax, 5, 10.4, 5, 9.8)

box(ax, 2, 9.0, 6, 0.8, "Reshape to 2D grid (32 x 32)", fontsize=9)
arrow(ax, 5, 9.0, 5, 8.4)

box(ax, 2, 7.6, 6, 0.8, "Bilinear upsample to image size\n(32x32  ->  512x512)",
    fontsize=8.5)
arrow(ax, 5, 7.6, 5, 7.0)

box(ax, 2, 6.2, 6, 0.8, "Normalize to [0, 1]", fontsize=9)
arrow(ax, 5, 6.2, 5, 5.6)

box(ax, 2, 4.8, 6, 0.8, "Optional: subtract positional baseline\n(--raw-attention skips this)",
    fontsize=8, fontstyle="italic")
arrow(ax, 5, 4.8, 5, 4.2)

box(ax, 2, 3.4, 6, 0.8, "Self-attention heatmap\n(jet colormap: blue to red)",
    bg="#E8EAF6", border="#5C6BC0", fontsize=9, fontweight="bold")

# ═══════════════════════════════════════════════════════════════════════
# COLUMN 3: Cross-Attention -> Heatmap + Co-attention
# ═══════════════════════════════════════════════════════════════════════
ax = axes[2]
ax.set_title("Cross-attention  ->  heatmap", fontsize=13,
             fontweight="bold", color=GREEN_BORDER, pad=15)

box(ax, 2, 14.2, 6, 0.8, "Intercepted softmax probs\n(expert_Q_len, prefix_K_len)",
    bg=GREEN_BG, border=GREEN_BORDER, fontsize=8.5)
arrow(ax, 5, 14.2, 5, 13.6, color=GREEN_BORDER)

box(ax, 2, 12.8, 6, 0.8, "Slice vision-token columns only\n(expert_Q_len, 64)",
    bg=GREEN_BG, border=GREEN_BORDER, fontsize=8.5)
arrow(ax, 5, 12.8, 5, 12.2, color=GREEN_BORDER)

box(ax, 2, 11.4, 6, 0.8, "Average across expert layers\nand query positions  ->  (64,)",
    bg=GREEN_BG, border=GREEN_BORDER, fontsize=8.5)
arrow(ax, 5, 11.4, 5, 10.8, color=GREEN_BORDER)

box(ax, 2, 10.0, 6, 0.8, "Undo pixel shuffle: reshape\n64 tokens  ->  8x8 grid",
    bg=GREEN_BG, border=GREEN_BORDER, fontsize=8.5)
arrow(ax, 5, 10.0, 5, 9.4, color=GREEN_BORDER)

box(ax, 2, 8.6, 6, 0.8, "Bilinear upsample to image size\n(8x8  ->  512x512)",
    bg=GREEN_BG, border=GREEN_BORDER, fontsize=8.5)
arrow(ax, 5, 8.6, 5, 8.0, color=GREEN_BORDER)

box(ax, 2, 7.2, 6, 0.8, "Normalize to [0, 1]",
    bg=GREEN_BG, border=GREEN_BORDER, fontsize=9)
arrow(ax, 5, 7.2, 5, 6.6, color=GREEN_BORDER)

box(ax, 2, 5.8, 6, 0.8, "Cross-attention heatmap\n(Greens colormap)",
    bg=GREEN_BG, border=GREEN_BORDER, fontsize=9, fontweight="bold")

# Co-attention
arrow(ax, 5, 5.8, 5, 5.3, color=CYAN_BORDER)

# Show the multiplication — centered at x=5
ax.text(3.8, 4.9, "self-attn", ha="center", va="center", fontsize=8,
        color=BLUE_BORDER, fontweight="bold")
ax.text(5.0, 4.9, " x ", ha="center", va="center", fontsize=10,
        color=TEXT_COLOR, fontweight="bold")
ax.text(6.2, 4.9, "cross-attn", ha="center", va="center", fontsize=8,
        color=GREEN_BORDER, fontweight="bold")

arrow(ax, 5, 4.6, 5, 4.2, color=CYAN_BORDER)

box(ax, 2, 3.4, 6, 0.8, "Co-attention heatmap\n(cyan colormap: black -> cyan -> white)",
    bg=CYAN_BG, border=CYAN_BORDER, fontsize=9, fontweight="bold")

ax.text(5, 2.8, "Bright cyan = both visually salient\nAND action-relevant",
        ha="center", va="center", fontsize=8, color=CYAN_BORDER, fontstyle="italic")

plt.tight_layout(w_pad=2)
plt.savefig("/Users/subirmansukhani/Desktop/smolvla-inspect/assets/how_it_works_architecture.png",
            dpi=150, bbox_inches="tight", facecolor="white")
plt.close()
print("Saved how_it_works_architecture.png")