| <!doctype html> |
| <html lang="en"> |
| <head> |
| <meta charset="utf-8" /> |
| <meta name="viewport" content="width=device-width, initial-scale=1" /> |
| <meta name="color-scheme" content="light" /> |
| <meta name="theme-color" content="#eef2ee" /> |
| <title>The Physical AI Inference Gap in Batch-1 LLM Decode</title> |
| <meta name="description" content="An interactive companion to 'Memory-Bound but Not Bandwidth-Limited: The Physical AI Inference Gap in Batch-1 LLM Decode' by Josef Chen (KAIKAKU). Why faster GPUs realise less of their bandwidth on single-stream decode, and what to do about it." /> |
| <meta property="og:type" content="website" /> |
| <meta property="og:title" content="The Physical AI Inference Gap in Batch-1 LLM Decode" /> |
| <meta property="og:description" content="Physical AI is runtime-poor, not compute-poor. A $0.30/hr L4 serves a 7B model 7.9x cheaper per token than a $3.50/hr H100 with 11x the bandwidth. 44 measured cells." /> |
| <meta property="og:image" content="https://josefchen-physical-ai-inference-gap.static.hf.space/thumbnail.png" /> |
| <meta property="og:image:width" content="1200" /> |
| <meta property="og:image:height" content="630" /> |
| <meta name="twitter:card" content="summary_large_image" /> |
| <meta name="twitter:title" content="The Physical AI Inference Gap in Batch-1 LLM Decode" /> |
| <meta name="twitter:description" content="Runtime-poor, not compute-poor. A $0.30/hr L4 beats a $3.50/hr H100 per token at batch 1. 44 measured cells." /> |
| <meta name="twitter:image" content="https://josefchen-physical-ai-inference-gap.static.hf.space/thumbnail.png" /> |
| <link rel="preconnect" href="https://fonts.googleapis.com"> |
| <link rel="preconnect" href="https://fonts.gstatic.com" crossorigin> |
| <link href="https://fonts.googleapis.com/css2?family=Fraunces:opsz,wght@9..144,400;9..144,500;9..144,600&family=Inter:wght@400;500;600&family=IBM+Plex+Mono:wght@400;500;600&display=swap" rel="stylesheet"> |
| <script src="https://cdnjs.cloudflare.com/ajax/libs/three.js/r128/three.min.js"></script> |
| <style> |
| :root{ |
| --paper:#eef2ee; --paper2:#f7faf8; --card:#ffffff; --line:#d3e0d7; --line2:#e3ece6; |
| --ink:#15251d; --muted:#566b60; --faint:#7d9389; |
| --green:#0e7a52; --green2:#11936a; --greenDk:#0a4d34; |
| --L4:#1f9d6b; --L40S:#7f9c32; --A100:#cc7a2b; --H100:#a87c16; |
| --shadow:0 1px 2px rgba(20,50,35,.05),0 8px 24px rgba(20,50,35,.06); |
| } |
| *{box-sizing:border-box} |
| html{scroll-behavior:smooth} |
| body{margin:0;background:var(--paper);color:var(--ink); |
| font-family:"Inter",-apple-system,BlinkMacSystemFont,sans-serif; |
| font-size:18px;line-height:1.72;-webkit-font-smoothing:antialiased} |
| body::before{content:"";position:fixed;inset:0;pointer-events:none;z-index:0;opacity:.22; |
| background-image:repeating-linear-gradient(180deg,transparent 0 38px,#dde7e0 38px 39px); |
| -webkit-mask-image:radial-gradient(150% 100% at 50% 0%,#000 40%,transparent 92%); |
| mask-image:radial-gradient(150% 100% at 50% 0%,#000 40%,transparent 92%);} |
| .mono{font-family:"IBM Plex Mono",ui-monospace,Menlo,monospace} |
| .caps{text-transform:uppercase;letter-spacing:.13em} |
| .serif{font-family:"Fraunces",Georgia,serif} |
| h1,h2,.headline,.eyebrow,.sec-eyebrow{text-wrap:balance} |
| .lede,.h2sub,.note,.prose p{text-wrap:pretty} |
| a{color:var(--green);text-decoration:none;border-bottom:1px solid rgba(14,122,82,.3)} |
| a:hover{border-bottom-color:var(--green)} |
|
|
| /* sticky header + tabs */ |
| header{position:sticky;top:0;z-index:50;background:rgba(238,242,238,.86); |
| backdrop-filter:saturate(140%) blur(10px);border-bottom:1px solid var(--line)} |
| .bar{max-width:1080px;margin:0 auto;padding:11px 24px;display:flex;align-items:center;gap:16px} |
| .wordmark{font-family:"Fraunces",serif;font-weight:600;font-size:18px;letter-spacing:-.01em;color:var(--ink)} |
| .wordmark b{color:var(--green)} |
| .barmeta{font-size:11px;color:var(--muted)} |
| .barlinks{margin-left:auto;display:flex;gap:16px;font-size:13px} |
| .barlinks a{border:0;color:var(--muted)} |
| .barlinks a:hover{color:var(--green)} |
| nav.tabs{max-width:1080px;margin:0 auto;padding:0 24px;display:flex;gap:4px;flex-wrap:wrap} |
| .tab{appearance:none;background:none;border:0;border-bottom:2px solid transparent; |
| font-family:"IBM Plex Mono",monospace;font-size:12px;letter-spacing:.08em;text-transform:uppercase; |
| color:var(--muted);padding:11px 14px;cursor:pointer} |
| .tab:hover{color:var(--ink)} |
| .tab.active{color:var(--green);border-bottom-color:var(--green)} |
|
|
| .wrap{position:relative;z-index:1;max-width:1080px;margin:0 auto;padding:0 24px} |
| .prose{max-width:720px} |
| .prose p{margin:0 0 18px} |
|
|
| /* hero */ |
| .hero{padding:52px 0 48px;display:grid;grid-template-columns:1.05fr 0.95fr;gap:48px;align-items:center} |
| @media(max-width:860px){.hero{grid-template-columns:1fr;gap:20px}} |
| .heroL{min-width:0} |
| .heroR{position:relative;min-width:0;height:430px} |
| #heroviz{width:100%;height:100%;display:block} |
| .herocap{position:absolute;left:0;right:0;bottom:2px;text-align:center;font-size:10px;letter-spacing:.14em;color:var(--faint)} |
| .eyebrow{font-family:"IBM Plex Mono",monospace;font-size:12px;letter-spacing:.16em;text-transform:uppercase;color:var(--green);margin:0 0 18px} |
| h1{font-family:"Fraunces",serif;font-weight:600;font-size:50px;line-height:1.05;letter-spacing:-.025em;margin:0 0 22px} |
| .lede{font-size:19px;color:var(--muted);margin:0 0 28px;line-height:1.55} |
| .herostats{display:flex;gap:34px;flex-wrap:wrap;margin:26px 0 4px} |
| .hstat .n{font-family:"Fraunces",serif;font-size:34px;font-weight:600;color:var(--ink);line-height:1} |
| .hstat .l{font-family:"IBM Plex Mono",monospace;font-size:11px;color:var(--muted);text-transform:uppercase;letter-spacing:.08em;margin-top:5px} |
| .reslinks{display:flex;gap:10px;flex-wrap:wrap;margin-top:26px} |
| .reslink{font-family:"IBM Plex Mono",monospace;font-size:12px;border:1px solid var(--line);border-radius:999px; |
| padding:7px 14px;color:var(--ink);background:var(--card)} |
| .reslink:hover{border-color:var(--green);color:var(--green)} |
|
|
| /* sections / panels */ |
| .panel-wrap{padding:76px 0} |
| .panel-wrap + .panel-wrap{border-top:1px solid var(--line2)} |
| h2{font-family:"Fraunces",serif;font-weight:600;font-size:36px;letter-spacing:-.02em;margin:12px 0 10px;max-width:760px} |
| .sec-eyebrow{font-family:"IBM Plex Mono",monospace;font-size:11.5px;letter-spacing:.16em;text-transform:uppercase;color:var(--green)} |
| .h2sub{color:var(--muted);max-width:660px;margin:10px 0 30px;font-size:17.5px;line-height:1.55} |
| .card{background:var(--card);border:1px solid var(--line2);border-radius:16px;padding:30px;box-shadow:0 1px 2px rgba(20,50,35,.04),0 18px 50px rgba(20,50,35,.07);margin-top:8px} |
| .note{font-size:12.5px;color:var(--faint);margin-top:18px;line-height:1.65;max-width:680px} |
| .note b{color:var(--muted)} |
|
|
| /* controls */ |
| .controls{display:grid;grid-template-columns:repeat(4,1fr);gap:14px} |
| @media(max-width:760px){.controls{grid-template-columns:repeat(2,1fr)}} |
| label{display:block;font-family:"IBM Plex Mono",monospace;font-size:10.5px;letter-spacing:.1em;text-transform:uppercase;color:var(--muted);margin:0 0 6px} |
| select,input[type=number],input[type=text],textarea{width:100%;background:var(--paper2);color:var(--ink); |
| border:1px solid var(--line);border-radius:8px;padding:9px 11px;font-size:14px;font-family:"IBM Plex Mono",monospace} |
| select:focus,input:focus,textarea:focus{outline:2px solid rgba(14,122,82,.25);border-color:var(--green)} |
| .ratebadge{font-size:10px;color:var(--faint);margin-top:5px;display:block;font-family:"IBM Plex Mono",monospace} |
| .levers{display:flex;gap:18px;flex-wrap:wrap;align-items:center;margin-top:16px;font-size:13px;color:var(--muted)} |
| .levers label{display:inline-flex;align-items:center;gap:7px;color:var(--ink);text-transform:none;letter-spacing:0;font-size:13px;margin:0;cursor:pointer} |
| .pill{display:inline-block;font-family:"IBM Plex Mono",monospace;font-size:10.5px;padding:3px 9px;border-radius:999px;border:1px solid var(--line);color:var(--muted)} |
| .pill.measured{color:var(--green);border-color:rgba(14,122,82,.4)} |
| .pill.estimate{color:var(--A100);border-color:rgba(204,122,43,.4)} |
|
|
| .cards{display:grid;grid-template-columns:repeat(4,1fr);gap:14px;margin-top:18px} |
| @media(max-width:760px){.cards{grid-template-columns:repeat(2,1fr)}} |
| .kpi{border:1px solid var(--line);border-radius:10px;padding:14px;background:var(--paper2)} |
| .kpi .k{font-family:"IBM Plex Mono",monospace;font-size:9.5px;letter-spacing:.08em;text-transform:uppercase;color:var(--muted)} |
| .kpi .v{font-family:"Fraunces",serif;font-size:30px;font-weight:600;margin-top:5px;line-height:1} |
| .kpi .v small{font-size:14px;color:var(--muted)} |
| .kpi .u{font-family:"IBM Plex Mono",monospace;font-size:10px;color:var(--faint);margin-top:4px} |
| .verdict{margin-top:18px;padding:13px 16px;border:1px solid var(--line);border-left:3px solid var(--green);border-radius:8px;background:rgba(14,122,82,.04);font-size:14.5px} |
| .verdict b{color:var(--green)} |
|
|
| .headline{font-family:"Fraunces",serif;font-size:28px;font-weight:600;letter-spacing:-.01em;margin:18px 0 2px;line-height:1.12} |
| .statblock{display:grid;grid-template-columns:repeat(3,1fr);margin-top:18px;gap:0} |
| .statblock .s{padding:0 20px;border-left:1px solid var(--line)} |
| .statblock .s:first-child{border-left:0;padding-left:0} |
| .statblock .num{font-family:"Fraunces",serif;font-size:42px;font-weight:600;line-height:1} |
| .statblock .num small{font-size:17px;color:var(--muted)} |
| .statblock .cap{font-family:"IBM Plex Mono",monospace;font-size:10px;color:var(--green);margin-top:4px;letter-spacing:.06em;text-transform:uppercase} |
|
|
| .vizbox{width:100%;overflow:visible} |
| svg{display:block;width:100%;height:auto;overflow:visible} |
| .toggles{display:flex;gap:9px;flex-wrap:wrap;margin-bottom:8px;align-items:center} |
| .tbtn{font-family:"IBM Plex Mono",monospace;font-size:11px;letter-spacing:.04em;background:var(--paper2); |
| color:var(--muted);border:1px solid var(--line);border-radius:7px;padding:6px 12px;cursor:pointer} |
| .tbtn.on{color:#fff;background:var(--green);border-color:var(--green)} |
| .tbtn:hover{border-color:var(--green)} |
| .twocol{display:grid;grid-template-columns:1fr 1fr;gap:18px} |
| @media(max-width:820px){.twocol{grid-template-columns:1fr}} |
| input[type=range]{accent-color:var(--green)} |
|
|
| /* sandbox lanes */ |
| .lane{border:1px solid var(--line);border-radius:10px;padding:13px 15px;margin-top:12px;background:var(--paper2)} |
| .lane .top{display:flex;justify-content:space-between;align-items:baseline;gap:10px;flex-wrap:wrap} |
| .lane .name{font-family:"IBM Plex Mono",monospace;font-size:12px;font-weight:600} |
| .lane .stat{font-family:"IBM Plex Mono",monospace;font-size:11px;color:var(--muted)} |
| .stream{font-family:"IBM Plex Mono",monospace;font-size:13px;line-height:1.7;margin-top:9px;min-height:46px; |
| color:var(--ink);word-break:break-word;background:var(--card);border:1px solid var(--line2);border-radius:7px;padding:9px 11px} |
| .tok{opacity:0;animation:pop .18s ease forwards} |
| @keyframes pop{from{opacity:0}to{opacity:1}} |
| .bargauge{height:5px;background:var(--line2);border-radius:3px;margin-top:9px;overflow:hidden} |
| .bargauge > i{display:block;height:100%;width:0;border-radius:3px;transition:width .12s linear} |
|
|
| .tabpanel{display:none} |
| .tabpanel.active{display:block} |
|
|
| footer{position:relative;z-index:1;border-top:1px solid var(--line);margin-top:10px;background:var(--paper2)} |
| .footin{max-width:1080px;margin:0 auto;padding:34px 24px;font-size:13px;color:var(--muted);line-height:1.7} |
| .footin b{color:var(--ink)} |
| .footcols{display:flex;gap:60px;flex-wrap:wrap;margin-bottom:22px} |
| .footcols h4{font-family:"IBM Plex Mono",monospace;font-size:11px;letter-spacing:.1em;text-transform:uppercase;color:var(--green);margin:0 0 8px} |
| .footcols a{display:block;color:var(--ink);border:0;font-size:13.5px;margin-bottom:5px} |
| .codeblock{background:#0c1a14;color:#cfe8dc;border-radius:10px;padding:15px 17px;overflow:auto; |
| font-family:"IBM Plex Mono",monospace;font-size:12.5px;line-height:1.6;margin-top:12px} |
| .codeblock .c{color:#6fae93} |
| .codeblock .k{color:#7fd7b0} |
| </style> |
| </head> |
| <body> |
|
|
| <header> |
| <div class="bar"> |
| <span class="wordmark">THE PHYSICAL AI <b>INFERENCE GAP</b></span> |
| <span class="barmeta mono">josefchen · KAIKAKU</span> |
| <span class="barlinks"> |
| <a id="arxivTop" href="#" target="_blank">arXiv ↗</a> |
| <a id="pdfTop" href="#" target="_blank">PDF ↗</a> |
| <a id="hfTop" href="#" target="_blank">Space ↗</a> |
| </span> |
| </div> |
| <nav class="tabs"> |
| <button class="tab active" data-tab="intro">Intro</button> |
| <button class="tab" data-tab="mechanism">Mechanism</button> |
| <button class="tab" data-tab="methods">Methods</button> |
| <button class="tab" data-tab="sandbox">Sandbox</button> |
| </nav> |
| </header> |
|
|
| <div class="wrap"> |
|
|
| |
| <div class="tabpanel active" id="tab-intro"> |
|
|
| <section class="hero"> |
| <div class="heroL"> |
| <p class="eyebrow">Physical AI Inference · Batch-1 Decode</p> |
| <h1 class="serif">The faster the silicon, the less of its bandwidth ever reaches the work.</h1> |
| <p class="lede">A robot serves one token at a time, not a crowd. Across four NVIDIA GPUs and 44 measured cells, the realised fraction of peak HBM bandwidth <i>falls</i> as peak bandwidth rises, and the cheapest GPU often serves the stream cheapest.</p> |
| <div class="herostats"> |
| <div class="hstat"><div class="n serif">0.27</div><div class="l">H100 floor used</div></div> |
| <div class="hstat"><div class="n serif">0.81</div><div class="l">L4 floor used</div></div> |
| <div class="hstat"><div class="n serif">7.9×</div><div class="l">cheaper per token</div></div> |
| </div> |
| <div class="reslinks"> |
| <a class="reslink" id="arxivHero" href="#" target="_blank">arXiv paper ↗</a> |
| <a class="reslink" id="pdfHero" href="#" target="_blank">PDF ↗</a> |
| <a class="reslink" href="#sandbox" data-jump="sandbox">Open the sandbox →</a> |
| </div> |
| </div> |
| <div class="heroR"><canvas id="heroviz"></canvas><div class="herocap mono">ONE TOKEN AT A TIME · AUTOREGRESSIVE DECODE</div></div> |
| </section> |
|
|
| <section class="panel-wrap"> |
| <p class="sec-eyebrow">The workload</p> |
| <h2 class="serif">One token at a time, one stream at a time</h2> |
| <div class="prose"> |
| <p>Cloud serving batches a crowd to hide the cost of memory. A robot can't: it serves one stream, and the metric is the latency between tokens. The textbook says step time should track peak HBM bandwidth, so a faster GPU should be a faster robot. We measured it. It is right on slow silicon and badly wrong on fast silicon.</p> |
| </div> |
| <div class="card"> |
| <div class="toggles"> |
| <button class="tbtn on" data-wl="dc">Datacenter · a crowd</button> |
| <button class="tbtn" data-wl="robot">Robot · batch 1</button> |
| </div> |
| <div class="vizbox"><svg id="wlviz" viewBox="0 0 1000 300" preserveAspectRatio="xMidYMid meet"></svg></div> |
| <div class="note" id="wlNote"></div> |
| </div> |
| </section> |
|
|
| <section class="panel-wrap"> |
| <p class="sec-eyebrow">Explore · interactive</p> |
| <h2 class="serif">Pick a cell, watch the bandwidth split</h2> |
| <p class="h2sub">Every number below is measured, not modelled. R<sub>floor</sub> = analytic memory floor ÷ observed step time; it is recomputed from first principles and matches the paper's tables to three decimals.</p> |
| <div class="card"> |
| <div class="controls"> |
| <div><label>Model</label><select id="selModel"></select></div> |
| <div><label>GPU</label><select id="selGpu"></select></div> |
| <div><label>Context length</label><select id="selCtx"></select></div> |
| <div><label>GPU rate ($/hr)</label><input type="number" id="selRate" step="0.01" min="0"><span class="ratebadge" id="rateBadge"></span></div> |
| </div> |
| <div class="levers"> |
| <span class="pill mono caps" style="border:0;color:var(--muted)">Lever</span> |
| <label><input type="radio" name="lever" value="none" checked> bf16 / eager</label> |
| <label><input type="radio" name="lever" value="best"> best lever per GPU</label> |
| <span class="pill" id="leverInfo"></span> |
| </div> |
| <div class="cards"> |
| <div class="kpi"><div class="k">Step time</div><div class="v" id="cStep"></div><div class="u" id="cStepU"></div></div> |
| <div class="kpi"><div class="k">Analytic HBM floor</div><div class="v" id="cFloor"></div><div class="u">(W+K)/B_peak</div></div> |
| <div class="kpi"><div class="k">% peak HBM realised</div><div class="v" id="cR"></div><div class="u">R_floor</div></div> |
| <div class="kpi"><div class="k">Serving cost</div><div class="v" id="cCost"></div><div class="u">$ / Mtok</div></div> |
| </div> |
| <div class="verdict" id="verdict"></div> |
| </div> |
|
|
| <div class="card"> |
| <div class="vizbox"><canvas id="flow" style="width:100%;display:block"></canvas></div> |
| <div class="headline serif" id="flowHeadline">Most of the bandwidth never reaches the work.</div> |
| <div class="statblock" id="flowStats"></div> |
| <div class="note">Bright ribbon = useful bandwidth (R_floor × peak GB/s) for the selected cell; faded stream peeling below = bandwidth lost, dominated by CPU launch latency on fast silicon. The H100 pours in 11× the L4's bandwidth and delivers barely 3.7× the useful traffic.</div> |
| </div> |
| </section> |
| </div> |
|
|
| |
| <div class="tabpanel" id="tab-mechanism"> |
|
|
| <section class="panel-wrap"> |
| <p class="sec-eyebrow">Mechanism · §1</p> |
| <h2 class="serif">Anatomy of one decode step</h2> |
| <p class="h2sub">Where the bytes go. One Qwen-2.5-7B decoder block, drawn with each kernel's HBM weight traffic to scale. The SwiGLU MLP moves roughly 407 MB per block, nearly seven times the attention projections, and every block fires the same short kernels back to back. Twenty-eight of them per token sets the memory floor.</p> |
| <div class="card"> |
| <div class="vizbox"><svg id="arch" viewBox="0 0 1000 360" preserveAspectRatio="xMidYMid meet"></svg></div> |
| <div class="note">Box width is HBM bytes moved by that kernel at bf16 (weights), the dominant traffic at batch 1. KV read at ctx 2048 is about 4.2 MB per block, small but growing linearly with context. Twenty-eight blocks total about 13.16 GB per step; over the H100's 3.35 TB/s that is a 3.93 ms floor, against a measured 14.83 ms (eager), so R_floor 0.27. The kernels are tiny and many, which is exactly why per-kernel launch cost, not bandwidth, dominates on fast silicon.</div> |
| </div> |
| </section> |
|
|
| <section class="panel-wrap"> |
| <p class="sec-eyebrow">Mechanism · §2</p> |
| <h2 class="serif">Two clocks: memory cost vs launch cost</h2> |
| <p class="h2sub">Every step pays two bills: the <b style="color:var(--green)">memory clock</b> (streaming weights + KV, the analytic floor) and the <b style="color:var(--A100)">overhead clock</b> (everything above the floor). The slower clock sets the pace. Change GPU in the Intro tab and watch which one binds flip.</p> |
| <div class="card"> |
| <div class="toggles"> |
| <span class="pill mono" style="border:0;color:var(--muted)">Follows the Intro selection ·</span> |
| <button class="tbtn" id="clkCG">CUDA GRAPHS: OFF</button> |
| </div> |
| <div class="vizbox"><svg id="clocks" viewBox="0 0 1000 420" preserveAspectRatio="xMidYMid meet"></svg></div> |
| <div class="note" id="clkNote"></div> |
| </div> |
| </section> |
|
|
| <section class="panel-wrap"> |
| <p class="sec-eyebrow">Mechanism · §3</p> |
| <h2 class="serif">The empty highway</h2> |
| <p class="h2sub">One decoder block, ten kernels per layer. On slow silicon the kernels are a solid highway of compute; on fast silicon the same kernels are islands in a sea of CPU launch latency. CUDA Graphs removes the launch tax. Watch the gaps collapse.</p> |
| <div class="card"> |
| <div class="toggles"> |
| <button class="tbtn on" data-hw="H100">H100 · 3350</button> |
| <button class="tbtn" data-hw="A100-80GB">A100 · 2039</button> |
| <button class="tbtn" data-hw="L40S">L40S · 864</button> |
| <button class="tbtn" data-hw="L4">L4 · 300</button> |
| <button class="tbtn" id="cgToggle" style="margin-left:auto">CUDA GRAPHS: OFF</button> |
| </div> |
| <div class="vizbox"><svg id="highway" viewBox="0 0 1000 250" preserveAspectRatio="xMidYMid meet"></svg></div> |
| <div class="note" id="hwNote"></div> |
| </div> |
| </section> |
|
|
| <section class="panel-wrap"> |
| <p class="sec-eyebrow">Mechanism · §4</p> |
| <h2 class="serif">The dexterous barrier</h2> |
| <p class="h2sub">Pure-bandwidth scaling says decision rate should climb with HBM bandwidth. It doesn't. Drag the required control rate and feel how few GPUs clear it.</p> |
| <div class="card"> |
| <div class="toggles"> |
| <span class="pill mono" style="border:0;color:var(--muted)">Action tokens / chunk</span> |
| <button class="tbtn" data-tok="1">1</button> |
| <button class="tbtn on" data-tok="7">7 · OpenVLA</button> |
| <button class="tbtn" data-tok="16">16</button> |
| <span class="pill mono" id="targetReadout" style="margin-left:auto"></span> |
| </div> |
| <div style="display:flex;align-items:center;gap:12px;margin:4px 0 2px"> |
| <label style="margin:0;white-space:nowrap">Required control rate</label> |
| <input type="range" id="hzTarget" min="1" max="50" value="30" step="1" style="flex:1"> |
| <span class="mono" id="hzVal" style="color:var(--green);min-width:54px;text-align:right">30 Hz</span> |
| </div> |
| <div class="vizbox"><svg id="barrier" viewBox="0 0 1000 460" preserveAspectRatio="xMidYMid meet"></svg></div> |
| <div class="note" id="barrierNote"></div> |
| </div> |
| </section> |
| </div> |
|
|
| |
| <div class="tabpanel" id="tab-methods"> |
|
|
| <section class="panel-wrap"> |
| <p class="sec-eyebrow">Methods · the map</p> |
| <h2 class="serif">All 44 cells at once</h2> |
| <p class="h2sub">Three 7-8B GQA models × four GPUs × four context lengths. Each cell is the median of 30 measured decode steps after 5 warmup, bf16, sdpa, batch 1, on Modal cloud hosts. Darker is closer to the memory floor. Four L4 cells OOM at long context.</p> |
| <div class="card"> |
| <div class="vizbox"><svg id="heatmap" viewBox="0 0 1000 360" preserveAspectRatio="xMidYMid meet"></svg></div> |
| <div class="note">R_floor is monotone in peak bandwidth at fixed context, and falls with context at fixed GPU (the KV term grows faster than launch overhead). The L4 column blazes; the H100 column is dim.</div> |
| </div> |
| </section> |
|
|
| <section class="panel-wrap"> |
| <p class="sec-eyebrow">Methods · the surface</p> |
| <h2 class="serif">The efficiency surface</h2> |
| <p class="h2sub">The same R_floor values as a terrain. Height is the fraction of the memory floor a cell actually reaches. A bright high plateau on the cheap, short-context corner collapses into a dark trench where the silicon is fastest and the context longest. Follows the model selected in the Intro console.</p> |
| <div class="card"> |
| <div class="vizbox"><div id="surface" style="width:100%;height:540px;background:radial-gradient(120% 100% at 50% 30%,#0a2a22,#04140f);border-radius:10px"></div></div> |
| <div class="note">Drag to orbit, scroll to zoom. Peak bandwidth runs along the long axis (L4 300 to H100 3350 GB/s), context length along the depth axis (2048 to 16384), height is R_floor. The bright green plateau is the L4 near its floor; the deep red basin is the H100 sinking to 27% and below. Macro height follows the 44 measured cells; the fine contour relief and the basin are stylised for legibility, the ground truth is the cell values.</div> |
| </div> |
| </section> |
|
|
| <section class="panel-wrap"> |
| <p class="sec-eyebrow">Methods · the proof</p> |
| <h2 class="serif">A pre-registered falsification, not a story</h2> |
| <p class="h2sub">The load-bearing claim is mechanistic: the H100 gap is per-kernel CPU launch overhead. It was tested with a CUDA Graphs A/B that touches the launch term and nothing else, with kill-conditions stated in advance.</p> |
| <div class="card"> |
| <div class="vizbox"><svg id="falsify" viewBox="0 0 1000 300" preserveAspectRatio="xMidYMid meet"></svg></div> |
| <div class="note"><b>Pre-registered:</b> an H100 speedup under ~1.15× would have killed the launch-tax claim; an L4 speedup over ~1.15× would also have killed it. Neither occurred: H100 measured 1.259× (95% bootstrap CI [1.253, 1.267], N=10, cross-session CV 0.9%); L4 measured a null 1.028×. <b>An honest caveat:</b> the A/B only proves the graph-removable slice is launch (~3.05 ms on H100). The remaining ~7.2 ms above the floor survives graphing and is not, by this evidence, attributable to launch.</div> |
| </div> |
| </section> |
|
|
| <section class="panel-wrap"> |
| <div class="twocol"> |
| <div> |
| <p class="sec-eyebrow">Methods · §A</p> |
| <h2 class="serif">Quant: kernel, not bit-width</h2> |
| <p class="h2sub">Qwen-2.5-7B, ctx 2048, L4. The 4× weight-traffic saving only lands when the int4 kernel is tuned for Ada SM89.</p> |
| <div class="card"><div class="vizbox"><svg id="quant" viewBox="0 0 500 300" preserveAspectRatio="xMidYMid meet"></svg></div></div> |
| </div> |
| <div> |
| <p class="sec-eyebrow">Methods · §B</p> |
| <h2 class="serif">Attention backends</h2> |
| <p class="h2sub">Per-layer p50, Llama-3-8B decode shape, H100. Default SDPA beats every pinned backend; cuDNN rejects the shape.</p> |
| <div class="card"><div class="vizbox"><svg id="attn" viewBox="0 0 500 300" preserveAspectRatio="xMidYMid meet"></svg></div></div> |
| </div> |
| </div> |
| </section> |
|
|
| <section class="panel-wrap"> |
| <p class="sec-eyebrow">Methods · §C</p> |
| <h2 class="serif">The cost-per-token inversion</h2> |
| <p class="h2sub">Each GPU runs its best measured lever (Qwen-2.5-7B, ctx 2048). Cost uses the editable rates from the Intro console.</p> |
| <div class="card"> |
| <div class="vizbox"><svg id="cost" viewBox="0 0 1000 300" preserveAspectRatio="xMidYMid meet"></svg></div> |
| <div class="statblock" id="costStats"></div> |
| <div class="note" id="costNote"></div> |
| </div> |
| </section> |
|
|
| <section class="panel-wrap"> |
| <p class="sec-eyebrow">Methods · reproduce</p> |
| <h2 class="serif">The measurement protocol</h2> |
| <div class="prose"><p>Each cell loads the model in bf16, runs a fixed prefill to populate the KV cache, then times 5 warmup decode steps and 30 measured single-token decode steps at batch 1. Step time is the median. The directly-measured ratio is the only number that matters:</p></div> |
| <div class="codeblock"><span class="c"># R_floor = analytic memory floor / observed step time</span> |
| t_floor = (W + K) / B_peak <span class="c"># W = bf16 weight bytes; K = 2·n_layers·n_kv_heads·head_dim·ctx·2</span> |
| R_floor = t_floor / t_obs <span class="c"># t_obs = median of 30 measured decode steps, batch 1</span> |
| <span class="c"># A purely HBM-bandwidth-bound decode would sit at R_floor = 1.</span> |
| <span class="c"># H100 ctx2048 Qwen: t_floor 4.58 ms, t_obs 16.97 ms -> R_floor 0.27</span></div> |
| </section> |
| </div> |
|
|
| |
| <div class="tabpanel" id="tab-sandbox"> |
|
|
| <section class="panel-wrap"> |
| <p class="sec-eyebrow">Sandbox · the bill</p> |
| <h2 class="serif">What your fleet costs to serve</h2> |
| <p class="h2sub">At batch 1 every stream needs its own GPU. Per token, a $0.30/hr L4 with an Ada-tuned int4 runtime serves a 7B model 7.9× cheaper than a $3.50/hr H100. Slide your monthly token volume and see the bill.</p> |
| <div class="card"> |
| <label>Tokens served per month, across your fleet</label> |
| <div style="display:flex;align-items:center;gap:16px;margin-top:4px"> |
| <input type="range" id="calcTok" min="6" max="12" step="0.05" value="9" style="flex:1"> |
| <span class="mono" id="calcTokVal" style="color:var(--green);min-width:150px;text-align:right;font-size:14px"></span> |
| </div> |
| <div class="cards" style="grid-template-columns:repeat(3,1fr)"> |
| <div class="kpi"><div class="k">H100 · $3.50/hr</div><div class="v" id="calcH"></div><div class="u">CUDA Graphs · 11.78 ms/tok</div></div> |
| <div class="kpi"><div class="k">L4 · $0.30/hr</div><div class="v" id="calcL" style="color:var(--green)"></div><div class="u">ExLlamaV2 · 17.36 ms/tok</div></div> |
| <div class="kpi" style="border-color:rgba(14,122,82,.4);background:rgba(14,122,82,.05)"><div class="k">You save</div><div class="v" id="calcSave" style="color:var(--green)"></div><div class="u" id="calcRatio"></div></div> |
| </div> |
| <div class="verdict" id="calcVerdict"></div> |
| <div class="note">Cost = monthly tokens × measured ms/token × GPU rate. Per-token rates: H100 $11.45 / Mtok, L4 $1.45 / Mtok (Modal published rates, May 2026). Excludes idle, networking, storage and batching; the ratio is the point.</div> |
| </div> |
| </section> |
|
|
| <section class="panel-wrap"> |
| <p class="sec-eyebrow">Sandbox · the feel</p> |
| <h2 class="serif">Runtime-poor, not compute-poor</h2> |
| <p class="h2sub">A robot serves one token at a time, not a crowd. Stream the same sentence at each GPU's <i>measured</i> ms/token and watch two things: the same $0.30/hr L4 goes from slow to fast on a runtime swap alone (bf16 to ExLlamaV2 int4), and that cheap L4 then serves each token cheaper than the $3.50/hr H100 with 11x the bandwidth. (Qwen-2.5-7B, ctx 2048, batch 1.)</p> |
| <div class="card"> |
| <label>Prompt to stream</label> |
| <textarea id="sbPrompt" rows="2">The robot locates the red block, grasps it, and places it on the blue platform.</textarea> |
| <div class="toggles" style="margin-top:12px"> |
| <button class="tbtn on" id="sbRun">▶ Stream on all three</button> |
| <button class="tbtn" id="sbReset">Reset</button> |
| <span class="pill mono" style="border:0;color:var(--muted)">real time · 1000 / (tokens × step_ms)</span> |
| </div> |
| <div id="sbLanes"></div> |
| <div class="verdict" id="sbVerdict" style="display:none"></div> |
| <div class="note">Tokens are split on whitespace and punctuation as a proxy for model tokens; each lane emits one every <i>measured</i> ms/step. H100 + CUDA Graphs 11.78 ms, L4 + ExLlamaV2 17.36 ms, L4 bf16 63.15 ms. Same sentence, very different felt latency, and the cheap GPU with the right kernel nearly keeps up with the flagship.</div> |
| </div> |
| </section> |
| </div> |
|
|
| </div> |
|
|
| <footer> |
| <div class="footin"> |
| <div class="footcols"> |
| <div> |
| <h4>Resources</h4> |
| <a id="arxivFoot" href="#" target="_blank">arXiv paper ↗</a> |
| <a id="pdfFoot" href="#" target="_blank">PDF ↗</a> |
| <a id="hfFoot" href="#" target="_blank">This Space ↗</a> |
| </div> |
| <div> |
| <h4>Sections</h4> |
| <a href="#" data-jump="intro">Intro</a> |
| <a href="#" data-jump="mechanism">Mechanism</a> |
| <a href="#" data-jump="methods">Methods</a> |
| <a href="#" data-jump="sandbox">Sandbox</a> |
| </div> |
| <div> |
| <h4>The study</h4> |
| <span style="display:block;margin-bottom:5px">44 cells · 4 GPUs · 3 models</span> |
| <span style="display:block;margin-bottom:5px">bf16 · sdpa · batch 1</span> |
| <span style="display:block">Modal cloud hosts · May 2026</span> |
| </div> |
| </div> |
| <b>Data provenance.</b> All step times are medians of 30 measured single-token decode steps after 5 warmup, batch 1, bf16, sdpa. Peak HBM: H100 3350, A100-80GB 2039, L40S 864, L4 300 GB/s. R_floor recomputed from W (weight_bytes) and K; matches the paper to 3 dp. <b>Schematic:</b> the highway splits each block into active-compute vs launch-gap using the measured active fraction (= R_floor); the 10-kernel decomposition is illustrative. <b>Rates:</b> only H100 ($3.50/hr) and L4 ($0.30/hr) defaults are paper-sourced (Modal, May 2026); A100/L40S are editable placeholders. Not affiliated with NVIDIA; model names are trademarks of their respective owners. © 2026 Josef Chen · KAIKAKU. |
| </div> |
| </footer> |
|
|
| <script> |
| const DATA = {"meta": {"paper": "Memory-Bound but Not Bandwidth-Limited: The Physical AI Inference Gap in Batch-1 LLM Decode", "author": "Josef Chen, KAIKAKU", "workload": "batch-1 single-stream autoregressive decode, bf16, sdpa"}, "gpus": {"H100": {"bw_gbs": 3350, "rate_hr": 3.5, "rate_src": "paper"}, "A100-80GB": {"bw_gbs": 2039, "rate_hr": 2.1, "rate_src": "editable"}, "L40S": {"bw_gbs": 864, "rate_hr": 1.95, "rate_src": "editable"}, "L4": {"bw_gbs": 300, "rate_hr": 0.3, "rate_src": "paper"}}, "cells": [{"arch": "Llama-3.1-8B", "gpu": "H100", "ctx": 2048, "t_obs": 16.13, "t_floor": 4.87, "R_floor": 0.302, "W_gb": 16.06}, {"arch": "Llama-3.1-8B", "gpu": "H100", "ctx": 4096, "t_obs": 15.98, "t_floor": 4.95, "R_floor": 0.31, "W_gb": 16.06}, {"arch": "Llama-3.1-8B", "gpu": "H100", "ctx": 8192, "t_obs": 18.33, "t_floor": 5.11, "R_floor": 0.279, "W_gb": 16.06}, {"arch": "Llama-3.1-8B", "gpu": "H100", "ctx": 16384, "t_obs": 26.08, "t_floor": 5.44, "R_floor": 0.208, "W_gb": 16.06}, {"arch": "Llama-3.1-8B", "gpu": "A100-80GB", "ctx": 2048, "t_obs": 19.32, "t_floor": 8.01, "R_floor": 0.414, "W_gb": 16.06}, {"arch": "Llama-3.1-8B", "gpu": "A100-80GB", "ctx": 4096, "t_obs": 22.54, "t_floor": 8.14, "R_floor": 0.361, "W_gb": 16.06}, {"arch": "Llama-3.1-8B", "gpu": "A100-80GB", "ctx": 8192, "t_obs": 29.02, "t_floor": 8.4, "R_floor": 0.29, "W_gb": 16.06}, {"arch": "Llama-3.1-8B", "gpu": "A100-80GB", "ctx": 16384, "t_obs": 41.54, "t_floor": 8.93, "R_floor": 0.215, "W_gb": 16.06}, {"arch": "Llama-3.1-8B", "gpu": "L40S", "ctx": 2048, "t_obs": 26.46, "t_floor": 18.9, "R_floor": 0.714, "W_gb": 16.06}, {"arch": "Llama-3.1-8B", "gpu": "L40S", "ctx": 4096, "t_obs": 28.74, "t_floor": 19.21, "R_floor": 0.668, "W_gb": 16.06}, {"arch": "Llama-3.1-8B", "gpu": "L40S", "ctx": 8192, "t_obs": 38.94, "t_floor": 19.83, "R_floor": 0.509, "W_gb": 16.06}, {"arch": "Llama-3.1-8B", "gpu": "L40S", "ctx": 16384, "t_obs": 57.28, "t_floor": 21.07, "R_floor": 0.368, "W_gb": 16.06}, {"arch": "Llama-3.1-8B", "gpu": "L4", "ctx": 2048, "t_obs": 69.93, "t_floor": 54.43, "R_floor": 0.778, "W_gb": 16.06}, {"arch": "Llama-3.1-8B", "gpu": "L4", "ctx": 4096, "t_obs": 82.99, "t_floor": 55.32, "R_floor": 0.667, "W_gb": 16.06}, {"arch": "Mistral-7B", "gpu": "H100", "ctx": 2048, "t_obs": 18.17, "t_floor": 4.41, "R_floor": 0.243, "W_gb": 14.5}, {"arch": "Mistral-7B", "gpu": "H100", "ctx": 4096, "t_obs": 21.73, "t_floor": 4.49, "R_floor": 0.207, "W_gb": 14.5}, {"arch": "Mistral-7B", "gpu": "H100", "ctx": 8192, "t_obs": 18.54, "t_floor": 4.65, "R_floor": 0.251, "W_gb": 14.5}, {"arch": "Mistral-7B", "gpu": "H100", "ctx": 16384, "t_obs": 26.13, "t_floor": 4.97, "R_floor": 0.19, "W_gb": 14.5}, {"arch": "Mistral-7B", "gpu": "A100-80GB", "ctx": 2048, "t_obs": 27.95, "t_floor": 7.24, "R_floor": 0.259, "W_gb": 14.5}, {"arch": "Mistral-7B", "gpu": "A100-80GB", "ctx": 4096, "t_obs": 34.04, "t_floor": 7.37, "R_floor": 0.217, "W_gb": 14.5}, {"arch": "Mistral-7B", "gpu": "A100-80GB", "ctx": 8192, "t_obs": 29.76, "t_floor": 7.64, "R_floor": 0.257, "W_gb": 14.5}, {"arch": "Mistral-7B", "gpu": "A100-80GB", "ctx": 16384, "t_obs": 42.53, "t_floor": 8.16, "R_floor": 0.192, "W_gb": 14.5}, {"arch": "Mistral-7B", "gpu": "L40S", "ctx": 2048, "t_obs": 32.67, "t_floor": 17.09, "R_floor": 0.523, "W_gb": 14.5}, {"arch": "Mistral-7B", "gpu": "L40S", "ctx": 4096, "t_obs": 31.76, "t_floor": 17.4, "R_floor": 0.548, "W_gb": 14.5}, {"arch": "Mistral-7B", "gpu": "L40S", "ctx": 8192, "t_obs": 38.3, "t_floor": 18.02, "R_floor": 0.471, "W_gb": 14.5}, {"arch": "Mistral-7B", "gpu": "L40S", "ctx": 16384, "t_obs": 56.62, "t_floor": 19.26, "R_floor": 0.34, "W_gb": 14.5}, {"arch": "Mistral-7B", "gpu": "L4", "ctx": 2048, "t_obs": 66.65, "t_floor": 49.21, "R_floor": 0.738, "W_gb": 14.5}, {"arch": "Mistral-7B", "gpu": "L4", "ctx": 4096, "t_obs": 79.9, "t_floor": 50.11, "R_floor": 0.627, "W_gb": 14.5}, {"arch": "Mistral-7B", "gpu": "L4", "ctx": 8192, "t_obs": 108.61, "t_floor": 51.9, "R_floor": 0.478, "W_gb": 14.5}, {"arch": "Mistral-7B", "gpu": "L4", "ctx": 16384, "t_obs": 156.93, "t_floor": 55.48, "R_floor": 0.354, "W_gb": 14.5}, {"arch": "Qwen-2.5-7B", "gpu": "H100", "ctx": 2048, "t_obs": 16.97, "t_floor": 4.58, "R_floor": 0.27, "W_gb": 15.23}, {"arch": "Qwen-2.5-7B", "gpu": "H100", "ctx": 4096, "t_obs": 17.06, "t_floor": 4.62, "R_floor": 0.271, "W_gb": 15.23}, {"arch": "Qwen-2.5-7B", "gpu": "H100", "ctx": 8192, "t_obs": 17.1, "t_floor": 4.69, "R_floor": 0.274, "W_gb": 15.23}, {"arch": "Qwen-2.5-7B", "gpu": "H100", "ctx": 16384, "t_obs": 20.58, "t_floor": 4.83, "R_floor": 0.235, "W_gb": 15.23}, {"arch": "Qwen-2.5-7B", "gpu": "A100-80GB", "ctx": 2048, "t_obs": 24.24, "t_floor": 7.53, "R_floor": 0.311, "W_gb": 15.23}, {"arch": "Qwen-2.5-7B", "gpu": "A100-80GB", "ctx": 4096, "t_obs": 20.55, "t_floor": 7.59, "R_floor": 0.369, "W_gb": 15.23}, {"arch": "Qwen-2.5-7B", "gpu": "A100-80GB", "ctx": 8192, "t_obs": 24.66, "t_floor": 7.7, "R_floor": 0.312, "W_gb": 15.23}, {"arch": "Qwen-2.5-7B", "gpu": "A100-80GB", "ctx": 16384, "t_obs": 32.66, "t_floor": 7.93, "R_floor": 0.243, "W_gb": 15.23}, {"arch": "Qwen-2.5-7B", "gpu": "L40S", "ctx": 2048, "t_obs": 24.58, "t_floor": 17.76, "R_floor": 0.723, "W_gb": 15.23}, {"arch": "Qwen-2.5-7B", "gpu": "L40S", "ctx": 4096, "t_obs": 25.19, "t_floor": 17.9, "R_floor": 0.711, "W_gb": 15.23}, {"arch": "Qwen-2.5-7B", "gpu": "L40S", "ctx": 8192, "t_obs": 27.28, "t_floor": 18.17, "R_floor": 0.666, "W_gb": 15.23}, {"arch": "Qwen-2.5-7B", "gpu": "L40S", "ctx": 16384, "t_obs": 38.43, "t_floor": 18.72, "R_floor": 0.487, "W_gb": 15.23}, {"arch": "Qwen-2.5-7B", "gpu": "L4", "ctx": 2048, "t_obs": 63.15, "t_floor": 51.16, "R_floor": 0.81, "W_gb": 15.23}, {"arch": "Qwen-2.5-7B", "gpu": "L4", "ctx": 4096, "t_obs": 71.42, "t_floor": 51.55, "R_floor": 0.722, "W_gb": 15.23}], "quant_l4_qwen2048": [{"name": "bf16", "step_ms": 62.32, "t_floor": 51.17, "R": 0.821}, {"name": "bnb-nf4", "step_ms": 59.36, "t_floor": 13.09, "R": 0.22}, {"name": "AutoAWQ+Marlin", "step_ms": 45.24, "t_floor": 13.09, "R": 0.289}, {"name": "GPTQ+ExLlamaV2 4.25bpw", "step_ms": 17.36, "t_floor": 13.09, "R": 0.754}], "cudagraphs": {"H100_qwen_2048": {"eager_ms": 14.83, "graphed_ms": 11.78, "speedup": 1.259, "ci": [1.253, 1.267]}, "L4_qwen_2048": {"eager_ms": 64.52, "graphed_ms": 62.77, "speedup": 1.028}}, "attention_h100": [{"backend": "default SDPA", "us": 36.05, "ms32": 1.15, "vs": 1.0}, {"backend": "SDPA FLASH_ATTENTION", "us": 44.35, "ms32": 1.42, "vs": 0.81}, {"backend": "FlashInfer single_decode", "us": 48.2, "ms32": 1.54, "vs": 0.75}, {"backend": "FlashAttention-3", "us": 79.25, "ms32": 2.54, "vs": 0.45}, {"backend": "SDPA EFFICIENT", "us": 89.72, "ms32": 2.87, "vs": 0.4}, {"backend": "SDPA MATH", "us": 177.55, "ms32": 5.68, "vs": 0.2}, {"backend": "SDPA CUDNN", "us": null, "ms32": null, "vs": null, "note": "not supported for this shape"}]}; |
| const ARXIV_URL = "https://arxiv.org/abs/2605.30571"; |
| const PDF_URL = "https://arxiv.org/pdf/2605.30571"; |
| |
| const PAL={phos:"#0e7a52",phos2:"#11936a",dim:"#566b60",ink:"#15251d",contour:"#cdded4", |
| mint:"#1f9d6b",aqua:"#0f8a78",lime:"#7f9c32",peach:"#cc7a2b",cream:"#a87c16",panel:"#dbe7df",faint:"#7d9389"}; |
| const GPU_ORDER=["L4","L40S","A100-80GB","H100"]; |
| const GPU_COL={"L4":PAL.mint,"L40S":PAL.lime,"A100-80GB":PAL.peach,"H100":PAL.cream}; |
| const SVGNS="http://www.w3.org/2000/svg"; |
| function el(t,a={},x){const e=document.createElementNS(SVGNS,t);for(const k in a)e.setAttribute(k,a[k]);if(x!=null)e.textContent=x;return e;} |
| function clear(s){while(s.firstChild)s.removeChild(s.firstChild);} |
| function cell(a,g,c){return DATA.cells.find(x=>x.arch===a&&x.gpu===g&&x.ctx==c);} |
| function fmt(x,d=2){return x==null?"-":Number(x).toFixed(d);} |
| function costPerMtok(ms,rate){return (ms/1000)*(rate/3600)*1e6;} |
| function bestLever(a,g,c){ |
| if(a==="Qwen-2.5-7B"&&c==2048){ |
| if(g==="H100")return{step:DATA.cudagraphs.H100_qwen_2048.graphed_ms,label:"CUDA Graphs",measured:true}; |
| if(g==="L4")return{step:DATA.quant_l4_qwen2048.find(q=>q.name.startsWith("GPTQ")).step_ms,label:"GPTQ+ExLlamaV2",measured:true};} |
| return null;} |
| |
| |
| const HF_URL="https://huggingface.co/spaces/josefchen/physical-ai-inference-gap"; |
| ["arxivTop","arxivHero","arxivFoot"].forEach(id=>{const e=document.getElementById(id);if(e)e.href=ARXIV_URL;}); |
| ["pdfTop","pdfHero","pdfFoot"].forEach(id=>{const e=document.getElementById(id);if(e)e.href=PDF_URL;}); |
| ["hfTop","hfFoot"].forEach(id=>{const e=document.getElementById(id);if(e)e.href=HF_URL;}); |
| |
| |
| const archs=[...new Set(DATA.cells.map(c=>c.arch))]; |
| const ctxs=[...new Set(DATA.cells.map(c=>c.ctx))].sort((a,b)=>a-b); |
| const selModel=document.getElementById("selModel"),selGpu=document.getElementById("selGpu"),selCtx=document.getElementById("selCtx"),selRate=document.getElementById("selRate"); |
| archs.forEach(a=>selModel.add(new Option(a,a)));GPU_ORDER.forEach(g=>selGpu.add(new Option(g,g)));ctxs.forEach(c=>selCtx.add(new Option(c.toLocaleString(),c))); |
| selModel.value="Qwen-2.5-7B";selGpu.value="H100";selCtx.value="2048"; |
| function refreshRate(){const gi=DATA.gpus[selGpu.value];selRate.value=gi.rate_hr; |
| document.getElementById("rateBadge").textContent=gi.rate_src==="paper"?"paper-sourced (editable)":"placeholder, edit to your rate";} |
| |
| function renderConsole(){ |
| const a=selModel.value,g=selGpu.value,c=parseInt(selCtx.value); |
| const cl=cell(a,g,c),rate=parseFloat(selRate.value)||0; |
| const lever=document.querySelector('input[name=lever]:checked').value; |
| const li=document.getElementById("leverInfo"),$=id=>document.getElementById(id); |
| if(!cl){$("cStep").innerHTML="OOM";$("cStepU").textContent="not a valid cell";$("cFloor").textContent="-";$("cR").textContent="-";$("cCost").textContent="-"; |
| $("verdict").innerHTML="Out-of-memory: a 7-8B bf16 model at this context does not fit the L4's 24 GB. The paper reports 4 such L4 OOMs.";li.textContent="";li.className="pill";buildFlow(a,c);buildClocks();return;} |
| let step=cl.t_obs,mode="bf16 / eager",leverLabel=null; |
| if(lever==="best"){const bl=bestLever(a,g,c);if(bl){step=bl.step;leverLabel=bl.label;mode=bl.label;}else leverLabel="none measured";} |
| const R=cl.t_floor/step,cost=costPerMtok(step,rate); |
| $("cStep").innerHTML=fmt(step,2)+" <small>ms</small>";$("cStepU").textContent=mode; |
| $("cFloor").innerHTML=fmt(cl.t_floor,2)+" <small>ms</small>";$("cR").innerHTML=(R*100).toFixed(0)+" <small>%</small>";$("cCost").innerHTML="$"+cost.toFixed(2); |
| if(lever==="best"){if(leverLabel&&leverLabel!=="none measured"){li.textContent=leverLabel+" (measured)";li.className="pill measured";}else{li.textContent="no measured lever, bf16/eager";li.className="pill estimate";}}else{li.textContent="";li.className="pill";} |
| let v=`On <b>${g}</b>, ${a} at ctx ${c.toLocaleString()} realises <b>${(R*100).toFixed(0)}%</b> of peak HBM bandwidth. `; |
| v+= R<0.4?`Most of the step is <b>not</b> memory traffic. Launch overhead dominates, which CUDA Graphs recovers on fast silicon.`:`This GPU sits near its memory floor. The lever is int4 weight compression, not launch removal.`; |
| $("verdict").innerHTML=v; |
| buildFlow(a,c);buildClocks();drawCost();drawSurface(); |
| } |
| |
| |
| const flowCanvas=document.getElementById("flow"),fctx=flowCanvas.getContext("2d"); |
| let FLOW={squares:[],W:1660,H:900,arch:"Qwen-2.5-7B",ctx:2048,phase:0,lanes:[]}; |
| const PIX=8; |
| function hexToRgb(h){const n=parseInt(h.slice(1),16);return[n>>16&255,n>>8&255,n&255];} |
| const COLRGB={};GPU_ORDER.forEach(g=>COLRGB[g]=hexToRgb(GPU_COL[g])); |
| const WASTE_RGB=[150,170,158]; |
| function buildFlow(arch,ctx){ |
| FLOW.arch=arch;FLOW.ctx=ctx;FLOW.squares=[]; |
| const H=FLOW.H,xIn=180,xSplit=560,xOut=1490,topMargin=84,botMargin=120,usable=H-topMargin-botMargin; |
| const th=g=>34+165*(DATA.gpus[g].bw_gbs/3350);const ths=GPU_ORDER.map(th);const gap=(usable-ths.reduce((a,b)=>a+b,0))/3; |
| let y=topMargin;FLOW.lanes=[]; |
| GPU_ORDER.forEach((g,i)=>{ |
| const cl=cell(arch,g,ctx),laneH=ths[i],laneTop=y,laneBot=y+laneH,gapBelow=(i<3)?gap:botMargin*1.4; |
| const R=cl?cl.R_floor:0,peak=DATA.gpus[g].bw_gbs,useful=Math.round(R*peak),wasted=peak-useful; |
| FLOW.lanes.push({g,laneTop,laneBot,laneH,R,peak,useful,wasted,oom:!cl,xIn,xSplit,xOut,gapBelow}); |
| if(cl){const cr=COLRGB[g]; |
| for(let x=xIn;x<xOut;x+=PIX){const rows=Math.max(1,Math.round(laneH/PIX)); |
| for(let r=0;r<rows;r++){const fy=r/rows;let sy,col,baseA,kind; |
| if(x<xSplit){sy=laneTop+fy*laneH;col=cr;baseA=0.92;kind="in";} |
| else if(fy<R){sy=laneTop+fy*laneH;col=cr;baseA=1.0;kind="use";} |
| else{const t=(x-xSplit)/(xOut-xSplit),wfrac=(R<1)?(fy-R)/(1-R):0,plume=Math.min(gapBelow*0.98,(0.12+t*1.25)*gapBelow); |
| sy=laneBot+plume*(0.18+0.82*wfrac)+(Math.random()-0.5)*plume*0.5;col=WASTE_RGB;baseA=Math.max(0,0.5*(1-t*0.7));kind="waste"; |
| if(Math.random()<0.45*t)continue;} |
| FLOW.squares.push({x,y:sy,col,baseA,seed:Math.random()*6.28});}}}y=laneBot+gap;}); |
| const h100=FLOW.lanes.find(l=>l.g==="H100"),l4=FLOW.lanes.find(l=>l.g==="L4"); |
| const ratio=(h100&&l4&&l4.useful)?(h100.useful/l4.useful):0; |
| document.getElementById("flowStats").innerHTML= |
| `<div class="s"><div class="num">${h100?h100.useful:'-'} <small>GB/s</small></div><div class="cap">H100 useful</div></div>`+ |
| `<div class="s"><div class="num">${l4?l4.useful:'-'} <small>GB/s</small></div><div class="cap">L4 useful</div></div>`+ |
| `<div class="s"><div class="num">${ratio?ratio.toFixed(1)+'×':'-'}</div><div class="cap">H100 useful over L4 · peak gap 11.2×</div></div>`; |
| resizeFlow(); |
| } |
| function resizeFlow(){const cssW=flowCanvas.clientWidth||1000;if(cssW<10)return;const dpr=Math.min(2,window.devicePixelRatio||1); |
| FLOW.scale=cssW/FLOW.W;flowCanvas.width=cssW*dpr;flowCanvas.height=FLOW.H*FLOW.scale*dpr; |
| flowCanvas.style.height=(FLOW.H*FLOW.scale)+"px";fctx.setTransform(dpr*FLOW.scale,0,0,dpr*FLOW.scale,0,0);} |
| function renderFlow(){FLOW.phase+=0.05;fctx.clearRect(0,0,FLOW.W,FLOW.H); |
| fctx.fillStyle=PAL.phos;fctx.font="500 18px 'IBM Plex Mono',monospace"; |
| fctx.fillText("PHYSICAL AI INFERENCE . BANDWIDTH FLOW",24,30); |
| fctx.fillStyle=PAL.dim;fctx.font="14px 'IBM Plex Mono',monospace"; |
| fctx.fillText(FLOW.arch.toUpperCase()+" . CTX "+FLOW.ctx.toLocaleString()+" . BATCH 1 . BF16 . 4 GPUS",24,50); |
| const ph=FLOW.phase; |
| for(let i=0;i<FLOW.squares.length;i++){const s=FLOW.squares[i]; |
| const wave=0.78+0.22*Math.sin(ph-s.x*0.016+s.seed);let a=s.baseA*wave;if(a<=0.03)continue; |
| fctx.globalAlpha=Math.min(1,a);fctx.fillStyle=`rgb(${s.col[0]},${s.col[1]},${s.col[2]})`;fctx.fillRect(s.x,s.y,PIX-1.5,PIX-1.5);} |
| fctx.globalAlpha=1; |
| FLOW.lanes.forEach(l=>{const midY=(l.laneTop+l.laneBot)/2; |
| fctx.fillStyle=PAL.ink;fctx.font="600 15px 'IBM Plex Mono',monospace";fctx.textAlign="left"; |
| fctx.fillText(l.g+" . "+l.peak+" GB/s",24,midY+4); |
| if(l.oom){fctx.fillStyle="#c0632e";fctx.fillText("OOM",560,midY);return;} |
| fctx.fillStyle="rgb("+COLRGB[l.g].join(",")+")";fctx.font="600 14px 'IBM Plex Mono',monospace"; |
| fctx.fillText("USEFUL . "+l.useful+" GB/s",1512,l.laneTop+l.R*l.laneH*0.5+6); |
| fctx.fillStyle=PAL.dim;fctx.font="13px 'IBM Plex Mono',monospace"; |
| fctx.fillText("WASTED . "+l.wasted+" GB/s"+(l.g==="H100"?" . LAUNCH-BOUND":""),l.xSplit+90,Math.min(FLOW.H-8,l.laneBot+l.gapBelow*0.78));}); |
| fctx.textAlign="left";requestAnimationFrame(renderFlow);} |
| |
| |
| let CLK={mem:null,launch:null,memPeriod:1,launchPeriod:1,angM:0,angL:0,last:0},CLK_CG=false; |
| function clockData(a,g,c){const cl=cell(a,g,c);if(!cl)return null; |
| const isG=(a==="Qwen-2.5-7B"&&c==2048&&(g==="H100"||g==="L4"));let obs; |
| if(isG){const cg=DATA.cudagraphs[g==="H100"?"H100_qwen_2048":"L4_qwen_2048"];obs=CLK_CG?cg.graphed_ms:cg.eager_ms;}else obs=cl.t_obs; |
| const floor=cl.t_floor,overhead=Math.max(0.01,obs-floor);return {floor,overhead,obs,isG};} |
| function dial(s,cx,cy,r,color,label,timeMs,binding){ |
| for(let i=0;i<48;i++){const ang=i/48*2*Math.PI-Math.PI/2,major=(i%4===0),rr=r-(major?2:6),sz=major?6:3,x=cx+Math.cos(ang)*rr,y=cy+Math.sin(ang)*rr; |
| s.appendChild(el("rect",{x:x-sz/2,y:y-sz/2,width:sz,height:sz,fill:color,opacity:major?0.85:0.4}));} |
| if(binding)s.appendChild(el("circle",{cx,cy,r:r+10,fill:"none",stroke:color,"stroke-width":1.5,opacity:.55,"stroke-dasharray":"2 5"})); |
| s.appendChild(el("circle",{cx,cy,r:7,fill:"none",stroke:color,"stroke-width":1.5})); |
| const hand=el("line",{x1:cx,y1:cy,x2:cx,y2:cy-(r-22),stroke:color,"stroke-width":binding?4:2.5,"stroke-linecap":"round",opacity:binding?1:.8}); |
| s.appendChild(hand); |
| s.appendChild(el("text",{x:cx,y:cy-r-22,fill:color,"font-size":13,"text-anchor":"middle","font-family":"IBM Plex Mono","font-weight":600},label)); |
| s.appendChild(el("text",{x:cx,y:cy+r+26,fill:PAL.ink,"font-size":24,"text-anchor":"middle","font-family":"Fraunces,serif","font-weight":600},timeMs.toFixed(2)+" ms")); |
| s.appendChild(el("text",{x:cx,y:cy+r+44,fill:binding?color:PAL.faint,"font-size":10.5,"text-anchor":"middle","font-family":"IBM Plex Mono","font-weight":binding?600:400},binding?"◄ SETS THE PACE ►":"not binding")); |
| return hand;} |
| function buildClocks(){ |
| const a=selModel.value,g=selGpu.value,c=parseInt(selCtx.value),s=document.getElementById("clocks");clear(s); |
| const d=clockData(a,g,c); |
| s.appendChild(el("text",{x:0,y:18,fill:PAL.phos,"font-size":11,"font-family":"IBM Plex Mono"},"MEMORY COST VS OVERHEAD · "+a.toUpperCase()+" · "+g+" · CTX "+c.toLocaleString())); |
| const cgBtn=document.getElementById("clkCG");cgBtn.style.opacity=(d&&d.isG)?1:.4;cgBtn.disabled=!(d&&d.isG); |
| if(!d){s.appendChild(el("text",{x:500,y:210,fill:"#c0632e","font-size":20,"text-anchor":"middle","font-family":"IBM Plex Mono"},"OOM, no measured cell"));document.getElementById("clkNote").innerHTML="L4 OOM; pick a valid cell.";CLK.mem=CLK.launch=null;return;} |
| const memBinds=d.floor>=d.overhead; |
| CLK.mem=dial(s,290,180,118,PAL.phos,"MEMORY CLOCK",d.floor,memBinds); |
| CLK.launch=dial(s,710,180,118,PAL.peach,"OVERHEAD CLOCK",d.overhead,!memBinds); |
| CLK.memPeriod=Math.min(6,Math.max(0.5,d.floor/9));CLK.launchPeriod=Math.min(6,Math.max(0.5,d.overhead/9)); |
| s.appendChild(el("text",{x:500,y:150,fill:PAL.dim,"font-size":11,"text-anchor":"middle","font-family":"IBM Plex Mono"},"STEP =")); |
| s.appendChild(el("text",{x:500,y:184,fill:PAL.ink,"font-size":26,"text-anchor":"middle","font-family":"Fraunces,serif","font-weight":600},d.obs.toFixed(2))); |
| s.appendChild(el("text",{x:500,y:202,fill:PAL.dim,"font-size":11,"text-anchor":"middle","font-family":"IBM Plex Mono"},"ms / token")); |
| s.appendChild(el("text",{x:500,y:224,fill:PAL.phos,"font-size":10,"text-anchor":"middle","font-family":"IBM Plex Mono"},"= MEM + OVERHEAD")); |
| const pct=Math.round((memBinds?d.floor:d.overhead)/d.obs*100); |
| let proven="";if(d.isG&&g==="H100"){proven=` Of the overhead, CUDA Graphs proves ${(DATA.cudagraphs.H100_qwen_2048.eager_ms-DATA.cudagraphs.H100_qwen_2048.graphed_ms).toFixed(2)} ms is launch (the 1.259× A/B); the rest survives graphing and is not isolated to a mechanism.`;} |
| document.getElementById("clkNote").innerHTML=(memBinds |
| ?`On <b>${g}</b> the <b style="color:var(--green)">memory clock</b> binds: ${d.floor.toFixed(2)} ms of irreducible HBM traffic vs ${d.overhead.toFixed(2)} ms above the floor. Bandwidth-bound, so the lever is int4 weight compression.` |
| :`On <b>${g}</b> the <b style="color:var(--A100)">overhead clock</b> binds: ${d.overhead.toFixed(2)} ms above the floor vs only ${d.floor.toFixed(2)} ms of memory traffic. Fast silicon is overhead-bound.`) |
| +` The binding clock is ${pct}% of the step.`+proven;} |
| function animateClocks(ts){if(!CLK.last)CLK.last=ts;const dt=(ts-CLK.last)/1000;CLK.last=ts; |
| if(CLK.mem){CLK.angM=(CLK.angM+dt/CLK.memPeriod*360)%360;CLK.mem.setAttribute("transform",`rotate(${CLK.angM} 290 180)`);} |
| if(CLK.launch){CLK.angL=(CLK.angL+dt/CLK.launchPeriod*360)%360;CLK.launch.setAttribute("transform",`rotate(${CLK.angL} 710 180)`);} |
| requestAnimationFrame(animateClocks);} |
| |
| |
| let HW_GPU="H100",HW_CG=false;const KERNELS=["QKV","RoPE","ATTN","O_PROJ","RMS","GATE","UP","SiLU","DOWN","ADD"]; |
| function drawHighway(){ |
| const s=document.getElementById("highway");clear(s); |
| const W=1000,trackX=20,trackW=W-40,laneY=120,laneH=46,a="Qwen-2.5-7B",cl=cell(a,HW_GPU,2048); |
| let active=cl?cl.R_floor:0.3,stepms=cl?cl.t_obs:0,mode="bf16 eager"; |
| if(HW_CG){if(HW_GPU==="H100"){const g=DATA.cudagraphs.H100_qwen_2048;active=cl.t_floor/g.graphed_ms;stepms=g.graphed_ms;mode="CUDA Graphs";} |
| else if(HW_GPU==="L4"){const g=DATA.cudagraphs.L4_qwen_2048;stepms=g.graphed_ms;active=cl.t_floor/g.graphed_ms;mode="CUDA Graphs";}else mode="bf16 eager (no graphs measured)";} |
| const blockMs=stepms/28,n=KERNELS.length,activeW=trackW*active,gapW=trackW*(1-active),kActive=activeW/n,kGap=gapW/n; |
| s.appendChild(el("text",{x:0,y:20,fill:PAL.phos,"font-size":11,"font-family":"IBM Plex Mono"},"KERNEL TIMELINE · ONE "+a.toUpperCase()+" DECODER BLOCK · "+HW_GPU+" · "+mode)); |
| s.appendChild(el("text",{x:0,y:laneY-14,fill:PAL.dim,"font-size":10,"font-family":"IBM Plex Mono"},HW_GPU+": ~"+Math.round(blockMs*1000)+" us / block · "+Math.round(active*100)+"% GPU active")); |
| s.appendChild(el("rect",{x:trackX,y:laneY,width:trackW,height:laneH,fill:"#e9f1ec",stroke:PAL.panel,"stroke-width":1,rx:3})); |
| let x=trackX;const col=GPU_COL[HW_GPU]; |
| for(let i=0;i<n;i++){const kw=Math.max(2.5,kActive); |
| s.appendChild(el("rect",{x,y:laneY+6,width:kw,height:laneH-12,fill:col,opacity:.95,rx:1.5})); |
| if(kw>14||i%2===0)s.appendChild(el("text",{x:x+kw/2,y:laneY-2,fill:PAL.dim,"font-size":8,"text-anchor":"middle","font-family":"IBM Plex Mono"},KERNELS[i])); |
| x+=kw;if(kGap>0.5){s.appendChild(el("rect",{x,y:laneY+6,width:kGap,height:laneH-12,fill:"url(#hatch2)",opacity:.85}));x+=kGap;}} |
| s.appendChild(el("text",{x:trackX,y:laneY+laneH+26,fill:PAL.phos,"font-size":10,"font-family":"IBM Plex Mono"},active<0.6?"↑ STIPPLED = CPU LAUNCH LATENCY (wasted)":"↑ SOLID HIGHWAY, compute-bound, little wasted launch latency")); |
| s.appendChild(el("text",{x:0,y:225,fill:PAL.dim,"font-size":10,"font-family":"IBM Plex Mono"},"REPEATED 28× PER DECODE STEP · STEP = "+stepms.toFixed(2)+" ms")); |
| const defs=el("defs",{}),pat=el("pattern",{id:"hatch2",width:6,height:6,patternUnits:"userSpaceOnUse",patternTransform:"rotate(45)"}); |
| pat.appendChild(el("line",{x1:0,y1:0,x2:0,y2:6,stroke:"#b9ccc1","stroke-width":1.6}));defs.appendChild(pat);s.appendChild(defs); |
| const note=document.getElementById("hwNote"); |
| if(HW_GPU==="H100")note.innerHTML=`H100 active fraction rises from <b>27%</b> (eager) to <b>39%</b> (CUDA Graphs). The gaps collapse, not the kernels. Measured N=10 speedup 1.259×.`; |
| else if(HW_GPU==="L4")note.innerHTML=`L4 is already <b>81%</b> active: the highway is full, so CUDA Graphs buys almost nothing (1.028×). The L4 lever is int4 weight traffic.`; |
| else note.innerHTML=`${HW_GPU}: intermediate active fraction. CUDA Graphs was measured only on H100 and L4.`;} |
| |
| |
| let TOK=7,TARGET_HZ=30; |
| function drawBarrier(){ |
| const s=document.getElementById("barrier");clear(s); |
| const W=1000,H=460,padL=70,padR=40,padB=70,padT=40,x0=padL,x1=W-padR,y0=H-padB,y1=padT,bwMin=300,bwMax=3350,hzMin=1,hzMax=50; |
| const lx=v=>x0+(Math.log10(v)-Math.log10(bwMin))/(Math.log10(bwMax)-Math.log10(bwMin))*(x1-x0); |
| const ly=v=>y0-(Math.log10(v)-Math.log10(hzMin))/(Math.log10(hzMax)-Math.log10(hzMin))*(y0-y1); |
| s.appendChild(el("line",{x1:x0,y1:y0,x2:x1,y2:y0,stroke:PAL.dim,"stroke-width":1,opacity:.5})); |
| s.appendChild(el("line",{x1:x0,y1:y0,x2:x0,y2:y1,stroke:PAL.dim,"stroke-width":1,opacity:.5})); |
| GPU_ORDER.forEach(g=>{const X=lx(DATA.gpus[g].bw_gbs); |
| s.appendChild(el("text",{x:X,y:y0+18,fill:PAL.ink,"font-size":10,"text-anchor":"middle","font-family":"IBM Plex Mono"},g)); |
| s.appendChild(el("text",{x:X,y:y0+31,fill:PAL.dim,"font-size":8.5,"text-anchor":"middle","font-family":"IBM Plex Mono"},DATA.gpus[g].bw_gbs));}); |
| [1,3,10,30,50].forEach(v=>s.appendChild(el("text",{x:x0-10,y:ly(v)+3,fill:PAL.dim,"font-size":9,"text-anchor":"end","font-family":"IBM Plex Mono"},v))); |
| s.appendChild(el("text",{x:x0-44,y:(y0+y1)/2,fill:PAL.dim,"font-size":10,"font-family":"IBM Plex Mono","transform":`rotate(-90 ${x0-44} ${(y0+y1)/2})`,"text-anchor":"middle"},"ACTION CHUNK RATE (Hz)")); |
| s.appendChild(el("text",{x:0,y:20,fill:PAL.phos,"font-size":11,"font-family":"IBM Plex Mono"},"POLICY DECISION RATE LADDER · "+TOK+" TOKENS / ACTION CHUNK")); |
| const hz=ms=>1000/(TOK*ms); |
| [[1,"TELEOP / LANGUAGE · 1 Hz",.4],[10,"NAVIGATION · 10 Hz",.5],[30,"DEXTEROUS · 30 Hz",.6]].forEach(([v,lab,op])=>{if(v>hzMax)return;const Y=ly(v); |
| s.appendChild(el("line",{x1:x0,y1:Y,x2:x1,y2:Y,stroke:PAL.phos,"stroke-width":1,"stroke-dasharray":"3 5",opacity:op})); |
| s.appendChild(el("text",{x:x1,y:Y-5,fill:PAL.phos,"font-size":8.5,"text-anchor":"end","font-family":"IBM Plex Mono",opacity:Math.min(1,op+.2)},lab));}); |
| const l4=cell("Qwen-2.5-7B","L4",2048),idealAtL4=hz(l4.t_obs),idealAtH100=idealAtL4*(3350/300); |
| s.appendChild(el("line",{x1:lx(300),y1:ly(idealAtL4),x2:lx(3350),y2:ly(Math.min(hzMax,idealAtH100)),stroke:PAL.dim,"stroke-width":1.2,"stroke-dasharray":"3 4",opacity:.7})); |
| s.appendChild(el("text",{x:lx(900),y:ly(idealAtL4*3)-4,fill:PAL.dim,"font-size":8.5,"font-family":"IBM Plex Mono",opacity:.85},"IDEAL · pure-bandwidth scaling")); |
| const pts=[];GPU_ORDER.forEach(g=>{const c=cell("Qwen-2.5-7B",g,2048);if(c)pts.push({g,ms:c.t_obs,lab:g,lever:false});}); |
| pts.push({g:"H100",ms:DATA.cudagraphs.H100_qwen_2048.graphed_ms,lab:"H100 +Graphs",lever:true}); |
| pts.push({g:"L4",ms:DATA.quant_l4_qwen2048.find(q=>q.name.startsWith("GPTQ")).step_ms,lab:"L4 +ExLlamaV2",lever:true}); |
| const bestHz={};pts.forEach(p=>{const h=hz(p.ms);if(!(p.g in bestHz)||h>bestHz[p.g])bestHz[p.g]=h;}); |
| const target=TARGET_HZ,passCount=GPU_ORDER.filter(g=>bestHz[g]>=target).length; |
| if(target<=hzMax&&target>=hzMin){const Yt=ly(target); |
| s.appendChild(el("rect",{x:x0,y:Yt,width:x1-x0,height:y0-Yt,fill:"#c0632e",opacity:.07})); |
| s.appendChild(el("line",{x1:x0,y1:Yt,x2:x1,y2:Yt,stroke:PAL.peach,"stroke-width":2,"stroke-dasharray":"7 4"})); |
| s.appendChild(el("text",{x:x0+6,y:Yt-6,fill:PAL.peach,"font-size":11,"font-family":"IBM Plex Mono","font-weight":600},"REQUIRED · "+target+" Hz"));} |
| pts.forEach(p=>{const X=lx(DATA.gpus[p.g].bw_gbs),Y=ly(Math.max(hzMin,hz(p.ms))),c=GPU_COL[p.g],h=hz(p.ms),pass=h>=target,grp=el("g",{});grp.style.cursor="pointer"; |
| grp.appendChild(el("title",{},`${p.lab}\nstep ${p.ms.toFixed(2)} ms/token\n${h.toFixed(2)} Hz @ ${TOK} tok/chunk\n${pass?"CLEARS":"BELOW"} ${target} Hz`)); |
| for(let k=0;k<6;k++){const dx=(k%3-1)*6,dy=(Math.floor(k/3)-0.5)*6;grp.appendChild(el("rect",{x:X+dx-2.5,y:Y+dy-2.5,width:5,height:5,fill:c,opacity:p.lever?1:.8,rx:1}));} |
| if(!pass)grp.appendChild(el("circle",{cx:X,cy:Y,r:13,fill:"none",stroke:"#c0632e","stroke-width":1,opacity:.5})); |
| grp.appendChild(el("text",{x:X,y:Y-13,fill:p.lever?PAL.phos:PAL.dim,"font-size":8.5,"text-anchor":"middle","font-family":"IBM Plex Mono"},p.lab+" · "+h.toFixed(1)+"Hz")); |
| s.appendChild(grp);}); |
| const ro=document.getElementById("targetReadout");ro.textContent=passCount+" / 4 CLEAR "+target+" Hz"; |
| ro.style.color=passCount?PAL.phos:"#c0632e";ro.style.borderColor=passCount?"rgba(14,122,82,.4)":"rgba(192,99,46,.5)"; |
| const tier=target<=1?"teleop / language":target<=10?"navigation":target<=30?"dexterous manipulation":"beyond dexterous"; |
| document.getElementById("barrierNote").innerHTML=`At ${TOK} tokens/chunk, <b>${passCount} of 4</b> GPUs (best measured lever) clear <b>${target} Hz</b> for <b>${tier}</b>. H100+Graphs tops out at ${bestHz["H100"].toFixed(1)} Hz; L4+ExLlamaV2 at ${bestHz["L4"].toFixed(1)} Hz. Hover any point for detail.`;} |
| |
| |
| function rateFor(g){return (selGpu.value===g?parseFloat(selRate.value):DATA.gpus[g].rate_hr)||DATA.gpus[g].rate_hr;} |
| function drawCost(){ |
| const s=document.getElementById("cost");clear(s); |
| const steps={"H100":DATA.cudagraphs.H100_qwen_2048.graphed_ms,"A100-80GB":cell("Qwen-2.5-7B","A100-80GB",2048).t_obs,"L40S":cell("Qwen-2.5-7B","L40S",2048).t_obs,"L4":DATA.quant_l4_qwen2048.find(q=>q.name.startsWith("GPTQ")).step_ms}; |
| const lever={"H100":"CUDA Graphs","A100-80GB":"bf16 eager","L40S":"bf16 eager","L4":"ExLlamaV2"}; |
| const vals=GPU_ORDER.map(g=>costPerMtok(steps[g],rateFor(g))),maxV=Math.max(...vals),W=1000,padL=120,padR=50,top=24,bh=42,gapy=18; |
| GPU_ORDER.forEach((g,i)=>{const y=top+i*(bh+gapy),w=(W-padL-padR)*(vals[i]/maxV); |
| s.appendChild(el("text",{x:padL-12,y:y+bh/2+1,fill:PAL.ink,"font-size":11,"text-anchor":"end","font-family":"IBM Plex Mono"},g)); |
| s.appendChild(el("text",{x:padL-12,y:y+bh/2+14,fill:PAL.dim,"font-size":8.5,"text-anchor":"end","font-family":"IBM Plex Mono"},lever[g])); |
| const r=el("rect",{x:padL,y,width:0,height:bh,fill:GPU_COL[g],opacity:.92,rx:2});r.style.transition="width .8s cubic-bezier(.2,.8,.2,1)";s.appendChild(r); |
| requestAnimationFrame(()=>r.setAttribute("width",Math.max(2,w))); |
| s.appendChild(el("text",{x:padL+Math.max(2,w)+8,y:y+bh/2+4,fill:PAL.ink,"font-size":14,"font-weight":600,"font-family":"Fraunces,serif"},"$"+vals[i].toFixed(2)));}); |
| s.appendChild(el("text",{x:padL,y:top+4*(bh+gapy)+4,fill:PAL.dim,"font-size":9.5,"font-family":"IBM Plex Mono"},"$ / MILLION TOKENS →")); |
| const ratio=costPerMtok(steps.H100,rateFor("H100"))/costPerMtok(steps.L4,rateFor("L4")); |
| document.getElementById("costStats").innerHTML= |
| `<div class="s"><div class="num">11.2×</div><div class="cap">peak HBM ratio H100:L4</div></div>`+ |
| `<div class="s"><div class="num">${(steps.L4/steps.H100).toFixed(2)}×</div><div class="cap">step-time ratio L4:H100</div></div>`+ |
| `<div class="s"><div class="num">${ratio.toFixed(1)}×</div><div class="cap">cost ratio (L4 wins)</div></div>`; |
| document.getElementById("costNote").innerHTML=`At current rates the L4 (ExLlamaV2) serves this stream at <b>${ratio.toFixed(1)}×</b> lower $/Mtok than the H100 (CUDA Graphs). A100/L40S use bf16 eager (no measured int4/graphs lever); edit their rates in the Intro console.`;} |
| |
| |
| function hbars(svgId,rows,maxKeyFn,colorFn,fmtFn){ |
| const s=document.getElementById(svgId);clear(s);const W=500,padL=150,padR=44,top=18,bh=30,gapy=14,maxV=Math.max(...rows.map(maxKeyFn)); |
| rows.forEach((r,i)=>{const y=top+i*(bh+gapy),v=maxKeyFn(r),w=(W-padL-padR)*(v/maxV); |
| s.appendChild(el("text",{x:padL-10,y:y+bh/2+4,fill:PAL.ink,"font-size":9.5,"text-anchor":"end","font-family":"IBM Plex Mono"},r.__label)); |
| const rect=el("rect",{x:padL,y,width:0,height:bh,fill:colorFn(r),opacity:.92,rx:2});rect.style.transition="width .8s";s.appendChild(rect); |
| requestAnimationFrame(()=>rect.setAttribute("width",Math.max(2,w))); |
| s.appendChild(el("text",{x:padL+Math.max(2,w)+7,y:y+bh/2+4,fill:PAL.ink,"font-size":12,"font-weight":600,"font-family":"Fraunces,serif"},fmtFn(r)));});} |
| function drawQuant(){hbars("quant",DATA.quant_l4_qwen2048.map(q=>({...q,__label:q.name})),r=>r.step_ms,r=>r.name.startsWith("GPTQ")?PAL.mint:r.name==="bf16"?PAL.dim:PAL.peach,r=>r.step_ms.toFixed(1)+" ms");} |
| function drawAttn(){hbars("attn",DATA.attention_h100.filter(x=>x.us!=null).map(x=>({...x,__label:x.backend})),r=>r.us,r=>r.backend==="default SDPA"?PAL.mint:PAL.aqua,r=>r.us.toFixed(1)+" µs");} |
| |
| |
| function drawArch(){ |
| const s=document.getElementById("arch");clear(s); |
| const x0=70,xMax=930,yTrack=150,h=72; |
| const kernels=[{n:"QKV proj",mb:33.0,grp:"attn"},{n:"O proj",mb:25.7,grp:"attn"}, |
| {n:"Gate",mb:135.8,grp:"mlp"},{n:"Up",mb:135.8,grp:"mlp"},{n:"Down",mb:135.8,grp:"mlp"}]; |
| const total=kernels.reduce((a,k)=>a+k.mb,0),gap=10,avail=(xMax-x0)-gap*(kernels.length-1),scale=avail/total; |
| s.appendChild(el("text",{x:x0,y:26,fill:PAL.phos,"font-size":11,"font-family":"IBM Plex Mono"},"ONE QWEN-2.5-7B DECODER BLOCK · KERNEL HBM TRAFFIC PER DECODE STEP (BF16)")); |
| s.appendChild(el("text",{x:x0,y:44,fill:PAL.dim,"font-size":10,"font-family":"IBM Plex Mono"},"28 query heads · 4 KV heads · head_dim 128 · hidden 3584 · MLP 18944 · RMSNorm/RoPE/SiLU move negligible bytes")); |
| let x=x0;const gb={attn:[1e9,-1e9],mlp:[1e9,-1e9]}; |
| kernels.forEach(k=>{const w=k.mb*scale,col=k.grp==="attn"?PAL.mint:PAL.peach; |
| s.appendChild(el("rect",{x,y:yTrack,width:w,height:h,fill:col,opacity:.9,rx:3})); |
| s.appendChild(el("text",{x:x+w/2,y:yTrack-8,fill:PAL.ink,"font-size":10,"text-anchor":"middle","font-family":"IBM Plex Mono"},k.n)); |
| s.appendChild(el("text",{x:x+w/2,y:yTrack+h/2+4,fill:"#0a2018","font-size":11,"text-anchor":"middle","font-family":"IBM Plex Mono","font-weight":600},k.mb.toFixed(0)+" MB")); |
| gb[k.grp][0]=Math.min(gb[k.grp][0],x);gb[k.grp][1]=Math.max(gb[k.grp][1],x+w);x+=w+gap;}); |
| const bracket=(b,label,col)=>{const my=yTrack+h+22; |
| s.appendChild(el("line",{x1:b[0],y1:my,x2:b[1],y2:my,stroke:col,"stroke-width":1.5})); |
| s.appendChild(el("line",{x1:b[0],y1:my-5,x2:b[0],y2:my,stroke:col,"stroke-width":1.5})); |
| s.appendChild(el("line",{x1:b[1],y1:my-5,x2:b[1],y2:my,stroke:col,"stroke-width":1.5})); |
| s.appendChild(el("text",{x:(b[0]+b[1])/2,y:my+17,fill:col,"font-size":11,"text-anchor":"middle","font-family":"IBM Plex Mono","font-weight":600},label));}; |
| bracket(gb.attn,"ATTENTION · 59 MB",PAL.mint); |
| bracket(gb.mlp,"SwiGLU MLP · 407 MB",PAL.peach); |
| s.appendChild(el("text",{x:x0,y:330,fill:PAL.ink,"font-size":12,"font-family":"IBM Plex Mono"},"× 28 blocks ≈ 13.16 GB / step ÷ 3.35 TB/s (H100) = 3.93 ms floor · measured 14.83 ms · R_floor 0.27")); |
| } |
| |
| |
| function drawHeatmap(){ |
| const s=document.getElementById("heatmap");clear(s); |
| const W=1000,padL=150,padT=46,cw=190,chGap=6,rowH=22; |
| const ctxsLocal=[2048,4096,8192,16384];const models=["Qwen-2.5-7B","Mistral-7B","Llama-3.1-8B"]; |
| GPU_ORDER.forEach((g,ci)=>{const cx=padL+ci*((W-padL-20)/4); |
| s.appendChild(el("text",{x:cx+ (W-padL-20)/8,y:padT-16,fill:PAL.ink,"font-size":11,"text-anchor":"middle","font-family":"IBM Plex Mono","font-weight":600},g)); |
| s.appendChild(el("text",{x:cx+ (W-padL-20)/8,y:padT-3,fill:PAL.dim,"font-size":8.5,"text-anchor":"middle","font-family":"IBM Plex Mono"},DATA.gpus[g].bw_gbs+" GB/s"));}); |
| let row=0; |
| models.forEach(m=>{ |
| ctxsLocal.forEach((ctx,ri)=>{const y=padT+row*(rowH+3); |
| s.appendChild(el("text",{x:padL-12,y:y+rowH-6,fill:ri===0?PAL.ink:PAL.dim,"font-size":ri===0?10:9,"text-anchor":"end","font-family":"IBM Plex Mono"},ri===0?m:("/ ctx "+ctx))); |
| if(ri===0)s.appendChild(el("text",{x:padL-12,y:y+rowH+5,fill:PAL.dim,"font-size":8,"text-anchor":"end","font-family":"IBM Plex Mono"},"ctx "+ctx)); |
| GPU_ORDER.forEach((g,ci)=>{const cl=cell(m,g,ctx),cellW=(W-padL-20)/4-8,cx=padL+ci*((W-padL-20)/4); |
| if(!cl){s.appendChild(el("rect",{x:cx,y,width:cellW,height:rowH,fill:"#e6ece8",stroke:"#d2ddd6",rx:3})); |
| s.appendChild(el("text",{x:cx+cellW/2,y:y+rowH-6,fill:"#c0632e","font-size":9,"text-anchor":"middle","font-family":"IBM Plex Mono"},"OOM"));return;} |
| const R=cl.R_floor,rgb=hexToRgb(GPU_COL[g]); |
| s.appendChild(el("rect",{x:cx,y,width:cellW,height:rowH,fill:`rgb(${rgb[0]},${rgb[1]},${rgb[2]})`,opacity:0.18+0.82*R,rx:3})); |
| s.appendChild(el("text",{x:cx+cellW/2,y:y+rowH-6,fill:R>0.55?"#0a2018":"#15251d","font-size":10,"text-anchor":"middle","font-family":"IBM Plex Mono","font-weight":600},R.toFixed(2)));}); |
| row++;});});} |
| |
| |
| function _h2(x,y){const n=Math.sin(x*127.1+y*311.7)*43758.5453;return n-Math.floor(n);} |
| function _vnoise(x,y){const xi=Math.floor(x),yi=Math.floor(y),xf=x-xi,yf=y-yi; |
| const u=xf*xf*(3-2*xf),v=yf*yf*(3-2*yf); |
| const tl=_h2(xi,yi),tr=_h2(xi+1,yi),bl=_h2(xi,yi+1),br=_h2(xi+1,yi+1); |
| return (tl*(1-u)+tr*u)*(1-v)+(bl*(1-u)+br*u)*v;} |
| function _fbm(x,y){let a=0,amp=0.5,f=1;for(let o=0;o<5;o++){a+=amp*_vnoise(x*f,y*f);f*=2;amp*=0.5;}return a;} |
| function surfaceGrid(arch){ |
| const gpus=GPU_ORDER, ctxs=[2048,4096,8192,16384]; |
| |
| const K=gpus.map(g=>{ |
| const row=ctxs.map(c=>{const cl=cell(arch,g,c);return cl?cl.R_floor:null;}); |
| const known=row.map((v,i)=>v==null?null:[i,v]).filter(Boolean); |
| if(known.length>=2&&known.length<row.length){ |
| const a=known[0],b=known[known.length-1],slope=(b[1]-a[1])/((b[0]-a[0])||1); |
| for(let j=0;j<row.length;j++) if(row[j]==null) row[j]=Math.max(0.12,Math.min(0.9,a[1]+slope*(j-a[0]))); |
| } else if(known.length===1){ for(let j=0;j<row.length;j++) if(row[j]==null) row[j]=known[0][1]; } |
| return row; |
| }); |
| |
| |
| const Nx=210,Ny=136,z=[],xs=[],ys=[],smooth=t=>t*t*(3-2*t); |
| for(let fj=0;fj<Ny;fj++){ |
| const gj=fj/(Ny-1)*3, j0=Math.floor(gj), j1=Math.min(3,j0+1), tj=smooth(gj-j0); ys.push(gj); |
| const rowz=[]; |
| for(let fi=0;fi<Nx;fi++){ |
| const gi=fi/(Nx-1)*3, i0=Math.floor(gi), i1=Math.min(3,i0+1), ti=smooth(gi-i0); |
| if(fj===0)xs.push(gi); |
| const a=K[i0][j0]*(1-ti)+K[i1][j0]*ti, b=K[i0][j1]*(1-ti)+K[i1][j1]*ti; |
| const dataH=a*(1-tj)+b*tj; |
| const ui=gi/3, vj=gj/3, ss=(e0,e1,x)=>{const t=Math.max(0,Math.min(1,(x-e0)/(e1-e0)));return t*t*(3-2*t);}; |
| |
| const mesa=0.95*ss(0.66,0.12, Math.hypot(ui*1.05, vj*0.85)); |
| |
| const ridge=(_fbm(ui*2.2+11,vj*2.2+7)-0.5)*0.05; |
| |
| const crater=-0.55*Math.exp(-(((ui-1.0)*(ui-1.0))/0.11 + ((vj-0.78)*(vj-0.78))/0.22)); |
| const h=mesa*0.55 + dataH*0.45 + ridge + crater; |
| rowz.push(Math.max(0.05, Math.min(0.99, h))); |
| } |
| z.push(rowz); |
| } |
| return {z,xs,ys,K,gpus}; |
| } |
| |
| let SURF=null; |
| const SURF_RMIN=0.0, SURF_RMAX=1.0, SURF_HY=4.3, SURF_WX=15.5, SURF_WZ=7.8; |
| function surfShader(){ |
| return new THREE.ShaderMaterial({side:THREE.DoubleSide, extensions:{derivatives:true}, |
| vertexShader:`attribute float aH;varying float vH; |
| void main(){vH=aH;gl_Position=projectionMatrix*modelViewMatrix*vec4(position,1.0);}`, |
| fragmentShader:`precision highp float;varying float vH; |
| vec3 ramp(float t){vec3 a=vec3(0.55,0.04,0.07),b=vec3(0.92,0.20,0.10),c=vec3(0.99,0.55,0.13),d=vec3(0.97,0.82,0.24),e=vec3(0.55,0.92,0.34),f2=vec3(0.30,1.0,0.74); |
| if(t<0.2)return mix(a,b,t/0.2);if(t<0.4)return mix(b,c,(t-0.2)/0.2);if(t<0.6)return mix(c,d,(t-0.4)/0.2);if(t<0.8)return mix(d,e,(t-0.6)/0.2);return mix(e,f2,(t-0.8)/0.2);} |
| void main(){float t=clamp(vH,0.0,1.0); |
| vec3 col=ramp(t); |
| float f=t*70.0; // dense contour intervals = many layers |
| float di=min(fract(f),1.0-fract(f)); |
| float df=fwidth(f); |
| float pres=smoothstep(0.006,0.045,df); // only draw lines where the ground actually slopes (flat ground = no contour) |
| float fade=1.0-smoothstep(0.45,1.1,df); // and fade where lines pack tighter than pixels |
| float mask=pres*fade; |
| float w=max(df*1.6,0.012); |
| float line=(1.0-smoothstep(0.0, w, di))*mask; // crisp anti-aliased lines |
| float glow=(1.0-smoothstep(0.0, df*9.0+0.04, di))*mask; // soft halo |
| vec3 fill=col*0.22; // dim colored body |
| vec3 lc=min(col*2.3+0.14, vec3(1.0)); // bright glowing line core |
| vec3 outc=mix(fill, lc, line) + col*glow*0.40; // strong halo for bloom feel |
| gl_FragColor=vec4(outc,1.0);}`}); |
| } |
| function initSurface(){ |
| const cont=document.getElementById("surface"); cont.style.position="relative"; |
| const W=cont.clientWidth||900, H=cont.clientHeight||540; |
| const scene=new THREE.Scene(); |
| const cam=new THREE.PerspectiveCamera(40,W/H,0.1,300); |
| const renderer=new THREE.WebGLRenderer({antialias:true,alpha:true}); |
| renderer.setPixelRatio(Math.min(2,window.devicePixelRatio||1)); renderer.setSize(W,H); |
| cont.appendChild(renderer.domElement); |
| const labels=[0,1,2,3].map(()=>{const d=document.createElement("div"); |
| d.style.cssText="position:absolute;transform:translate(-50%,-100%);pointer-events:none;font-family:'IBM Plex Mono',monospace;font-size:11px;white-space:nowrap;text-shadow:0 1px 3px #000;"; |
| cont.appendChild(d);return d;}); |
| SURF={cont,scene,cam,renderer,mat:surfShader(),mesh:null,curtain:null,labels,anchors:[], |
| yaw:-0.86,pitch:0.6,dist:12.4,drag:false,px:0,py:0,auto:true,arch:null}; |
| window.SURF=SURF; |
| const dom=renderer.domElement; |
| dom.style.cursor="grab"; |
| dom.addEventListener("pointerdown",e=>{SURF.drag=true;SURF.auto=false;SURF.px=e.clientX;SURF.py=e.clientY;dom.style.cursor="grabbing";}); |
| window.addEventListener("pointerup",()=>{SURF.drag=false;dom.style.cursor="grab";}); |
| window.addEventListener("pointermove",e=>{if(!SURF.drag)return; |
| SURF.yaw-=(e.clientX-SURF.px)*0.006; SURF.pitch=Math.max(0.08,Math.min(0.95,SURF.pitch+(e.clientY-SURF.py)*0.004)); |
| SURF.px=e.clientX;SURF.py=e.clientY;}); |
| dom.addEventListener("wheel",e=>{e.preventDefault();SURF.dist=Math.max(11,Math.min(30,SURF.dist+(e.deltaY>0?1:-1)*1.1));},{passive:false}); |
| animateSurface(); |
| } |
| function buildSurfaceMesh(arch){ |
| const g=surfaceGrid(arch), Ny=g.z.length, Nx=g.z[0].length; |
| if(SURF.mesh){SURF.scene.remove(SURF.mesh);SURF.mesh.geometry.dispose();} |
| if(SURF.curtain){SURF.scene.remove(SURF.curtain);SURF.curtain.geometry.dispose();} |
| const norm=R=>(R-SURF_RMIN)/(SURF_RMAX-SURF_RMIN); |
| const yoff=-SURF_HY*0.42; |
| const wx=(i)=>(i/(Nx-1)-0.5)*SURF_WX, wz=(j)=>(j/(Ny-1)-0.5)*SURF_WZ, wy=(R)=>norm(R)*SURF_HY+yoff; |
| const pos=new Float32Array(Nx*Ny*3), aH=new Float32Array(Nx*Ny), id=(i,j)=>j*Nx+i; |
| for(let j=0;j<Ny;j++)for(let i=0;i<Nx;i++){const R=g.z[j][i],k=id(i,j); |
| pos[k*3]=wx(i);pos[k*3+1]=wy(R);pos[k*3+2]=wz(j);aH[k]=norm(R);} |
| const indices=[]; |
| for(let j=0;j<Ny-1;j++)for(let i=0;i<Nx-1;i++){const a=id(i,j),b=id(i+1,j),c=id(i+1,j+1),d=id(i,j+1);indices.push(a,b,d,b,c,d);} |
| const geo=new THREE.BufferGeometry(); |
| geo.setAttribute("position",new THREE.BufferAttribute(pos,3)); |
| geo.setAttribute("aH",new THREE.BufferAttribute(aH,1)); |
| geo.setIndex(indices); geo.computeVertexNormals(); |
| SURF.mesh=new THREE.Mesh(geo,SURF.mat); SURF.scene.add(SURF.mesh); |
| |
| const base=yoff-0.95; |
| const perim=[]; for(let i=0;i<Nx;i++)perim.push([i,0]); for(let j=1;j<Ny;j++)perim.push([Nx-1,j]); |
| for(let i=Nx-2;i>=0;i--)perim.push([i,Ny-1]); for(let j=Ny-2;j>=1;j--)perim.push([0,j]); |
| const wp=[],wd=[],wu=[],widx=[]; let vi=0; |
| for(let p=0;p<perim.length-1;p++){ |
| const [i0,j0]=perim[p],[i1,j1]=perim[p+1]; |
| const R0=g.z[j0][i0],R1=g.z[j1][i1]; |
| |
| wp.push(wx(i0),wy(R0),wz(j0)); wd.push(0); wu.push(p); |
| wp.push(wx(i0),base,wz(j0)); wd.push(1); wu.push(p); |
| wp.push(wx(i1),wy(R1),wz(j1)); wd.push(0); wu.push(p+1); |
| wp.push(wx(i1),base,wz(j1)); wd.push(1); wu.push(p+1); |
| widx.push(vi,vi+1,vi+2, vi+1,vi+3,vi+2); vi+=4; |
| } |
| const cg=new THREE.BufferGeometry(); |
| cg.setAttribute("position",new THREE.BufferAttribute(new Float32Array(wp),3)); |
| cg.setAttribute("wd",new THREE.BufferAttribute(new Float32Array(wd),1)); |
| cg.setAttribute("wu",new THREE.BufferAttribute(new Float32Array(wu),1)); |
| cg.setIndex(widx); |
| const wallMat=new THREE.ShaderMaterial({side:THREE.DoubleSide, |
| vertexShader:`attribute float wd;attribute float wu;varying float vD;varying float vU; |
| void main(){vD=wd;vU=wu;gl_Position=projectionMatrix*modelViewMatrix*vec4(position,1.0);}`, |
| fragmentShader:`varying float vD;varying float vU; |
| void main(){vec3 top=vec3(0.13,0.42,0.32),bot=vec3(0.02,0.09,0.07); |
| vec3 col=mix(top,bot,vD); |
| float s=fract(vU*0.5);float dl=min(s,1.0-s); |
| float line=1.0-smoothstep(0.0,0.12,dl); |
| col=mix(col,col*1.8,line*(1.0-vD)*0.5); |
| gl_FragColor=vec4(col,1.0);}`}); |
| SURF.curtain=new THREE.Mesh(cg,wallMat); SURF.scene.add(SURF.curtain); |
| |
| SURF.anchors=g.gpus.map((gp,gi)=>{const i=Math.round(gi/3*(Nx-1)),R=g.z[0][i]; |
| return {name:gp,R:g.K[gi][0],col:gi<2?"#bff5cc":(gi<3?"#e8c879":"#ef8a5a"),v:new THREE.Vector3(wx(i),wy(R)+0.25,wz(0))};}); |
| SURF.arch=arch; |
| } |
| function animateSurface(){ |
| requestAnimationFrame(animateSurface); if(!SURF||!SURF.mesh)return; |
| if(SURF.auto)SURF.yaw=-0.8+0.34*Math.sin(performance.now()*0.00022); |
| const c=SURF.cam, p=SURF.pitch, y=SURF.yaw, d=SURF.dist; |
| c.position.set(d*Math.cos(p)*Math.sin(y), d*Math.sin(p), d*Math.cos(p)*Math.cos(y)); c.lookAt(0,0,0); |
| SURF.renderer.render(SURF.scene,c); |
| const W=SURF.cont.clientWidth,H=SURF.cont.clientHeight; |
| SURF.anchors.forEach((a,k)=>{const v=a.v.clone().project(c);const lab=SURF.labels[k]; |
| if(v.z>1){lab.style.display="none";return;} lab.style.display="block"; |
| const lx=Math.max(44,Math.min(W-44,(v.x*0.5+0.5)*W)), ly=Math.max(16,Math.min(H-12,(-v.y*0.5+0.5)*H)); |
| lab.style.left=lx+"px"; lab.style.top=ly+"px"; |
| lab.style.color=a.col; lab.innerHTML=a.name+" . "+a.R.toFixed(2);}); |
| } |
| function drawSurface(){ |
| if(typeof THREE==="undefined"){setTimeout(drawSurface,250);return;} |
| if(!SURF)initSurface(); |
| buildSurfaceMesh(selModel.value); resizeSurface(); |
| } |
| function resizeSurface(){if(!SURF)return;const W=SURF.cont.clientWidth||900,H=SURF.cont.clientHeight||540; |
| SURF.cam.aspect=W/H;SURF.cam.updateProjectionMatrix();SURF.renderer.setSize(W,H);} |
| |
| |
| function drawFalsify(){ |
| const s=document.getElementById("falsify");clear(s); |
| const W=1000,H=300,padL=70,padR=200,padB=50,padT=30,x0=padL,x1=W-padR,y0=H-padB,y1=padT,yMin=0.9,yMax=2.6; |
| const ly=v=>y0-(v-yMin)/(yMax-yMin)*(y0-y1); |
| s.appendChild(el("text",{x:0,y:18,fill:PAL.phos,"font-size":11,"font-family":"IBM Plex Mono"},"CUDA GRAPHS A/B · PRE-REGISTERED FALSIFICATION · N=10")); |
| s.appendChild(el("line",{x1:x0,y1:y0,x2:x1,y2:y0,stroke:PAL.dim,"stroke-width":1,opacity:.5})); |
| [1.0,1.5,2.0,2.5].forEach(v=>{s.appendChild(el("text",{x:x0-10,y:ly(v)+3,fill:PAL.dim,"font-size":9,"text-anchor":"end","font-family":"IBM Plex Mono"},v.toFixed(1)+"×")); |
| s.appendChild(el("line",{x1:x0,y1:ly(v),x2:x1,y2:ly(v),stroke:PAL.line,"stroke-width":1,opacity:.6}));}); |
| |
| const yk=ly(1.15); |
| s.appendChild(el("rect",{x:x0,y:yk,width:x1-x0,height:y0-yk,fill:"#c0632e",opacity:.06})); |
| s.appendChild(el("line",{x1:x0,y1:yk,x2:x1,y2:yk,stroke:"#c0632e","stroke-width":1.6,"stroke-dasharray":"6 4"})); |
| s.appendChild(el("text",{x:x1+8,y:yk+4,fill:"#c0632e","font-size":9.5,"font-family":"IBM Plex Mono"},"PRE-REG KILL LINE 1.15×")); |
| s.appendChild(el("text",{x:x1+8,y:yk+18,fill:PAL.dim,"font-size":8.5,"font-family":"IBM Plex Mono"},"H100 below / L4 above")); |
| s.appendChild(el("text",{x:x1+8,y:yk+30,fill:PAL.dim,"font-size":8.5,"font-family":"IBM Plex Mono"},"would falsify")); |
| |
| const items=[{x:x0+(x1-x0)*0.32,g:"H100",val:1.259,ci:[1.253,1.267],stamp:"NOT FALSIFIED",col:PAL.mint}, |
| {x:x0+(x1-x0)*0.7,g:"L4",val:1.028,ci:null,stamp:"NULL CONFIRMED",col:PAL.peach}]; |
| items.forEach(it=>{const Y=ly(it.val); |
| if(it.ci){const yhi=ly(it.ci[1]),ylo=ly(it.ci[0]);s.appendChild(el("line",{x1:it.x,y1:yhi,x2:it.x,y2:ylo,stroke:it.col,"stroke-width":2})); |
| s.appendChild(el("line",{x1:it.x-7,y1:yhi,x2:it.x+7,y2:yhi,stroke:it.col,"stroke-width":2}));s.appendChild(el("line",{x1:it.x-7,y1:ylo,x2:it.x+7,y2:ylo,stroke:it.col,"stroke-width":2}));} |
| for(let k=0;k<9;k++){const dx=(k%3-1)*7,dy=(Math.floor(k/3)-1)*7;s.appendChild(el("rect",{x:it.x+dx-3,y:Y+dy-3,width:6,height:6,fill:it.col,rx:1}));} |
| s.appendChild(el("text",{x:it.x,y:Y-26,fill:PAL.ink,"font-size":18,"text-anchor":"middle","font-family":"Fraunces,serif","font-weight":600},it.val.toFixed(3)+"×")); |
| s.appendChild(el("text",{x:it.x,y:y0+18,fill:PAL.ink,"font-size":11,"text-anchor":"middle","font-family":"IBM Plex Mono","font-weight":600},it.g)); |
| s.appendChild(el("text",{x:it.x,y:y0+31,fill:it.col,"font-size":9,"text-anchor":"middle","font-family":"IBM Plex Mono","font-weight":600},it.stamp));});} |
| |
| |
| const SB_LANES=[ |
| {name:"H100",sub:"CUDA Graphs · 3350 GB/s",ms:DATA.cudagraphs.H100_qwen_2048.graphed_ms,col:"H100",rate:3.50}, |
| {name:"L4",sub:"ExLlamaV2 int4 · 300 GB/s",ms:DATA.quant_l4_qwen2048.find(q=>q.name.startsWith("GPTQ")).step_ms,col:"L4",rate:0.30}, |
| {name:"L4",sub:"bf16 default runtime · 300 GB/s",ms:cell("Qwen-2.5-7B","L4",2048).t_obs,col:"L40S",rate:0.30}]; |
| const SB_COST=SB_LANES.map(L=>costPerMtok(L.ms,L.rate)); |
| const SB_WIN=SB_COST.indexOf(Math.min(...SB_COST)); |
| let sbTimers=[]; |
| function sbBuild(){ |
| const wrap=document.getElementById("sbLanes");wrap.innerHTML=""; |
| SB_LANES.forEach((L,i)=>{ |
| const win=(i===SB_WIN); |
| wrap.insertAdjacentHTML("beforeend", |
| `<div class="lane" style="${win?'border-color:var(--green);border-width:1.5px':''}"> |
| <div class="top"> |
| <span class="name" style="color:${GPU_COL[L.col]}">${L.name} · $${L.rate.toFixed(2)}/hr <span style="color:var(--muted);font-weight:400">· ${L.sub}</span> ${win?'<span class="pill measured" style="margin-left:6px">CHEAPEST / TOKEN</span>':''}</span> |
| <span class="name" id="sbcost${i}" style="color:${GPU_COL[L.col]}">$${SB_COST[i].toFixed(2)} / Mtok</span> |
| </div> |
| <div class="stream" id="sbout${i}"></div> |
| <div class="bargauge"><i id="sbbar${i}" style="background:${GPU_COL[L.col]}"></i></div> |
| <div class="stat" id="sbstat${i}" style="margin-top:7px">${L.ms.toFixed(2)} ms/token · idle</div> |
| </div>`);}); |
| } |
| function sbReset(){sbTimers.forEach(t=>clearTimeout(t));sbTimers=[]; |
| const vd=document.getElementById("sbVerdict");if(vd)vd.style.display="none"; |
| SB_LANES.forEach((L,i)=>{const o=document.getElementById("sbout"+i);if(o)o.innerHTML=""; |
| const st=document.getElementById("sbstat"+i);if(st)st.textContent=`${L.ms.toFixed(2)} ms/token · idle`; |
| const bar=document.getElementById("sbbar"+i);if(bar)bar.style.width="0";});} |
| function sbRun(){ |
| sbReset(); |
| const vd=document.getElementById("sbVerdict"); |
| const h=SB_LANES[0].ms,exl=SB_LANES[1].ms,bf=SB_LANES[2].ms; |
| const rate=(h/exl*100).toFixed(0),costR=((h*3.50)/(exl*0.30)).toFixed(1),runtime=(bf/exl).toFixed(1); |
| vd.style.display="block"; |
| vd.innerHTML=`<b>Runtime-poor, not compute-poor:</b> on the <i>same</i> $0.30/hr L4, switching the runtime from bf16 to ExLlamaV2 int4 cuts latency <b>${runtime}×</b> (${bf.toFixed(1)} → ${exl.toFixed(1)} ms/token). No new hardware. <br><b>The inversion:</b> that L4 then serves each token <b>${costR}× cheaper</b> than the $3.50/hr H100, which has 11× its bandwidth, while still running at <b>${rate}%</b> of the H100's token rate. One token at a time, the cheap chip with the right runtime wins.`; |
| const prompt=document.getElementById("sbPrompt").value.trim()||"hello world"; |
| const toks=prompt.match(/\s+|[^\s]+/g)||[prompt]; |
| const nTok=toks.filter(t=>t.trim().length).length; |
| SB_LANES.forEach((L,i)=>{ |
| const out=document.getElementById("sbout"+i),st=document.getElementById("sbstat"+i),bar=document.getElementById("sbbar"+i); |
| const t0=performance.now();let emitted=0; |
| toks.forEach((tk,k)=>{ |
| sbTimers.push(setTimeout(()=>{ |
| const span=document.createElement("span");span.className="tok";span.textContent=tk;out.appendChild(span); |
| if(tk.trim().length)emitted++; |
| const el2=performance.now()-t0,tps=emitted/(el2/1000||1); |
| st.textContent=`${L.ms.toFixed(2)} ms/token · ${(el2/1000).toFixed(2)}s · ${tps.toFixed(0)} tok/s`; |
| bar.style.width=(100*(k+1)/toks.length)+"%"; |
| if(k===toks.length-1)st.textContent=`done · ${nTok} tokens in ${(el2/1000).toFixed(2)}s · ${(nTok/(el2/1000)).toFixed(0)} tok/s`; |
| },k*L.ms));});});} |
| |
| |
| const hv=document.getElementById("heroviz"); let hctx,hW=0,hH=0,hcells=[],htok=[],hnext=0,hpct=27; |
| function roundRect(c,x,y,w,h,r){c.beginPath();c.moveTo(x+r,y);c.arcTo(x+w,y,x+w,y+h,r);c.arcTo(x+w,y+h,x,y+h,r);c.arcTo(x,y+h,x,y,r);c.arcTo(x,y,x+w,y,r);c.closePath();} |
| function heroResize(){if(!hv)return;const r=hv.getBoundingClientRect();if(r.width<10)return; |
| const dpr=Math.min(2,window.devicePixelRatio||1);hv.width=r.width*dpr;hv.height=r.height*dpr; |
| hctx=hv.getContext("2d");hctx.setTransform(dpr,0,0,dpr,0,0);hW=r.width;hH=r.height;hbuild();} |
| let hcols=18,hrows=12; |
| function hbuild(){const cols=18,rows=12,pad=16,gap=6,topY=44;hcols=cols;hrows=rows; |
| const gw=hW-pad*2, gh=hH-topY-26, cw=(gw-(cols-1)*gap)/cols, ch=(gh-(rows-1)*gap)/rows; |
| hcells=[];for(let r=0;r<rows;r++)for(let c=0;c<cols;c++) |
| hcells.push({x:pad+c*(cw+gap),y:topY+r*(ch+gap),w:Math.max(2,cw),h:Math.max(2,ch), |
| diag:(c/(cols-1)+r/(rows-1))/2, seed:Math.random()*6.283, b:0});} |
| let hprev=0; |
| function heroDraw(ts){requestAnimationFrame(heroDraw);if(!hctx||hW<10)return;ts=ts||0; |
| const PERIOD=2600, sw=(ts%PERIOD)/PERIOD; |
| if(sw<hprev){htok.push({x:hW*0.5,y:38,a:1});} hprev=sw; |
| hctx.clearRect(0,0,hW,hH); |
| hcells.forEach(c=>{const d=sw-c.diag; |
| let target=(d>=0&&d<0.42)?Math.exp(-(d*d)/0.010):0; |
| target=Math.max(target, 0.05*(0.5+0.5*Math.sin(ts*0.0011+c.seed))); |
| c.b += (target-c.b)*0.18; |
| const f=Math.min(1,c.b); |
| const r=Math.round(224+(31-224)*f),g=Math.round(234+(157-234)*f),b=Math.round(227+(107-227)*f); |
| hctx.fillStyle=`rgb(${r},${g},${b})`;roundRect(hctx,c.x,c.y,c.w,c.h,2.5);hctx.fill();}); |
| htok.forEach(t=>{t.y-=0.8;t.a*=0.987;});htok=htok.filter(t=>t.a>0.05); |
| htok.forEach(t=>{hctx.globalAlpha=t.a*0.3;hctx.fillStyle="#1f9d6b";hctx.beginPath();hctx.arc(t.x,t.y,9,0,6.283);hctx.fill(); |
| hctx.globalAlpha=t.a;hctx.beginPath();hctx.arc(t.x,t.y,3.6,0,6.283);hctx.fill();}); |
| hctx.globalAlpha=1; |
| hctx.fillStyle="#566b60";hctx.font="11px 'IBM Plex Mono',monospace";hctx.textAlign="left";hctx.fillText("H100 . ONE DECODE STEP",16,26); |
| hctx.textAlign="right";hctx.fillStyle="#0e7a52";hctx.fillText("27% ACTIVE . 73% IDLE",hW-16,26);hctx.textAlign="left";} |
| |
| |
| function drawWL(mode){ |
| const s=document.getElementById("wlviz");if(!s)return;clear(s); |
| const dc=mode==="dc", util=dc?0.92:0.27, nReq=dc?14:1; |
| const cx0=560,cy0=58,cw=320,ch=188,pad=12,gap=4,cols=15,rows=9; |
| s.appendChild(el("rect",{x:cx0,y:cy0,width:cw,height:ch,rx:12,fill:"#fff",stroke:PAL.line,"stroke-width":1.5})); |
| const gw=cw-pad*2,gh=ch-pad*2,w=(gw-(cols-1)*gap)/cols,h=(gh-(rows-1)*gap)/rows; |
| for(let r=0;r<rows;r++)for(let c=0;c<cols;c++){const v=((c*31+r*17)%100)/100, lit=v<util; |
| s.appendChild(el("rect",{x:cx0+pad+c*(w+gap),y:cy0+pad+r*(h+gap),width:w,height:h,rx:2, |
| fill:lit?PAL.mint:"#e6ece8"}));} |
| s.appendChild(el("text",{x:cx0+cw/2,y:cy0-14,fill:PAL.ink,"font-size":13,"text-anchor":"middle","font-family":"IBM Plex Mono","font-weight":600},"H100 · "+Math.round(util*100)+"% USED")); |
| |
| const lx=78,target={x:cx0,y:cy0+ch/2};const ys=[]; |
| for(let i=0;i<nReq;i++)ys.push(dc?(70+i*(160/(nReq-1))):cy0+ch/2); |
| ys.forEach(y=>{s.appendChild(el("line",{x1:lx,y1:y,x2:target.x,y2:target.y,stroke:PAL.mint,"stroke-width":dc?1.1:2,opacity:dc?0.38:0.85})); |
| s.appendChild(el("circle",{cx:lx,cy:y,r:dc?3.5:5,fill:PAL.mint}));}); |
| s.appendChild(el("text",{x:lx-6,y:38,fill:PAL.ink,"font-size":13,"font-family":"IBM Plex Mono","font-weight":600},dc?(nReq+" STREAMS BATCHED"):"1 STREAM · A ROBOT")); |
| s.appendChild(el("text",{x:lx-6,y:54,fill:PAL.muted,"font-size":10.5,"font-family":"IBM Plex Mono"},dc?"cost shared across the crowd":"nothing to share the cost")); |
| |
| s.appendChild(el("text",{x:cx0+cw+18,y:cy0+ch/2-6,fill:PAL.muted,"font-size":11,"font-family":"IBM Plex Mono"},"COST / STREAM")); |
| s.appendChild(el("text",{x:cx0+cw+18,y:cy0+ch/2+18,fill:dc?PAL.mint:"#c0632e","font-size":20,"font-family":"Fraunces,serif","font-weight":600},dc?"LOW":"FULL")); |
| document.getElementById("wlNote").innerHTML=dc |
| ? `Many streams ride one GPU, so each token's memory cost is split across the crowd. The H100 runs near full, and the price per stream is low. This is the job GPUs were built for.` |
| : `A robot is alone. One stream cannot fill the chip: <b>73% of the H100 sits idle</b>, and that single stream pays the whole bill. The faster the silicon, the more of it is wasted.`; |
| } |
| |
| |
| function fmtMoney(x){if(x>=1e6)return "$"+(x/1e6).toFixed(2)+"M";if(x>=1e3)return "$"+(x/1e3).toFixed(1)+"k";return "$"+x.toFixed(0);} |
| function fmtTokens(n){if(n>=1e12)return (n/1e12).toFixed(1)+"T";if(n>=1e9)return (n/1e9).toFixed(1)+"B";if(n>=1e6)return (n/1e6).toFixed(0)+"M";return Math.round(n).toLocaleString();} |
| function calcUpdate(){const el=document.getElementById("calcTok");if(!el)return; |
| const tok=Math.pow(10,parseFloat(el.value)); |
| const cH=costPerMtok(11.78,3.50)/1e6*tok, cL=costPerMtok(17.36,0.30)/1e6*tok; |
| document.getElementById("calcTokVal").textContent=fmtTokens(tok)+" tok / month"; |
| document.getElementById("calcH").textContent=fmtMoney(cH)+" /mo"; |
| document.getElementById("calcL").textContent=fmtMoney(cL)+" /mo"; |
| document.getElementById("calcSave").textContent=fmtMoney(cH-cL)+" /mo"; |
| document.getElementById("calcRatio").textContent=(cH/cL).toFixed(1)+"× cheaper"; |
| document.getElementById("calcVerdict").innerHTML=`At <b>${fmtTokens(tok)} tokens a month</b>, an H100 fleet costs <b>${fmtMoney(cH)}</b> and an L4 fleet <b>${fmtMoney(cL)}</b>. Same model, same tokens: <b>${fmtMoney(cH-cL)}</b> saved every month by serving on cheaper silicon with the right runtime.`;} |
| |
| |
| function showTab(name){ |
| document.querySelectorAll(".tab").forEach(t=>t.classList.toggle("active",t.dataset.tab===name)); |
| document.querySelectorAll(".tabpanel").forEach(p=>p.classList.toggle("active",p.id==="tab-"+name)); |
| if(name==="intro"){requestAnimationFrame(()=>{resizeFlow();heroResize();});} |
| if(name==="methods"){requestAnimationFrame(()=>{drawSurface();resizeSurface();});} |
| window.scrollTo({top:0,behavior:"instant"}); |
| } |
| document.querySelectorAll(".tab").forEach(t=>t.addEventListener("click",()=>showTab(t.dataset.tab))); |
| document.querySelectorAll("[data-jump]").forEach(a=>a.addEventListener("click",e=>{e.preventDefault();showTab(a.dataset.jump);})); |
| |
| |
| [selModel,selGpu,selCtx].forEach(e=>e.addEventListener("change",()=>{if(e===selGpu)refreshRate();renderConsole();})); |
| selRate.addEventListener("input",()=>{renderConsole();}); |
| document.querySelectorAll('input[name=lever]').forEach(r=>r.addEventListener("change",renderConsole)); |
| document.querySelectorAll('[data-hw]').forEach(b=>b.addEventListener("click",()=>{HW_GPU=b.dataset.hw;document.querySelectorAll('[data-hw]').forEach(x=>x.classList.toggle("on",x===b));drawHighway();})); |
| document.getElementById("cgToggle").addEventListener("click",function(){HW_CG=!HW_CG;this.textContent="CUDA GRAPHS: "+(HW_CG?"ON":"OFF");this.classList.toggle("on",HW_CG);drawHighway();}); |
| document.getElementById("clkCG").addEventListener("click",function(){if(this.disabled)return;CLK_CG=!CLK_CG;this.textContent="CUDA GRAPHS: "+(CLK_CG?"ON":"OFF");this.classList.toggle("on",CLK_CG);buildClocks();}); |
| document.querySelectorAll('[data-tok]').forEach(b=>b.addEventListener("click",()=>{TOK=parseInt(b.dataset.tok);document.querySelectorAll('[data-tok]').forEach(x=>x.classList.toggle("on",x===b));drawBarrier();})); |
| document.getElementById("hzTarget").addEventListener("input",function(){TARGET_HZ=parseInt(this.value);document.getElementById("hzVal").textContent=TARGET_HZ+" Hz";drawBarrier();}); |
| document.getElementById("sbRun").addEventListener("click",sbRun); |
| document.getElementById("sbReset").addEventListener("click",sbReset); |
| document.getElementById("calcTok").addEventListener("input",calcUpdate); |
| document.querySelectorAll('[data-wl]').forEach(b=>b.addEventListener("click",()=>{document.querySelectorAll('[data-wl]').forEach(x=>x.classList.toggle("on",x===b));drawWL(b.dataset.wl);})); |
| window.addEventListener("resize",()=>{resizeFlow();resizeSurface();heroResize();}); |
| |
| refreshRate();renderConsole();drawArch();drawHighway();drawBarrier();drawQuant();drawAttn();drawHeatmap();drawSurface();drawFalsify();sbBuild();calcUpdate();drawWL("dc"); |
| requestAnimationFrame(renderFlow);requestAnimationFrame(animateClocks); |
| heroResize();requestAnimationFrame(heroDraw); |
| </script> |
| </body> |
| </html> |
|
|
