Preload image descriptions for reader mode

Co-authored-by: Codex <codex@openai.com>

FIELD_NOTES.md

@@ -44,3 +44,5 @@ Keyboard map:
 ## Fallback Strategy
 
 Live accessibility demos need dependable behavior. If llama.cpp is not available, the app uses deterministic local narration prefixes. If Kokoro is not installed or cannot load a voice, the app returns a short silent WAV while keeping the transcript visible. If the VLM integration is unavailable, image ids map to cached alt text.
+
+The app preloads image descriptions through `/api/image-descriptions`, caches them in the frontend, and writes the descriptions back into each image's `alt` attribute. That keeps the visible article, transcript, and spoken image narration aligned.

README.md

@@ -50,7 +50,7 @@ Start the app:
 python app.py
 ```
 
-Open the local URL printed by Gradio. The custom frontend calls `/api/reader-brain`, `/api/describe-image`, `/api/speak`, and `/api/generate-image`.
+Open the local URL printed by Gradio. The custom frontend calls `/api/reader-brain`, `/api/image-descriptions`, `/api/describe-image`, `/api/speak`, and `/api/generate-image`.
 
 Useful environment variables:
 
@@ -85,3 +85,5 @@ The frontend builds a reading queue from semantic article nodes. When screen-rea
 Each readable node is sent to the reader brain for concise narration, then Kokoro generates speech. If a model is unavailable, the app uses deterministic fallbacks so the demo remains navigable.
 
 The session panel keeps a transcript of recent narration with copy and clear controls, making the spoken path inspectable during demos and useful for the Field Notes write-up.
+
+Image descriptions are preloaded into a local cache and written into the page's real `img alt` attributes. When the planned MiniCPM-V runtime is unavailable, deterministic alt-text fallbacks keep the screen-reader path usable.

app.py

@@ -53,6 +53,27 @@ MODEL_MANIFEST: dict[str, dict[str, str]] = {
     },
 }
 
+ARTICLE_IMAGES: list[dict[str, str]] = [
+    {
+        "id": "desk-reader",
+        "asset_url": "/static/generated/desk-reader.svg",
+        "caption": "The article view doubles as the demo surface, so every feature has a real reading task.",
+        "prompt": "Accessibility article reader with highlighted paragraph and narration controls.",
+    },
+    {
+        "id": "model-map",
+        "asset_url": "/static/generated/model-map.svg",
+        "caption": "Each model stays at or below four billion parameters for Tiny Titan eligibility.",
+        "prompt": "Diagram of four small AI models working together in an accessibility reader.",
+    },
+    {
+        "id": "field-notes",
+        "asset_url": "/static/generated/field-notes.svg",
+        "caption": "Field notes document the choices behind the screen-reader behavior.",
+        "prompt": "Notebook page with model sizes, keyboard controls, and accessibility checks.",
+    },
+]
+
 ARTICLE_MANIFEST: dict[str, Any] = {
     "title": "A tiny model reader that turns articles into guided narration",
     "reader_controls": [
@@ -66,6 +87,7 @@ ARTICLE_MANIFEST: dict[str, Any] = {
         {"key": "Esc", "action": "Stop audio"},
     ],
     "bonus_targets": ["Tiny Titan", "Llama Champion", "Off-Brand", "Field Notes"],
+    "images": ARTICLE_IMAGES,
     "models": MODEL_MANIFEST,
 }
 
@@ -209,6 +231,23 @@ def describe_image_core(image_id: str, caption: str | None, prompt: str | None)
     }
 
 
+def describe_article_images_core() -> dict[str, Any]:
+    descriptions = []
+    for image in ARTICLE_IMAGES:
+        description = describe_image_core(
+            image["id"],
+            caption=image.get("caption"),
+            prompt=image.get("prompt"),
+        )
+        descriptions.append({**image, **description})
+    return {
+        "ok": True,
+        "runtime": "MiniCPM-V placeholder",
+        "model": "OpenBMB MiniCPM-V-2",
+        "descriptions": descriptions,
+    }
+
+
 def _silent_wav(path: Path, seconds: float = 0.35, sample_rate: int = 24000) -> None:
     frames = int(seconds * sample_rate)
     with wave.open(str(path), "wb") as wav:
@@ -290,6 +329,11 @@ async def describe_image_endpoint(payload: ImageDescriptionRequest) -> JSONRespo
     return _json(describe_image_core(payload.image_id, payload.caption, payload.prompt))
 
 
+@app.get("/api/image-descriptions")
+async def image_descriptions_endpoint() -> JSONResponse:
+    return _json(describe_article_images_core())
+
+
 @app.post("/api/speak")
 async def speak_endpoint(payload: SpeechRequest) -> JSONResponse:
     return _json(speak_core(payload.text, payload.voice, payload.speed))
@@ -310,6 +354,11 @@ def describe_image_api(image_id: str, caption: str = "", prompt: str = "") -> st
     return json.dumps(describe_image_core(image_id, caption, prompt))
 
 
+@app.api(name="describe_article_images")
+def describe_article_images_api() -> str:
+    return json.dumps(describe_article_images_core())
+
+
 @app.api(name="speak")
 def speak_api(text: str, voice: str = "af_heart", speed: float = 1.0) -> str:
     return json.dumps(speak_core(text, voice, speed))

scripts/verify.py

@@ -54,6 +54,10 @@ def verify_core_fallbacks() -> None:
     assert_true(description["ok"], "Image description did not return ok")
     assert_true("small AI models" in description["alt_text"], "Image description fallback changed unexpectedly")
 
+    article_descriptions = app.describe_article_images_core()
+    assert_true(article_descriptions["ok"], "Article image descriptions did not return ok")
+    assert_true(len(article_descriptions["descriptions"]) == 3, "Article should expose three image descriptions")
+
     speech = app.speak_core("Tiny Narrator verification.", voice="af_heart", speed=1.0)
     assert_true(speech["ok"], "Speech path did not return ok")
     audio_path = ROOT / speech["audio_url"].lstrip("/")
@@ -71,6 +75,7 @@ def verify_routes() -> None:
     assert_true("Tiny Narrator" in home.text, "Home route should include app title")
     assert_true("readerToggle" in home.text, "Home route should include reader toggle")
     assert_true("summaryButton" in home.text, "Home route should include summary control")
+    assert_true("imageStatus" in home.text, "Home route should include image status")
     assert_true("transcriptLog" in home.text, "Home route should include transcript log")
 
     health = client.get("/api/health")
@@ -95,6 +100,15 @@ def verify_routes() -> None:
         "Manifest should document Kokoro speech model",
     )
 
+    image_descriptions = client.get("/api/image-descriptions")
+    assert_true(image_descriptions.status_code == 200, "Image descriptions route should return 200")
+    image_payload = image_descriptions.json()
+    assert_true(image_payload["model"] == "OpenBMB MiniCPM-V-2", "Image route should document MiniCPM-V model")
+    assert_true(
+        {item["id"] for item in image_payload["descriptions"]} == {"desk-reader", "model-map", "field-notes"},
+        "Image route should describe all article images",
+    )
+
 
 def main() -> None:
     py_compile.compile(str(ROOT / "app.py"), doraise=True)

static/app.js

@@ -4,6 +4,7 @@ const modeStatus = document.querySelector("#modeStatus");
 const currentStatus = document.querySelector("#currentStatus");
 const runtimeStatus = document.querySelector("#runtimeStatus");
 const modelStatus = document.querySelector("#modelStatus");
+const imageStatus = document.querySelector("#imageStatus");
 const liveNarration = document.querySelector("#liveNarration");
 const audio = document.querySelector("#speechAudio");
 const speedControl = document.querySelector("#speedControl");
@@ -35,6 +36,7 @@ let currentIndex = -1;
 let playing = false;
 let requestSerial = 0;
 let transcriptEntries = [];
+let imageDescriptions = new Map();
 
 function escapeHtml(value) {
   return value.replace(/[&<>"']/g, (char) => ({
@@ -90,6 +92,27 @@ async function loadManifest() {
   }
 }
 
+async function loadImageDescriptions() {
+  imageStatus.textContent = "Describing";
+  try {
+    const payload = await postJson("/api/image-descriptions");
+    imageDescriptions = new Map(
+      payload.descriptions.map((description) => [description.id, description]),
+    );
+    for (const node of nodes.filter((item) => item.type === "image")) {
+      const description = imageDescriptions.get(node.imageId);
+      const image = node.element.querySelector("img");
+      if (description?.alt_text && image) {
+        image.alt = description.alt_text;
+        node.element.dataset.altReady = "true";
+      }
+    }
+    imageStatus.textContent = `${imageDescriptions.size} ready`;
+  } catch {
+    imageStatus.textContent = "Fallback on demand";
+  }
+}
+
 function setEnabled(nextValue) {
   enabled = nextValue;
   toggle.setAttribute("aria-pressed", String(enabled));
@@ -151,10 +174,13 @@ async function narrate(index) {
   try {
     let sourceText = node.text;
     if (node.type === "image") {
-      const description = await postJson("/api/describe-image", {
-        image_id: node.imageId,
-        caption: node.text,
-      });
+      let description = imageDescriptions.get(node.imageId);
+      if (!description) {
+        description = await postJson("/api/describe-image", {
+          image_id: node.imageId,
+          caption: node.text,
+        });
+      }
       sourceText = description.alt_text;
     }
 
@@ -302,6 +328,7 @@ controls.play.addEventListener("click", () => {
 });
 
 loadManifest();
+loadImageDescriptions();
 
 audio.addEventListener("ended", () => {
   playing = false;

static/index.html

@@ -96,6 +96,10 @@
             <dt>Model Stack</dt>
             <dd id="modelStatus">Loading</dd>
           </div>
+          <div>
+            <dt>Image Alt</dt>
+            <dd id="imageStatus">Waiting</dd>
+          </div>
         </dl>
         <p id="liveNarration" class="live-narration" aria-live="polite">
           Turn on screen reader mode to begin.