Sync from GitHub 97ec386

README.md

@@ -48,8 +48,10 @@ Space (no external endpoints).
   chapters plus a per-request fuzzy shortlist of fine parse headings
   (`app/core/sections.py`).
 
-**Retrieval is fused** — the search tool needs both indexes of a manual, so a
-manual must be indexed both ways:
+**Two indexes, one picked per turn** — the **Search index** setting chooses
+which one the search tool ranks against (default **parsed**). A manual must be
+indexed both ways regardless: the parsed index also supplies the page text the
+agent reads and circles on.
 
 - **Visual** — every page is embedded as an image with
   [Nemotron ColEmbed v2](https://huggingface.co/nvidia/nemotron-colembed-vl-4b-v2)
@@ -58,13 +60,16 @@ manual must be indexed both ways:
   disk via numpy memmap) to shortlist candidate pages — the top page is shown.
   (Late-interaction visual ranking beat a 1B text rerank of the shortlist in the
   eval, 0.84 vs 0.68 hit@1, so the search tool takes ColEmbed's top page.)
+  Strongest on diagrams.
 - **Parsed** — pages are parsed with
   [Nemotron Parse v1.2](https://huggingface.co/nvidia/NVIDIA-Nemotron-Parse-v1.2),
   figures and tables are described by MiniCPM-V, and heading-based section
   chunks are embedded with
-  [Llama Nemotron Embed VL 1B v2](https://huggingface.co/nvidia/llama-nemotron-embed-vl-1b-v2)
-  (used by the offline answer eval). On the Space the parsed pages supply the
-  **whole-page text** the agent reasons over (`app/core/page_context.py`).
+  [Llama Nemotron Embed VL 1B v2](https://huggingface.co/nvidia/llama-nemotron-embed-vl-1b-v2).
+  The search tool scores the query against those chunks by dense cosine and
+  votes their parent pages — strongest on spec/table lookups, the default. The
+  parsed pages also supply the **whole-page text** the agent reasons over
+  (`app/core/page_context.py`).
 
 ## Indexing (offline only)
 

app.py

@@ -61,6 +61,7 @@ from huggingface_hub import HfApi, snapshot_download
 from core.constants import (
     AGENT_MODELS,
     DEFAULT_AGENT_MODEL,
+    DEFAULT_RETRIEVAL_MODE,
     DEFAULT_TOP_K,
     GROUND_ENABLE_THINKING,
     LIBRARY_DATASET_ID,
@@ -70,6 +71,7 @@ from core.constants import (
     PARSED_SUBDIR,
     PREINDEXED_DIR,
     RENDER_DPI,
+    RETRIEVAL_MODES,
     VISUAL_SUBDIR,
 )
 from core.pdf import pdf_outline, render_page_png
@@ -120,6 +122,8 @@ def _build_libraries():
 VISUAL_STORE, PARSED_STORE, PIPELINE = _build_libraries()
 # Valid agent-brain keys, for validating the per-request `agent_model`.
 _AGENT_MODEL_KEYS = {m["key"] for m in AGENT_MODELS}
+# Valid search-index keys, for validating the per-request `retrieval_mode`.
+_RETRIEVAL_MODE_KEYS = {m["key"] for m in RETRIEVAL_MODES}
 # method -> store, for the picker / pdf lookups. In mock both keys map to the
 # one MockStore, so every mock manual reads as indexed under both methods.
 _METHOD_STORES = {"visual": VISUAL_STORE, "parsed": PARSED_STORE}
@@ -268,6 +272,7 @@ def api_find(
     history: list = None,
     think: bool = GROUND_ENABLE_THINKING,
     agent_model: str = DEFAULT_AGENT_MODEL,
+    retrieval_mode: str = DEFAULT_RETRIEVAL_MODE,
     vram_log: bool = False,
     session_id: str = "",
 ) -> dict:  # the per-yield type: Server.api infers outputs from this annotation
@@ -305,16 +310,18 @@ def api_find(
     # the log reflects what actually ran).
     if agent_model not in _AGENT_MODEL_KEYS:
         agent_model = DEFAULT_AGENT_MODEL
+    if retrieval_mode not in _RETRIEVAL_MODE_KEYS:
+        retrieval_mode = DEFAULT_RETRIEVAL_MODE
     log.info(
-        "find: manual=%s k=%s think=%s model=%s viewer=%s hist=%d q=%r",
-        manual, k, bool(think), agent_model, viewer, len(history or []),
-        request[:200],
+        "find: manual=%s k=%s think=%s model=%s search=%s viewer=%s hist=%d q=%r",
+        manual, k, bool(think), agent_model, retrieval_mode, viewer,
+        len(history or []), request[:200],
     )
     try:
         events = PIPELINE.run_find(
             VISUAL_STORE, PARSED_STORE, request, [manual], int(k),
-            viewer, history, bool(think), agent_model, bool(vram_log),
-            str(session_id or ""),
+            viewer, history, bool(think), agent_model, retrieval_mode,
+            bool(vram_log), str(session_id or ""),
         )
         for ev in events:
             if ev.get("type") == "tool_result" and "gallery" in ev:
@@ -365,6 +372,14 @@ def index():
             json.dumps([{"key": m["key"], "label": m["label"]} for m in AGENT_MODELS]),
         )
         .replace("__AGENT_MODEL__", DEFAULT_AGENT_MODEL)
+        # Search-index picker: which index the search tool ranks against
+        # (visual / parsed) and the default, so the settings dropdown needs no
+        # extra round-trip on load.
+        .replace(
+            "__RETRIEVAL_MODES_JSON__",
+            json.dumps([{"key": m["key"], "label": m["label"]} for m in RETRIEVAL_MODES]),
+        )
+        .replace("__RETRIEVAL_MODE__", DEFAULT_RETRIEVAL_MODE)
         # Cache-bust key for /page images: changing the render DPI changes the
         # served page size, so it must change the URL too — otherwise a browser
         # could keep an old-resolution page (cached up to a day) under the same

core/constants.py

@@ -171,6 +171,17 @@ AGENT_MAX_NEW_TOKENS = 128
 AGENT_MAX_STEPS = 6
 # ColEmbed shortlist size the search tool retrieves (the eval default).
 AGENT_SEARCH_CANDIDATES = 5
+# Which index the search tool ranks against. Both retrievers return the same
+# (doc, page, score) shape, so the agent loop is identical either way:
+#   visual — ColEmbed late-interaction MaxSim over page-image embeddings
+#   parsed — Nemotron dense cosine over parsed section/figure/table chunks
+# Exposed as a UI setting (settings panel); the parsed index wins on spec/table
+# lookups, so it is the default for now.
+RETRIEVAL_MODES = [
+    {"key": "parsed", "label": "Parsed (text)"},
+    {"key": "visual", "label": "Visual (ColEmbed)"},
+]
+DEFAULT_RETRIEVAL_MODE = RETRIEVAL_MODES[0]["key"]
 # Past turns of conversation fed back as memory (resolve "the other one", "go
 # back"); the live turn carries the full current page text (no table of contents).
 AGENT_HISTORY_TURNS = 6

frontend/index.html

@@ -355,6 +355,21 @@
         <p class="mt-1 text-xs text-brand-400">The model that finds pages and points. Switching loads it fresh — the first turn after a change is slower.</p>
       </div>
 
+      <!-- search index: which index the search tool ranks the query against -->
+      <div class="mt-5">
+        <label class="block text-xs font-semibold uppercase tracking-wide text-brand-500/80 mb-1.5">Search index</label>
+        <div class="relative">
+          <select x-model="retrievalMode"
+            class="w-full appearance-none rounded-xl border border-brand-200 bg-brand-50/50 px-3.5 py-2.5 pr-9 text-sm font-medium text-navy focus:border-brand-400 focus:ring-2 focus:ring-brand-100 outline-none transition">
+            <template x-for="m in retrievalModes" :key="m.key">
+              <option :value="m.key" x-text="m.label"></option>
+            </template>
+          </select>
+          <i data-lucide="chevron-down" class="pointer-events-none absolute right-3 top-1/2 -translate-y-1/2 w-4 h-4 text-brand-400"></i>
+        </div>
+        <p class="mt-1 text-xs text-brand-400">What the search tool ranks against. Parsed (text) wins on specs and tables; Visual (ColEmbed) on diagrams.</p>
+      </div>
+
       <!-- think: let the VLM reason before committing to a circle box -->
       <div class="mt-5 flex items-start justify-between gap-3">
         <div>
@@ -442,6 +457,10 @@ function repairGuy() {
     // the previous from VRAM server-side (load-on-switch), so the FIRST turn
     // after a change is slower while the new model loads.
     agentModels:__AGENT_MODELS_JSON__, agentModel:'__AGENT_MODEL__',
+    // The search-index picker: which index the search tool ranks against
+    // (visual / parsed), sent as `retrieval_mode` each turn. Parsed wins on
+    // spec/table lookups, visual on diagrams; initialized from the server default.
+    retrievalModes:__RETRIEVAL_MODES_JSON__, retrievalMode:'__RETRIEVAL_MODE__',
     // Diagnostics: when on, the server logs a GPU-memory snapshot at each step of
     // the turn (resident models, brain evict/load, the grounding spike). Sent as
     // `vram_log`; off by default since it's only for debugging the VRAM budget.
@@ -728,6 +747,7 @@ function repairGuy() {
       try{
         const job = this.client.submit('/find', {
           request:q, manual:this.manual, k:this.k, think:this.think, agent_model:this.agentModel,
+          retrieval_mode:this.retrievalMode,
           session_id:this.sessionId,
           page: this.viewDoc ? this.viewPage : 0, section: this.sectionTitle(this.viewPage),
           pages: this.spreadPages,  // every page currently on screen — the agent may circle on any

pipelines/agent_ask.py

@@ -9,15 +9,18 @@ Flow (one @spaces.GPU call, streamed as events):
   figures/tables as their descriptions). No table of contents is injected. Then
   loop, up to AGENT_MAX_STEPS:
     decide → ONE tool:
-      search(query)      ColEmbed (visual) top-N → show the best page; its
-                         text is fed back so the agent can then circle on it (continues)
+      search(query)      retrieve top-N → show the best page; its text is fed
+                         back so the agent can then circle on it          (continues)
       circle(target)     ground the target on the CURRENT page (VLM) and
                          circle it                                        (terminal)
       done(message)      nothing to do / not in the manual                (terminal)
 
-Retrieval is VISUAL: search uses ColEmbed page embeddings (visual store). The
-parsed store still supplies the page text the agent reads and circles on, so a
-manual must be indexed both ways.
+The search tool ranks against whichever index `retrieval_mode` picks (a UI
+setting): "visual" — ColEmbed late interaction over page-image embeddings;
+"parsed" — Nemotron dense cosine over parsed chunks. Both retrievers return the
+same (doc, page, score) shape, so the loop is identical either way. The parsed
+store ALSO supplies the page text the agent reads and circles on, so a manual
+must be indexed both ways regardless of mode.
 
 History is used only to resolve references, never to restate answers. Each turn
 is otherwise grounded in the viewer state the client sends (current page +
@@ -45,6 +48,7 @@ from core import tracing
 from core.constants import (
     AGENT_HISTORY_TURNS,
     AGENT_MAX_STEPS,
+    DEFAULT_RETRIEVAL_MODE,
     FIND_GPU_DURATION,
 )
 from core.page_context import index_pages, page_to_text
@@ -52,6 +56,7 @@ from core.pdf import page_count, render_page
 from core.vram import log_vram, reset_peak, set_enabled
 from models import minicpm, minicpm_agent
 from models.colembed import maxsim_search
+from pipelines.parsed_ask import retrieve_pages
 
 log = logging.getLogger("repairguy.agent")
 
@@ -91,15 +96,18 @@ def agent_events(
     history: list | None = None,
     ground_thinking: bool | None = None,
     agent_model: str | None = None,
+    retrieval_mode: str | None = None,
     vram_log: bool = False,
     session_id: str | None = None,
 ):
     """Yield the events of one agent turn (see module docstring). The agent gets
-    no table of contents — it works from search (visual retrieval) and the
-    current page's text. ground_thinking toggles MiniCPM-V's
-    reasoning for the circle grounding (None → server default); agent_model picks
-    which brain drives the loop (None → default), loaded on switch inside this
-    GPU window. vram_log enables the per-turn VRAM probe (UI setting)."""
+    no table of contents — it works from search and the current page's text.
+    retrieval_mode picks the search index ("visual" | "parsed", None → default).
+    ground_thinking toggles MiniCPM-V's reasoning for the circle grounding (None →
+    server default); agent_model picks which brain drives the loop (None →
+    default), loaded on switch inside this GPU window. vram_log enables the
+    per-turn VRAM probe (UI setting)."""
+    retrieval_mode = retrieval_mode if retrieval_mode in ("visual", "parsed") else DEFAULT_RETRIEVAL_MODE
     # Apply the UI's VRAM-logging toggle for this turn. Done here (inside the GPU
     # worker) rather than in the parent so a reused ZeroGPU worker always honors
     # the current request's setting. When off, every log_vram/reset_peak below is
@@ -269,14 +277,20 @@ def agent_events(
                              [(p, round(s, 3)) for _, p, s in hits])
                 else:
                     yield {"type": "status", "text": f"Searching for “{query}”…"}
-                    # k (the viewer's slider) is the shortlist size; ColEmbed's top
-                    # page is the one shown. A 1B text rerank measured WORSE than raw
-                    # ColEmbed top-1 (0.68 vs 0.84 hit@1) — visual late interaction
-                    # already ranks these (figure-heavy) pages better than re-judging
-                    # from page text.
+                    # k (the viewer's slider) is the shortlist size; the top page is
+                    # the one shown. Two indexes can answer the query (UI setting):
+                    #   visual — ColEmbed late interaction over page images. A 1B text
+                    #     rerank measured WORSE than raw ColEmbed top-1 (0.68 vs 0.84
+                    #     hit@1), so the search tool takes ColEmbed's top page as-is.
+                    #   parsed — Nemotron dense cosine over chunks → parent pages
+                    #     (wins on spec/table lookups). Same (doc, page, score) shape.
+                    retriever_name = "colembed" if retrieval_mode == "visual" else "nemotron-embed"
                     with tracing.retriever("search", input=query,
-                                           metadata={"retriever": "colembed", "k": int(top_k)}) as rsp:
-                        hits = maxsim_search(query, visual_store, doc_ids, top_k)
+                                           metadata={"retriever": retriever_name, "k": int(top_k)}) as rsp:
+                        if retrieval_mode == "visual":
+                            hits = maxsim_search(query, visual_store, doc_ids, top_k)
+                        else:
+                            hits = retrieve_pages(query, parsed_store, doc_ids, top_k)
                         if rsp is not None:
                             rsp.update(output=[{"page": p, "score": round(float(s), 4)}
                                                for _, p, s in hits])
@@ -375,12 +389,14 @@ class AgentPipeline:
         history: list | None = None,
         ground_thinking: bool | None = None,
         agent_model: str | None = None,
+        retrieval_mode: str | None = None,
         vram_log: bool = False,
         session_id: str | None = None,
     ):
         """One streamed agent turn (the event generator of agent_events).
-        vram_log forwards the UI's VRAM-logging toggle to the probe; session_id
-        groups a page session's turns together in Langfuse."""
+        retrieval_mode picks the search index ("visual" | "parsed"); vram_log
+        forwards the UI's VRAM-logging toggle to the probe; session_id groups a
+        page session's turns together in Langfuse."""
         request = (request or "").strip()
         if not request:
             raise ValueError("Tell me what to find.")
@@ -391,5 +407,5 @@ class AgentPipeline:
         return agent_events(
             request, visual_store, parsed_store, doc_ids or list(names),
             int(top_k), names, viewer, history, ground_thinking,
-            agent_model, vram_log, session_id,
+            agent_model, retrieval_mode, vram_log, session_id,
         )

pipelines/mock_ask.py

@@ -143,13 +143,14 @@ class MockAskPipeline:
         history: list | None = None,
         ground_thinking: bool | None = None,
         agent_model: str | None = None,
+        retrieval_mode: str | None = None,
         vram_log: bool = False,
         session_id: str | None = None,
     ):
         """Yield the same event sequence as pipelines/agent_ask.py, with a
         keyword-driven stand-in for the agent's tool choice. Both stores are the
-        one MockStore; history, ground_thinking, agent_model, vram_log and
-        session_id are ignored. MOCK_DELAY (seconds) paces events."""
+        one MockStore; history, ground_thinking, agent_model, retrieval_mode,
+        vram_log and session_id are ignored. MOCK_DELAY (seconds) paces events."""
         request = (request or "").strip()
         if not request:
             raise ValueError("Tell me what to find.")