Sync from GitHub 8a93941

app.py

@@ -57,6 +57,7 @@ from huggingface_hub import HfApi, snapshot_download
 
 from core.constants import (
     DEFAULT_TOP_K,
+    GROUND_ENABLE_THINKING,
     LIBRARY_DATASET_ID,
     MAX_TOP_K,
     MOCK_MODELS,
@@ -291,6 +292,7 @@ def api_find(
     section: str = "",
     pages: list = None,
     history: list = None,
+    think: bool = GROUND_ENABLE_THINKING,
 ) -> dict:  # the per-yield type: Server.api infers outputs from this annotation
     """One agent turn (one ZeroGPU call), streamed as events (see
     pipelines/agent_ask.py for the protocol). page/section are what the viewer
@@ -324,13 +326,13 @@ def api_find(
     viewer = {"page": int(page or 0), "section": str(section or ""), "pages": shown}
     options = _router_options(manual, request)
     log.info(
-        "find: manual=%s k=%s viewer=%s hist=%d opts=%d q=%r",
-        manual, k, viewer, len(history or []), len(options), request[:200],
+        "find: manual=%s k=%s think=%s viewer=%s hist=%d opts=%d q=%r",
+        manual, k, bool(think), viewer, len(history or []), len(options), request[:200],
     )
     try:
         events = PIPELINE.run_find(
             VISUAL_STORE, PARSED_STORE, request, [manual], int(k), options,
-            viewer, history,
+            viewer, history, bool(think),
         )
         for ev in events:
             if ev.get("type") == "tool_result" and "gallery" in ev:
@@ -365,12 +367,15 @@ _FRONTEND_DIR = os.path.join(os.path.dirname(__file__), "frontend")
 @app.get("/")
 def index():
     """Serve the single-page UI, injecting the small bit of server config the
-    frontend needs (default/max k) so it needs no extra round-trip on load."""
+    frontend needs (default/max k, default grounding-thinking) so it needs no
+    extra round-trip on load."""
     with open(os.path.join(_FRONTEND_DIR, "index.html")) as f:
         html = f.read()
     html = (
         html.replace("__DEFAULT_K__", str(DEFAULT_TOP_K))
         .replace("__MAX_K__", str(MAX_TOP_K))
+        # Initial state of the settings-panel "thinking" toggle (a JS bool).
+        .replace("__GROUND_THINK__", "true" if GROUND_ENABLE_THINKING else "false")
         # Cache-bust key for /page images: changing the render DPI changes the
         # served page size, so it must change the URL too — otherwise a browser
         # could keep an old-resolution page (cached up to a day) under the same

core/constants.py

@@ -16,6 +16,23 @@ MINICPM_REVISION = os.environ.get(
 )
 ANSWER_MAX_NEW_TOKENS = 2048
 
+# Let MiniCPM-V reason (legend → callout-number → leader-line → part) before
+# committing to a grounding box for "circle the <thing>". This is only the
+# DEFAULT — the UI settings panel sends a per-request override (see api_find's
+# `think`). Off by default: it roughly multiplies grounding latency (64 → ~512
+# generated tokens) and mainly helps exploded-diagram callouts, not the
+# dense-table wrong-row misses.
+GROUND_ENABLE_THINKING = os.environ.get("GROUND_ENABLE_THINKING", "").lower() in (
+    "1",
+    "true",
+    "yes",
+)
+# Token budgets for one grounding generation. A bare <box> fits in 64; a think
+# trace does not (it gets cut off before the box), so the budget tracks whether
+# thinking is on for that call.
+GROUND_BOX_MAX_NEW_TOKENS = 64
+GROUND_THINK_MAX_NEW_TOKENS = 512
+
 # Agent brain: MiniCPM5-1B — a standard LlamaForCausalLM (no trust_remote_code),
 # 131k context. The small TEXT model that drives the find-and-point loop: each
 # step it picks ONE tool from the conversation so far, the manual's table of

frontend/index.html

@@ -345,6 +345,20 @@
                class="w-full accent-brand-600">
       </div>
 
+      <!-- think: let the VLM reason before committing to a circle box -->
+      <div class="mt-5 flex items-start justify-between gap-3">
+        <div>
+          <label class="text-xs font-semibold uppercase tracking-wide text-brand-500/80">Careful pointing</label>
+          <p class="mt-0.5 text-xs text-brand-400">VLM reasons before circling — more accurate on diagrams, but slower.</p>
+        </div>
+        <button type="button" role="switch" :aria-checked="think" @click="think=!think"
+                :class="think ? 'bg-brand-600' : 'bg-brand-200'"
+                class="relative inline-flex h-6 w-11 shrink-0 items-center rounded-full transition mt-0.5">
+          <span :class="think ? 'translate-x-5' : 'translate-x-1'"
+                class="inline-block h-4 w-4 rounded-full bg-white shadow transition"></span>
+        </button>
+      </div>
+
       <div class="flex gap-2 mt-6">
         <button @click="refresh()" class="flex-1 rounded-xl border border-brand-200 px-4 py-2.5 text-sm font-medium text-navy hover:bg-brand-50 transition flex items-center justify-center gap-1.5">
           <i data-lucide="refresh-cw" class="w-4 h-4"></i>
@@ -378,6 +392,10 @@ function repairGuy() {
     client:null, ready:false,
     manuals:[], manual:'',
     k:__DEFAULT_K__, maxK:__MAX_K__,
+    // When on, MiniCPM-V reasons before committing to a circle box (slower,
+    // helps diagram-callout targets). Sent as `think` with each /find turn;
+    // initialized from the server default.
+    think:__GROUND_THINK__,
     input:'', loading:false, status:'',
     settingsOpen:false,
     ICONS,
@@ -627,7 +645,7 @@ function repairGuy() {
         mono:this.history.map((h,i)=>`${i+1}. asked: ${h.request}\n   did:   ${h.action}`).join('\n')});
       try{
         const job = this.client.submit('/find', {
-          request:q, manual:this.manual, k:this.k,
+          request:q, manual:this.manual, k:this.k, think:this.think,
           page: this.viewDoc ? this.viewPage : 0, section: this.sectionTitle(this.viewPage),
           pages: this.spreadPages,  // every page currently on screen — the agent may circle on any
           history: this.history,

models/minicpm.py

@@ -28,6 +28,9 @@ from transformers import AutoModel, AutoProcessor, AutoTokenizer
 from core.constants import (
     ANSWER_MAX_NEW_TOKENS,
     DESCRIBE_MAX_NEW_TOKENS,
+    GROUND_BOX_MAX_NEW_TOKENS,
+    GROUND_ENABLE_THINKING,
+    GROUND_THINK_MAX_NEW_TOKENS,
     MINICPM_MODEL_ID,
     MINICPM_REVISION,
 )
@@ -155,11 +158,17 @@ def generate_answer(question: str, pages: list[tuple[str, Image.Image]]) -> str:
 
 
 def ground_box(
-    image: Image.Image, query: str
+    image: Image.Image, query: str, enable_thinking: bool | None = None
 ) -> tuple[tuple[float, float, float, float] | None, str]:
     """(bbox, raw reply): the bounding box of the described object on a page
     image, in that image's pixel coordinates — or None when the model can't
-    place it (no box in the reply, or a degenerate one). Must run on GPU."""
+    place it (no box in the reply, or a degenerate one). Must run on GPU.
+
+    enable_thinking lets the model reason (legend → callout-number →
+    leader-line → part) before committing to a box. None defers to the
+    GROUND_ENABLE_THINKING default; the UI settings panel passes an explicit
+    bool per request."""
+    think = GROUND_ENABLE_THINKING if enable_thinking is None else enable_thinking
     with torch.no_grad():
         out = _MODEL.chat(
             msgs=[
@@ -169,17 +178,28 @@ def ground_box(
                 }
             ],
             tokenizer=_TOKENIZER,
-            enable_thinking=False,
-            max_new_tokens=64,
+            # The think trace needs room (a bare box fits in 64 tokens; a think
+            # trace does not) or it gets cut off before emitting the box — so
+            # the token budget tracks the flag.
+            enable_thinking=think,
+            max_new_tokens=(
+                GROUND_THINK_MAX_NEW_TOKENS if think else GROUND_BOX_MAX_NEW_TOKENS
+            ),
         )
     raw = str(out).strip()
-    if "NOT FOUND" in raw.upper():
+    # The thinking trace can mention "not found" mid-reasoning ("at first this
+    # looked not found, but…"), so test only the FINAL answer after </think>,
+    # not the whole reply.
+    final = raw.rsplit("</think>", 1)[-1]
+    if "NOT FOUND" in final.upper():
         return None, raw
     # Read the coordinates from the <box>…</box> tag specifically. MiniCPM-V
     # may prefix a <ref>…</ref> (e.g. echoing "5. Rod"); that digit would
     # otherwise be grabbed as the first coordinate and shift the whole box.
-    m = re.search(r"<box>(.*?)</box>", raw, re.IGNORECASE | re.DOTALL)
-    nums = re.findall(r"\d+(?:\.\d+)?", m.group(1) if m else raw)
+    # Search `final` so digits inside the reasoning can't be mistaken for
+    # coordinates.
+    m = re.search(r"<box>(.*?)</box>", final, re.IGNORECASE | re.DOTALL)
+    nums = re.findall(r"\d+(?:\.\d+)?", m.group(1) if m else final)
     if len(nums) < 4:
         return None, raw
     x1, y1, x2, y2 = (min(1000.0, max(0.0, float(n))) for n in nums[:4])

pipelines/agent_ask.py

@@ -87,10 +87,12 @@ def agent_events(
     sections: list[dict],
     viewer: dict | None = None,
     history: list | None = None,
+    ground_thinking: bool | None = None,
 ):
     """Yield the events of one agent turn (see module docstring). sections is the
     numbered table of contents shown to the agent ([{title, page}]); the agent's
-    go_to_section index is 1-based into it."""
+    go_to_section index is 1-based into it. ground_thinking toggles MiniCPM-V's
+    reasoning for the circle grounding (None → server default)."""
     doc_id = doc_ids[0]
     manual = names[doc_id]
     viewer = viewer or {}
@@ -269,7 +271,7 @@ def agent_events(
             yield {"type": "step", "tool": "circle", "target": target, "page": page}
             yield {"type": "status", "text": "Pinning it down…"}
             img = render_page(visual_store.pdf_path(doc_id), page)
-            box, braw = minicpm.ground_box(img, target)
+            box, braw = minicpm.ground_box(img, target, enable_thinking=ground_thinking)
             log.info("ground_box(%r) on p.%d → %s | raw=%r",
                      target, page, box, braw[:200])
             # The VLM couldn't find the target on this page — almost always
@@ -332,6 +334,7 @@ class AgentPipeline:
         sections: list[dict],
         viewer: dict | None = None,
         history: list | None = None,
+        ground_thinking: bool | None = None,
     ):
         """One streamed agent turn (the event generator of agent_events)."""
         request = (request or "").strip()
@@ -343,5 +346,5 @@ class AgentPipeline:
         names = {d["doc_id"]: d["name"] for d in docs}
         return agent_events(
             request, visual_store, parsed_store, doc_ids or list(names),
-            int(top_k), names, sections, viewer, history,
+            int(top_k), names, sections, viewer, history, ground_thinking,
         )

pipelines/mock_ask.py

@@ -143,10 +143,12 @@ class MockAskPipeline:
         sections: list[dict],
         viewer: dict | None = None,
         history: list | None = None,
+        ground_thinking: bool | None = None,
     ):
         """Yield the same event sequence as pipelines/agent_ask.py, with a
         keyword-driven stand-in for the agent's tool choice. Both stores are the
-        one MockStore; history is ignored. MOCK_DELAY (seconds) paces events."""
+        one MockStore; history and ground_thinking are ignored. MOCK_DELAY
+        (seconds) paces events."""
         request = (request or "").strip()
         if not request:
             raise ValueError("Tell me what to find.")