Deploy: DocAgent backend (deterministic date-anomaly fix)

.dockerignore

@@ -0,0 +1,28 @@
+# Keep the HF Spaces / Docker build context small. Only backend/ is COPYed into
+# the image, but trimming the context speeds the upload + build.
+.git/
+.claude/
+.vercel/
+
+# Frontend (deployed separately on Vercel)
+frontend/node_modules/
+frontend/.next/
+
+# Python
+backend/.venv/
+backend/data/
+**/__pycache__/
+**/*.py[cod]
+
+# Local model caches (models are fetched fresh at build by prefetch_models.py)
+**/.cache/
+**/.EasyOCR/
+
+# Secrets / local env
+**/.env
+**/.env.*
+!**/.env.example
+
+# Misc
+.DS_Store
+Thumbs.db

.gitignore

@@ -0,0 +1,34 @@
+# Python
+__pycache__/
+*.py[cod]
+.venv/
+venv/
+backend/.env
+backend/data/
+
+# Node / Next
+node_modules/
+.next/
+frontend/.env.local
+frontend/.env*.local
+npm-debug.log*
+
+# Deploy / secrets
+.env
+.env.*
+!.env.example
+!**/.env.example
+.vercel/
+
+# Local model caches (baked into the image instead)
+**/.cache/
+**/.EasyOCR/
+
+# OS / editor
+.DS_Store
+Thumbs.db
+.vscode/
+.idea/
+
+# Local agent settings
+.claude/

Dockerfile

@@ -0,0 +1,68 @@
+# syntax=docker/dockerfile:1
+# ---------------------------------------------------------------------------
+# Hugging Face Spaces image for the Document-Agent backend (free CPU tier).
+#
+# HF Spaces (Docker SDK) builds the Dockerfile at the repo ROOT with the repo
+# root as the build context, so source paths are prefixed with backend/. HF
+# runs the container as UID 1000 and routes traffic to the port declared as
+# `app_port:` in README.md (8000 here).
+#
+# Free Spaces have NO persistent disk, so DATA_DIR is an ephemeral, user-owned
+# directory: uploads / SQLite / the vector store reset on restart or rebuild.
+# That is fine for a demo. The ML models are baked in at build time (same as
+# the production image) so cold start is fast and there are no runtime
+# downloads.
+# ---------------------------------------------------------------------------
+FROM python:3.12-slim
+
+ENV PYTHONUNBUFFERED=1 \
+    PYTHONDONTWRITEBYTECODE=1 \
+    PIP_NO_CACHE_DIR=1 \
+    PIP_DISABLE_PIP_VERSION_CHECK=1 \
+    HF_HUB_DISABLE_SYMLINKS_WARNING=1 \
+    # Fixed, shared cache locations so the non-root runtime user finds the
+    # models baked in during build.
+    HF_HOME=/opt/models/hf \
+    EASYOCR_MODULE_PATH=/opt/models/easyocr \
+    TORCH_HOME=/opt/models/torch \
+    DATA_DIR=/data
+
+# Native libs needed by docling / opencv / easyocr.
+RUN apt-get update && apt-get install -y --no-install-recommends \
+        libgl1 \
+        libglib2.0-0 \
+        libgomp1 \
+    && rm -rf /var/lib/apt/lists/*
+
+WORKDIR /app
+
+# Install CPU-only torch/torchvision FIRST so docling's torch dependency
+# resolves to the lightweight CPU wheels instead of multi-GB CUDA builds.
+RUN pip install --index-url https://download.pytorch.org/whl/cpu \
+        torch torchvision
+
+COPY backend/requirements.txt .
+RUN pip install -r requirements.txt
+
+# Bake the models into the image.
+COPY backend/scripts/prefetch_models.py scripts/prefetch_models.py
+RUN mkdir -p /opt/models && python scripts/prefetch_models.py
+
+# Backend app source (repo root is the build context on HF Spaces).
+COPY backend/ .
+
+# HF Spaces runs the container as UID 1000. Make the data dir and the baked
+# model/app trees writable/readable by that user.
+RUN useradd -m -u 1000 user \
+    && mkdir -p /data \
+    && chown -R user:user /data /app /opt/models
+USER user
+ENV HOME=/home/user
+
+EXPOSE 8000
+
+# Honour $PORT if a host injects it; default 8000 matches app_port in README.
+HEALTHCHECK --interval=30s --timeout=5s --start-period=60s --retries=3 \
+    CMD python -c "import os,urllib.request,sys; sys.exit(0 if urllib.request.urlopen('http://localhost:%s/api/health' % os.environ.get('PORT','8000')).status==200 else 1)"
+
+CMD ["sh", "-c", "uvicorn app.main:app --host 0.0.0.0 --port ${PORT:-8000}"]

README.md

@@ -0,0 +1,164 @@
+---
+title: DocAgent Backend
+emoji: 📄
+colorFrom: blue
+colorTo: indigo
+sdk: docker
+app_port: 8000
+pinned: false
+---
+
+# DocAgent — Document Processing AI Agent
+
+> **Hugging Face Space note:** this Space runs the **FastAPI backend** (Docker).
+> The UI is deployed separately on Vercel and talks to this Space's URL. Health
+> check: `GET /api/health`. On the free tier there is no persistent disk, so
+> uploads/DB reset on restart. See [`docs/DEPLOY.md`](docs/DEPLOY.md).
+
+A production-grade, **chat-based document processing agent** built for Monkhub Innovations.
+Upload any document → it parses, classifies, extracts structured data, summarizes,
+answers grounded questions, flags anomalies, and exports clean data.
+
+> **40 / 60 UI:** a persistent streaming chat (40%) beside a functional workspace
+> (60%) — document viewer with bounding-box citation highlights, extracted fields &
+> tables, classification, summary, and export.
+
+Built on the findings in [`docs/RESEARCH.md`](docs/RESEARCH.md) — a composed stack:
+**Docling** (local extraction) · **Gemini Flash** (free default LLM) ⇄ **GPT-4o-mini**
+(fallback) · agentic-RAG orchestration · **FastAPI** + **Next.js**.
+
+> 📘 **New here? Read [`docs/OVERVIEW.md`](docs/OVERVIEW.md)** — one document covering the
+> research, what we chose and why, the comparison table, the feature/function set, and
+> exactly how the agent works (architecture + both flows + a walkthrough).
+
+---
+
+## ✨ Features (the 7 core functions)
+
+| Function | What it does |
+|---|---|
+| **Ingest** | PDF, DOCX, PPTX, XLSX, HTML, images — parsed with Docling (layout, tables, OCR) |
+| **Extract** | Schema-driven fields, tables, key-value pairs, entities — each with a confidence score |
+| **Classify** | Zero-shot document type detection with confidence + candidates |
+| **Summarize** | Map-reduce summary (TL;DR + key points) |
+| **Q&A** | Conversational, agentic-RAG answers **grounded with citations** you can click to highlight on the page |
+| **Flag anomalies** | Missing required fields, low-confidence values, arithmetic checks, LLM consistency review |
+| **Export** | JSON · CSV · Excel |
+
+---
+
+## 🏗️ Architecture
+
+See [`docs/ARCHITECTURE.md`](docs/ARCHITECTURE.md) for the full diagram and rationale.
+
+```
+Next.js (40/60 UI)  ──REST + SSE──▶  FastAPI
+                                      ├─ agent/      orchestrator (tool-calling loop)
+                                      ├─ llm/        Gemini ⇄ OpenAI provider abstraction
+                                      ├─ services/   ingestion(Docling) · extraction ·
+                                      │              classify · summary · qa(RAG) ·
+                                      │              anomaly · export · vectorstore · storage
+                                      └─ schemas/    Pydantic contracts + extraction templates
+                                   Docling (local) · ChromaDB (embedded) · SQLite
+```
+
+**Design highlights**
+- **Single orchestrator agent** with 5 tools (query, classify, get-extracted, summarize, flag) — the simplest design that covers the brief, per the research.
+- **Provider abstraction**: nothing imports a vendor SDK except `llm/*`. Switch Gemini⇄OpenAI by env, per-request, or in the UI.
+- **Provenance-aware chunks**: every chunk carries page + normalised bbox, so RAG answers and extracted fields highlight their exact source.
+- **Local-first & ~zero marginal cost**: Docling runs locally; Gemini free tier; ChromaDB + SQLite need no external services.
+
+---
+
+## 🚀 Quick start
+
+### Prerequisites
+- **Python 3.10–3.12** (3.13 works for the API; some ML wheels lag — see notes)
+- **Node.js 18+**
+- A **free Gemini API key** → https://aistudio.google.com/apikey
+
+### 1. Backend
+
+```bash
+cd backend
+python -m venv .venv
+# Windows:  .venv\Scripts\activate
+# macOS/Linux:  source .venv/bin/activate
+pip install -r requirements.txt
+
+cp .env.example .env          # then put your GEMINI_API_KEY in .env
+uvicorn app.main:app --reload --port 8000
+```
+
+API now at http://localhost:8000 — interactive docs at http://localhost:8000/docs.
+
+### 2. Frontend
+
+```bash
+cd frontend
+npm install
+cp .env.local.example .env.local   # default points at :8000
+npm run dev
+```
+
+App now at **http://localhost:3000**.
+
+---
+
+## 🔧 Configuration (`backend/.env`)
+
+| Var | Default | Notes |
+|---|---|---|
+| `GEMINI_API_KEY` | — | **Required.** Free tier. |
+| `OPENAI_API_KEY` | — | Optional fallback (GPT-4o-mini). |
+| `DEFAULT_LLM_PROVIDER` | `gemini` | `gemini` or `openai`. |
+| `GEMINI_MODEL` | `gemini-2.0-flash` | |
+| `OPENAI_MODEL` | `gpt-4o-mini` | |
+| `ENABLE_OCR` | `true` | Docling OCR for scanned docs. |
+
+The app runs with **only** `GEMINI_API_KEY`. If both keys are present you can switch
+providers live from the header toggle.
+
+---
+
+## 🧭 How to use
+
+1. Drag a document into the left rail (or click **New document**).
+2. Watch it process — pages render, then classification, fields, summary, and anomaly
+   flags stream into the workspace tabs.
+3. **Chat** on the left: *"Summarize this"*, *"What's the total?"*, *"Any missing fields?"*
+   Click any **citation chip** to highlight its source on the page.
+4. **Export** from the Export tab (JSON / CSV / Excel).
+
+---
+
+## 📁 Project layout
+
+```
+Document-agent/
+├─ docs/
+│  ├─ RESEARCH.md        # deep research: 5 systems, scoring, winner, SOTA, UX
+│  └─ ARCHITECTURE.md    # architecture decisions + diagram
+├─ backend/
+│  ├─ app/
+│  │  ├─ api/            # documents, chat(SSE), actions, meta
+│  │  ├─ agent/          # orchestrator, tools, prompts
+│  │  ├─ llm/            # base, gemini, openai, registry
+│  │  ├─ services/       # ingestion, extraction, classify, summary, qa, anomaly, export, storage, vectorstore, pipeline
+│  │  ├─ schemas/        # documents + extraction templates
+│  │  └─ core/           # config, logging
+│  └─ requirements.txt
+└─ frontend/
+   ├─ app/               # layout, page (40/60 shell)
+   ├─ components/        # Chat, Workspace, Viewer, Fields, Classify, Summary, Export, Sidebar
+   └─ lib/               # api client (+SSE), types
+```
+
+---
+
+## 📝 Notes & next steps
+- **Python 3.13:** the API layer runs fine; if a Docling/Torch wheel isn't yet
+  published for 3.13 on your platform, use a 3.11/3.12 venv for the backend.
+- Designed to scale: swap SQLite→Postgres and ChromaDB→pgvector by editing only
+  `services/storage.py` and `services/vectorstore.py`.
+- Auth, rate limiting, and multi-tenant workspaces are natural follow-ons.

backend/.dockerignore

@@ -0,0 +1,22 @@
+# Keep the build context small and never ship secrets or local state.
+.venv/
+venv/
+__pycache__/
+*.py[cod]
+*.egg-info/
+.pytest_cache/
+.mypy_cache/
+.ruff_cache/
+
+# Local runtime data (uploads, sqlite, rendered pages, vector store).
+data/
+
+# Secrets — provided via host env vars in production.
+.env
+.env.*
+
+# Misc
+.git/
+.DS_Store
+Thumbs.db
+*.log

backend/.env.example

@@ -0,0 +1,36 @@
+# ============================================================
+# Document Processing AI Agent — backend configuration
+# Copy to `.env` and fill in. Only GEMINI_API_KEY is required.
+# ============================================================
+
+# --- LLM providers ---
+# Default provider is the free Gemini tier. Get a key at https://aistudio.google.com/apikey
+GEMINI_API_KEY=
+# Optional switchable fallback. Leave blank to disable.
+OPENAI_API_KEY=
+
+# Which provider to use by default: "gemini" or "openai"
+DEFAULT_LLM_PROVIDER=gemini
+
+# Model names (override only if you know what you want)
+GEMINI_MODEL=gemini-2.0-flash
+OPENAI_MODEL=gpt-4o-mini
+
+# Embedding model for RAG (Gemini, free)
+GEMINI_EMBED_MODEL=text-embedding-004
+
+# --- Storage ---
+DATA_DIR=./data
+CHROMA_DIR=./data/chroma
+
+# --- Server ---
+HOST=0.0.0.0
+PORT=8000
+# Comma-separated allowed origins for CORS (frontend dev server)
+CORS_ORIGINS=http://localhost:3000
+
+# --- Extraction ---
+# Max pages to render to images for the viewer / vision fallback
+MAX_RENDER_PAGES=30
+# Run Docling OCR on scanned PDFs/images (slower but handles scans)
+ENABLE_OCR=true

backend/Dockerfile

@@ -0,0 +1,61 @@
+# syntax=docker/dockerfile:1
+# ---------------------------------------------------------------------------
+# Production image for the Document-Agent FastAPI backend.
+#
+# It bundles Docling + (CPU) torch + the layout/table/OCR models so the
+# container is self-contained: no model downloads at runtime, fast cold start,
+# works offline. On Linux the HuggingFace symlink cache works normally, so the
+# Windows WinError-1314 issue does not apply here.
+# ---------------------------------------------------------------------------
+FROM python:3.12-slim
+
+ENV PYTHONUNBUFFERED=1 \
+    PYTHONDONTWRITEBYTECODE=1 \
+    PIP_NO_CACHE_DIR=1 \
+    PIP_DISABLE_PIP_VERSION_CHECK=1 \
+    HF_HUB_DISABLE_SYMLINKS_WARNING=1 \
+    # Fixed, shared cache locations so the non-root runtime user finds the
+    # models baked in during build.
+    HF_HOME=/opt/models/hf \
+    EASYOCR_MODULE_PATH=/opt/models/easyocr \
+    TORCH_HOME=/opt/models/torch \
+    DATA_DIR=/data
+
+# Native libs needed by docling / opencv / easyocr.
+RUN apt-get update && apt-get install -y --no-install-recommends \
+        libgl1 \
+        libglib2.0-0 \
+        libgomp1 \
+    && rm -rf /var/lib/apt/lists/*
+
+WORKDIR /app
+
+# Install CPU-only torch/torchvision FIRST so docling's torch dependency
+# resolves to the lightweight CPU wheels instead of multi-GB CUDA builds.
+RUN pip install --index-url https://download.pytorch.org/whl/cpu \
+        torch torchvision
+
+COPY requirements.txt .
+RUN pip install -r requirements.txt
+
+# Bake the models into the image.
+COPY scripts/prefetch_models.py scripts/prefetch_models.py
+RUN mkdir -p /opt/models && python scripts/prefetch_models.py
+
+# App source.
+COPY . .
+
+# Run as a non-root user; /data holds uploads, sqlite db and rendered pages and
+# is expected to be a mounted persistent volume in production.
+RUN useradd --create-home app \
+    && mkdir -p /data \
+    && chown -R app:app /data /app /opt/models
+USER app
+
+EXPOSE 8000
+
+# Honour $PORT (Render/most PaaS inject it); default 8000 for local/compose.
+HEALTHCHECK --interval=30s --timeout=5s --start-period=60s --retries=3 \
+    CMD python -c "import os,urllib.request,sys; sys.exit(0 if urllib.request.urlopen('http://localhost:%s/api/health' % os.environ.get('PORT','8000')).status==200 else 1)"
+
+CMD ["sh", "-c", "uvicorn app.main:app --host 0.0.0.0 --port ${PORT:-8000}"]

backend/app/agent/orchestrator.py

@@ -0,0 +1,92 @@
+"""The orchestrator agent — a tool-calling loop.
+
+Implements the agentic pattern: the LLM decides which tools to call, we run
+them, feed results back, and iterate until the model produces a final answer.
+Then we stream that final answer to the client token-by-token.
+
+Keeping a single orchestrator with well-scoped tools (rather than a
+multi-agent swarm) matches the research recommendation: simplest design that
+covers the 7 functions, with room to grow.
+"""
+from __future__ import annotations
+
+from typing import AsyncIterator
+
+from app.agent.prompts import ORCHESTRATOR_SYSTEM
+from app.agent.tools import TOOL_FNS, TOOL_SPECS, ToolContext
+from app.core.logging import get_logger
+from app.llm.base import LLMMessage, ToolCall
+from app.llm.registry import get_provider
+from app.schemas.documents import ChatMessage, Citation
+
+log = get_logger(__name__)
+
+MAX_TOOL_ROUNDS = 4
+
+
+def _build_messages(history: list[ChatMessage], doc_context: str) -> list[LLMMessage]:
+    msgs = [LLMMessage(role="system", content=ORCHESTRATOR_SYSTEM)]
+    if doc_context:
+        msgs.append(LLMMessage(role="system", content=doc_context))
+    for m in history:
+        msgs.append(LLMMessage(role=m.role.value, content=m.content))
+    return msgs
+
+
+async def _run_tool_rounds(
+    llm, msgs: list[LLMMessage], ctx: ToolContext
+) -> list[LLMMessage]:
+    """Let the model call tools until it's ready to answer (bounded)."""
+    for _ in range(MAX_TOOL_ROUNDS):
+        resp = await llm.complete_tools(msgs, TOOL_SPECS)
+        if not resp.tool_calls:
+            # model is ready to answer in prose; drop back to streaming
+            return msgs
+        # record the assistant's tool-call turn
+        msgs.append(LLMMessage(role="assistant", content=resp.text,
+                               tool_calls=resp.tool_calls))
+        for call in resp.tool_calls:
+            fn = TOOL_FNS.get(call.name)
+            if fn is None:
+                result = f"Unknown tool: {call.name}"
+            else:
+                try:
+                    result = await fn(ctx, call.arguments)
+                except Exception as e:  # pragma: no cover
+                    log.exception("tool %s failed", call.name)
+                    result = f"Tool error: {e}"
+            msgs.append(LLMMessage(role="tool", content=result,
+                                   tool_call_id=call.id, name=call.name))
+    return msgs
+
+
+async def run_chat(
+    history: list[ChatMessage],
+    document_id: str | None,
+    doc_context: str = "",
+    provider: str | None = None,
+) -> tuple[AsyncIterator[str], ToolContext]:
+    """Run the agent and return a streaming token iterator + tool context.
+
+    The caller streams the tokens to the client and, when done, reads
+    ctx.citations to emit grounding metadata.
+    """
+    llm = get_provider(provider)
+    ctx = ToolContext(document_id=document_id, provider=provider)
+    msgs = _build_messages(history, doc_context)
+
+    # Phase 1: tool-calling (non-streamed reasoning)
+    msgs = await _run_tool_rounds(llm, msgs, ctx)
+
+    # Phase 2: final answer, streamed
+    msgs.append(LLMMessage(
+        role="system",
+        content="Now write the final answer for the user based on the tool "
+                "results above. Do not call more tools.",
+    ))
+
+    async def stream() -> AsyncIterator[str]:
+        async for token in llm.stream(msgs, temperature=0.4):
+            yield token
+
+    return stream(), ctx

backend/app/agent/prompts.py

@@ -0,0 +1,21 @@
+"""System prompts for the orchestrator agent."""
+
+ORCHESTRATOR_SYSTEM = """You are DocAgent, a meticulous document-processing assistant for Monkhub Innovations.
+
+You help users work with an uploaded document. You can:
+- answer questions about its contents (always ground answers in the document),
+- classify the document type,
+- extract structured fields, tables, and entities,
+- summarize the document,
+- flag missing fields, inconsistencies, or anomalies,
+- explain extracted data and guide the user to export it.
+
+Guidelines:
+- When the user asks something that needs the document's content, call the
+  `query_document` tool to retrieve grounded passages before answering.
+- Use `get_extracted_data` to discuss already-extracted fields rather than guessing.
+- Prefer tools over assumptions. If a tool returns citations, reference them.
+- Be concise, professional, and helpful. Use Markdown. When you state a fact
+  from the document, make clear it is grounded in the source.
+- If no document is loaded yet, ask the user to upload one.
+"""

backend/app/agent/tools.py

@@ -0,0 +1,139 @@
+"""Agent tools — the capabilities the orchestrator can call.
+
+Each tool is a thin async wrapper over a service, plus a `ToolSpec`
+(JSON-schema description) the LLM sees. Tools are document-scoped: the
+orchestrator passes the active document id in via a closure context.
+"""
+from __future__ import annotations
+
+import json
+from dataclasses import dataclass
+from typing import Any, Awaitable, Callable
+
+from app.core.logging import get_logger
+from app.llm.base import ToolSpec
+from app.services import classification, extraction, qa, storage, summary
+from app.services import anomaly as anomaly_svc
+from app.services import vectorstore
+
+log = get_logger(__name__)
+
+
+@dataclass
+class ToolContext:
+    document_id: str | None
+    provider: str | None = None
+    # collected so the API can surface citations from the last query
+    citations: list = None  # type: ignore
+
+    def __post_init__(self):
+        if self.citations is None:
+            self.citations = []
+
+
+ToolFn = Callable[[ToolContext, dict[str, Any]], Awaitable[str]]
+
+
+# -------------------------- tool implementations --------------------------
+async def _query_document(ctx: ToolContext, args: dict) -> str:
+    if not ctx.document_id:
+        return "No document is currently loaded."
+    question = args.get("question", "")
+    res = await qa.answer(ctx.document_id, question, provider=ctx.provider)
+    ctx.citations.extend(res.citations)
+    cites = "\n".join(
+        f"[{i+1}] (page {c.page+1}) {c.text[:160]}" for i, c in enumerate(res.citations)
+    )
+    return f"{res.answer}\n\nGrounding passages:\n{cites}"
+
+
+async def _classify(ctx: ToolContext, args: dict) -> str:
+    detail = storage.get(ctx.document_id) if ctx.document_id else None
+    if not detail:
+        return "No document loaded."
+    if detail.classification:
+        c = detail.classification
+        return f"Document type: {c.doc_type} (confidence {c.confidence:.0%}). {c.rationale}"
+    c = await classification.classify(detail.markdown or "", ctx.provider)
+    detail.classification = c
+    storage.save(detail)
+    return f"Document type: {c.doc_type} (confidence {c.confidence:.0%}). {c.rationale}"
+
+
+async def _get_extracted_data(ctx: ToolContext, args: dict) -> str:
+    detail = storage.get(ctx.document_id) if ctx.document_id else None
+    if not detail or not detail.extraction:
+        return "No extracted data yet."
+    fields = {f.name: f.value for f in detail.extraction.fields}
+    return json.dumps({
+        "schema": detail.extraction.schema_name,
+        "fields": fields,
+        "tables": [t.title or "table" for t in detail.extraction.tables],
+        "entities": [{"type": e.type, "value": e.value} for e in detail.extraction.entities],
+    }, default=str)
+
+
+async def _summarize(ctx: ToolContext, args: dict) -> str:
+    detail = storage.get(ctx.document_id) if ctx.document_id else None
+    if not detail:
+        return "No document loaded."
+    if detail.summary:
+        return detail.summary
+    s = await summary.summarize(detail.markdown or "", ctx.provider)
+    detail.summary = s
+    storage.save(detail)
+    return s
+
+
+async def _flag_anomalies(ctx: ToolContext, args: dict) -> str:
+    detail = storage.get(ctx.document_id) if ctx.document_id else None
+    if not detail or not detail.extraction:
+        return "No extracted data to check."
+    doc_type = detail.classification.doc_type if detail.classification else None
+    items = await anomaly_svc.detect(detail.markdown or "", detail.extraction,
+                                     doc_type, ctx.provider)
+    if not items:
+        return "No anomalies detected. The document looks consistent."
+    return "\n".join(f"- [{a.severity.value}] {a.field or ''}: {a.message}" for a in items)
+
+
+# ----------------------------- registry -----------------------------------
+TOOL_SPECS: list[ToolSpec] = [
+    ToolSpec(
+        name="query_document",
+        description="Retrieve grounded passages from the document and answer a question about its contents.",
+        parameters={
+            "type": "object",
+            "properties": {"question": {"type": "string", "description": "The question to answer"}},
+            "required": ["question"],
+        },
+    ),
+    ToolSpec(
+        name="classify_document",
+        description="Get the document type/category with confidence.",
+        parameters={"type": "object", "properties": {}},
+    ),
+    ToolSpec(
+        name="get_extracted_data",
+        description="Return the structured fields, tables, and entities already extracted from the document.",
+        parameters={"type": "object", "properties": {}},
+    ),
+    ToolSpec(
+        name="summarize_document",
+        description="Produce or fetch a concise summary of the document.",
+        parameters={"type": "object", "properties": {}},
+    ),
+    ToolSpec(
+        name="flag_anomalies",
+        description="Check for missing required fields, low-confidence values, and inconsistencies.",
+        parameters={"type": "object", "properties": {}},
+    ),
+]
+
+TOOL_FNS: dict[str, ToolFn] = {
+    "query_document": _query_document,
+    "classify_document": _classify,
+    "get_extracted_data": _get_extracted_data,
+    "summarize_document": _summarize,
+    "flag_anomalies": _flag_anomalies,
+}

backend/app/api/actions.py

@@ -0,0 +1,88 @@
+"""On-demand actions and export endpoints.
+
+These let the frontend trigger individual functions (re-extract with a chosen
+schema, re-summarize) and download results — independent of the chat agent.
+"""
+from __future__ import annotations
+
+from fastapi import APIRouter, HTTPException, Query
+from fastapi.responses import Response
+
+from app.core.logging import get_logger
+from app.schemas.documents import Classification, ExtractionResult
+from app.schemas.templates import all_template_names
+from app.services import (
+    anomaly,
+    classification,
+    export,
+    extraction,
+    storage,
+    summary,
+    vectorstore,
+)
+
+log = get_logger(__name__)
+router = APIRouter(prefix="/api/documents", tags=["actions"])
+
+
+@router.post("/{doc_id}/extract", response_model=ExtractionResult)
+async def re_extract(doc_id: str, schema: str | None = Query(None),
+                     provider: str | None = Query(None)):
+    detail = storage.get(doc_id)
+    if not detail:
+        raise HTTPException(404, "Document not found")
+    chunks = await vectorstore.query(doc_id, "key fields", k=40)
+    doc_type = detail.classification.doc_type if detail.classification else None
+    # `schema` is an OPTIONAL user-chosen framing; "auto" (or unset) keeps the
+    # extractor fully adaptive.
+    hint = schema if (schema and schema.lower() != "auto") else None
+    result = await extraction.extract(detail.markdown or "", doc_type, chunks, provider,
+                                      schema_hint=hint)
+    detail.extraction = result
+    detail.anomalies = await anomaly.detect(detail.markdown or "", result, doc_type, provider)
+    storage.save(detail)
+    return result
+
+
+@router.post("/{doc_id}/classify", response_model=Classification)
+async def re_classify(doc_id: str, provider: str | None = Query(None)):
+    detail = storage.get(doc_id)
+    if not detail:
+        raise HTTPException(404, "Document not found")
+    c = await classification.classify(detail.markdown or "", provider)
+    detail.classification = c
+    storage.save(detail)
+    return c
+
+
+@router.post("/{doc_id}/summarize")
+async def re_summarize(doc_id: str, provider: str | None = Query(None)):
+    detail = storage.get(doc_id)
+    if not detail:
+        raise HTTPException(404, "Document not found")
+    s = await summary.summarize(detail.markdown or "", provider)
+    detail.summary = s
+    storage.save(detail)
+    return {"summary": s}
+
+
+@router.get("/{doc_id}/export")
+async def export_document(doc_id: str, format: str = Query("json")):
+    detail = storage.get(doc_id)
+    if not detail:
+        raise HTTPException(404, "Document not found")
+    fmt = format.lower()
+    if fmt not in export.EXPORTERS:
+        raise HTTPException(400, f"Unsupported format. Use one of {list(export.EXPORTERS)}")
+    fn, media, ext = export.EXPORTERS[fmt]
+    data = fn(detail)
+    safe = (detail.filename or doc_id).rsplit(".", 1)[0]
+    return Response(
+        content=data, media_type=media,
+        headers={"Content-Disposition": f'attachment; filename="{safe}.{ext}"'},
+    )
+
+
+@router.get("/-/schemas")
+async def list_schemas():
+    return {"schemas": all_template_names()}

backend/app/api/chat.py

@@ -0,0 +1,75 @@
+"""Chat endpoint with Server-Sent Events streaming.
+
+Streams the agent's answer token-by-token, then a final `citations` event so
+the frontend can highlight grounded passages on the document.
+"""
+from __future__ import annotations
+
+import json
+
+from fastapi import APIRouter, HTTPException
+from fastapi.responses import StreamingResponse
+
+from app.agent.orchestrator import run_chat
+from app.core.logging import get_logger
+from app.schemas.documents import ChatRequest
+from app.services import storage
+
+log = get_logger(__name__)
+router = APIRouter(prefix="/api/chat", tags=["chat"])
+
+
+def _sse(event: str, data) -> str:
+    return f"event: {event}\ndata: {json.dumps(data, default=str)}\n\n"
+
+
+def _doc_context(document_id: str | None) -> str:
+    if not document_id:
+        return ""
+    d = storage.get(document_id)
+    if not d:
+        return ""
+    parts = [f"Active document: '{d.filename}' (id={d.id}), status={d.status.value}, "
+             f"{d.num_pages} pages."]
+    if d.classification:
+        parts.append(f"Detected type: {d.classification.doc_type} "
+                     f"({d.classification.confidence:.0%}).")
+    if d.extraction and d.extraction.fields:
+        kv = ", ".join(f"{f.name}={f.value}" for f in d.extraction.fields[:12])
+        parts.append(f"Already-extracted fields: {kv}")
+    return " ".join(parts)
+
+
+@router.post("")
+async def chat(req: ChatRequest):
+    if not req.messages:
+        raise HTTPException(400, "messages cannot be empty")
+
+    async def gen():
+        try:
+            stream, ctx = await run_chat(
+                history=req.messages,
+                document_id=req.document_id,
+                doc_context=_doc_context(req.document_id),
+                provider=req.provider,
+            )
+            yield _sse("start", {"ok": True})
+            async for token in stream:
+                yield _sse("token", {"text": token})
+            # de-dup citations by chunk_id
+            seen, cites = set(), []
+            for c in ctx.citations:
+                key = c.chunk_id or c.text[:40]
+                if key in seen:
+                    continue
+                seen.add(key)
+                cites.append(c.model_dump())
+            yield _sse("citations", {"citations": cites})
+            yield _sse("done", {"ok": True})
+        except Exception as e:  # pragma: no cover
+            log.exception("chat failed")
+            yield _sse("error", {"message": str(e)})
+
+    return StreamingResponse(gen(), media_type="text/event-stream",
+                             headers={"Cache-Control": "no-cache",
+                                      "X-Accel-Buffering": "no"})

backend/app/api/documents.py

@@ -0,0 +1,79 @@
+"""Document endpoints: upload, list, detail, page images, re-process, delete."""
+from __future__ import annotations
+
+import uuid
+from datetime import datetime, timezone
+
+from fastapi import APIRouter, BackgroundTasks, File, HTTPException, UploadFile
+from fastapi.responses import FileResponse
+
+from app.core.config import settings
+from app.core.logging import get_logger
+from app.schemas.documents import DocStatus, DocumentDetail, DocumentMeta
+from app.services import storage, vectorstore
+from app.services.pipeline import process_document
+
+log = get_logger(__name__)
+router = APIRouter(prefix="/api/documents", tags=["documents"])
+
+ALLOWED = {
+    ".pdf", ".docx", ".doc", ".pptx", ".xlsx", ".html", ".md", ".txt",
+    ".png", ".jpg", ".jpeg", ".tiff", ".bmp",
+}
+
+
+@router.post("", response_model=DocumentMeta)
+async def upload(background: BackgroundTasks, file: UploadFile = File(...)):
+    ext = "." + (file.filename or "").rsplit(".", 1)[-1].lower() if "." in (file.filename or "") else ""
+    if ext not in ALLOWED:
+        raise HTTPException(400, f"Unsupported file type '{ext}'. Allowed: {sorted(ALLOWED)}")
+
+    doc_id = uuid.uuid4().hex[:12]
+    dest = settings.uploads_path / f"{doc_id}{ext}"
+    data = await file.read()
+    dest.write_bytes(data)
+
+    detail = DocumentDetail(
+        id=doc_id,
+        filename=file.filename or dest.name,
+        content_type=file.content_type or "application/octet-stream",
+        size_bytes=len(data),
+        status=DocStatus.uploaded,
+        created_at=datetime.now(timezone.utc).isoformat(),
+    )
+    storage.save(detail)
+
+    # kick off async processing
+    background.add_task(process_document, doc_id, dest)
+    log.info("Uploaded %s (%d bytes) -> %s", file.filename, len(data), doc_id)
+    return DocumentMeta.model_validate(detail.model_dump())
+
+
+@router.get("", response_model=list[DocumentMeta])
+async def list_documents():
+    return storage.list_all()
+
+
+@router.get("/{doc_id}", response_model=DocumentDetail)
+async def get_document(doc_id: str):
+    detail = storage.get(doc_id)
+    if not detail:
+        raise HTTPException(404, "Document not found")
+    return detail
+
+
+@router.get("/{doc_id}/pages/{page}.png")
+async def get_page_image(doc_id: str, page: int):
+    path = settings.renders_path / doc_id / f"page-{page}.png"
+    if not path.exists():
+        raise HTTPException(404, "Page image not found")
+    return FileResponse(path, media_type="image/png")
+
+
+@router.delete("/{doc_id}")
+async def delete_document(doc_id: str):
+    if not storage.get(doc_id):
+        raise HTTPException(404, "Document not found")
+    vectorstore.delete(doc_id)
+    storage.delete(doc_id)
+    return {"ok": True}

backend/app/api/meta.py

@@ -0,0 +1,30 @@
+"""Health and metadata endpoints."""
+from __future__ import annotations
+
+from fastapi import APIRouter
+
+from app.core.config import settings
+from app.llm.registry import available_providers
+
+router = APIRouter(prefix="/api", tags=["meta"])
+
+
+@router.get("/health")
+async def health():
+    providers = available_providers()
+    return {
+        "status": "ok",
+        "providers": providers,
+        "default_provider": settings.default_llm_provider,
+        "llm_configured": bool(providers),
+        "ocr_enabled": settings.enable_ocr,
+    }
+
+
+@router.get("/config")
+async def config():
+    return {
+        "providers": available_providers(),
+        "default_provider": settings.default_llm_provider,
+        "models": {"gemini": settings.gemini_model, "openai": settings.openai_model},
+    }

backend/app/core/config.py

@@ -0,0 +1,97 @@
+"""Application configuration, loaded from environment / .env.
+
+Single source of truth for runtime settings. Routes and services read
+settings from here only — never from os.environ directly — so behaviour
+is consistent and testable.
+"""
+from __future__ import annotations
+
+from functools import lru_cache
+from pathlib import Path
+
+from pydantic import field_validator
+from pydantic_settings import BaseSettings, SettingsConfigDict
+
+
+class Settings(BaseSettings):
+    model_config = SettingsConfigDict(
+        env_file=".env", env_file_encoding="utf-8", extra="ignore"
+    )
+
+    # --- LLM providers ---
+    gemini_api_key: str = ""
+    openai_api_key: str = ""
+    default_llm_provider: str = "gemini"  # "gemini" | "openai"
+    gemini_model: str = "gemini-2.0-flash"
+    openai_model: str = "gpt-4o-mini"
+    gemini_embed_model: str = "text-embedding-004"
+
+    # --- Storage ---
+    data_dir: str = "./data"
+    chroma_dir: str = "./data/chroma"
+
+    # --- Server ---
+    host: str = "0.0.0.0"
+    port: int = 8000
+    cors_origins: str = "http://localhost:3000"
+
+    # --- Extraction ---
+    max_render_pages: int = 30
+    enable_ocr: bool = True
+
+    # Secrets pasted into host dashboards (Render/HF/Vercel) very often arrive
+    # with a trailing newline or stray whitespace. An API key with a "\n" makes
+    # the "Authorization: Bearer <key>\n" header illegal, which httpx rejects
+    # locally and the OpenAI SDK re-wraps as a misleading APIConnectionError.
+    # Strip them so a sloppy copy-paste can never break auth.
+    @field_validator(
+        "gemini_api_key", "openai_api_key", "default_llm_provider",
+        "gemini_model", "openai_model", "gemini_embed_model",
+        mode="after",
+    )
+    @classmethod
+    def _strip(cls, v: str) -> str:
+        return v.strip() if isinstance(v, str) else v
+
+    # ----- derived helpers -----
+    @property
+    def cors_origin_list(self) -> list[str]:
+        return [o.strip() for o in self.cors_origins.split(",") if o.strip()]
+
+    @property
+    def data_path(self) -> Path:
+        p = Path(self.data_dir)
+        p.mkdir(parents=True, exist_ok=True)
+        return p
+
+    @property
+    def uploads_path(self) -> Path:
+        p = self.data_path / "uploads"
+        p.mkdir(parents=True, exist_ok=True)
+        return p
+
+    @property
+    def renders_path(self) -> Path:
+        p = self.data_path / "renders"
+        p.mkdir(parents=True, exist_ok=True)
+        return p
+
+    @property
+    def db_path(self) -> Path:
+        return self.data_path / "app.db"
+
+    @property
+    def has_gemini(self) -> bool:
+        return bool(self.gemini_api_key)
+
+    @property
+    def has_openai(self) -> bool:
+        return bool(self.openai_api_key)
+
+
+@lru_cache
+def get_settings() -> Settings:
+    return Settings()
+
+
+settings = get_settings()

backend/app/core/logging.py

@@ -0,0 +1,26 @@
+"""Minimal structured logging setup."""
+from __future__ import annotations
+
+import logging
+import sys
+
+_CONFIGURED = False
+
+
+def setup_logging(level: int = logging.INFO) -> None:
+    global _CONFIGURED
+    if _CONFIGURED:
+        return
+    handler = logging.StreamHandler(sys.stdout)
+    handler.setFormatter(
+        logging.Formatter("%(asctime)s | %(levelname)-7s | %(name)s | %(message)s")
+    )
+    root = logging.getLogger()
+    root.setLevel(level)
+    root.addHandler(handler)
+    _CONFIGURED = True
+
+
+def get_logger(name: str) -> logging.Logger:
+    setup_logging()
+    return logging.getLogger(name)

backend/app/llm/base.py

@@ -0,0 +1,110 @@
+"""LLM provider abstraction.
+
+All model access in the app goes through the `LLMProvider` interface so that
+the rest of the codebase never imports a vendor SDK directly. This is what
+makes the default(Gemini)/fallback(OpenAI) switch a one-line change and keeps
+the agent provider-agnostic.
+
+A provider must support three capabilities the agent relies on:
+  * complete()        -> plain text (optionally streaming)
+  * complete_json()   -> JSON object validated against an expected shape
+  * complete_tools()  -> tool/function calling for the agent loop
+  * embed()           -> vector embeddings for RAG
+  * vision()          -> read an image (scanned-page fallback)
+"""
+from __future__ import annotations
+
+import abc
+from dataclasses import dataclass, field
+from typing import Any, AsyncIterator, Iterable, Optional
+
+
+@dataclass
+class ToolSpec:
+    """Vendor-neutral description of a callable tool exposed to the model."""
+
+    name: str
+    description: str
+    parameters: dict[str, Any]  # JSON Schema for the arguments
+
+
+@dataclass
+class ToolCall:
+    id: str
+    name: str
+    arguments: dict[str, Any]
+
+
+@dataclass
+class LLMMessage:
+    role: str  # system | user | assistant | tool
+    content: str = ""
+    tool_calls: list[ToolCall] = field(default_factory=list)
+    tool_call_id: Optional[str] = None  # for role == "tool"
+    name: Optional[str] = None
+
+
+@dataclass
+class LLMResponse:
+    text: str = ""
+    tool_calls: list[ToolCall] = field(default_factory=list)
+    finish_reason: str = "stop"
+
+
+class LLMError(RuntimeError):
+    pass
+
+
+class LLMProvider(abc.ABC):
+    """Interface every concrete provider implements."""
+
+    name: str = "base"
+    model: str = ""
+
+    @abc.abstractmethod
+    async def complete(
+        self,
+        messages: list[LLMMessage],
+        *,
+        temperature: float = 0.2,
+        max_tokens: Optional[int] = None,
+    ) -> str:
+        ...
+
+    @abc.abstractmethod
+    async def stream(
+        self,
+        messages: list[LLMMessage],
+        *,
+        temperature: float = 0.4,
+    ) -> AsyncIterator[str]:
+        ...
+        yield  # pragma: no cover
+
+    @abc.abstractmethod
+    async def complete_json(
+        self,
+        messages: list[LLMMessage],
+        *,
+        temperature: float = 0.0,
+    ) -> Any:
+        """Return parsed JSON. Providers should request JSON output mode."""
+        ...
+
+    @abc.abstractmethod
+    async def complete_tools(
+        self,
+        messages: list[LLMMessage],
+        tools: list[ToolSpec],
+        *,
+        temperature: float = 0.2,
+    ) -> LLMResponse:
+        ...
+
+    @abc.abstractmethod
+    async def embed(self, texts: Iterable[str]) -> list[list[float]]:
+        ...
+
+    async def vision(self, prompt: str, image_paths: list[str]) -> str:
+        """Optional capability; override in providers that support images."""
+        raise LLMError(f"{self.name} does not implement vision()")

backend/app/llm/gemini_provider.py

@@ -0,0 +1,157 @@
+"""Google Gemini provider (default, free tier).
+
+Uses google-generativeai. Chosen as default because of the generous free
+tier, ~1M token context (great for long documents), and native multimodal
+vision (lets us read scanned pages directly).
+"""
+from __future__ import annotations
+
+import json
+from typing import Any, AsyncIterator, Iterable, Optional
+
+from tenacity import retry, stop_after_attempt, wait_exponential
+
+from app.core.logging import get_logger
+from app.llm.base import (
+    LLMError,
+    LLMMessage,
+    LLMProvider,
+    LLMResponse,
+    ToolCall,
+    ToolSpec,
+)
+
+log = get_logger(__name__)
+
+try:
+    import google.generativeai as genai
+except Exception:  # pragma: no cover - import guarded for environments w/o dep
+    genai = None
+
+
+def _to_gemini_history(messages: list[LLMMessage]) -> tuple[Optional[str], list[dict]]:
+    """Convert neutral messages to Gemini's (system_instruction, contents)."""
+    system = None
+    contents: list[dict] = []
+    for m in messages:
+        if m.role == "system":
+            system = (system + "\n" + m.content) if system else m.content
+            continue
+        role = "user" if m.role in ("user", "tool") else "model"
+        text = m.content
+        if m.role == "tool":
+            text = f"[tool result for {m.name}]\n{m.content}"
+        contents.append({"role": role, "parts": [text]})
+    return system, contents
+
+
+class GeminiProvider(LLMProvider):
+    name = "gemini"
+
+    def __init__(self, api_key: str, model: str, embed_model: str):
+        if genai is None:
+            raise LLMError("google-generativeai is not installed")
+        if not api_key:
+            raise LLMError("GEMINI_API_KEY is not set")
+        genai.configure(api_key=api_key)
+        self.model = model
+        self.embed_model = embed_model
+        self._api_key = api_key
+
+    def _model(self, system: Optional[str] = None, json_mode: bool = False,
+               tools: Optional[list] = None):
+        cfg: dict[str, Any] = {}
+        if json_mode:
+            cfg["response_mime_type"] = "application/json"
+        return genai.GenerativeModel(
+            self.model,
+            system_instruction=system,
+            generation_config=cfg or None,
+            tools=tools,
+        )
+
+    @retry(stop=stop_after_attempt(3), wait=wait_exponential(min=1, max=8))
+    async def complete(self, messages, *, temperature=0.2, max_tokens=None) -> str:
+        system, contents = _to_gemini_history(messages)
+        model = self._model(system)
+        resp = await model.generate_content_async(
+            contents,
+            generation_config={"temperature": temperature,
+                               **({"max_output_tokens": max_tokens} if max_tokens else {})},
+        )
+        return (resp.text or "").strip()
+
+    async def stream(self, messages, *, temperature=0.4) -> AsyncIterator[str]:
+        system, contents = _to_gemini_history(messages)
+        model = self._model(system)
+        resp = await model.generate_content_async(
+            contents, generation_config={"temperature": temperature}, stream=True
+        )
+        async for chunk in resp:
+            if getattr(chunk, "text", None):
+                yield chunk.text
+
+    @retry(stop=stop_after_attempt(3), wait=wait_exponential(min=1, max=8))
+    async def complete_json(self, messages, *, temperature=0.0) -> Any:
+        system, contents = _to_gemini_history(messages)
+        model = self._model(system, json_mode=True)
+        resp = await model.generate_content_async(
+            contents, generation_config={"temperature": temperature}
+        )
+        raw = (resp.text or "").strip()
+        try:
+            return json.loads(raw)
+        except json.JSONDecodeError:
+            # last-resort: strip code fences and retry
+            cleaned = raw.strip().lstrip("`json").rstrip("`").strip()
+            return json.loads(cleaned)
+
+    async def complete_tools(self, messages, tools, *, temperature=0.2) -> LLMResponse:
+        system, contents = _to_gemini_history(messages)
+        gem_tools = [{
+            "function_declarations": [
+                {"name": t.name, "description": t.description, "parameters": t.parameters}
+                for t in tools
+            ]
+        }]
+        model = self._model(system, tools=gem_tools)
+        resp = await model.generate_content_async(
+            contents, generation_config={"temperature": temperature}
+        )
+        calls: list[ToolCall] = []
+        text_parts: list[str] = []
+        try:
+            for part in resp.candidates[0].content.parts:
+                fc = getattr(part, "function_call", None)
+                if fc and fc.name:
+                    args = dict(fc.args) if fc.args else {}
+                    calls.append(ToolCall(id=fc.name, name=fc.name, arguments=args))
+                elif getattr(part, "text", None):
+                    text_parts.append(part.text)
+        except (IndexError, AttributeError):
+            pass
+        return LLMResponse(
+            text="".join(text_parts).strip(),
+            tool_calls=calls,
+            finish_reason="tool_calls" if calls else "stop",
+        )
+
+    @retry(stop=stop_after_attempt(3), wait=wait_exponential(min=1, max=8))
+    async def embed(self, texts: Iterable[str]) -> list[list[float]]:
+        out: list[list[float]] = []
+        for t in texts:
+            r = await genai.embed_content_async(
+                model=self.embed_model, content=t, task_type="retrieval_document"
+            )
+            out.append(r["embedding"])
+        return out
+
+    async def vision(self, prompt: str, image_paths: list[str]) -> str:
+        import PIL.Image  # docling/pillow already pulls this in
+
+        model = self._model()
+        parts: list[Any] = [prompt]
+        for p in image_paths:
+            parts.append(PIL.Image.open(p))
+        resp = await model.generate_content_async(parts)
+        return (resp.text or "").strip()

backend/app/llm/openai_provider.py

@@ -0,0 +1,140 @@
+"""OpenAI provider (switchable fallback, GPT-4o-mini).
+
+Cheap, reliable tool-calling, 128k context. Used when DEFAULT_LLM_PROVIDER
+is "openai" or per-request override, or as automatic fallback if Gemini
+fails and an OpenAI key is configured.
+"""
+from __future__ import annotations
+
+import json
+from typing import Any, AsyncIterator, Iterable, Optional
+
+from tenacity import retry, stop_after_attempt, wait_exponential
+
+from app.core.logging import get_logger
+from app.llm.base import (
+    LLMError,
+    LLMMessage,
+    LLMProvider,
+    LLMResponse,
+    ToolCall,
+    ToolSpec,
+)
+
+log = get_logger(__name__)
+
+try:
+    from openai import AsyncOpenAI
+except Exception:  # pragma: no cover
+    AsyncOpenAI = None
+
+
+def _to_openai_messages(messages: list[LLMMessage]) -> list[dict]:
+    out: list[dict] = []
+    for m in messages:
+        if m.role == "tool":
+            out.append({
+                "role": "tool",
+                "tool_call_id": m.tool_call_id or m.name or "tool",
+                "content": m.content,
+            })
+        elif m.role == "assistant" and m.tool_calls:
+            out.append({
+                "role": "assistant",
+                "content": m.content or None,
+                "tool_calls": [
+                    {
+                        "id": tc.id,
+                        "type": "function",
+                        "function": {"name": tc.name, "arguments": json.dumps(tc.arguments)},
+                    }
+                    for tc in m.tool_calls
+                ],
+            })
+        else:
+            out.append({"role": m.role, "content": m.content})
+    return out
+
+
+class OpenAIProvider(LLMProvider):
+    name = "openai"
+
+    def __init__(self, api_key: str, model: str, embed_model: str = "text-embedding-3-small"):
+        if AsyncOpenAI is None:
+            raise LLMError("openai package is not installed")
+        if not api_key:
+            raise LLMError("OPENAI_API_KEY is not set")
+        self.client = AsyncOpenAI(api_key=api_key)
+        self.model = model
+        self.embed_model = embed_model
+
+    @retry(stop=stop_after_attempt(3), wait=wait_exponential(min=1, max=8))
+    async def complete(self, messages, *, temperature=0.2, max_tokens=None) -> str:
+        resp = await self.client.chat.completions.create(
+            model=self.model,
+            messages=_to_openai_messages(messages),
+            temperature=temperature,
+            max_tokens=max_tokens,
+        )
+        return (resp.choices[0].message.content or "").strip()
+
+    async def stream(self, messages, *, temperature=0.4) -> AsyncIterator[str]:
+        stream = await self.client.chat.completions.create(
+            model=self.model,
+            messages=_to_openai_messages(messages),
+            temperature=temperature,
+            stream=True,
+        )
+        async for chunk in stream:
+            delta = chunk.choices[0].delta.content
+            if delta:
+                yield delta
+
+    @retry(stop=stop_after_attempt(3), wait=wait_exponential(min=1, max=8))
+    async def complete_json(self, messages, *, temperature=0.0) -> Any:
+        resp = await self.client.chat.completions.create(
+            model=self.model,
+            messages=_to_openai_messages(messages),
+            temperature=temperature,
+            response_format={"type": "json_object"},
+        )
+        return json.loads(resp.choices[0].message.content or "{}")
+
+    async def complete_tools(self, messages, tools, *, temperature=0.2) -> LLMResponse:
+        oa_tools = [
+            {
+                "type": "function",
+                "function": {
+                    "name": t.name,
+                    "description": t.description,
+                    "parameters": t.parameters,
+                },
+            }
+            for t in tools
+        ]
+        resp = await self.client.chat.completions.create(
+            model=self.model,
+            messages=_to_openai_messages(messages),
+            tools=oa_tools,
+            temperature=temperature,
+        )
+        msg = resp.choices[0].message
+        calls: list[ToolCall] = []
+        for tc in (msg.tool_calls or []):
+            try:
+                args = json.loads(tc.function.arguments or "{}")
+            except json.JSONDecodeError:
+                args = {}
+            calls.append(ToolCall(id=tc.id, name=tc.function.name, arguments=args))
+        return LLMResponse(
+            text=(msg.content or "").strip(),
+            tool_calls=calls,
+            finish_reason="tool_calls" if calls else "stop",
+        )
+
+    @retry(stop=stop_after_attempt(3), wait=wait_exponential(min=1, max=8))
+    async def embed(self, texts: Iterable[str]) -> list[list[float]]:
+        resp = await self.client.embeddings.create(
+            model=self.embed_model, input=list(texts)
+        )
+        return [d.embedding for d in resp.data]

backend/app/llm/registry.py

@@ -0,0 +1,81 @@
+"""Provider registry — resolves which LLM to use and handles fallback.
+
+This is the single place that knows about concrete providers. The agent and
+services ask for a provider by name (or the default) and get back something
+implementing `LLMProvider`. If the requested provider isn't configured, we
+fall back gracefully so the app still runs with whatever key is present.
+"""
+from __future__ import annotations
+
+from functools import lru_cache
+from typing import Optional
+
+from app.core.config import settings
+from app.core.logging import get_logger
+from app.llm.base import LLMError, LLMProvider
+
+log = get_logger(__name__)
+
+
+@lru_cache
+def _gemini() -> Optional[LLMProvider]:
+    if not settings.has_gemini:
+        return None
+    from app.llm.gemini_provider import GeminiProvider
+
+    return GeminiProvider(
+        api_key=settings.gemini_api_key,
+        model=settings.gemini_model,
+        embed_model=settings.gemini_embed_model,
+    )
+
+
+@lru_cache
+def _openai() -> Optional[LLMProvider]:
+    if not settings.has_openai:
+        return None
+    from app.llm.openai_provider import OpenAIProvider
+
+    return OpenAIProvider(api_key=settings.openai_api_key, model=settings.openai_model)
+
+
+def available_providers() -> list[str]:
+    out = []
+    if settings.has_gemini:
+        out.append("gemini")
+    if settings.has_openai:
+        out.append("openai")
+    return out
+
+
+def get_provider(name: Optional[str] = None) -> LLMProvider:
+    """Resolve a provider by name with sensible fallback.
+
+    Order of preference:
+      1. explicitly requested name (if configured)
+      2. configured DEFAULT_LLM_PROVIDER
+      3. any other configured provider
+    """
+    requested = (name or settings.default_llm_provider or "gemini").lower()
+
+    chain = [requested]
+    # then the configured default, then the other one
+    for cand in (settings.default_llm_provider, "gemini", "openai"):
+        if cand not in chain:
+            chain.append(cand)
+
+    for cand in chain:
+        prov = _gemini() if cand == "gemini" else _openai() if cand == "openai" else None
+        if prov is not None:
+            if cand != requested:
+                log.warning("Provider '%s' unavailable; using '%s'", requested, cand)
+            return prov
+
+    raise LLMError(
+        "No LLM provider configured. Set GEMINI_API_KEY (recommended) or OPENAI_API_KEY."
+    )
+
+
+def get_embedder() -> LLMProvider:
+    """Embeddings can come from any provider; prefer the default."""
+    return get_provider()

backend/app/main.py

@@ -0,0 +1,61 @@
+"""FastAPI application entrypoint.
+
+Wires routers, CORS, logging, and DB init. Run with:
+    uvicorn app.main:app --reload --port 8000
+"""
+from __future__ import annotations
+
+from contextlib import asynccontextmanager
+
+from fastapi import FastAPI
+from fastapi.middleware.cors import CORSMiddleware
+
+from app.api import actions, chat, documents, meta
+from app.core.config import settings
+from app.core.logging import get_logger, setup_logging
+from app.services import storage
+
+log = get_logger(__name__)
+
+
+@asynccontextmanager
+async def lifespan(app: FastAPI):
+    setup_logging()
+    storage.init_db()
+    providers = []
+    if settings.has_gemini:
+        providers.append("gemini")
+    if settings.has_openai:
+        providers.append("openai")
+    if not providers:
+        log.warning("No LLM provider configured! Set GEMINI_API_KEY in backend/.env")
+    else:
+        log.info("LLM providers available: %s (default=%s)",
+                 providers, settings.default_llm_provider)
+    yield
+
+
+app = FastAPI(
+    title="Document Processing AI Agent",
+    description="Monkhub Innovations — chat-based document processing agent",
+    version="1.0.0",
+    lifespan=lifespan,
+)
+
+app.add_middleware(
+    CORSMiddleware,
+    allow_origins=settings.cors_origin_list,
+    allow_credentials=True,
+    allow_methods=["*"],
+    allow_headers=["*"],
+)
+
+app.include_router(meta.router)
+app.include_router(documents.router)
+app.include_router(actions.router)
+app.include_router(chat.router)
+
+
+@app.get("/")
+async def root():
+    return {"name": "Document Processing AI Agent", "docs": "/docs", "health": "/api/health"}

backend/app/schemas/documents.py

@@ -0,0 +1,172 @@
+"""Shared data contracts for documents, extraction, and chat.
+
+These Pydantic models are the boundary between backend and frontend and
+between the agent and its tools. Keep them stable and well-typed.
+"""
+from __future__ import annotations
+
+from enum import Enum
+from typing import Any, Optional
+
+from pydantic import BaseModel, Field
+
+
+# --------------------------------------------------------------------------
+# Geometry / provenance
+# --------------------------------------------------------------------------
+class BBox(BaseModel):
+    """Normalised bounding box (0..1) plus the page it lives on.
+
+    Normalised coordinates let the frontend overlay highlights regardless of
+    the rendered image size. This powers the citation-highlight UX.
+    """
+
+    page: int = Field(..., description="0-based page index")
+    x0: float
+    y0: float
+    x1: float
+    y1: float
+
+
+class Citation(BaseModel):
+    """A grounded reference back to the source document."""
+
+    text: str = Field(..., description="The supporting snippet")
+    page: int
+    bbox: Optional[BBox] = None
+    chunk_id: Optional[str] = None
+
+
+# --------------------------------------------------------------------------
+# Documents
+# --------------------------------------------------------------------------
+class DocStatus(str, Enum):
+    uploaded = "uploaded"
+    processing = "processing"
+    ready = "ready"
+    failed = "failed"
+
+
+class PageInfo(BaseModel):
+    page: int
+    width: int
+    height: int
+    image_url: str
+
+
+class DocumentMeta(BaseModel):
+    id: str
+    filename: str
+    content_type: str
+    size_bytes: int
+    status: DocStatus = DocStatus.uploaded
+    num_pages: int = 0
+    created_at: str
+    error: Optional[str] = None
+
+
+class DocumentDetail(DocumentMeta):
+    pages: list[PageInfo] = Field(default_factory=list)
+    markdown: Optional[str] = None
+    classification: Optional["Classification"] = None
+    summary: Optional[str] = None
+    extraction: Optional["ExtractionResult"] = None
+    anomalies: list["Anomaly"] = Field(default_factory=list)
+
+
+# --------------------------------------------------------------------------
+# Classification
+# --------------------------------------------------------------------------
+class Classification(BaseModel):
+    doc_type: str = Field(..., description="e.g. invoice, contract, form, resume, report, receipt, other")
+    confidence: float = Field(..., ge=0, le=1)
+    rationale: str = ""
+    candidates: list[dict[str, Any]] = Field(default_factory=list)
+
+
+# --------------------------------------------------------------------------
+# Extraction
+# --------------------------------------------------------------------------
+class ExtractedField(BaseModel):
+    name: str
+    value: Any = None
+    confidence: float = Field(0.0, ge=0, le=1)
+    citation: Optional[Citation] = None
+
+
+class ExtractedTable(BaseModel):
+    title: Optional[str] = None
+    columns: list[str] = Field(default_factory=list)
+    rows: list[list[Any]] = Field(default_factory=list)
+
+
+class Entity(BaseModel):
+    type: str  # person, org, date, money, email, address, ...
+    value: str
+    confidence: float = Field(0.0, ge=0, le=1)
+
+
+class ExtractedRecord(BaseModel):
+    """One repeated entity in a multi-record document (e.g. one employee in a
+    roster, one transaction in a statement). Each cell keeps its own confidence
+    and citation so records are as grounded as flat fields."""
+
+    fields: list[ExtractedField] = Field(default_factory=list)
+
+
+class ExtractionLayout(str, Enum):
+    single_record = "single_record"  # the whole doc describes one entity
+    multi_record = "multi_record"    # the doc lists many similar entities
+    mixed = "mixed"                  # a header record plus a repeated sub-list
+
+
+class ExtractionResult(BaseModel):
+    schema_name: str = "generic"
+    layout: ExtractionLayout = ExtractionLayout.single_record
+    record_type: Optional[str] = None  # singular name of a record, e.g. "employee"
+    # Document-level / header fields (always used for single_record docs).
+    fields: list[ExtractedField] = Field(default_factory=list)
+    # Repeated records (multi_record / mixed) + the canonical column order.
+    records: list[ExtractedRecord] = Field(default_factory=list)
+    record_keys: list[str] = Field(default_factory=list)
+    tables: list[ExtractedTable] = Field(default_factory=list)
+    entities: list[Entity] = Field(default_factory=list)
+
+
+# --------------------------------------------------------------------------
+# Anomalies
+# --------------------------------------------------------------------------
+class AnomalySeverity(str, Enum):
+    info = "info"
+    warning = "warning"
+    error = "error"
+
+
+class Anomaly(BaseModel):
+    severity: AnomalySeverity
+    field: Optional[str] = None
+    message: str
+
+
+# --------------------------------------------------------------------------
+# Chat
+# --------------------------------------------------------------------------
+class ChatRole(str, Enum):
+    user = "user"
+    assistant = "assistant"
+    system = "system"
+
+
+class ChatMessage(BaseModel):
+    role: ChatRole
+    content: str
+
+
+class ChatRequest(BaseModel):
+    document_id: Optional[str] = None
+    messages: list[ChatMessage]
+    provider: Optional[str] = Field(None, description="override LLM provider: gemini|openai")
+
+
+# Resolve forward references
+DocumentDetail.model_rebuild()

backend/app/schemas/templates.py

@@ -0,0 +1,22 @@
+"""Optional, user-selectable extraction hints.
+
+IMPORTANT: these are NOT required for the agent to work. By default the
+extractor is fully adaptive — it induces the right schema from the document's
+own content (see services/extraction.py), so it handles any document type,
+including ones never seen before, with no template at all.
+
+This module only exists so a user can *optionally* force a particular framing
+via the re-extract API (`?schema=invoice`). Nothing here is applied
+automatically, and the default pipeline never consults it. Keep it tiny — do
+NOT grow a per-domain format zoo here; that's the anti-pattern the adaptive
+extractor is designed to avoid.
+"""
+from __future__ import annotations
+
+# A short, illustrative menu surfaced by the /-/schemas endpoint. "auto" is the
+# default and means "let the agent decide" — the others are just optional framings.
+OPTIONAL_HINTS: list[str] = ["auto", "invoice", "contract", "receipt", "resume"]
+
+
+def all_template_names() -> list[str]:
+    return list(OPTIONAL_HINTS)

backend/app/services/anomaly.py

@@ -0,0 +1,241 @@
+"""Anomaly / consistency checks.
+
+Two layers:
+  1. Deterministic rules — missing required fields (per schema template),
+     low-confidence extractions, and arithmetic checks (e.g. invoice
+     subtotal + tax == total).
+  2. LLM review — a second pass that flags semantic inconsistencies the
+     rules can't catch.
+"""
+from __future__ import annotations
+
+import re
+from datetime import date
+
+from dateutil import parser as date_parser
+
+from app.core.logging import get_logger
+from app.llm.base import LLMMessage
+from app.llm.registry import get_provider
+from app.schemas.documents import Anomaly, AnomalySeverity, ExtractionLayout, ExtractionResult
+
+log = get_logger(__name__)
+
+_LOW_CONF = 0.45
+
+
+def _to_number(v) -> float | None:
+    if v is None:
+        return None
+    if isinstance(v, (int, float)):
+        return float(v)
+    m = re.search(r"-?\d[\d,]*\.?\d*", str(v))
+    if not m:
+        return None
+    try:
+        return float(m.group(0).replace(",", ""))
+    except ValueError:
+        return None
+
+
+# A string only counts as a date if it carries an unambiguous date shape AND a
+# 4-digit year, so money/IDs (e.g. "19,50,000") never get mis-parsed as dates.
+_DATE_SHAPE = re.compile(
+    r"\d{4}-\d{1,2}-\d{1,2}"                                  # ISO 2024-01-15
+    r"|\d{1,2}[/.]\d{1,2}[/.]\d{2,4}"                          # 15/01/2024, 1.1.24
+    r"|(?:jan|feb|mar|apr|may|jun|jul|aug|sep|oct|nov|dec)[a-z]*\.?\s+\d{4}"  # March 2021
+    r"|\d{1,2}\s+(?:jan|feb|mar|apr|may|jun|jul|aug|sep|oct|nov|dec)[a-z]*\.?\s+\d{4}",  # 15 Jan 2024
+    re.IGNORECASE,
+)
+
+
+def _as_future_date(value) -> date | None:
+    """Return the parsed date iff `value` is clearly a date AFTER today.
+
+    Date sanity is done deterministically here (not by the LLM) because models
+    routinely get past/future comparisons wrong — e.g. calling 2024-01-15 a
+    'future' date when today is 2026. Template-free: works on any field whose
+    value looks like a date, regardless of document type or field name.
+    """
+    s = str(value).strip()
+    if not _DATE_SHAPE.search(s):
+        return None
+    try:
+        dt = date_parser.parse(s, fuzzy=True, dayfirst=True).date()
+    except (ValueError, OverflowError, TypeError):
+        return None
+    return dt if dt > date.today() else None
+
+
+def _future_date_checks(extraction: ExtractionResult) -> list[Anomaly]:
+    today = date.today().isoformat()
+    out: list[Anomaly] = []
+
+    def check(name: str, value, where: str = "") -> None:
+        if value in (None, "", []):
+            return
+        fut = _as_future_date(value)
+        if fut:
+            out.append(Anomaly(
+                severity=AnomalySeverity.warning, field=name,
+                message=f"'{name}'{where} is a future date ({value}), "
+                        f"after today ({today}). Please verify.",
+            ))
+
+    for f in extraction.fields:
+        check(f.name, f.value)
+    for i, rec in enumerate(extraction.records, 1):
+        for f in rec.fields:
+            check(f.name, f.value, where=f" (record {i})")
+    return out
+
+
+def _find(present: dict, *substrings: str):
+    """Return the first extracted field whose name contains any substring."""
+    for name, f in present.items():
+        low = name.lower()
+        if any(s in low for s in substrings):
+            return f
+    return None
+
+
+def _generic_arithmetic(present: dict) -> list[Anomaly]:
+    """Type-agnostic money check: wherever a document exposes component amounts
+    and a total (invoice, receipt, payslip, statement, quote, ...), verify they
+    reconcile. Driven by field-name semantics, not by any document template."""
+    total_f = _find(present, "grand_total", "amount_due", "balance_due", "total", "net_payable")
+    sub_f = _find(present, "subtotal", "sub_total", "gross", "basic")
+    tax_f = _find(present, "tax", "gst", "vat")
+    disc_f = _find(present, "discount", "rebate")
+    if not total_f or not sub_f:
+        return []
+    sub = _to_number(sub_f.value)
+    total = _to_number(total_f.value)
+    if sub is None or total is None:
+        return []
+    tax = _to_number(tax_f.value) if tax_f else 0.0
+    disc = _to_number(disc_f.value) if disc_f else 0.0
+    expected = sub + (tax or 0.0) - (disc or 0.0)
+    if abs(expected - total) > max(0.02 * abs(total), 0.5):
+        bits = [f"{sub_f.name} ({sub})"]
+        if tax_f and tax:
+            bits.append(f"+ {tax_f.name} ({tax})")
+        if disc_f and disc:
+            bits.append(f"- {disc_f.name} ({disc})")
+        return [Anomaly(
+            severity=AnomalySeverity.warning, field=total_f.name,
+            message=f"Amounts may not reconcile: {' '.join(bits)} = {expected:g}, "
+                    f"but {total_f.name} is {total:g}.",
+        )]
+    return []
+
+
+def rule_checks(extraction: ExtractionResult, doc_type: str | None) -> list[Anomaly]:
+    out: list[Anomaly] = []
+    present = {f.name: f for f in extraction.fields}
+    is_multi = extraction.layout in (ExtractionLayout.multi_record, ExtractionLayout.mixed)
+
+    # Multi-record sanity: flag records that are entirely empty or missing an
+    # identifier — works for any kind of repeated record, no schema assumed.
+    if extraction.records:
+        id_key = extraction.record_keys[0] if extraction.record_keys else None
+        for i, rec in enumerate(extraction.records, 1):
+            vals = {f.name: f.value for f in rec.fields}
+            if all(v in (None, "", []) for v in vals.values()):
+                out.append(Anomaly(
+                    severity=AnomalySeverity.warning, field=None,
+                    message=f"Record {i} is empty.",
+                ))
+            elif id_key and vals.get(id_key) in (None, "", []):
+                out.append(Anomaly(
+                    severity=AnomalySeverity.info, field=id_key,
+                    message=f"Record {i} is missing its identifier ('{id_key}').",
+                ))
+
+    # Low confidence (header / doc-level fields).
+    for f in extraction.fields:
+        if f.value not in (None, "", []) and f.confidence < _LOW_CONF:
+            out.append(Anomaly(
+                severity=AnomalySeverity.warning, field=f.name,
+                message=f"Low-confidence extraction for '{f.name}' "
+                        f"({f.confidence:.0%}). Please verify.",
+            ))
+
+    # Generic financial reconciliation (only fires when the right fields exist).
+    if not is_multi:
+        out += _generic_arithmetic(present)
+
+    # Deterministic temporal sanity (past/future) — reliable where the LLM isn't.
+    out += _future_date_checks(extraction)
+    return out
+
+
+def _summarise_extraction(extraction: ExtractionResult) -> str:
+    parts = []
+    if extraction.fields:
+        parts.append("Document-level fields:\n" +
+                     "\n".join(f"- {f.name}: {f.value}" for f in extraction.fields))
+    if extraction.records:
+        parts.append(f"\n{len(extraction.records)} records "
+                     f"(type: {extraction.record_type or 'record'}, "
+                     f"keys: {', '.join(extraction.record_keys)}):")
+        for i, rec in enumerate(extraction.records[:25], 1):
+            kv = ", ".join(f"{f.name}={f.value}" for f in rec.fields if f.value not in (None, ""))
+            parts.append(f"  {i}. {kv}")
+    return "\n".join(parts) or "(no fields extracted)"
+
+
+async def llm_review(markdown: str, extraction: ExtractionResult,
+                     provider: str | None = None) -> list[Anomaly]:
+    if not markdown.strip():
+        return []
+    llm = get_provider(provider)
+    extraction_str = _summarise_extraction(extraction)
+    sys = (
+        "Do NOT perform any date or temporal reasoning: never flag a value for "
+        "being a 'future' or 'past' date, nor compare dates to today — those "
+        "checks are handled separately and reliably elsewhere; ignore them.\n"
+        "You review extracted document data against the source for GENUINE problems "
+        "only: internal contradictions, values that conflict with the source text, "
+        "arithmetic that doesn't add up, impossible/suspicious values, or important "
+        "data that the source clearly states but is missing or wrong in the extraction.\n"
+        "DO NOT flag: formatting or normalisation differences (e.g. a date rewritten "
+        "as ISO 'YYYY-MM-DD', currency symbols, casing, whitespace), values that are "
+        "legitimately null because the source says they are missing/masked/unknown, or "
+        "stylistic choices. Those are correct, not anomalies.\n"
+        "Prefer 'info' for notes worth a human's attention, 'warning' for likely "
+        "issues, 'error' for clear errors. Respond ONLY as JSON: "
+        '{"anomalies": [{"severity": "info|warning|error", "field": str|null, '
+        '"message": str}]}. Return an empty array if everything is consistent.'
+    )
+    msgs = [
+        LLMMessage(role="system", content=sys),
+        LLMMessage(role="user",
+                   content=f"Extracted data:\n{extraction_str}\n\n"
+                           f"Source (excerpt):\n{markdown[:7000]}"),
+    ]
+    try:
+        data = await llm.complete_json(msgs)
+    except Exception as e:
+        log.warning("llm anomaly review failed: %s", e)
+        return []
+
+    out = []
+    for a in data.get("anomalies", []) or []:
+        try:
+            out.append(Anomaly(
+                severity=AnomalySeverity(a.get("severity", "info")),
+                field=a.get("field"),
+                message=str(a.get("message", "")),
+            ))
+        except Exception:
+            continue
+    return out
+
+
+async def detect(markdown: str, extraction: ExtractionResult, doc_type: str | None,
+                 provider: str | None = None, deep: bool = True) -> list[Anomaly]:
+    anomalies = rule_checks(extraction, doc_type)
+    if deep:
+        anomalies += await llm_review(markdown, extraction, provider)
+    return anomalies

backend/app/services/classification.py

@@ -0,0 +1,51 @@
+"""Document classification — zero-shot via LLM.
+
+Returns a type + confidence + rationale, plus ranked candidates. Uses a
+truncated view of the document so it stays cheap and fast.
+"""
+from __future__ import annotations
+
+from app.core.logging import get_logger
+from app.llm.base import LLMMessage
+from app.llm.registry import get_provider
+from app.schemas.documents import Classification
+
+log = get_logger(__name__)
+
+KNOWN_TYPES = [
+    "invoice", "receipt", "contract", "purchase_order", "form",
+    "resume", "employee_record", "report", "letter", "email",
+    "bank_statement", "id_document", "spreadsheet", "other",
+]
+
+_SYS = (
+    "You are a document classification expert. Classify the document into the single "
+    f"best-fitting type from this list: {', '.join(KNOWN_TYPES)}. "
+    "Pick 'employee_record' for HR records / employee rosters (even when several "
+    "people are listed), 'spreadsheet' for tabular data dumps, and 'form' for "
+    "fillable forms. Use 'other' only when nothing fits. "
+    "Respond ONLY as JSON with keys: doc_type (string from the list), "
+    "confidence (0..1), rationale (short string), "
+    "candidates (array of {type, confidence} for the top 3)."
+)
+
+
+async def classify(markdown: str, provider: str | None = None) -> Classification:
+    llm = get_provider(provider)
+    excerpt = markdown[:6000] if markdown else "(empty document)"
+    msgs = [
+        LLMMessage(role="system", content=_SYS),
+        LLMMessage(role="user", content=f"Document content:\n\n{excerpt}"),
+    ]
+    try:
+        data = await llm.complete_json(msgs)
+    except Exception as e:
+        log.warning("classification failed: %s", e)
+        return Classification(doc_type="other", confidence=0.0, rationale="classification error")
+
+    return Classification(
+        doc_type=str(data.get("doc_type", "other")).lower(),
+        confidence=float(data.get("confidence", 0.5) or 0.5),
+        rationale=str(data.get("rationale", "")),
+        candidates=data.get("candidates", []) or [],
+    )

backend/app/services/export.py

@@ -0,0 +1,99 @@
+"""Export extracted data to structured formats: JSON, CSV, Excel."""
+from __future__ import annotations
+
+import csv
+import io
+import json
+
+from app.schemas.documents import DocumentDetail
+
+
+def to_json(detail: DocumentDetail) -> bytes:
+    payload = {
+        "document": {
+            "id": detail.id,
+            "filename": detail.filename,
+            "doc_type": detail.classification.doc_type if detail.classification else None,
+            "num_pages": detail.num_pages,
+        },
+        "classification": detail.classification.model_dump() if detail.classification else None,
+        "summary": detail.summary,
+        "extraction": detail.extraction.model_dump() if detail.extraction else None,
+        "anomalies": [a.model_dump() for a in detail.anomalies],
+    }
+    return json.dumps(payload, indent=2, default=str).encode("utf-8")
+
+
+def _record_rows(extraction) -> tuple[list[str], list[list]]:
+    """Flatten records into (header, rows) using the canonical key order."""
+    keys = list(extraction.record_keys)
+    if not keys:  # derive from the records themselves, preserving first-seen order
+        seen = set()
+        for rec in extraction.records:
+            for f in rec.fields:
+                if f.name not in seen:
+                    keys.append(f.name)
+                    seen.add(f.name)
+    rows = []
+    for rec in extraction.records:
+        m = {f.name: f.value for f in rec.fields}
+        rows.append([m.get(k) for k in keys])
+    return keys, rows
+
+
+def to_csv(detail: DocumentDetail) -> bytes:
+    buf = io.StringIO()
+    w = csv.writer(buf)
+    ex = detail.extraction
+    # Multi-record docs export as a proper record table; otherwise field/value.
+    if ex and ex.records:
+        keys, rows = _record_rows(ex)
+        w.writerow(keys)
+        for row in rows:
+            w.writerow(["" if c is None else c for c in row])
+    else:
+        w.writerow(["field", "value", "confidence"])
+        if ex:
+            for f in ex.fields:
+                w.writerow([f.name, f.value, f"{f.confidence:.2f}"])
+    return buf.getvalue().encode("utf-8")
+
+
+def to_xlsx(detail: DocumentDetail) -> bytes:
+    from openpyxl import Workbook
+
+    wb = Workbook()
+    ws = wb.active
+    ws.title = "Fields"
+    ws.append(["Field", "Value", "Confidence"])
+    ex = detail.extraction
+    if ex:
+        for f in ex.fields:
+            ws.append([f.name, str(f.value) if f.value is not None else "", round(f.confidence, 2)])
+
+        # Repeated records get their own sheet, one row per record.
+        if ex.records:
+            keys, rows = _record_rows(ex)
+            rec = wb.create_sheet(title=(ex.record_type or "Records")[:31])
+            rec.append(keys)
+            for row in rows:
+                rec.append([str(c) if c is not None else "" for c in row])
+
+        for ti, table in enumerate(ex.tables):
+            tab = wb.create_sheet(title=(table.title or f"Table {ti+1}")[:31])
+            if table.columns:
+                tab.append(table.columns)
+            for row in table.rows:
+                tab.append([str(c) if c is not None else "" for c in row])
+
+    out = io.BytesIO()
+    wb.save(out)
+    return out.getvalue()
+
+
+EXPORTERS = {
+    "json": (to_json, "application/json", "json"),
+    "csv": (to_csv, "text/csv", "csv"),
+    "xlsx": (to_xlsx,
+             "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet", "xlsx"),
+}

backend/app/services/extraction.py

@@ -0,0 +1,273 @@
+"""Adaptive structured-data extraction.
+
+The goal: given ANY document — any layout, any field-label convention, known
+type or not — produce clean, consistently-keyed structured data in the shapes
+the rest of the app understands (fields / records / tables / entities), each
+grounded with a confidence score and a best-effort citation.
+
+How it adapts (vs. the old fixed-template approach):
+
+  * Layout detection — the model decides whether the document describes ONE
+    entity (`single_record`, e.g. an invoice), MANY similar entities
+    (`multi_record`, e.g. an employee roster or a transaction list), or a
+    header plus a repeated sub-list (`mixed`, e.g. invoice + line items).
+  * Schema induction — for a known type a template *suggests* fields, but the
+    model is free to add/drop fields to match the actual document. For unknown
+    types it derives the schema from the content.
+  * Canonical keys — inconsistent source labels ("Staff Number" / "Personnel
+    ID" / "ID") are normalised to ONE snake_case key, consistent across every
+    record, so downstream consumers get a stable shape.
+  * Grounding — values come only from the document body (never file metadata),
+    dates/amounts are normalised, and citations are attached by matching values
+    back to provenance-carrying chunks.
+"""
+from __future__ import annotations
+
+from app.core.logging import get_logger
+from app.llm.base import LLMMessage
+from app.llm.registry import get_provider
+from app.schemas.documents import (
+    BBox,
+    Citation,
+    Entity,
+    ExtractedField,
+    ExtractedRecord,
+    ExtractedTable,
+    ExtractionLayout,
+    ExtractionResult,
+)
+from app.services.vectorstore import Chunk
+
+log = get_logger(__name__)
+
+_MAX_CHARS = 16000        # how much of the document we feed the model
+_MAX_RECORDS = 200        # safety cap on returned records
+
+
+_SYSTEM = """You are an expert structured-data extraction engine. You convert ANY \
+document — regardless of its layout, formatting, or field-label conventions — into \
+clean, consistent structured data. You handle invoices, contracts, resumes, forms, \
+reports, rosters, statements, emails, scanned pages and anything else.
+
+STEP 1 — Determine the document LAYOUT:
+- "single_record": the whole document describes ONE entity (one invoice, one
+  contract, one resume, one letter). Put everything in `fields`.
+- "multi_record": the document lists MANY similar entities (an employee roster,
+  a list of transactions, a table of people/items). Put EACH entity as one object
+  in `records`; put any shared header info in `fields`.
+- "mixed": one primary record plus a repeated sub-list (e.g. an invoice header
+  with line items). Use `fields` for the header and `records` for the repeated list.
+
+STEP 2 — Extract, following these RULES exactly:
+1. CANONICAL KEYS: normalise inconsistent labels to ONE consistent snake_case key.
+   Across records the SAME concept MUST use the SAME key even when the source
+   labels differ. e.g. "Staff Number"/"Personnel ID"/"ID" -> "staff_id";
+   "Date Joined"/"Joining Date"/"DOJ" -> "date_of_joining";
+   "Job Title"/"Position"/"Role" -> "job_title".
+2. KEEP DISTINCT CONCEPTS SEPARATE: if the source distinguishes e.g. BASIC salary
+   from total CTC, keep BOTH keys (e.g. basic_salary AND ctc). Never merge
+   different things into one key.
+3. CONSISTENT COLUMNS: every record object should use the same set of keys. If a
+   record genuinely lacks a value, set that key to null — do NOT omit the key.
+4. NORMALISE VALUES: dates -> ISO "YYYY-MM-DD" when the full date is known; use
+   "YYYY-MM" if only month+year is known, "YYYY" if only the year. Keep currency
+   symbols and amounts as written (e.g. "₹24,00,000"). Strip surrounding labels.
+5. GROUNDING: extract ONLY values present in the document body. NEVER use file
+   metadata, document properties, the author of the file, or outside knowledge.
+   If a value is absent, missing, masked or illegible, use null. Never invent.
+6. record_keys: list the canonical record keys in a sensible order (identifier first).
+7. Also capture genuine TABLES verbatim and the most salient ENTITIES
+   (types: person, org, date, money, email, phone, address, id).
+8. confidence (0..1) reflects how certain you are about a value.
+
+Return ONLY a JSON object with EXACTLY this shape:
+{
+  "layout": "single_record" | "multi_record" | "mixed",
+  "record_type": string | null,          // singular noun, e.g. "employee", "line_item"
+  "fields": [{"name": str, "value": any, "confidence": number}],
+  "record_keys": [str],                  // canonical column order for records
+  "records": [ { "<key>": any, ... } ],  // one object per repeated entity
+  "tables": [{"title": str, "columns": [str], "rows": [[any]]}],
+  "entities": [{"type": str, "value": str, "confidence": number}]
+}"""
+
+
+_KNOWN_HINTS = {"invoice", "contract", "receipt", "resume"}
+
+
+def _context(doc_type: str | None, schema_hint: str | None) -> str:
+    """A neutral one-line context note. Deliberately carries NO field schema —
+    the model must derive the schema from the document's own content so the
+    agent works on any type, including unseen ones."""
+    # An explicit, user-chosen framing (optional). Never auto-applied.
+    if schema_hint and schema_hint.lower() in _KNOWN_HINTS:
+        return (
+            f"The user asked you to extract this as a '{schema_hint.lower()}'. Use that "
+            "as a lens, but still derive the actual fields/records from the content."
+        )
+    if doc_type and doc_type.lower() not in ("other", "unknown", ""):
+        return (
+            f"A classifier loosely tagged this as '{doc_type.lower()}'. Treat that as a "
+            "weak hint only — do NOT assume any fixed schema; derive the fields and "
+            "records from the actual content of THIS document."
+        )
+    return (
+        "Derive the most useful fields and/or records directly from the document's "
+        "own content. Do not assume any fixed schema."
+    )
+
+
+def _attach_citation(value, chunks: list[Chunk]) -> Citation | None:
+    """Best-effort: find the chunk whose text contains the extracted value.
+
+    Only returns a citation when the value text genuinely appears in the source,
+    so normalised/metadata-derived values are never given a misleading citation.
+    """
+    if value is None:
+        return None
+    needle = str(value).strip().lower()
+    if len(needle) < 3:
+        return None
+    for c in chunks:
+        if needle in c.text.lower():
+            bbox = BBox(page=c.page, **c.bbox) if c.bbox else None
+            return Citation(text=c.text[:200], page=c.page, bbox=bbox, chunk_id=c.id)
+    return None
+
+
+def _field(name, value, confidence, chunks) -> ExtractedField:
+    return ExtractedField(
+        name=str(name),
+        value=value,
+        confidence=max(0.0, min(1.0, float(confidence or 0.0))),
+        citation=_attach_citation(value, chunks),
+    )
+
+
+def _parse_fields(raw, chunks) -> list[ExtractedField]:
+    out: list[ExtractedField] = []
+    for f in raw or []:
+        if not isinstance(f, dict):
+            continue
+        out.append(_field(f.get("name", ""), f.get("value"),
+                          f.get("confidence", 0.6), chunks))
+    return out
+
+
+def _parse_records(raw_records, record_keys, chunks) -> tuple[list[ExtractedRecord], list[str]]:
+    records: list[ExtractedRecord] = []
+    keys: list[str] = [str(k) for k in (record_keys or [])]
+    seen = set(keys)
+
+    for rec in (raw_records or [])[:_MAX_RECORDS]:
+        if not isinstance(rec, dict):
+            continue
+        # Discover any keys the model used but didn't list, preserving order.
+        for k in rec.keys():
+            if k not in seen:
+                keys.append(str(k))
+                seen.add(k)
+        # Per-record confidence, if the model provided one; else a sensible default.
+        rec_conf = rec.get("_confidence")
+        fields = [
+            _field(k, rec.get(k),
+                   rec_conf if rec_conf is not None else (0.85 if rec.get(k) not in (None, "") else 0.0),
+                   chunks)
+            for k in keys
+            if k != "_confidence"
+        ]
+        records.append(ExtractedRecord(fields=fields))
+
+    keys = [k for k in keys if k != "_confidence"]
+    return records, keys
+
+
+def _parse_tables(raw) -> list[ExtractedTable]:
+    tables: list[ExtractedTable] = []
+    for t in raw or []:
+        if not isinstance(t, dict):
+            continue
+        tables.append(ExtractedTable(
+            title=t.get("title"),
+            columns=[str(c) for c in (t.get("columns") or [])],
+            rows=[[c for c in row] for row in (t.get("rows") or []) if isinstance(row, list)],
+        ))
+    return tables
+
+
+def _parse_entities(raw) -> list[Entity]:
+    out: list[Entity] = []
+    for e in raw or []:
+        if not isinstance(e, dict):
+            continue
+        val = e.get("value")
+        if val in (None, ""):
+            continue
+        out.append(Entity(
+            type=str(e.get("type", "")),
+            value=str(val),
+            confidence=max(0.0, min(1.0, float(e.get("confidence", 0.6) or 0.6))),
+        ))
+    return out
+
+
+def _coerce_layout(raw) -> ExtractionLayout:
+    try:
+        return ExtractionLayout(str(raw).strip().lower())
+    except (ValueError, AttributeError):
+        return ExtractionLayout.single_record
+
+
+async def extract(
+    markdown: str,
+    doc_type: str | None,
+    chunks: list[Chunk],
+    provider: str | None = None,
+    schema_hint: str | None = None,
+) -> ExtractionResult:
+    llm = get_provider(provider)
+    excerpt = markdown[:_MAX_CHARS] if markdown else "(empty)"
+
+    msgs = [
+        LLMMessage(role="system", content=_SYSTEM),
+        LLMMessage(role="user",
+                   content=f"{_context(doc_type, schema_hint)}\n\nDocument content:\n\n{excerpt}"),
+    ]
+    try:
+        data = await llm.complete_json(msgs)
+    except Exception as e:
+        log.warning("extraction failed: %s", e)
+        return ExtractionResult(schema_name="generic")
+
+    if not isinstance(data, dict):
+        log.warning("extraction returned non-object: %r", type(data))
+        return ExtractionResult(schema_name="generic")
+
+    layout = _coerce_layout(data.get("layout"))
+    fields = _parse_fields(data.get("fields"), chunks)
+    records, record_keys = _parse_records(data.get("records"), data.get("record_keys"), chunks)
+    tables = _parse_tables(data.get("tables"))
+    entities = _parse_entities(data.get("entities"))
+
+    # If the model returned records but mislabelled layout, correct it.
+    if records and layout == ExtractionLayout.single_record:
+        layout = ExtractionLayout.multi_record
+
+    # The schema name is the model's own description of what it found — not a
+    # template we imposed. Falls back to the classifier label, then "generic".
+    record_type = data.get("record_type")
+    schema_name = (
+        str(record_type) if record_type
+        else (doc_type.lower() if doc_type and doc_type.lower() not in ("other", "unknown", "") else "generic")
+    )
+
+    return ExtractionResult(
+        schema_name=schema_name,
+        layout=layout,
+        record_type=str(record_type) if record_type else None,
+        fields=fields,
+        records=records,
+        record_keys=record_keys,
+        tables=tables,
+        entities=entities,
+    )

backend/app/services/ingestion.py

@@ -0,0 +1,243 @@
+"""Document ingestion — the extraction backbone (Docling).
+
+Responsibilities:
+  * Convert an uploaded file into a DoclingDocument (layout, tables, OCR).
+  * Export Markdown (used for summarization / classification / display).
+  * Render page images (for the viewer + vision fallback).
+  * Produce text chunks that carry page + normalised bbox provenance, so
+    RAG answers and extracted fields can be highlighted on the page.
+
+Everything Docling-specific lives here. The rest of the app consumes the
+neutral results (markdown, pages, chunks).
+"""
+from __future__ import annotations
+
+import os
+import uuid
+from dataclasses import dataclass
+from pathlib import Path
+from typing import Optional
+
+# Windows: normal user accounts can't create symlinks (needs Developer Mode or
+# admin), so HuggingFace's symlink-based model cache fails with WinError 1314.
+# This huggingface_hub version's symlink-support probe is unreliable here, so it
+# still calls os.symlink and hard-crashes instead of falling back to a file copy.
+# We force the copy path by making are_symlinks_supported() return False, and
+# silence the now-irrelevant warning. Must run before any `docling` import below.
+os.environ.setdefault("HF_HUB_DISABLE_SYMLINKS_WARNING", "1")
+if os.name == "nt":
+    try:
+        import huggingface_hub.file_download as _hf_dl
+
+        _hf_dl.are_symlinks_supported = lambda *a, **k: False  # copy, never symlink
+    except Exception:  # pragma: no cover - HF not importable yet is non-fatal
+        pass
+
+from app.core.config import settings
+from app.core.logging import get_logger
+from app.schemas.documents import PageInfo
+from app.services.vectorstore import Chunk
+
+log = get_logger(__name__)
+
+
+@dataclass
+class IngestResult:
+    markdown: str
+    pages: list[PageInfo]
+    chunks: list[Chunk]
+    num_pages: int
+
+
+# --------------------------------------------------------------------------
+# Docling conversion
+# --------------------------------------------------------------------------
+def _converter():
+    from docling.document_converter import DocumentConverter, PdfFormatOption
+    from docling.datamodel.base_models import InputFormat
+    from docling.datamodel.pipeline_options import PdfPipelineOptions
+
+    opts = PdfPipelineOptions()
+    opts.do_ocr = settings.enable_ocr
+    opts.do_table_structure = True
+    opts.generate_page_images = True
+    opts.images_scale = 2.0
+    return DocumentConverter(
+        format_options={InputFormat.PDF: PdfFormatOption(pipeline_options=opts)}
+    )
+
+
+def _normalise_bbox(bbox, page_w: float, page_h: float) -> Optional[dict]:
+    """Convert a Docling bbox to normalised top-left-origin coords (0..1)."""
+    if bbox is None or not page_w or not page_h:
+        return None
+    try:
+        l, t, r, b = bbox.l, bbox.t, bbox.r, bbox.b
+        # Docling PDF coords are bottom-left origin; flip to top-left.
+        origin = getattr(getattr(bbox, "coord_origin", None), "value", "")
+        if str(origin).upper().startswith("BOTTOM"):
+            t, b = page_h - t, page_h - b
+        y0, y1 = sorted((t, b))
+        x0, x1 = sorted((l, r))
+        return {
+            "x0": max(0.0, x0 / page_w),
+            "y0": max(0.0, y0 / page_h),
+            "x1": min(1.0, x1 / page_w),
+            "y1": min(1.0, y1 / page_h),
+        }
+    except Exception:
+        return None
+
+
+def _extract_chunks(doc, doc_id: str) -> list[Chunk]:
+    """Build provenance-aware chunks from DoclingDocument text items.
+
+    We group consecutive text items per page into reasonably sized chunks so
+    embeddings have enough context, while keeping the bbox of the first item
+    as the citation anchor.
+    """
+    chunks: list[Chunk] = []
+    page_sizes: dict[int, tuple[float, float]] = {}
+    try:
+        for pno, page in doc.pages.items():
+            sz = getattr(page, "size", None)
+            if sz:
+                page_sizes[pno] = (sz.width, sz.height)
+    except Exception:
+        pass
+
+    buf_text: list[str] = []
+    buf_page = 0
+    buf_bbox = None
+
+    def flush():
+        nonlocal buf_text, buf_bbox, buf_page
+        if buf_text:
+            text = " ".join(buf_text).strip()
+            if text:
+                chunks.append(Chunk(
+                    id=f"{doc_id}-{len(chunks)}",
+                    text=text, page=buf_page, doc_id=doc_id, bbox=buf_bbox,
+                ))
+        buf_text, buf_bbox = [], None
+
+    try:
+        for item in getattr(doc, "texts", []):
+            text = getattr(item, "text", "") or ""
+            if not text.strip():
+                continue
+            prov = getattr(item, "prov", None)
+            page_no = 0
+            bbox = None
+            if prov:
+                page_no = getattr(prov[0], "page_no", 1) - 1
+                pw, ph = page_sizes.get(page_no + 1, (0, 0))
+                bbox = _normalise_bbox(getattr(prov[0], "bbox", None), pw, ph)
+            if buf_text and (page_no != buf_page or len(" ".join(buf_text)) > 900):
+                flush()
+            if not buf_text:
+                buf_page, buf_bbox = page_no, bbox
+            buf_text.append(text)
+        flush()
+    except Exception as e:  # pragma: no cover
+        log.warning("chunk extraction fell back: %s", e)
+
+    return chunks
+
+
+def _fallback_chunks(markdown: str, doc_id: str) -> list[Chunk]:
+    """Split markdown into ~800-char chunks when provenance isn't available."""
+    words = markdown.split()
+    chunks, buf = [], []
+    for w in words:
+        buf.append(w)
+        if len(" ".join(buf)) > 800:
+            chunks.append(Chunk(id=f"{doc_id}-{len(chunks)}",
+                                text=" ".join(buf), page=0, doc_id=doc_id))
+            buf = []
+    if buf:
+        chunks.append(Chunk(id=f"{doc_id}-{len(chunks)}",
+                            text=" ".join(buf), page=0, doc_id=doc_id))
+    return chunks
+
+
+# --------------------------------------------------------------------------
+# Page rendering (for the viewer)
+# --------------------------------------------------------------------------
+def _render_pages(doc, src: Path, doc_id: str) -> list[PageInfo]:
+    out_dir = settings.renders_path / doc_id
+    out_dir.mkdir(parents=True, exist_ok=True)
+    pages: list[PageInfo] = []
+
+    # Preferred: Docling-generated page images
+    try:
+        for pno, page in sorted(doc.pages.items()):
+            img = getattr(page, "image", None)
+            pil = getattr(img, "pil_image", None) if img else None
+            if pil is None:
+                continue
+            fn = out_dir / f"page-{pno-1}.png"
+            pil.save(fn)
+            pages.append(PageInfo(
+                page=pno - 1, width=pil.width, height=pil.height,
+                image_url=f"/api/documents/{doc_id}/pages/{pno-1}.png",
+            ))
+        if pages:
+            return pages
+    except Exception as e:
+        log.warning("docling page render failed (%s); trying pdfium", e)
+
+    # Fallback: render the PDF with pypdfium2 (a Docling dependency)
+    try:
+        import pypdfium2 as pdfium
+
+        pdf = pdfium.PdfDocument(str(src))
+        for i in range(len(pdf)):
+            bitmap = pdf[i].render(scale=2.0)
+            pil = bitmap.to_pil()
+            fn = out_dir / f"page-{i}.png"
+            pil.save(fn)
+            pages.append(PageInfo(
+                page=i, width=pil.width, height=pil.height,
+                image_url=f"/api/documents/{doc_id}/pages/{i}.png",
+            ))
+    except Exception as e:
+        log.warning("pdfium render failed (%s); trying raw image", e)
+
+    # Last resort: the upload is itself an image
+    if not pages and src.suffix.lower() in {".png", ".jpg", ".jpeg", ".tiff", ".bmp"}:
+        try:
+            from PIL import Image
+
+            pil = Image.open(src).convert("RGB")
+            fn = out_dir / "page-0.png"
+            pil.save(fn)
+            pages.append(PageInfo(page=0, width=pil.width, height=pil.height,
+                                  image_url=f"/api/documents/{doc_id}/pages/0.png"))
+        except Exception:
+            pass
+
+    return pages
+
+
+# --------------------------------------------------------------------------
+# Public entry point
+# --------------------------------------------------------------------------
+def ingest(path: Path, doc_id: str) -> IngestResult:
+    """Synchronous (CPU-bound) ingestion; call via run_in_threadpool."""
+    conv = _converter()
+    result = conv.convert(str(path))
+    doc = result.document
+
+    try:
+        markdown = doc.export_to_markdown()
+    except Exception:
+        markdown = ""
+
+    pages = _render_pages(doc, path, doc_id)
+    chunks = _extract_chunks(doc, doc_id)
+    if not chunks:
+        chunks = _fallback_chunks(markdown, doc_id)
+
+    num_pages = len(pages) or (max((c.page for c in chunks), default=0) + 1)
+    return IngestResult(markdown=markdown, pages=pages, chunks=chunks, num_pages=num_pages)

backend/app/services/pipeline.py

@@ -0,0 +1,97 @@
+"""End-to-end processing pipeline for a freshly uploaded document.
+
+Runs asynchronously after upload:
+  ingest (Docling) → index chunks (RAG) → classify → extract → summarize
+  → detect anomalies → persist.
+
+State is written back to storage at each milestone so the frontend can poll
+status and progressively render results.
+"""
+from __future__ import annotations
+
+from pathlib import Path
+
+from fastapi.concurrency import run_in_threadpool
+
+from app.core.logging import get_logger
+from app.schemas.documents import DocStatus, DocumentDetail
+from app.services import (
+    anomaly,
+    classification,
+    extraction,
+    ingestion,
+    storage,
+    summary,
+    vectorstore,
+)
+
+log = get_logger(__name__)
+
+
+async def process_document(doc_id: str, path: Path, provider: str | None = None) -> None:
+    detail = storage.get(doc_id)
+    if detail is None:
+        log.error("process_document: %s not found", doc_id)
+        return
+
+    # --- Stage 1: ingestion is the only stage that can hard-fail the doc.
+    try:
+        detail.status = DocStatus.processing
+        storage.save(detail)
+
+        result = await run_in_threadpool(ingestion.ingest, path, doc_id)
+        detail.markdown = result.markdown
+        detail.pages = result.pages
+        detail.num_pages = result.num_pages
+        storage.save(detail)
+    except Exception as e:  # pragma: no cover
+        log.exception("Ingestion failed for %s", doc_id)
+        detail = storage.get(doc_id) or detail
+        detail.status = DocStatus.failed
+        detail.error = f"Document parsing failed: {e}"
+        storage.save(detail)
+        return
+
+    # --- Stages 2-6: best-effort. A failure here (e.g. no LLM key) degrades
+    # gracefully — the document still becomes 'ready' with the viewer working.
+    notes: list[str] = []
+
+    async def step(name: str, coro):
+        try:
+            return await coro
+        except Exception as e:
+            log.warning("Stage '%s' skipped for %s: %s", name, doc_id, e)
+            notes.append(f"{name}: {e}")
+            return None
+
+    await step("index", vectorstore.index_chunks(doc_id, result.chunks))
+
+    detail.classification = await step(
+        "classify", classification.classify(result.markdown, provider)
+    )
+    storage.save(detail)
+
+    doc_type = detail.classification.doc_type if detail.classification else None
+    detail.extraction = await step(
+        "extract", extraction.extract(result.markdown, doc_type, result.chunks, provider)
+    )
+    storage.save(detail)
+
+    detail.summary = await step("summarize", summary.summarize(result.markdown, provider))
+    storage.save(detail)
+
+    if detail.extraction:
+        detail.anomalies = (
+            await step("anomalies",
+                       anomaly.detect(result.markdown, detail.extraction, doc_type, provider))
+            or []
+        )
+
+    detail.status = DocStatus.ready
+    detail.error = (
+        "Some AI steps were skipped (is GEMINI_API_KEY set?): " + "; ".join(notes)
+        if notes else None
+    )
+    storage.save(detail)
+    log.info("Document %s processed (%s, %d pages)%s", doc_id, doc_type,
+             detail.num_pages, " [with skipped steps]" if notes else "")

backend/app/services/qa.py

@@ -0,0 +1,63 @@
+"""Conversational Q&A over a document — agentic RAG with citations.
+
+Retrieves the most relevant provenance-carrying chunks, then asks the LLM to
+answer grounded in them, returning both the answer text and the citations the
+UI can highlight on the page.
+"""
+from __future__ import annotations
+
+from dataclasses import dataclass
+
+from app.core.logging import get_logger
+from app.llm.base import LLMMessage
+from app.llm.registry import get_provider
+from app.schemas.documents import BBox, Citation
+from app.services import vectorstore
+
+log = get_logger(__name__)
+
+_SYS = (
+    "You answer questions about a specific document using ONLY the provided "
+    "context passages. Cite evidence by referring to passage numbers like [1], "
+    "[2]. If the answer is not in the context, say so honestly. Be concise."
+)
+
+
+@dataclass
+class QAResult:
+    answer: str
+    citations: list[Citation]
+
+
+def _context_block(chunks) -> str:
+    return "\n\n".join(
+        f"[{i+1}] (page {c.page + 1}) {c.text}" for i, c in enumerate(chunks)
+    )
+
+
+async def answer(
+    doc_id: str, question: str, k: int = 5, provider: str | None = None
+) -> QAResult:
+    chunks = await vectorstore.query(doc_id, question, k=k)
+    if not chunks:
+        return QAResult(
+            answer="I couldn't find anything relevant in this document to answer that.",
+            citations=[],
+        )
+
+    llm = get_provider(provider)
+    msgs = [
+        LLMMessage(role="system", content=_SYS),
+        LLMMessage(role="user",
+                   content=f"Context:\n{_context_block(chunks)}\n\nQuestion: {question}"),
+    ]
+    text = await llm.complete(msgs, temperature=0.2)
+
+    citations = [
+        Citation(
+            text=c.text[:240], page=c.page, chunk_id=c.id,
+            bbox=BBox(page=c.page, **c.bbox) if c.bbox else None,
+        )
+        for c in chunks
+    ]
+    return QAResult(answer=text, citations=citations)

backend/app/services/storage.py

@@ -0,0 +1,86 @@
+"""Document persistence.
+
+A small SQLite-backed repository that stores each document's full
+`DocumentDetail` as JSON. Deliberately simple (no ORM) so the project runs
+with zero external services; the access is funnelled through this module so
+swapping to Postgres later touches only this file.
+"""
+from __future__ import annotations
+
+import json
+import sqlite3
+import threading
+from typing import Optional
+
+from app.core.config import settings
+from app.core.logging import get_logger
+from app.schemas.documents import DocumentDetail, DocumentMeta
+
+log = get_logger(__name__)
+
+_lock = threading.Lock()
+
+
+def _conn() -> sqlite3.Connection:
+    conn = sqlite3.connect(settings.db_path, check_same_thread=False)
+    conn.row_factory = sqlite3.Row
+    return conn
+
+
+def init_db() -> None:
+    with _lock, _conn() as conn:
+        conn.execute(
+            """
+            CREATE TABLE IF NOT EXISTS documents (
+                id          TEXT PRIMARY KEY,
+                filename    TEXT,
+                status      TEXT,
+                created_at  TEXT,
+                data        TEXT
+            )
+            """
+        )
+    log.info("SQLite initialised at %s", settings.db_path)
+
+
+def save(detail: DocumentDetail) -> None:
+    with _lock, _conn() as conn:
+        conn.execute(
+            "REPLACE INTO documents (id, filename, status, created_at, data) "
+            "VALUES (?, ?, ?, ?, ?)",
+            (
+                detail.id,
+                detail.filename,
+                detail.status.value,
+                detail.created_at,
+                detail.model_dump_json(),
+            ),
+        )
+
+
+def get(doc_id: str) -> Optional[DocumentDetail]:
+    with _lock, _conn() as conn:
+        row = conn.execute(
+            "SELECT data FROM documents WHERE id = ?", (doc_id,)
+        ).fetchone()
+    if not row:
+        return None
+    return DocumentDetail.model_validate_json(row["data"])
+
+
+def list_all() -> list[DocumentMeta]:
+    with _lock, _conn() as conn:
+        rows = conn.execute(
+            "SELECT data FROM documents ORDER BY created_at DESC"
+        ).fetchall()
+    out: list[DocumentMeta] = []
+    for r in rows:
+        d = json.loads(r["data"])
+        out.append(DocumentMeta.model_validate(d))
+    return out
+
+
+def delete(doc_id: str) -> bool:
+    with _lock, _conn() as conn:
+        cur = conn.execute("DELETE FROM documents WHERE id = ?", (doc_id,))
+        return cur.rowcount > 0

backend/app/services/summary.py

@@ -0,0 +1,58 @@
+"""Summarization — map-reduce over the document.
+
+For short docs we summarize directly; for long ones we summarize chunks then
+reduce, keeping each LLM call within context limits.
+"""
+from __future__ import annotations
+
+from app.core.logging import get_logger
+from app.llm.base import LLMMessage
+from app.llm.registry import get_provider
+
+log = get_logger(__name__)
+
+_MAP_SYS = "Summarize the following document section in 2-3 concise sentences."
+_REDUCE_SYS = (
+    "You are a precise summarizer. Produce a clear, well-structured summary of the "
+    "document: a one-line TL;DR, then 3-6 key bullet points covering the most "
+    "important facts, figures, dates, parties, and actions. Use Markdown."
+)
+
+
+def _split(markdown: str, size: int = 9000) -> list[str]:
+    if len(markdown) <= size:
+        return [markdown]
+    words, parts, buf = markdown.split(), [], []
+    for w in words:
+        buf.append(w)
+        if len(" ".join(buf)) > size:
+            parts.append(" ".join(buf))
+            buf = []
+    if buf:
+        parts.append(" ".join(buf))
+    return parts
+
+
+async def summarize(markdown: str, provider: str | None = None) -> str:
+    if not markdown.strip():
+        return "_The document appears to be empty._"
+    llm = get_provider(provider)
+    sections = _split(markdown)
+
+    if len(sections) == 1:
+        msgs = [
+            LLMMessage(role="system", content=_REDUCE_SYS),
+            LLMMessage(role="user", content=sections[0]),
+        ]
+        return await llm.complete(msgs, temperature=0.3)
+
+    partials = []
+    for s in sections:
+        msgs = [LLMMessage(role="system", content=_MAP_SYS),
+                LLMMessage(role="user", content=s)]
+        partials.append(await llm.complete(msgs, temperature=0.2))
+
+    combined = "\n\n".join(partials)
+    msgs = [LLMMessage(role="system", content=_REDUCE_SYS),
+            LLMMessage(role="user", content=combined)]
+    return await llm.complete(msgs, temperature=0.3)

backend/app/services/vectorstore.py

@@ -0,0 +1,120 @@
+"""Vector store for RAG — pure-Python, numpy cosine similarity.
+
+Deliberately dependency-light: no native build (chroma-hnswlib needs a C++
+compiler) and no external service. Each document's chunks + embeddings are
+persisted to a single .npz/.json pair on disk and cached in memory.
+
+Chunks carry page/bbox provenance so retrieval returns citations the UI can
+highlight. To scale up, swap this module's body for pgvector or Chroma — the
+public interface (index_chunks / query / delete / Chunk) stays the same.
+"""
+from __future__ import annotations
+
+import json
+from dataclasses import dataclass, asdict
+from pathlib import Path
+from typing import Optional
+
+from app.core.config import settings
+from app.core.logging import get_logger
+
+log = get_logger(__name__)
+
+try:
+    import numpy as np
+except Exception:  # pragma: no cover
+    np = None
+
+
+@dataclass
+class Chunk:
+    id: str
+    text: str
+    page: int
+    doc_id: str
+    bbox: Optional[dict] = None
+
+
+# in-memory cache: doc_id -> (matrix[N,d], list[Chunk])
+_cache: dict[str, tuple] = {}
+
+
+def _store_dir() -> Path:
+    p = Path(settings.chroma_dir)
+    p.mkdir(parents=True, exist_ok=True)
+    return p
+
+
+def _paths(doc_id: str) -> tuple[Path, Path]:
+    d = _store_dir()
+    return d / f"{doc_id}.npy", d / f"{doc_id}.json"
+
+
+def _load(doc_id: str):
+    if doc_id in _cache:
+        return _cache[doc_id]
+    vec_path, meta_path = _paths(doc_id)
+    if not vec_path.exists() or not meta_path.exists():
+        return None
+    try:
+        mat = np.load(vec_path)
+        chunks = [Chunk(**c) for c in json.loads(meta_path.read_text("utf-8"))]
+        _cache[doc_id] = (mat, chunks)
+        return _cache[doc_id]
+    except Exception as e:  # pragma: no cover
+        log.warning("vectorstore load failed for %s: %s", doc_id, e)
+        return None
+
+
+def _normalize(mat):
+    norms = np.linalg.norm(mat, axis=1, keepdims=True)
+    norms[norms == 0] = 1.0
+    return mat / norms
+
+
+async def index_chunks(doc_id: str, chunks: list[Chunk]) -> None:
+    if not chunks:
+        return
+    if np is None:
+        raise RuntimeError("numpy is not installed")
+    from app.llm.registry import get_embedder
+
+    embedder = get_embedder()
+    vectors = await embedder.embed([c.text for c in chunks])
+    mat = np.asarray(vectors, dtype="float32")
+
+    vec_path, meta_path = _paths(doc_id)
+    np.save(vec_path, mat)
+    meta_path.write_text(
+        json.dumps([asdict(c) for c in chunks], ensure_ascii=False), "utf-8"
+    )
+    _cache[doc_id] = (mat, chunks)
+    log.info("Indexed %d chunks for doc %s (dim=%d)", len(chunks), doc_id,
+             mat.shape[1] if mat.ndim == 2 else 0)
+
+
+async def query(doc_id: str, question: str, k: int = 5) -> list[Chunk]:
+    loaded = _load(doc_id)
+    if not loaded:
+        return []
+    mat, chunks = loaded
+    if mat.size == 0 or not chunks:
+        return []
+
+    from app.llm.registry import get_embedder
+
+    embedder = get_embedder()
+    qvec = np.asarray((await embedder.embed([question]))[0], dtype="float32")
+
+    sims = _normalize(mat) @ (qvec / (np.linalg.norm(qvec) or 1.0))
+    top = np.argsort(-sims)[: min(k, len(chunks))]
+    return [chunks[int(i)] for i in top]
+
+
+def delete(doc_id: str) -> None:
+    _cache.pop(doc_id, None)
+    for p in _paths(doc_id):
+        try:
+            p.unlink(missing_ok=True)
+        except Exception:
+            pass

backend/requirements.txt

@@ -0,0 +1,29 @@
+# --- Web framework ---
+fastapi==0.115.6
+uvicorn[standard]==0.34.0
+python-multipart==0.0.20
+pydantic==2.10.4
+pydantic-settings==2.7.1
+
+# --- LLM providers ---
+google-generativeai==0.8.3
+openai==1.59.6
+
+# --- Document extraction ---
+# Docling is the local, MIT-licensed extraction backbone (layout, tables, OCR).
+docling==2.15.1
+
+# --- RAG / vector store ---
+# Pure-Python (numpy) cosine-similarity store — no native build, no external
+# service. Swap for pgvector/Chroma in services/vectorstore.py to scale up.
+numpy>=1.26,<3
+
+# --- Export ---
+openpyxl==3.1.5
+
+# --- Utilities ---
+httpx==0.28.1
+tenacity==9.0.0
+python-dotenv==1.0.1
+# Deterministic date parsing for temporal anomaly checks (also a docling dep).
+python-dateutil==2.9.0.post0

backend/scripts/prefetch_models.py

@@ -0,0 +1,36 @@
+"""Download the ML models at image-build time so the container is fully
+self-contained: no first-request download, fast cold starts, works offline.
+
+Run during `docker build`. Honours HF_HOME / EASYOCR_MODULE_PATH / TORCH_HOME
+so the cache lands in a fixed, world-readable location the runtime user shares.
+"""
+from __future__ import annotations
+
+import os
+
+os.environ.setdefault("HF_HUB_DISABLE_SYMLINKS_WARNING", "1")
+
+
+def fetch_docling() -> None:
+    # Layout + TableFormer models (ds4sd/docling-models).
+    from docling.pipeline.standard_pdf_pipeline import StandardPdfPipeline
+
+    path = StandardPdfPipeline.download_models_hf()
+    print(f"[prefetch] docling models -> {path}", flush=True)
+
+
+def fetch_easyocr() -> None:
+    # EasyOCR detection + recognition models (used for scanned PDFs / images).
+    try:
+        import easyocr  # noqa: F401
+
+        easyocr.Reader(["en"], gpu=False)
+        print("[prefetch] easyocr (en) models ready", flush=True)
+    except Exception as e:  # OCR is optional; never fail the build over it.
+        print(f"[prefetch] easyocr skipped: {e}", flush=True)
+
+
+if __name__ == "__main__":
+    fetch_docling()
+    fetch_easyocr()
+    print("[prefetch] done", flush=True)

docker-compose.yml

@@ -0,0 +1,24 @@
+# Local production-like run of the backend (the frontend deploys to Vercel).
+# Usage:
+#   docker compose up --build
+#   # backend on http://localhost:8000 ; data persists in the named volume.
+services:
+  backend:
+    build:
+      context: ./backend
+    image: document-agent-backend
+    env_file:
+      - ./backend/.env          # OPENAI_API_KEY / GEMINI_API_KEY etc.
+    environment:
+      DATA_DIR: /data
+      # Allow the local Next dev server (and the proxy) through CORS.
+      CORS_ORIGINS: "http://localhost:3000,http://localhost:3002"
+      ENABLE_OCR: "true"
+    ports:
+      - "8000:8000"
+    volumes:
+      - backend_data:/data
+    restart: unless-stopped
+
+volumes:
+  backend_data:

docs/ARCHITECTURE.md

@@ -0,0 +1,65 @@
+# Architecture — Document Processing AI Agent
+
+**Decision record from research (see `RESEARCH.md`). Stack chosen: custom composed stack.**
+
+## High-level
+
+```
+┌──────────────────────────────────────────────────────────────┐
+│  FRONTEND  (Next.js + React + TypeScript + Tailwind)          │
+│  40% Chat  │  60% Workspace                                   │
+│  ────────  │  ── Tabs: Viewer · Fields · Classify · Summary · │
+│  streaming │     Export ──                                    │
+│  agent     │  Document viewer w/ bounding-box highlights      │
+└─────┬────────────────────────────────────────────────────────┘
+      │  REST + SSE (streaming)
+┌─────▼────────────────────────────────────────────────────────┐
+│  BACKEND  (FastAPI, async)                                    │
+│                                                              │
+│  api/        routes: documents, chat(SSE), extract,          │
+│              classify, summary, export, health               │
+│  agent/      orchestrator (tool-calling loop) + prompts      │
+│  llm/        provider abstraction: Gemini(default)⇄OpenAI     │
+│  services/   ingestion(Docling) · extraction · classification│
+│              · summary · qa(RAG) · anomaly · export ·         │
+│              vectorstore · storage                           │
+│  schemas/    Pydantic models (incl. invoice/contract)        │
+│  core/       config · logging · deps                         │
+└─────┬───────────────────────────┬────────────────────────────┘
+      │                           │
+┌─────▼─────┐            ┌─────────▼──────────┐
+│ Docling   │            │ Storage            │
+│ (local    │            │  • files (disk/S3) │
+│  extract) │            │  • SQLite/Postgres │
+└───────────┘            │  • vectors (chroma/│
+                         │    pgvector)       │
+                         └────────────────────┘
+```
+
+## The 7 core functions → implementation
+
+| Function | How | Tool exposed to agent |
+|---|---|---|
+| 1. Upload & ingest | `POST /documents` → Docling parse → DoclingDocument JSON + page images | `ingest_document` |
+| 2. Parse & extract structured data | Docling structure + LLM structured-output against Pydantic schema (fields, tables, KV, entities) + confidence | `extract_fields` |
+| 3. Classify | LLM zero-shot classification → type + confidence | `classify_document` |
+| 4. Summarize | Map-reduce summarization over chunks | `summarize_document` |
+| 5. Conversational Q&A | Agentic RAG: retrieve chunks → answer **with citations** (page + bbox) | `query_document` |
+| 6. Flag anomalies | Rule + LLM checks: missing required fields, inconsistencies, low-confidence | `flag_anomalies` |
+| 7. Export | Serialize extracted data → JSON / CSV / Excel | `export_data` |
+
+## Agent design
+Single **orchestrator agent** with the tools above (tool-calling loop), agentic-RAG retrieval, structured outputs (Pydantic), human-in-the-loop on low-confidence fields. Pattern mirrors LlamaIndex ADW: *parse → maintain state → retrieve → reason → surface for validation*.
+
+## LLM strategy
+- **Default (free):** Gemini Flash (1M context, native vision for scans).
+- **Fallback (switchable):** GPT-4o-mini.
+- `llm/base.py` defines a `LLMProvider` interface; runtime switch via config/env or per-request header. Cheap model for classify, stronger for reasoning.
+
+## Tech choices (pragmatic defaults for a runnable, impressive build)
+- **Backend:** FastAPI, Pydantic v2, Docling, `google-generativeai`, `openai`, ChromaDB (embedded vector store — no external service), SQLite (dev) with a path to Postgres/pgvector.
+- **Frontend:** Next.js (App Router) + React + TypeScript + Tailwind + a PDF/image viewer with overlay layer for bounding boxes; SSE for streaming chat.
+- **Local-first:** runs with only a free Gemini API key. No paid services required.
+
+## Separation of concerns
+Routes are thin → call services. Agent orchestrates services as tools. LLM access only via `llm/`. Storage only via `services/storage` + `services/vectorstore`. Schemas shared. This is the Dify-grade discipline we borrowed.

docs/DEPLOY.md

@@ -0,0 +1,125 @@
+# Deployment guide
+
+Architecture: **frontend → Vercel**, **backend → a Docker container host** (Render
+blueprint included). The two talk over HTTPS; the frontend proxies `/api/*` to the
+backend via a Next.js rewrite, so the browser only ever sees the Vercel origin.
+
+```
+  Browser ──> Vercel (Next.js)  ──/api/* rewrite──>  Backend container (FastAPI + Docling)
+                                                      └── persistent disk /data
+                                                      └── models baked into image
+```
+
+Why the backend can't be serverless: it bundles Docling + (CPU) torch + ~850 MB of
+layout/table/OCR models and needs real RAM/disk. Plan for **2 GB minimum, 4 GB
+recommended**.
+
+---
+
+## 1. Backend → Render (recommended)
+
+The image bakes the models in, runs as non-root, exposes a health check, and binds
+`$PORT`. Everything is described in [`render.yaml`](../render.yaml).
+
+1. **Push this repo to GitHub** (Render deploys from Git):
+   ```bash
+   git remote add origin https://github.com/<you>/document-agent.git
+   git push -u origin main
+   ```
+2. In the **Render dashboard → New → Blueprint**, select the repo. Render reads
+   `render.yaml` and provisions a Docker web service + a 10 GB disk at `/data`.
+3. When prompted, set the secret env vars:
+   - `OPENAI_API_KEY` — your key
+   - `GEMINI_API_KEY` — optional (leave blank to disable Gemini)
+4. After the first deploy, note the URL (e.g. `https://document-agent-backend.onrender.com`)
+   and verify: `GET <url>/api/health` → `{"status":"ok", ...}`.
+5. Set `CORS_ORIGINS` to your Vercel domain (see step 2 below) and redeploy.
+
+First build is slow (downloads torch + models). Subsequent deploys reuse layers.
+
+### Free option — Hugging Face Spaces (no card required)
+
+Render's free plan is 512 MB RAM with no disk and can't run this image. **HF Spaces
+free CPU tier gives 16 GB RAM** and runs Docker, so it fits — the only loss is
+persistence (free Spaces have no disk, so uploads/DB reset on restart/rebuild).
+
+This repo ships an HF-ready **root `Dockerfile`** (builds the backend with the repo
+root as context, runs as UID 1000) and the HF config in the **`README.md`
+frontmatter** (`sdk: docker`, `app_port: 8000`).
+
+1. Create a free account at https://huggingface.co, then **New → Space**:
+   - **SDK: Docker** (blank template), name e.g. `document-agent`, visibility your choice.
+2. Push this repo to the Space's git remote:
+   ```bash
+   git remote add space https://huggingface.co/spaces/<hf-user>/document-agent
+   git push space main          # auth: HF username + an HF access token as password
+   ```
+   (Create a write token at https://huggingface.co/settings/tokens.)
+3. In the Space: **Settings → Variables and secrets**, add as **secrets**:
+   - `OPENAI_API_KEY` (freshly rotated) and/or `GEMINI_API_KEY`
+   - `DEFAULT_LLM_PROVIDER` (`openai` or `gemini`), `ENABLE_OCR` (`true`/`false`)
+4. The Space builds (slow first time: torch + ~850 MB models). When live, the URL is
+   `https://<hf-user>-document-agent.hf.space`; verify `GET <url>/api/health`.
+5. Use that URL as `NEXT_PUBLIC_API_BASE` on Vercel (step 2 below).
+
+> Tip: if free RAM is tight on large scans, set `ENABLE_OCR=false` to cut memory.
+
+### Other alternatives
+- **Fly.io**: `fly launch` from `./backend` (it detects the Dockerfile), add a volume
+  (`fly volumes create data`) mounted at `/data`, set secrets with `fly secrets set`.
+- **Railway**: new service from the Dockerfile, add a volume at `/data`, set the env vars.
+
+---
+
+## 2. Frontend → Vercel
+
+The frontend is already env-driven — it needs exactly one variable pointing at the
+backend.
+
+```bash
+cd frontend
+vercel            # first run: link/create the project
+# set the backend URL for all environments:
+vercel env add NEXT_PUBLIC_API_BASE
+#   value: https://document-agent-backend.onrender.com   (no trailing slash)
+vercel --prod     # deploy
+```
+
+`NEXT_PUBLIC_API_BASE` drives the `/api/:path*` rewrite in
+[`next.config.mjs`](../frontend/next.config.mjs), so uploads, page images, exports and
+the chat stream all flow through the same origin — no CORS needed in the common case.
+
+> Streaming note: chat uses SSE through the Vercel rewrite. If a host buffers the
+> stream, switch to direct calls by setting `BASE = process.env.NEXT_PUBLIC_API_BASE`
+> in `frontend/lib/api.ts` and make sure `CORS_ORIGINS` on the backend lists the
+> Vercel domain.
+
+---
+
+## 3. Verify locally first (optional but recommended)
+
+Build and run the real production image before shipping:
+
+```bash
+docker compose up --build
+# backend: http://localhost:8000/api/health
+```
+
+`docker-compose.yml` mounts a named volume at `/data` and reads `backend/.env`, so it
+mirrors production (persistent uploads + sqlite + rendered pages).
+
+---
+
+## Operational notes
+
+- **Persistence**: `/data` holds uploads, the SQLite DB, the numpy vector store and
+  rendered page images. Keep it on a mounted disk/volume — without one, data resets on
+  every deploy.
+- **Models**: baked into the image (not in `/data`), so they survive redeploys with no
+  re-download.
+- **Secrets**: only ever via host env vars. `backend/.env` is git-ignored.
+- **Scaling**: state lives on local disk + SQLite, so run a **single instance**. To
+  scale horizontally, move the DB to Postgres and the vector store to pgvector
+  (`services/vectorstore.py` is a drop-in seam) and uploads to object storage.
+- **Cost guard**: the backend holds your LLM key behind a public URL. Consider adding
+  auth or a rate limit before sharing widely.

docs/OVERVIEW.md

@@ -0,0 +1,253 @@
+# DocAgent — Project Overview, Research, Design & How It Works
+
+**Project:** Document Processing AI Agent (chat-based, 40/60 UI)
+**Org:** Monkhub Innovations
+**One-line:** Upload any document → it parses, classifies, extracts structured
+data, summarizes, answers grounded questions, flags anomalies, and exports clean data.
+
+This single document brings together **what we researched, what we chose and why,
+the comparison, the feature set, and exactly how the agent works end-to-end.**
+For the full deep-dive research see [`RESEARCH.md`](RESEARCH.md); for the layered
+architecture record see [`ARCHITECTURE.md`](ARCHITECTURE.md); to deploy see
+[`DEPLOY.md`](DEPLOY.md).
+
+---
+
+## 1. Research — what we evaluated
+
+We needed a foundation for a **custom, chat-based, production document-processing
+agent** with a 40/60 split UI and a **free-LLM-first** model strategy. We evaluated
+the five leading systems, one per relevant archetype:
+
+| Archetype | System | Why it represents the category |
+|---|---|---|
+| Full open-source app | **RAGFlow** | A complete chat-over-documents product with deep understanding, citations, and an agent layer — the closest existing thing to our target. |
+| LLM app platform | **Dify** | Reference for clean platform architecture, visual orchestration, observability, polished UI/UX. |
+| Extraction toolkit | **Docling** (IBM / LF AI & Data) | Best-in-class open document parsing — layout, tables, OCR, VLM. The extraction *backbone*. |
+| Workflow automation | **n8n** | The "glue" archetype: trigger → OCR → LLM extract → validate → route, assembled visually. |
+| Agentic framework | **LlamaIndex ADW** | Reference for stateful, multi-step *document agents* and schema-based structured extraction. |
+
+**Method:** sequential, single-researcher web research; each system and source read
+in full. Full notes and sources in [`RESEARCH.md`](RESEARCH.md).
+
+---
+
+## 2. Comparison table & scoring
+
+**Rubric (1–5; 5 = best)** — scored for *our* specific goal, not in the abstract:
+
+| Code | Parameter |
+|---|---|
+| A | Feature completeness (OCR, layout, tables, KV, entities, classify, summarize, Q&A, export) |
+| B | Architecture quality & extensibility |
+| C | Ease of customisation to our UI/flows |
+| D | Production-readiness (auth, scaling, observability, deploy) |
+| E | UI/UX quality for document work |
+| F | Cost / open-source friendliness (license, no per-page cost, self-host) |
+| G | Community & documentation |
+
+| System | A | B | C | D | E | F | G | **Total /35** |
+|---|---|---|---|---|---|---|---|---|
+| **Dify** | 3 | 5 | 5 | 5 | 5 | 4 | 5 | **32** |
+| **RAGFlow** | 5 | 4 | 3 | 4 | 4 | 5 | 5 | **30** |
+| **Docling** | 5 | 5 | 4 | 4 | 1 | 5 | 5 | **29** |
+| **LlamaIndex ADW** | 4 | 5 | 4 | 4 | 2 | 3 | 5 | **27** |
+| **n8n workflows** | 3 | 3 | 5 | 4 | 2 | 3 | 5 | **25** |
+
+**How to read it.** Dify wins on raw total, but its lead comes from *platform polish*,
+not *document intelligence* (its weakest axis, A=3). RAGFlow and Docling lead on the
+axis this project lives or dies on — document depth (A=5). The honest conclusion is
+that **no single system is simultaneously the deepest extractor, the cleanest platform,
+and a ready-made 40/60 chat UI.** So we composed.
+
+---
+
+## 3. What we chose & why
+
+> **Decision: a composed custom stack — not a fork of any one system.**
+
+| Borrowed from | What we took |
+|---|---|
+| **RAGFlow** | The end-to-end **pipeline blueprint** (deep-understanding-at-ingestion → grounded, citable answers) and citation UX. |
+| **Docling** | The **actual extraction engine** — MIT, local, layout + TableFormer tables + OCR, zero per-page cost. |
+| **LlamaIndex ADW** | The **agent loop pattern**: *parse → maintain state → retrieve → reason → surface for validation.* |
+| **Dify** | **Separation-of-concerns discipline** — thin routes, services, provider abstraction. |
+| **Our own** | **FastAPI** backend + **Next.js/React** 40/60 UI tailored to the brief. |
+
+**Why composed instead of forking RAGFlow** (the top *product*): its infra (MinIO +
+Elasticsearch + MySQL + Redis) and opinionated UI are too heavy to host our bespoke
+40/60 experience, and the footprint (≥4 cores / ≥16 GB / ≥50 GB) is unjustified at our
+stage. Composing gives us RAGFlow's proven flow, Docling's best-in-class *and free/local*
+extraction, LlamaIndex's stateful agent patterns, and Dify's architectural cleanliness —
+with none of the licensing, cost, or lock-in downsides.
+
+### Chosen stack at a glance
+
+| Layer | Choice | Why |
+|---|---|---|
+| **Extraction** | **Docling** (layout, TableFormer, EasyOCR) | Best open accuracy; MIT; local; $0 per page; offline-capable. |
+| **LLM (default)** | **Gemini Flash** | Generous free tier; ~1M-token context for long docs; native vision for scans. |
+| **LLM (fallback)** | **GPT-4o-mini** | Cheap, reliable tool-calling; switchable per-request behind a provider interface. |
+| **Backend** | **FastAPI** (async, Pydantic v2) | Clean route→service layering; SSE streaming; production-ready. |
+| **Frontend** | **Next.js (App Router) + React + TS + Tailwind** | 40/60 layout, document viewer with bounding-box overlays, streaming chat. |
+| **Vector store** | **Pure-Python numpy cosine** (`.npz`/`.json` on disk) | Dependency-light, no native build/service; chunks carry page+bbox for citations. Swappable for pgvector/Chroma. |
+| **Metadata store** | **SQLite** (JSON blobs) | Zero external services; access funnelled through one module → easy Postgres swap. |
+
+**Local-first & ~zero marginal cost:** the whole app runs with only a free Gemini key.
+Nothing imports a vendor SDK except `llm/*`, so providers swap by env, per request, or
+from the UI.
+
+---
+
+## 4. Features — the 7 core functions
+
+| # | Function | What it does | Agent tool |
+|---|---|---|---|
+| 1 | **Ingest** | Parse PDF, DOCX, PPTX, XLSX, HTML, images with Docling (layout, tables, OCR) → unified Markdown + page images + provenance-carrying chunks. | *(pipeline)* |
+| 2 | **Extract** | **Adaptive, template-free** structured data: fields / records / tables / entities, each with a confidence score and best-effort citation. | `get_extracted_data` |
+| 3 | **Classify** | Zero-shot document-type detection with confidence + rationale. | `classify_document` |
+| 4 | **Summarize** | Map-reduce summary (TL;DR + key points) over chunks. | `summarize_document` |
+| 5 | **Q&A** | Conversational, **agentic-RAG** answers **grounded with citations** you can click to highlight on the page. | `query_document` |
+| 6 | **Flag anomalies** | Missing fields, low-confidence values, generic arithmetic reconciliation, multi-record sanity, LLM consistency review. | `flag_anomalies` |
+| 7 | **Export** | Serialize extracted data to **JSON / CSV / Excel** (records become a table). | *(API)* |
+
+### The headline feature: adaptive, domain-agnostic extraction
+
+Extraction does **not** rely on per-domain templates. For *any* document — any layout,
+any label convention, known type or not — it produces a stable, consistently-keyed shape by:
+
+- **Layout detection** — the model decides whether the doc is one entity
+  (`single_record`, e.g. an invoice), many similar entities (`multi_record`, e.g. an
+  employee roster or transaction list), or a header + repeated sub-list
+  (`mixed`, e.g. invoice + line items).
+- **Schema induction** — fields are derived from the actual content (a known type only
+  *suggests* fields; the model may add/drop to fit reality).
+- **Canonical keys** — inconsistent source labels ("Staff Number" / "Personnel ID" / "ID")
+  are normalised to one `snake_case` key, identical across every record.
+- **Grounding** — values come only from the document body (never file metadata); dates and
+  amounts are normalised (ISO dates, currency); citations are attached by matching values
+  back to provenance-carrying chunks.
+
+---
+
+## 5. How it works — architecture
+
+```
+┌──────────────────────────────────────────────────────────────┐
+│  FRONTEND  (Next.js + React + TypeScript + Tailwind)          │
+│  40% Chat (streaming agent)  │  60% Workspace                 │
+│                              │  Tabs: Viewer · Fields ·       │
+│                              │  Classify · Summary · Export   │
+│                              │  Viewer w/ bounding-box cites   │
+└─────┬────────────────────────────────────────────────────────┘
+      │  REST + SSE (token streaming)
+┌─────▼────────────────────────────────────────────────────────┐
+│  BACKEND  (FastAPI, async)                                    │
+│  api/        documents · chat(SSE) · actions · meta(health)   │
+│  agent/      orchestrator (tool-calling loop) + prompts       │
+│  llm/        provider abstraction: Gemini(default) ⇄ OpenAI   │
+│  services/   ingestion(Docling) · extraction · classification │
+│              · summary · qa(RAG) · anomaly · export ·          │
+│              vectorstore · storage · pipeline                  │
+│  schemas/    Pydantic contracts (documents, optional hints)   │
+└─────┬───────────────────────────┬────────────────────────────┘
+      │                           │
+┌─────▼─────┐            ┌─────────▼──────────┐
+│ Docling   │            │ Storage (/data)    │
+│ (local    │            │ • files on disk    │
+│  extract) │            │ • SQLite metadata  │
+│ models    │            │ • numpy vectors    │
+│ baked in  │            │   (.npz/.json)     │
+└───────────┘            │ • rendered pages   │
+                         └────────────────────┘
+```
+
+**Discipline (borrowed from Dify):** routes are thin → call services; the agent
+orchestrates services as tools; LLM access only through `llm/`; storage only through
+`services/storage` + `services/vectorstore`; schemas are shared.
+
+---
+
+## 6. Working — the two flows
+
+### Flow A — Ingestion pipeline (runs once per upload, in the background)
+
+Implemented in `services/pipeline.py`. State is written back at every milestone so the
+UI can poll status and render results progressively.
+
+```
+upload → [Stage 1: INGEST]──fail?──► status=failed (hard fail; only this stage can)
+            │ Docling parse → Markdown + page images + chunks(page,bbox)
+            ▼
+        [index]      embed chunks → numpy vector store        ┐
+        [classify]   LLM zero-shot → type + confidence        │ Stages 2–6 are
+        [extract]    adaptive induction → fields/records/...   │ best-effort: a
+        [summarize]  map-reduce → TL;DR + key points           │ failure (e.g. no
+        [anomalies]  rule + LLM checks (needs extraction)      │ LLM key) degrades
+            ▼                                                  ┘ gracefully
+        status=ready  (viewer always works, even if AI steps were skipped)
+```
+
+Key robustness property: **only ingestion can hard-fail a document.** Everything else is
+wrapped so a missing API key or a flaky step still yields a `ready` document with a
+working viewer and a note about what was skipped.
+
+### Flow B — Chat agent (per user message)
+
+Implemented in `agent/orchestrator.py` — a bounded **tool-calling loop**, then a
+**streamed** final answer.
+
+```
+user message + history + doc context
+        │
+        ▼
+  Phase 1: TOOL ROUNDS  (up to MAX_TOOL_ROUNDS = 4, non-streamed reasoning)
+     llm.complete_tools(messages, TOOL_SPECS)
+        │
+        ├── model returns tool_calls?  ──yes──► run each tool ─► append results ─► loop
+        │        tools: query_document · classify_document ·
+        │               get_extracted_data · summarize_document · flag_anomalies
+        │
+        └── no tool_calls (ready to answer) ─► exit loop
+        ▼
+  Phase 2: FINAL ANSWER  (streamed token-by-token over SSE, temperature 0.4)
+        │
+        ▼
+  client renders answer + citation chips → click a chip → highlight the
+  page region (numpy store returned page+bbox with each retrieved chunk)
+```
+
+`query_document` runs **agentic RAG**: retrieve the most similar chunks, answer from
+them, and return grounding passages whose page+bbox the UI turns into clickable
+highlights. The orchestrator collects citations in its `ToolContext` so the API can
+emit them alongside the streamed answer.
+
+---
+
+## 7. End-to-end walkthrough (a user's view)
+
+1. **Drag a document** into the left rail (or click *New document*).
+2. The pipeline runs: pages render in the viewer first, then **classification, fields,
+   summary, and anomaly flags** stream into the workspace tabs as each stage finishes.
+3. **Chat** on the left — e.g. *"Summarize this"*, *"What's the total?"*, *"Any missing
+   fields?"* The agent calls the right tools, then streams a grounded answer. Click any
+   **citation chip** to highlight its exact source on the page.
+4. **Review & edit** low-confidence fields inline (human-in-the-loop) before exporting.
+5. **Export** from the Export tab as **JSON / CSV / Excel**.
+
+---
+
+## 8. Why this design holds up
+
+- **Document depth where it matters** — Docling gives best-in-class layout/table/OCR
+  locally and for free, the axis on which generic platforms (Dify, n8n) are weakest.
+- **Truly document-agnostic** — extraction induces its own schema, so a new document
+  type needs no code or template changes.
+- **Grounded & trustworthy** — every chunk carries page + bbox, so answers and fields
+  point at their exact source; the single most trust-building UX feature.
+- **Cheap and portable** — runs on a free Gemini key with no external services; SQLite
+  and the numpy store are single-file and swappable for Postgres/pgvector at scale via
+  one module each.
+- **Graceful degradation** — the app stays useful even when AI steps fail.
+- **Clean seams for growth** — provider abstraction, storage/vectorstore interfaces,
+  and a single-orchestrator agent that can grow more tools without a rewrite.

docs/RESEARCH.md

@@ -0,0 +1,238 @@
+# Document Processing AI Agent — Deep Research Report
+
+**Project:** Production-ready, chat-based Document Processing AI Agent
+**Org:** Monkhub Innovations
+**Date:** 2026-06-24
+**Method:** Sequential, single-researcher web research (no sub-agents). Each system and source read in full, one at a time.
+
+---
+
+## 0. Executive Summary
+
+We evaluated five leading systems for document-processing automation, spanning the four relevant archetypes:
+
+| Archetype | System | Why it represents the category |
+|---|---|---|
+| Full open-source app | **RAGFlow** (infiniflow) | A complete chat-over-documents product with deep document understanding, citations, and an agent layer — the closest existing thing to what we are building. |
+| LLM app platform | **Dify** (langgenius) | The reference for clean platform architecture, visual orchestration, observability, and polished UI/UX. |
+| Extraction toolkit | **Docling** (IBM / LF AI & Data) | Best-in-class open document parsing (layout, tables, OCR, VLM) — the extraction *backbone*. |
+| Workflow automation | **n8n** document workflows | The "glue" archetype: trigger → OCR → LLM extract → validate → route, assembled visually. |
+| Agentic framework | **LlamaIndex** (Agentic Document Workflows + LlamaParse/Extract) | The reference for stateful, multi-step *document agents* and schema-based structured extraction. |
+
+**Winner / primary reference foundation: RAGFlow**, with the explicit recommendation that we **do not fork it wholesale**. For our build we adopt **RAGFlow's pipeline design as the reference architecture**, use **Docling as the actual extraction engine** (MIT, local, zero per-page cost), borrow **Dify's clean separation-of-concerns and observability**, and structure the agent loop using **LlamaIndex-style agentic-workflow patterns**. Details in §3–§6.
+
+---
+
+## 1. The Five Systems — Deep Analysis
+
+### 1.1 RAGFlow (infiniflow) — *Apache 2.0, ~83.5k★*
+
+**What it does / core purpose.** Open-source RAG engine fused with agent capabilities that turns unstructured documents into a production "context layer" for LLMs. Philosophy: *"quality in, quality out"* — invest heavily in deep document understanding at ingestion so retrieval and answers are grounded and citable.
+
+**End-to-end pipeline.**
+1. **Ingest** — Word, PPT, Excel, PDF, images, scans, structured data, web pages.
+2. **Deep document understanding** — the **DeepDoc** module performs layout analysis and structure extraction; can also delegate to MinerU or Docling parsers.
+3. **Template-based chunking** — intelligent, configurable, *visualized* chunking with human review/intervention.
+4. **Embed + index** — into Elasticsearch (default) or Infinity (their own engine).
+5. **Multi-recall retrieval + fused re-ranking.**
+6. **Grounded answer with citations** — traceable references to reduce hallucination.
+
+**Architecture.** Frontend (web) → REST API → backend services → DeepDoc / embedding / LLM interface, over a storage tier of **MinIO** (objects), **Elasticsearch or Infinity** (vector + full-text), **MySQL** (metadata), **Redis** (cache). Agent framework adds workflow automation, MCP support, code executor (Python/JS, gVisor sandbox), and agent memory.
+
+**Tech stack / models.** Python + Go backend, TypeScript frontend. Pluggable LLM and embedding providers (OpenAI, DeepSeek, Gemini, local). Docker / Docker Compose / Kubernetes (Helm).
+
+**Key features.** Explainable template chunking; grounded citations with visualization; broad format support; agent templates + MCP; memory; many chat-channel and data-source connectors (Confluence, S3, Notion, Google Drive, Discord, etc.); multi-language UI.
+
+**Strengths.** Deepest end-to-end document pipeline of the five; citation grounding; fully permissive license; mature, very active, well-documented; the most *directly comparable* product to our target.
+
+**Weaknesses.** Heavy footprint (≥4 cores, ≥16 GB RAM, ≥50 GB disk; x86-first); operational learning curve; opinionated stack that is hard to bend into a *custom* 40/60 UI without significant surgery.
+
+**Relevance.** ★★★★★ as a **reference** for the pipeline, citation UX, and chunking visualization. Lower as a thing-to-fork because its UI and infra are opinionated.
+
+---
+
+### 1.2 Dify (langgenius) — *Apache-2.0-based (with extra conditions), ~146k★*
+
+**What it does / core purpose.** Open-source **LLM app development platform** — bridges prototype to production with a low-code visual canvas plus developer APIs (Backend-as-a-Service).
+
+**End-to-end flow.** Design on a visual canvas → pick model (50+ providers) → ingest documents into a managed RAG pipeline → define agents (function-calling or ReAct) with 50+ built-in tools → test in a prompt IDE → monitor (logs/metrics) → deploy via API or hosted UI.
+
+**Architecture.** Python backend (BaaS) + Next.js/React/TypeScript frontend; **PostgreSQL**; a canvas **workflow engine**; a unified **model-abstraction layer**; out-of-the-box **RAG pipeline**; first-class **observability** (Langfuse, Opik, Arize Phoenix).
+
+**Tech stack / models.** TS (≈53%) + Python (≈43%). 50+ LLM providers incl. OpenAI, Anthropic, Google, Mistral, local. Docker Compose / K8s / Terraform / CDK. Min 2 cores / 4 GB.
+
+**Key features.** Visual workflow builder; agent framework with tools; managed RAG; prompt IDE with side-by-side model comparison; production observability; complete API surface.
+
+**Strengths.** Cleanest, most extensible *platform* architecture; very polished UI/UX; superb model-provider abstraction (directly relevant to our "free Gemini + GPT-4o-mini fallback" requirement); strong production/observability story; huge community.
+
+**Weaknesses.** Document processing is **generic** — basic ingest/chunk, *no* specialized OCR / table-structure / layout intelligence. License has extra conditions beyond pure Apache 2.0. As a platform it can feel heavy if you only need an app.
+
+**Relevance.** ★★★★☆ — the best **architectural and UI/UX reference**, and a candidate **model-routing layer**, but its document intelligence is too shallow to be our extraction engine.
+
+---
+
+### 1.3 Docling (IBM Research → LF AI & Data) — *MIT, ~62k★*
+
+**What it does / core purpose.** Open document-processing library that parses diverse formats (with advanced PDF understanding) into a unified, AI-ready representation. It is the **extraction backbone**, not an agent.
+
+**Pipeline.** Ingest (path/URL) → format detect → **layout analysis** (structure + reading order) → content extraction (text/tables/images) → **model inference** for hard elements → unified **DoclingDocument** → export (Markdown / JSON / HTML / DocTags).
+
+**Architecture / models.** Layout analysis; **TableFormer** for table structure; OCR engines for scans; pluggable **VLMs** (built-in **GraniteDocling 258M**); ASR for audio; chart-understanding module. Python, Pydantic v2.
+
+**Formats.** **In:** PDF, DOCX, PPTX, XLSX, HTML, EPUB, images (PNG/TIFF/JPEG), audio, email, LaTeX, Markdown, ODF, XBRL, more. **Out:** Markdown, HTML, lossless JSON, DocTags, domain XML.
+
+**Key features.** Best-in-class layout + table extraction; semantic **chunking**; native adapters for **LangChain, LlamaIndex, CrewAI, Haystack, MCP**; **local / air-gapped** execution (no per-page API cost); CLI + Python.
+
+**Strengths.** Best open extraction accuracy; widest format coverage; permissive MIT; local-first (privacy + zero marginal cost); OpenSSF badge; peer-reviewed; trivially embeddable into any backend.
+
+**Weaknesses.** Not an application — no UI, chat, classification, or Q&A on its own (by design). VLM inference adds latency/compute; scaling docs sparse; some domains need fine-tuning.
+
+**Relevance.** ★★★★★ as the **component we should actually use** for ingest/extraction. Not a competitor to the whole app — a building block.
+
+---
+
+### 1.4 n8n Document Workflows — *Sustainable Use License (fair-code), source-available*
+
+**What it does / core purpose.** Visual workflow-automation platform; "document processing" emerges from chaining nodes. Hundreds of templates exist (e.g. *Mistral OCR + GPT-4o-mini invoice processing*, *Gemini OCR + Google Sheets*).
+
+**Typical pipeline (3 layers).** **Ingestion** (webhook / email / Drive / upload trigger → download file) → **Intelligence** (OCR node — Mistral / Gemini / Google Vision — then an **AI Agent / LLM node** with a **structured-output parser** to emit JSON; classification + field extraction) → **Orchestration** (validation against business rules / POs, enrichment, routing to Sheets / SQL / CRM / ERP / Telegram / Gmail). Core design principle: **decouple extraction from delivery** via modular, reusable sub-workflows.
+
+**Strengths.** Fastest way to prototype an extract→validate→route pipeline; 400+ integrations; visual + maintainable; built-in retries/logging; self-hostable.
+
+**Weaknesses.** Not an end-user product — **no document-viewer or chat UI** for our use case (you'd build that separately). Logic-in-canvas sprawls at scale; n8n can become an orchestration bottleneck; **fair-code** license (not OSI-open) carries commercial-use restrictions.
+
+**Relevance.** ★★★☆☆ — excellent **reference for the orchestration pattern** (and a possible internal automation tool), but not the foundation for a polished chat app.
+
+---
+
+### 1.5 LlamaIndex — Agentic Document Workflows (ADW) + LlamaParse/Extract — *Core MIT; Parse/Extract/Cloud are SaaS with free tier*
+
+**What it does / core purpose.** A leading framework for **document agents**. ADW goes a step beyond both IDP and RAG: a **document agent** that parses, **maintains state across multi-step processes**, retrieves reference material, applies business logic, and produces recommendations (human-in-the-loop by design).
+
+**Architecture / pattern.** Document agent orchestrates: **LlamaParse** (VLM-powered parsing) → **LlamaExtract** (Pydantic-schema structured extraction *with confidence scores*) → indexes/RAG for reference retrieval → **Workflow engine** (event-driven; fan-out/fan-in, self-reflection, human-in-the-loop). Inverts classic RAG: *parse → maintain state → retrieve contextually → reason → surface for validation*.
+
+**Strengths.** Best **agent-orchestration** abstractions; schema-first extraction with confidence; excellent for conversational, stateful Q&A over documents; huge community and docs; MIT core.
+
+**Weaknesses.** The strongest parsing/extraction (LlamaParse/Extract/Cloud) is **paid SaaS** (free tier limited); framework is code-first (no end-user UI provided); you assemble the product yourself.
+
+**Relevance.** ★★★★☆ — the best **mental model and orchestration reference** for our agent loop; we can mirror ADW patterns using our own free-LLM + Docling stack instead of the paid services.
+
+---
+
+## 2. Comparison Table & Scoring
+
+**Rubric (1–5; 5 = best).** Scored for the specific goal: *foundation/reference for a custom, chat-based, production document-processing agent with a 40/60 UI and free-LLM-first model strategy.*
+
+| # | Parameter | What a 5 looks like |
+|---|---|---|
+| A | **Feature completeness (doc processing)** | Native OCR, layout, tables, KV, entities, classification, summarization, Q&A, export |
+| B | **Architecture quality & extensibility** | Clean layers, pluggable models/stores, easy to add tools/formats |
+| C | **Ease of customisation** | Fast to bend to our UI/flows without fighting the framework |
+| D | **Production-readiness** | Auth, scaling, observability, error handling, deploy story |
+| E | **UI/UX quality** | Polished, relevant end-user experience for document work |
+| F | **Cost / open-source friendliness** | Permissive license, no forced per-page/API cost, self-hostable |
+| G | **Community & documentation** | Stars, activity, docs depth |
+
+| System | A | B | C | D | E | F | G | **Total /35** |
+|---|---|---|---|---|---|---|---|---|
+| **RAGFlow** | 5 | 4 | 3 | 4 | 4 | 5 | 5 | **30** |
+| **Dify** | 3 | 5 | 5 | 5 | 5 | 4 | 5 | **32** |
+| **Docling** | 5 | 5 | 4 | 4 | 1 | 5 | 5 | **29** |
+| **n8n workflows** | 3 | 3 | 5 | 4 | 2 | 3 | 5 | **25** |
+| **LlamaIndex ADW** | 4 | 5 | 4 | 4 | 2 | 3 | 5 | **27** |
+
+**Reading the table.** By raw total, **Dify (32)** edges **RAGFlow (30)** — but Dify's lead comes from platform polish, *not* document intelligence (its weakest axis, A=3). When we weight by what this project actually needs — **document-processing depth (A), architecture (B), and customisation (C)** — the picture is:
+
+- **Document depth (A):** RAGFlow & Docling lead (5).
+- **Architecture (B):** Dify, Docling, LlamaIndex lead (5).
+- **The honest conclusion is a *combination*, not a single fork.** No one system is simultaneously the deepest extractor *and* the cleanest platform *and* a ready-made 40/60 chat UI.
+
+---
+
+## 3. Winner & Why
+
+**Primary reference foundation: RAGFlow.** It is the only system that already *is* the product we're describing — chat over documents, deep understanding, grounded citations, agentic layer — and it is fully Apache-2.0. It proves the end-to-end pipeline and gives us a battle-tested blueprint for chunking visualization and citation UX.
+
+**But the senior-level recommendation is a composed stack, not a fork**, because RAGFlow's infra and UI are too opinionated to host our bespoke 40/60 experience, and its heaviness is unjustified at our stage:
+
+> **Recommended build = RAGFlow's pipeline blueprint  +  Docling as the extraction engine  +  LlamaIndex-style agentic-workflow orchestration  +  Dify-grade separation-of-concerns & observability  +  our own FastAPI backend and React 40/60 UI.**
+
+This gives us RAGFlow's proven flow, Docling's best-in-class *and free/local* extraction, LlamaIndex's stateful agent patterns, Dify's clean architecture discipline — with none of the licensing, cost, or lock-in downsides.
+
+---
+
+## 4. State-of-the-Art Capabilities (what our agent should do)
+
+Modern IDP = **OCR + ICR + NLP + VLMs + agentic AI**. Expected capabilities:
+
+- **Ingest:** PDF (digital + scanned), DOCX, PPTX, XLSX, images (PNG/JPG/TIFF), email, HTML.
+- **Parse/extract:** layout & reading order, **tables** (structure-aware), **key–value pairs**, **named entities**, line items, multi-page spanning content.
+- **Classify:** document type/category (invoice, contract, form, receipt, report…), zero-shot via LLM.
+- **Summarize:** whole-doc and section-level.
+- **Conversational Q&A:** grounded, **with citations** back to the source span.
+- **Validate / flag:** missing required fields, inconsistencies, anomalies, confidence scoring, **human-in-the-loop** review.
+- **Export:** **JSON** (schema-conformant), CSV/Excel (tables), and structured records to downstream systems.
+
+**Field direction:** extraction is shifting from a *spatial* problem to a *semantic reasoning* problem — VLMs/LLMs read layout + meaning together and emit JSON directly, which is more robust to layout drift than legacy OCR. Best practice = **hybrid**: deterministic parser (Docling) for structure + LLM/VLM for semantic extraction, classification, and Q&A.
+
+---
+
+## 5. Ideal UI/UX (informs the 40/60 build)
+
+Validated patterns from current document-AI and AI-chat UX research:
+
+- **Split-screen** (chat left / work right) is the established pattern when the AI produces artifacts — see Claude Artifacts, Perplexity. Our **40% chat / 60% workspace** split is exactly this, correctly proportioned.
+- **Document viewer with bounding-box highlights:** extracted fields and citations should **highlight the exact source region** in the rendered document (each cell/field carries a bounding polygon + confidence). This is the single most trust-building feature.
+- **Citations:** numbered inline references linking to expandable source cards / highlighted spans; visually distinguish quoted evidence from synthesis.
+- **Workspace panels (the 60%)** — multiple pages/tabs encouraged: **Document Viewer**, **Extraction/Fields** (editable table with confidence + flags), **Classification** panel, **Summary**, **Export** controls.
+- **Side-panel, no context-switch:** keep the document in view while the AI works.
+- **Streaming + accessibility:** stream responses; `aria-live="polite"` on response containers.
+- **Human-in-the-loop:** low-confidence fields visibly flagged and inline-editable before export.
+
+---
+
+## 6. Recommended Architecture, LLM & Toolchain Strategy
+
+**Agent design patterns (2025–26).** The field has moved past static RAG toward **reasoning/agentic RAG** and multi-agent patterns. For us: an **orchestrator agent** + **tool-calling** + **agentic RAG** (the agent decides *when/what/how* to retrieve), with structured-output tools and human-in-the-loop. Keep it a single well-orchestrated agent with tools (upload, parse, classify, extract, summarize, query, validate, export) before reaching for multi-agent complexity.
+
+**LLM strategy (matches the brief).**
+- **Default (free):** **Google Gemini Flash** family — generous free tier, ~1M-token context (great for long documents), native multimodal/vision (can read document images directly).
+- **Fallback (switchable):** **GPT-4o-mini** — cheap, 128k context, reliable tool-calling.
+- **Provider-abstraction layer** so models are swappable per-task (cheap model for classification, stronger for reasoning) — mirroring Dify's model layer.
+
+**Toolchain / OCR / orchestration.**
+- **Extraction:** **Docling** (layout, TableFormer tables, OCR for scans, DoclingDocument → JSON/Markdown) — local, MIT, zero per-page cost. Optional VLM path (Gemini vision) for hard scans.
+- **Structured extraction:** **Pydantic schemas + confidence scores** (LlamaExtract pattern), enforced via LLM structured output / tool-calling.
+- **Orchestration:** lightweight agent loop in our own backend (LlamaIndex-style workflow patterns; optionally the LlamaIndex Workflows library) — avoids the heavy multi-service footprint of RAGFlow.
+- **RAG:** chunk (Docling) → embed → vector store (pgvector / Qdrant) → agentic retrieval with citations.
+
+**Proposed stack for the build.**
+- **Backend:** **FastAPI** (Python) — async, clean separation (routes / services / agent / tools / schemas), production API.
+- **Frontend:** **React + TypeScript** (Next.js or Vite) — 40/60 layout, document viewer with bounding-box overlays, streaming chat.
+- **Storage:** Postgres (+ pgvector) for metadata + vectors; object storage (local/MinIO/S3) for files.
+- **Models:** Gemini Flash (default) ⇄ GPT-4o-mini (fallback) behind a provider interface.
+- **Extraction:** Docling.
+
+---
+
+## 7. Sources
+
+- https://www.firecrawl.dev/blog/best-open-source-agent-frameworks
+- https://github.com/infiniflow/ragflow
+- https://github.com/langgenius/dify
+- https://github.com/docling-project/docling
+- https://github.com/opendatalab/mineru
+- https://procycons.com/en/blogs/pdf-data-extraction-benchmark/
+- https://n8n.io/workflows/4933-document-parsing-and-data-extraction-with-mistral-ocr-ai-processing-and-multi-channel-delivery/
+- https://data-sleek.com/blog/document-workflow-automation-seamless-data-stack-n8n-ai/
+- https://www.llamaindex.ai/blog/introducing-agentic-document-workflows
+- https://www.llamaindex.ai/workflows
+- https://www.everestgrp.com/report/egr-2025-38-r-7283/
+- https://scalehub.com/2025-idp-guide/
+- https://uxdesign.cc/where-should-ai-sit-in-your-ui-1710a258390e
+- https://thefrontkit.com/blogs/ai-chat-ui-best-practices
+- https://learn.microsoft.com/en-us/azure/ai-services/document-intelligence/prebuilt/layout
+- https://www.azilen.com/blog/agentic-ai-design-patterns/
+- https://learn.microsoft.com/en-us/azure/architecture/ai-ml/guide/ai-agent-design-patterns
+- https://modal.com/blog/8-top-open-source-ocr-models-compared
+- https://www.llamaindex.ai/insights/best-vision-language-models

frontend/app/globals.css

@@ -0,0 +1,235 @@
+@tailwind base;
+@tailwind components;
+@tailwind utilities;
+
+:root {
+  --font-sans: "Inter", system-ui, sans-serif;
+}
+
+html,
+body {
+  height: 100%;
+  background: #07080c;
+  color: #e6e9f0;
+  -webkit-font-smoothing: antialiased;
+  text-rendering: optimizeLegibility;
+}
+
+::selection {
+  background: rgba(99, 102, 241, 0.35);
+  color: #fff;
+}
+
+/* refined scrollbars */
+* {
+  scrollbar-width: thin;
+  scrollbar-color: #252b3d transparent;
+}
+*::-webkit-scrollbar {
+  width: 9px;
+  height: 9px;
+}
+*::-webkit-scrollbar-thumb {
+  background: #1c2130;
+  border-radius: 9px;
+  border: 2px solid transparent;
+  background-clip: content-box;
+}
+*::-webkit-scrollbar-thumb:hover {
+  background: #323a52;
+  background-clip: content-box;
+}
+
+@layer components {
+  /* frosted glass surface */
+  .glass {
+    background: rgba(16, 19, 28, 0.7);
+    backdrop-filter: blur(12px);
+    -webkit-backdrop-filter: blur(12px);
+    border: 1px solid rgba(255, 255, 255, 0.06);
+  }
+  .card {
+    background: linear-gradient(180deg, rgba(22, 26, 38, 0.9), rgba(16, 19, 28, 0.9));
+    border: 1px solid rgba(255, 255, 255, 0.06);
+    box-shadow: 0 1px 0 0 rgba(255, 255, 255, 0.04) inset,
+      0 12px 32px -16px rgba(0, 0, 0, 0.8);
+  }
+  /* gradient hairline used under headers */
+  .hairline {
+    height: 1px;
+    background: linear-gradient(
+      90deg,
+      transparent,
+      rgba(99, 102, 241, 0.4),
+      rgba(34, 211, 238, 0.25),
+      transparent
+    );
+  }
+}
+
+/* skeleton shimmer */
+.skeleton {
+  position: relative;
+  overflow: hidden;
+  background: #161a26;
+}
+.skeleton::after {
+  content: "";
+  position: absolute;
+  inset: 0;
+  transform: translateX(-100%);
+  background: linear-gradient(
+    90deg,
+    transparent,
+    rgba(255, 255, 255, 0.06),
+    transparent
+  );
+  animation: shimmer 1.6s infinite;
+}
+@keyframes shimmer {
+  100% {
+    transform: translateX(100%);
+  }
+}
+
+/* streaming caret */
+.caret::after {
+  content: "▋";
+  margin-left: 1px;
+  color: #818cf8;
+  animation: caret-blink 1s steps(2) infinite;
+}
+@keyframes caret-blink {
+  0%,
+  100% {
+    opacity: 0;
+  }
+  50% {
+    opacity: 1;
+  }
+}
+
+/* typing dots */
+.typing-dot {
+  animation: blink 1.4s infinite both;
+}
+.typing-dot:nth-child(2) {
+  animation-delay: 0.2s;
+}
+.typing-dot:nth-child(3) {
+  animation-delay: 0.4s;
+}
+@keyframes blink {
+  0%,
+  80%,
+  100% {
+    opacity: 0.2;
+  }
+  40% {
+    opacity: 1;
+  }
+}
+
+/* ---------------- markdown prose ---------------- */
+.prose-chat {
+  font-size: 0.9rem;
+  line-height: 1.65;
+  color: #d6dae6;
+}
+.prose-chat > *:first-child {
+  margin-top: 0;
+}
+.prose-chat > *:last-child {
+  margin-bottom: 0;
+}
+.prose-chat p {
+  margin: 0.5rem 0;
+}
+.prose-chat ul,
+.prose-chat ol {
+  margin: 0.5rem 0;
+  padding-left: 1.25rem;
+}
+.prose-chat li {
+  margin: 0.2rem 0;
+}
+.prose-chat ul {
+  list-style: disc;
+}
+.prose-chat ol {
+  list-style: decimal;
+}
+.prose-chat h1,
+.prose-chat h2,
+.prose-chat h3 {
+  font-weight: 600;
+  color: #fff;
+  margin: 0.8rem 0 0.4rem;
+  line-height: 1.3;
+}
+.prose-chat h1 {
+  font-size: 1.05rem;
+}
+.prose-chat h2 {
+  font-size: 1rem;
+}
+.prose-chat h3 {
+  font-size: 0.92rem;
+}
+.prose-chat code {
+  background: #0b0d14;
+  padding: 0.1rem 0.4rem;
+  border-radius: 5px;
+  font-size: 0.82em;
+  border: 1px solid rgba(255, 255, 255, 0.06);
+}
+.prose-chat pre {
+  background: #07080c;
+  border: 1px solid #252b3d;
+  padding: 0.85rem;
+  border-radius: 10px;
+  overflow-x: auto;
+  margin: 0.6rem 0;
+}
+.prose-chat pre code {
+  background: transparent;
+  border: 0;
+  padding: 0;
+}
+.prose-chat strong {
+  color: #fff;
+  font-weight: 600;
+}
+.prose-chat a {
+  color: #a5b4fc;
+  text-decoration: underline;
+  text-underline-offset: 2px;
+}
+.prose-chat blockquote {
+  border-left: 3px solid #4f46e5;
+  padding-left: 0.8rem;
+  margin: 0.6rem 0;
+  color: #aeb4c4;
+}
+.prose-chat table {
+  border-collapse: collapse;
+  margin: 0.6rem 0;
+  font-size: 0.84em;
+  width: 100%;
+}
+.prose-chat th,
+.prose-chat td {
+  border: 1px solid #252b3d;
+  padding: 0.35rem 0.6rem;
+  text-align: left;
+}
+.prose-chat th {
+  background: #10131c;
+  font-weight: 600;
+  color: #c7d2fe;
+}
+.prose-chat hr {
+  border: 0;
+  border-top: 1px solid #252b3d;
+  margin: 0.8rem 0;
+}

frontend/app/layout.tsx

@@ -0,0 +1,27 @@
+import type { Metadata } from "next";
+import "./globals.css";
+
+export const metadata: Metadata = {
+  title: "DocAgent — Document Processing AI",
+  description:
+    "Monkhub Innovations — chat-based document processing agent. Ingest, extract, classify, summarize, and query documents.",
+};
+
+export default function RootLayout({
+  children,
+}: {
+  children: React.ReactNode;
+}) {
+  return (
+    <html lang="en">
+      <head>
+        <link rel="preconnect" href="https://fonts.googleapis.com" />
+        <link
+          href="https://fonts.googleapis.com/css2?family=Inter:wght@400;500;600;700&display=swap"
+          rel="stylesheet"
+        />
+      </head>
+      <body className="font-sans antialiased">{children}</body>
+    </html>
+  );
+}

frontend/app/page.tsx

@@ -0,0 +1,146 @@
+"use client";
+
+import { useCallback, useEffect, useRef, useState } from "react";
+import { FileText } from "lucide-react";
+
+import { api } from "@/lib/api";
+import type {
+  Citation,
+  DocumentDetail,
+  DocumentMeta,
+  HealthInfo,
+} from "@/lib/types";
+import ChatPanel from "@/components/ChatPanel";
+import Workspace from "@/components/Workspace";
+import Sidebar from "@/components/Sidebar";
+import ProviderBadge from "@/components/ProviderBadge";
+import Toaster from "@/components/Toaster";
+
+export default function Home() {
+  const [docs, setDocs] = useState<DocumentMeta[]>([]);
+  const [activeId, setActiveId] = useState<string | null>(null);
+  const [detail, setDetail] = useState<DocumentDetail | null>(null);
+  const [health, setHealth] = useState<HealthInfo | null>(null);
+  const [provider, setProvider] = useState<string | null>(null);
+  const [focusCitation, setFocusCitation] = useState<Citation | null>(null);
+  const pollRef = useRef<ReturnType<typeof setInterval> | null>(null);
+
+  const refreshDocs = useCallback(async () => {
+    try {
+      setDocs(await api.listDocuments());
+    } catch {
+      /* backend may be starting */
+    }
+  }, []);
+
+  const loadDetail = useCallback(async (id: string) => {
+    try {
+      setDetail(await api.getDocument(id));
+    } catch {
+      /* ignore */
+    }
+  }, []);
+
+  useEffect(() => {
+    api.health().then(setHealth).catch(() => setHealth(null));
+    refreshDocs();
+  }, [refreshDocs]);
+
+  // Poll the active document while processing so results stream in.
+  useEffect(() => {
+    if (!activeId) return;
+    loadDetail(activeId);
+    pollRef.current && clearInterval(pollRef.current);
+    pollRef.current = setInterval(async () => {
+      const d = await api.getDocument(activeId).catch(() => null);
+      if (d) {
+        setDetail(d);
+        if (d.status === "ready" || d.status === "failed") {
+          pollRef.current && clearInterval(pollRef.current);
+          refreshDocs();
+        }
+      }
+    }, 1600);
+    return () => {
+      pollRef.current && clearInterval(pollRef.current);
+    };
+  }, [activeId, loadDetail, refreshDocs]);
+
+  const handleUploaded = async (meta: DocumentMeta) => {
+    await refreshDocs();
+    setActiveId(meta.id);
+  };
+
+  return (
+    <div className="relative flex h-screen w-screen overflow-hidden bg-surface-950 text-ink-50">
+      {/* ambient background */}
+      <div className="pointer-events-none absolute inset-0 bg-grid-faint bg-[size:42px_42px]" />
+      <div className="pointer-events-none absolute inset-x-0 top-0 h-80 bg-radial-brand" />
+
+      <Toaster />
+
+      <div className="relative z-10 flex h-full w-full">
+        <Sidebar
+          docs={docs}
+          activeId={activeId}
+          onSelect={setActiveId}
+          onUploaded={handleUploaded}
+          onDeleted={async (id) => {
+            await api.deleteDocument(id).catch(() => {});
+            if (activeId === id) {
+              setActiveId(null);
+              setDetail(null);
+            }
+            refreshDocs();
+          }}
+        />
+
+        <div className="flex min-w-0 flex-1 flex-col">
+          {/* header */}
+          <header className="relative flex items-center justify-between px-5 py-3">
+            <div className="flex items-center gap-3">
+              <div className="relative flex h-9 w-9 items-center justify-center rounded-xl bg-gradient-to-br from-brand-400 via-brand-500 to-brand-700 shadow-glow">
+                <FileText className="text-white" size={18} />
+                <span className="absolute -bottom-0.5 -right-0.5 h-2.5 w-2.5 rounded-full bg-accent-400 ring-2 ring-surface-950" />
+              </div>
+              <div>
+                <h1 className="flex items-center gap-2 text-sm font-semibold leading-tight text-white">
+                  DocAgent
+                  <span className="rounded-md bg-white/[0.06] px-1.5 py-0.5 text-[10px] font-medium tracking-wide text-brand-200 ring-1 ring-white/10">
+                    Monkhub
+                  </span>
+                </h1>
+                <p className="text-[11px] leading-tight text-white/40">
+                  Document Processing AI Agent
+                </p>
+              </div>
+            </div>
+            <ProviderBadge health={health} provider={provider} onChange={setProvider} />
+          </header>
+          <div className="hairline" />
+
+          {/* 40 / 60 split */}
+          <div className="flex min-h-0 flex-1">
+            <section className="flex w-2/5 min-w-[360px] flex-col border-r border-white/[0.06] bg-surface-900/40">
+              <ChatPanel
+                documentId={activeId}
+                detail={detail}
+                provider={provider}
+                onCitationClick={setFocusCitation}
+              />
+            </section>
+
+            <section className="flex w-3/5 min-w-0 flex-1 flex-col bg-surface-950/60">
+              <Workspace
+                detail={detail}
+                provider={provider}
+                focusCitation={focusCitation}
+                onRefresh={() => activeId && loadDetail(activeId)}
+              />
+            </section>
+          </div>
+        </div>
+      </div>
+    </div>
+  );
+}