Add Gradio NuExtract3 extraction app and launcher

.agents/.gitignore

@@ -0,0 +1 @@
+references

app/gradio.py

@@ -0,0 +1,236 @@
+from pathlib import Path
+
+import gradio as gr
+import torch
+from transformers import AutoModelForImageTextToText, AutoProcessor
+
+model_id = "numind/NuExtract3"
+
+structured_json_templates = {
+    "Basic receipt": """{
+  "merchant": "verbatim-string",
+  "purchase_date": "date",
+  "purchase_time": "time",
+  "total": "number",
+  "currency": "currency",
+  "item_count": "integer",
+  "paid": "boolean",
+  "payment_method": ["cash", "credit-card", "debit-card", "other"],
+  "items": ["verbatim-string"]
+}""",
+    "Invoice with line items": """{
+  "document_type": "verbatim-string",
+  "vendor": {
+    "name": "verbatim-string",
+    "address": "verbatim-string",
+    "email": "email-address",
+    "phone": "phone-number"
+  },
+  "invoice_number": "verbatim-string",
+  "invoice_date": "verbatim-string",
+  "line_items": [
+    {
+      "description": "verbatim-string",
+      "category": ["product", "service", "shipping", "tax", "other"],
+      "quantity": "integer",
+      "unit_price": "number",
+      "unit": "unit-code",
+      "amount": "number"
+    }
+  ],
+  "subtotal": "number",
+  "tax": "number",
+  "total": "number",
+  "currency": "currency"
+}""",
+    "Advanced global record": """{
+  "summary": "string",
+  "reference_text": "verbatim-string",
+  "sequence_number": "integer",
+  "amount": "number",
+  "effective_date": "date",
+  "effective_time": "time",
+  "signed_at": "date-time",
+  "term": "duration",
+  "active": "boolean",
+  "country": "country",
+  "currency": "currency",
+  "language": "language",
+  "language_tag": "language-tag",
+  "script": "script",
+  "website": "url",
+  "email": "email-address",
+  "phone": "phone-number",
+  "iban": "iban",
+  "bic": "bic",
+  "measurement_unit": "unit-code",
+  "regions": {
+    "us": "region:US",
+    "fr": "region:FR",
+    "ie": "region:IE",
+    "gb": "region:GB",
+    "it": "region:IT",
+    "es": "region:ES",
+    "de": "region:DE",
+    "pt": "region:PT",
+    "ca": "region:CA",
+    "mx": "region:MX",
+    "br": "region:BR",
+    "au": "region:AU",
+    "jp": "region:JP",
+    "kr": "region:KR"
+  },
+  "contacts": [
+    {
+      "name": "verbatim-string",
+      "role": "string",
+      "email": "email-address"
+    }
+  ],
+  "status": ["draft", "active", "expired", "unknown"],
+  "applicable_labels": [["legal", "financial", "technical", "personal"]],
+  "keywords": ["string"]
+}""",
+}
+
+default_structured_json_template = "Invoice with line items"
+
+
+def select_structured_json_template(name):
+    return structured_json_templates[name]
+
+processor = AutoProcessor.from_pretrained(
+    model_id,
+    trust_remote_code=True,
+)
+
+model = AutoModelForImageTextToText.from_pretrained(
+    model_id,
+    dtype=torch.bfloat16,
+    device_map="auto",
+    trust_remote_code=True,
+).eval()
+
+
+def run_nuextract(messages, **chat_template_kwargs):
+    inputs = processor.apply_chat_template(
+        messages,
+        add_generation_prompt=True,
+        tokenize=True,
+        return_dict=True,
+        return_tensors="pt",
+        **chat_template_kwargs,
+    ).to(model.device)
+
+    with torch.inference_mode():
+        generated_ids = model.generate(  # pyright: ignore[reportAttributeAccessIssue]
+            **inputs,
+            max_new_tokens=4096,
+            do_sample=False,
+        )
+
+    generated_ids = generated_ids[:, inputs.input_ids.shape[1] :]
+    return processor.batch_decode(
+        generated_ids,
+        skip_special_tokens=True,
+        clean_up_tokenization_spaces=False,
+    )[0].strip()
+
+
+def extract(image, text, mode, template, enable_thinking):
+    return run_nuextract(
+        [
+            {
+                "role": "user",
+                "content": [
+                    {"type": "image", "image": image},
+                    {"type": "text", "text": text},
+                ],
+            }
+        ],
+        mode=mode,
+        template=template,
+        enable_thinking=enable_thinking,
+    )
+
+
+with gr.Blocks(title="NuExtract3") as demo:
+    gr.Markdown(
+        "# NuExtract3\n"
+        "Extract Markdown or structured JSON from an image and optional text."
+    )
+    with gr.Row():
+        with gr.Column():
+            image = gr.Image(label="Document image", type="pil", height=430)
+            text = gr.Textbox(
+                label="Document text (optional)",
+                placeholder="Add text to process alongside the image",
+                lines=3,
+            )
+            mode = gr.Radio(
+                ["content", "structured"],
+                value="content",
+                label="Task",
+            )
+            template_preset = gr.Dropdown(
+                choices=list(structured_json_templates),
+                value=default_structured_json_template,
+                label="JSON template preset",
+                info="Choose a starting structure, then edit it below.",
+            )
+            template = gr.Textbox(
+                label="Structured JSON template",
+                value=structured_json_templates[default_structured_json_template],
+                lines=18,
+                info="Used for structured extraction. Edit the example to match your document.",
+            )
+            enable_thinking = gr.Checkbox(label="Enable thinking")
+            run = gr.Button("Extract", variant="primary")
+        output = gr.Textbox(label="Output", lines=28, buttons=["copy"])
+
+    gr.Examples(
+        examples=[
+            [
+                str(path),
+                "",
+                "content",
+                structured_json_templates[default_structured_json_template],
+                False,
+            ]
+            for path in [
+                Path(__file__).parents[1]
+                / "data"
+                / "samples"
+                / "receipt-ocr-original.webp",
+                Path(__file__).parents[1]
+                / "data"
+                / "samples"
+                / "invoice-with-items.png",
+                *sorted(
+                    path
+                    for path in (
+                        Path(__file__).parents[1] / "data" / "samples"
+                    ).glob("*.png")
+                    if path.name != "invoice-with-items.png"
+                ),
+            ]
+        ],
+        inputs=[image, text, mode, template, enable_thinking],
+    )
+
+    template_preset.change(
+        select_structured_json_template,
+        inputs=template_preset,
+        outputs=template,
+    )
+
+    run.click(
+        extract,
+        inputs=[image, text, mode, template, enable_thinking],
+        outputs=output,
+        api_name="extract",
+    )
+
+
+if __name__ == "__main__":
+    demo.launch()

gradio.stderr.log

@@ -0,0 +1,3 @@
+W0720 14:54:12.291000 61032 .venv\Lib\site-packages\torch\distributed\elastic\multiprocessing\redirects.py:29] NOTE: Redirects are currently not supported in Windows or MacOs.
+[transformers] The fast path is not available because one of the required library is not installed. Falling back to torch implementation. To install follow https://github.com/fla-org/flash-linear-attention#installation and https://github.com/Dao-AILab/causal-conv1d
+

launch.py

@@ -0,0 +1,28 @@
+import argparse
+import runpy
+
+
+MODULES = {
+    "gradio": "app.gradio",
+    "hf_space": "app.hf_space",
+    "nuextract": "app.nuextract",
+}
+
+
+def build_parser() -> argparse.ArgumentParser:
+    parser = argparse.ArgumentParser(description="Launch a NuMark application.")
+    parser.add_argument(
+        "app",
+        choices=MODULES,
+        help="Application to launch.",
+    )
+    return parser
+
+
+def main() -> None:
+    args = build_parser().parse_args()
+    runpy.run_module(MODULES[args.app], run_name="__main__")
+
+
+if __name__ == "__main__":
+    main()

mise.toml

@@ -5,3 +5,4 @@ uv = "latest"
 
 [env]
 UV_LINK_MODE="copy"
+PYTHONPATH=""

pyproject.toml

@@ -6,7 +6,7 @@ readme = "README.md"
 requires-python = ">=3.12,<3.13"
 dependencies = [
     "accelerate>=1.14.0",
-    "flash-attn",
+    "flash-linear-attention>=0.5.1",
     "gguf>=0.19.0",
     "gradio",
     "huggingface-hub",
@@ -14,8 +14,13 @@ dependencies = [
     "ipython>=9.15.0",
     "ipywidgets>=8.1.8",
     "openai>=2.46.0",
+    "pillow>=12.3.0",
+    "spaces>=0.51.0",
     "torch==2.9.1",
+    "torchaudio==2.9.1",
+    "torchvision",
     "transformers",
+    "triton-windows>=3.5.0.post21",
 ]
 
 [dependency-groups]
@@ -32,7 +37,8 @@ select = ["B", "E4", "E7", "E9", "F", "I", "SIM", "UP"]
 
 [tool.uv.sources]
 torch = { index = "pytorch-cu130" }
-flash-attn = { url = "https://github.com/mjun0812/flash-attention-prebuild-wheels/releases/download/v0.9.28/flash_attn-2.8.3+cu130torch2.9-cp312-cp312-win_amd64.whl" }
+torchaudio = { index = "pytorch-cu130" }
+torchvision = { index = "pytorch-cu130" }
 
 [[tool.uv.index]]
 name = "pytorch-cu130"

uv.lock

@@ -4,7 +4,8 @@ requires-python = "==3.12.*"
 resolution-markers = [
     "sys_platform == 'win32'",
     "sys_platform == 'emscripten'",
-    "sys_platform != 'emscripten' and sys_platform != 'win32'",
+    "(platform_machine != 'aarch64' and sys_platform == 'linux') or (platform_python_implementation != 'CPython' and sys_platform == 'linux') or (sys_platform != 'emscripten' and sys_platform != 'linux' and sys_platform != 'win32')",
+    "platform_machine == 'aarch64' and platform_python_implementation == 'CPython' and sys_platform == 'linux'",
 ]
 
 [[package]]
@@ -251,21 +252,28 @@ wheels = [
 ]
 
 [[package]]
-name = "flash-attn"
-version = "2.8.3+cu130torch2.9"
-source = { url = "https://github.com/mjun0812/flash-attention-prebuild-wheels/releases/download/v0.9.28/flash_attn-2.8.3+cu130torch2.9-cp312-cp312-win_amd64.whl" }
+name = "fla-core"
+version = "0.5.1"
+source = { registry = "https://pypi.org/simple" }
 dependencies = [
     { name = "einops" },
-    { name = "torch" },
 ]
+sdist = { url = "https://files.pythonhosted.org/packages/4e/62/99e149f19a447ce809d7f4fa64ae61c073da337505b9db7f389502470820/fla_core-0.5.1.tar.gz", hash = "sha256:7f3cf56edfbaa9115f4937d1181372e5c7b11809ad8eb2e411fffc3caf729f48", size = 498800, upload-time = "2026-06-18T18:17:15.377Z" }
 wheels = [
-    { url = "https://github.com/mjun0812/flash-attention-prebuild-wheels/releases/download/v0.9.28/flash_attn-2.8.3+cu130torch2.9-cp312-cp312-win_amd64.whl", hash = "sha256:89e22d6a1ff0f5c4e33f75fbce3e8135facae76a363aa0052bcc6a54289fa494" },
+    { url = "https://files.pythonhosted.org/packages/ce/78/a55ee7a62515dcb9220770dd99dfe59ac6599da8af84ad1d20f9f407df4d/fla_core-0.5.1-py3-none-any.whl", hash = "sha256:02150d34aa1e37f6b8ed9b2feec5d29af93680573e5077ed981238179c11fb06", size = 702955, upload-time = "2026-06-18T18:17:12.229Z" },
 ]
 
-[package.metadata]
-requires-dist = [
-    { name = "einops" },
-    { name = "torch" },
+[[package]]
+name = "flash-linear-attention"
+version = "0.5.1"
+source = { registry = "https://pypi.org/simple" }
+dependencies = [
+    { name = "fla-core" },
+    { name = "transformers" },
+]
+sdist = { url = "https://files.pythonhosted.org/packages/4c/e8/8f115be585a046795e4a1f7ade727889219bb66b8d96cc668b8c9e437c0e/flash_linear_attention-0.5.1.tar.gz", hash = "sha256:8840fd4c37de8b0612dc8fd493867f3d330672ba2f17c024a2ce37239634e247", size = 221733, upload-time = "2026-06-18T18:17:16.661Z" }
+wheels = [
+    { url = "https://files.pythonhosted.org/packages/68/3c/5819fb19dc071302ca818616a4e64d4454e1f1193929eb8365c8d38e9052/flash_linear_attention-0.5.1-py3-none-any.whl", hash = "sha256:9022862f0a238752372c81290694b8b2ed1cb2d13fc40d340ffeeb554d6bee5c", size = 403096, upload-time = "2026-06-18T18:17:14.025Z" },
 ]
 
 [[package]]
@@ -698,7 +706,7 @@ version = "0.1.0"
 source = { virtual = "." }
 dependencies = [
     { name = "accelerate" },
-    { name = "flash-attn" },
+    { name = "flash-linear-attention" },
     { name = "gguf" },
     { name = "gradio" },
     { name = "huggingface-hub" },
@@ -706,8 +714,15 @@ dependencies = [
     { name = "ipython" },
     { name = "ipywidgets" },
     { name = "openai" },
+    { name = "pillow" },
+    { name = "spaces" },
     { name = "torch" },
+    { name = "torchaudio", version = "2.9.1", source = { registry = "https://download.pytorch.org/whl/cu130" }, marker = "platform_machine == 'aarch64' and platform_python_implementation == 'CPython' and sys_platform == 'linux'" },
+    { name = "torchaudio", version = "2.9.1+cu130", source = { registry = "https://download.pytorch.org/whl/cu130" }, marker = "platform_machine != 'aarch64' or platform_python_implementation != 'CPython' or sys_platform != 'linux'" },
+    { name = "torchvision", version = "0.24.1", source = { registry = "https://download.pytorch.org/whl/cu130" }, marker = "platform_machine == 'aarch64' and platform_python_implementation == 'CPython' and sys_platform == 'linux'" },
+    { name = "torchvision", version = "0.24.1+cu130", source = { registry = "https://download.pytorch.org/whl/cu130" }, marker = "platform_machine != 'aarch64' or platform_python_implementation != 'CPython' or sys_platform != 'linux'" },
     { name = "transformers" },
+    { name = "triton-windows" },
 ]
 
 [package.dev-dependencies]
@@ -718,7 +733,7 @@ dev = [
 [package.metadata]
 requires-dist = [
     { name = "accelerate", specifier = ">=1.14.0" },
-    { name = "flash-attn", url = "https://github.com/mjun0812/flash-attention-prebuild-wheels/releases/download/v0.9.28/flash_attn-2.8.3+cu130torch2.9-cp312-cp312-win_amd64.whl" },
+    { name = "flash-linear-attention", specifier = ">=0.5.1" },
     { name = "gguf", specifier = ">=0.19.0" },
     { name = "gradio" },
     { name = "huggingface-hub" },
@@ -726,8 +741,13 @@ requires-dist = [
     { name = "ipython", specifier = ">=9.15.0" },
     { name = "ipywidgets", specifier = ">=8.1.8" },
     { name = "openai", specifier = ">=2.46.0" },
+    { name = "pillow", specifier = ">=12.3.0" },
+    { name = "spaces", specifier = ">=0.51.0" },
     { name = "torch", specifier = "==2.9.1", index = "https://download.pytorch.org/whl/cu130" },
+    { name = "torchaudio", specifier = "==2.9.1", index = "https://download.pytorch.org/whl/cu130" },
+    { name = "torchvision", index = "https://download.pytorch.org/whl/cu130" },
     { name = "transformers" },
+    { name = "triton-windows", specifier = ">=3.5.0.post21" },
 ]
 
 [package.metadata.requires-dev]
@@ -1365,6 +1385,23 @@ wheels = [
     { url = "https://files.pythonhosted.org/packages/e9/44/75a9c9421471a6c4805dbf2356f7c181a29c1879239abab1ea2cc8f38b40/sniffio-1.3.1-py3-none-any.whl", hash = "sha256:2f6da418d1f1e0fddd844478f41680e794e6051915791a034ff65e5f100525a2", size = 10235, upload-time = "2024-02-25T23:20:01.196Z" },
 ]
 
+[[package]]
+name = "spaces"
+version = "0.51.0"
+source = { registry = "https://pypi.org/simple" }
+dependencies = [
+    { name = "gradio" },
+    { name = "httpx" },
+    { name = "packaging" },
+    { name = "pydantic" },
+    { name = "requests" },
+    { name = "typing-extensions" },
+]
+sdist = { url = "https://files.pythonhosted.org/packages/d0/93/6066dcd0ee4eee2a6242de917b461e6779282cbe78fbf1d278b8042e7232/spaces-0.51.0.tar.gz", hash = "sha256:ff7700969ed3e3500eccae46b9f5f6daf00de182ee2f3ad51156f3ade471fd37", size = 88598, upload-time = "2026-07-10T16:08:46.921Z" }
+wheels = [
+    { url = "https://files.pythonhosted.org/packages/a1/ae/77e14929554053c2f5465a555fd0de0b2f26441442b555d677424d4ad0f0/spaces-0.51.0-py3-none-any.whl", hash = "sha256:a6783be3561d2a49a55c23cdbf2f88a1a8d922cf6884ca11a4d70a5ed501d0f6", size = 111642, upload-time = "2026-07-10T16:08:48.011Z" },
+]
+
 [[package]]
 name = "stack-data"
 version = "0.6.3"
@@ -1474,6 +1511,72 @@ wheels = [
     { url = "https://download-r2.pytorch.org/whl/cu130/torch-2.9.1%2Bcu130-cp312-cp312-win_amd64.whl", hash = "sha256:cd3232a562ad2a2699d48130255e1b24c07dfe694a40dcd24fad683c752de121", upload-time = "2026-01-26T16:37:50Z" },
 ]
 
+[[package]]
+name = "torchaudio"
+version = "2.9.1"
+source = { registry = "https://download.pytorch.org/whl/cu130" }
+resolution-markers = [
+    "platform_machine == 'aarch64' and platform_python_implementation == 'CPython' and sys_platform == 'linux'",
+]
+dependencies = [
+    { name = "torch", marker = "platform_machine == 'aarch64' and platform_python_implementation == 'CPython' and sys_platform == 'linux'" },
+]
+wheels = [
+    { url = "https://download-r2.pytorch.org/whl/cu130/torchaudio-2.9.1-cp312-cp312-manylinux_2_28_aarch64.whl", hash = "sha256:b3c75f87e325946276c952864dbce2c8fabc88a00d86730c3d5bc0999ebf7789", upload-time = "2025-11-11T19:17:11Z" },
+]
+
+[[package]]
+name = "torchaudio"
+version = "2.9.1+cu130"
+source = { registry = "https://download.pytorch.org/whl/cu130" }
+resolution-markers = [
+    "sys_platform == 'win32'",
+    "sys_platform == 'emscripten'",
+    "(platform_machine != 'aarch64' and sys_platform == 'linux') or (platform_python_implementation != 'CPython' and sys_platform == 'linux') or (sys_platform != 'emscripten' and sys_platform != 'linux' and sys_platform != 'win32')",
+]
+dependencies = [
+    { name = "torch", marker = "platform_machine != 'aarch64' or platform_python_implementation != 'CPython' or sys_platform != 'linux'" },
+]
+wheels = [
+    { url = "https://download-r2.pytorch.org/whl/cu130/torchaudio-2.9.1%2Bcu130-cp312-cp312-manylinux_2_28_x86_64.whl", hash = "sha256:a6c58d5e846da5a90d50bd425e2c24368747cd04297d95c6dd51d3f7f85fea26", upload-time = "2025-11-11T19:17:10Z" },
+    { url = "https://download-r2.pytorch.org/whl/cu130/torchaudio-2.9.1%2Bcu130-cp312-cp312-win_amd64.whl", hash = "sha256:7533a17bed21e5b86b8c49fd79656779779f2c991aef2804af6f318d2022ea6a", upload-time = "2025-11-11T19:17:10Z" },
+]
+
+[[package]]
+name = "torchvision"
+version = "0.24.1"
+source = { registry = "https://download.pytorch.org/whl/cu130" }
+resolution-markers = [
+    "platform_machine == 'aarch64' and platform_python_implementation == 'CPython' and sys_platform == 'linux'",
+]
+dependencies = [
+    { name = "numpy", marker = "platform_machine == 'aarch64' and platform_python_implementation == 'CPython' and sys_platform == 'linux'" },
+    { name = "pillow", marker = "platform_machine == 'aarch64' and platform_python_implementation == 'CPython' and sys_platform == 'linux'" },
+    { name = "torch", marker = "platform_machine == 'aarch64' and platform_python_implementation == 'CPython' and sys_platform == 'linux'" },
+]
+wheels = [
+    { url = "https://download-r2.pytorch.org/whl/cu130/torchvision-0.24.1-cp312-cp312-manylinux_2_28_aarch64.whl", hash = "sha256:89743dcee13e943f58b37c7647aff14b5bb24c11c84826376d457acf97586fec", upload-time = "2025-11-15T18:12:59Z" },
+]
+
+[[package]]
+name = "torchvision"
+version = "0.24.1+cu130"
+source = { registry = "https://download.pytorch.org/whl/cu130" }
+resolution-markers = [
+    "sys_platform == 'win32'",
+    "sys_platform == 'emscripten'",
+    "(platform_machine != 'aarch64' and sys_platform == 'linux') or (platform_python_implementation != 'CPython' and sys_platform == 'linux') or (sys_platform != 'emscripten' and sys_platform != 'linux' and sys_platform != 'win32')",
+]
+dependencies = [
+    { name = "numpy", marker = "platform_machine != 'aarch64' or platform_python_implementation != 'CPython' or sys_platform != 'linux'" },
+    { name = "pillow", marker = "platform_machine != 'aarch64' or platform_python_implementation != 'CPython' or sys_platform != 'linux'" },
+    { name = "torch", marker = "platform_machine != 'aarch64' or platform_python_implementation != 'CPython' or sys_platform != 'linux'" },
+]
+wheels = [
+    { url = "https://download-r2.pytorch.org/whl/cu130/torchvision-0.24.1%2Bcu130-cp312-cp312-manylinux_2_28_x86_64.whl", hash = "sha256:6939dd403cc28ab0a46f53e6c86e2e852cf65771c1b0ddd09c44c541a1cdbad9", upload-time = "2025-11-15T18:12:58Z" },
+    { url = "https://download-r2.pytorch.org/whl/cu130/torchvision-0.24.1%2Bcu130-cp312-cp312-win_amd64.whl", hash = "sha256:d31ceaded0d9b737471fa680ccd9e1acb6d5f0f70f03ef3a8d786a99c79da7cf", upload-time = "2025-11-15T20:02:26Z" },
+]
+
 [[package]]
 name = "tornado"
 version = "6.5.7"
@@ -1541,6 +1644,14 @@ wheels = [
     { url = "https://files.pythonhosted.org/packages/f2/50/9a8358d3ef58162c0a415d173cfb45b67de60176e1024f71fbc4d24c0b6d/triton-3.5.1-cp312-cp312-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:d2c6b915a03888ab931a9fd3e55ba36785e1fe70cbea0b40c6ef93b20fc85232", size = 170470207, upload-time = "2025-11-11T17:41:00.253Z" },
 ]
 
+[[package]]
+name = "triton-windows"
+version = "3.7.1.post27"
+source = { registry = "https://pypi.org/simple" }
+wheels = [
+    { url = "https://files.pythonhosted.org/packages/76/30/325b420efd0047e119679c646a9a410db216069800ec009fae3da26c69a3/triton_windows-3.7.1.post27-cp312-cp312-win_amd64.whl", hash = "sha256:f5406230d7dbf6965bc4051fcad27b81c39ba4a4bfde06f494dd7ff4eb325a9e", size = 49683004, upload-time = "2026-06-21T16:48:14.02Z" },
+]
+
 [[package]]
 name = "typer"
 version = "0.27.0"