File size: 4,038 Bytes
2a0825b | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 | """Blind handwriting transcription via an OpenAI-compatible VLM (Qwen3-VL on
Modal, see modal/ocr/modal_qwen3vl.py).
The one rule (specs/ocr.md §3): this module NEVER sees the reference text. It
asks the model for a faithful, uncorrected transcription of the page; grading
happens afterwards in ocr.grading. Note there is deliberately no ``reference``
parameter anywhere here."""
import base64
import json
import mimetypes
from pathlib import Path
# specs/ocr.md §10, hardened. The "word by word" + explicit spelling-error +
# umlaut-dot framing beat both the original prompt and a "letter by letter"
# variant on a real handwriting sample: on Qwen3-VL-8B it fixed gross misreads
# without adding noise. The one residual failure is the language-model prior
# auto-correcting missing umlauts on very common words (e.g. a deliberately
# written "Fruhstück" still came back "Frühstück") — see specs/ocr.md §9.
SYSTEM_PROMPT = (
"You are a forensic handwriting transcriber. Transcribe the page WORD BY "
"WORD, copying each written word exactly as it appears on the paper.\n"
"\n"
"This text was written in a SPELLING TEST and DELIBERATELY CONTAINS "
"SPELLING ERRORS. Reproduce every error faithfully. NEVER replace a word "
"with its dictionary-correct spelling. If a word is written wrong, "
"transcribe it exactly as wrongly written.\n"
"\n"
"Umlaut rule (critical): write a/o/u with two dots (a->ä, o->ö, u->ü) ONLY "
"when you can clearly see two dots above that exact letter. If a u, a or o "
"has NO dots above it, keep it plain (u, a, o) even if correct German would "
"use an umlaut. Never add dots that are not on the page; never drop dots "
"that are.\n"
"Do NOT change ß to ss or ss to ß. Do NOT translate. Do NOT fix spacing, "
"grammar, or capitalization.\n"
"Output ONLY the transcribed text, one line per written line."
)
USER_INSTRUCTION = (
"Transcribe this page word by word, reproducing every spelling error "
"exactly as written."
)
def guess_mime(path: str) -> str:
"""MIME type for an image path, defaulting to PNG."""
mime, _ = mimetypes.guess_type(path)
return mime or "image/png"
def data_uri(image_bytes: bytes, mime: str) -> str:
"""Encode raw image bytes as an OpenAI ``image_url`` data URI."""
b64 = base64.b64encode(image_bytes).decode("ascii")
return f"data:{mime};base64,{b64}"
def encode_image(path: str) -> str:
"""Read an image file and return its data URI."""
return data_uri(Path(path).read_bytes(), guess_mime(path))
def build_payload(image_uri: str, model: str, temperature: float = 0.0) -> dict:
"""Build the chat-completion request for a blind transcription. No
reference text appears anywhere (specs/ocr.md §3)."""
return {
"model": model,
"temperature": temperature,
"messages": [
{"role": "system", "content": SYSTEM_PROMPT},
{
"role": "user",
"content": [
{"type": "text", "text": USER_INSTRUCTION},
{"type": "image_url", "image_url": {"url": image_uri}},
],
},
],
}
def parse_response(data: dict) -> str:
"""Extract the transcription text from an OpenAI-style response dict."""
try:
return data["choices"][0]["message"]["content"].strip()
except (KeyError, IndexError, TypeError, AttributeError) as e:
raise ValueError(f"Unexpected VLM response: {json.dumps(data)[:500]}") from e
def transcribe_image(image_path: str, client, model: str = "qwen3-vl") -> str:
"""Transcribe a handwritten page blind.
``client`` is an OpenAI-compatible client pointed at the Modal VLM server
(build one with ``openai_client.make_client``). Cold-start retries live in
the client, not here."""
payload = build_payload(encode_image(image_path), model)
completion = client.chat.completions.create(**payload)
return parse_response(completion.model_dump())
|