Working pipeline: CPU landmarks, memory-aware resolution/decoding controls, verified end-to-end locally

app.py

@@ -1,395 +1,613 @@
-"""
-sign-language-bridge — ASL to English translation demo.
-
-Runs `mamounyosef/sign-language-bridge` (a multi-tier LoRA/RSLoRA adapter on
-Qwen3-VL-2B-Instruct) on an uploaded or webcam-recorded ASL clip.
-
-The adapter was trained with three *always-on* preprocessing stages, so this
-Space reproduces all of them before the model sees a frame — see
-`preprocessing.py`. The processed clip is returned alongside the translation so
-you can see exactly what the model was shown.
-"""
-
-import os
-
-# Must precede every CUDA-touching / native-library import.
-os.environ.setdefault("PYTORCH_CUDA_ALLOC_CONF", "expandable_segments:True")
-os.environ.setdefault("GLOG_minloglevel", "2")
-os.environ.setdefault("TF_CPP_MIN_LOG_LEVEL", "3")
-# torchvision is the most reliable decoder in the Spaces image.
-os.environ.setdefault("FORCE_QWENVL_VIDEO_READER", "torchvision")
-
-import spaces  # noqa: E402  — must come before torch
-
-import tempfile
-import time
-import traceback
-
-import gradio as gr
-import numpy as np
-import torch
-from peft import PeftModel
-from qwen_vl_utils import process_vision_info
-from transformers import AutoModelForImageTextToText, AutoProcessor
-
-import preprocessing as pp
-
-# ---------------------------------------------------------------------------
-# Configuration — mirrors the evaluated checkpoint's config exactly.
-# Source: saved_metrics/test_results_qwen3vl/step_4610_optimized_final/summary.txt
-# ---------------------------------------------------------------------------
-BASE_MODEL = "Qwen/Qwen3-VL-2B-Instruct"
-ADAPTER_REPO = "mamounyosef/sign-language-bridge"
-ADAPTER_SUBFOLDER = "adapter"
-
-VIDEO_FPS = 20
-VIDEO_MIN_PIXELS = 4 * 32 * 32          # 4096
-VIDEO_MAX_PIXELS = 180 * 32 * 32        # 184320
-VIDEO_TOTAL_PIXELS = 20480 * 32 * 32    # 20971520
-
-SYSTEM_PROMPT = "You are a sign language translator."
-USER_PROMPT = "Translate this American Sign Language video into English."
-
-GENERATION_KWARGS = dict(
-    max_new_tokens=32,
-    num_beams=5,
-    length_penalty=0.6,
-    no_repeat_ngram_size=4,
-    repetition_penalty=1.1,
-    do_sample=False,
-)
-
-MAX_CLIP_SECONDS = 15.0   # keeps preprocessing bounded; training clips were short
-BBOX_FRAME_STRIDE = 4     # the bbox pass only ever samples every 4th frame
-
-# Where the MediaPipe .task file is cached. Overridable so the same code runs
-# on a Space (ephemeral /tmp) and on a local machine (persistent, any drive).
-MODELS_DIR = os.environ.get(
-    "SLB_MODELS_DIR", os.path.join(tempfile.gettempdir(), "slb_models")
-)
-
-# ---------------------------------------------------------------------------
-# Load once, at module scope. ZeroGPU intercepts .to("cuda") here and streams
-# the weights into VRAM on the first @spaces.GPU entry.
-# ---------------------------------------------------------------------------
-print(f"Loading processor + base model: {BASE_MODEL}")
-processor = AutoProcessor.from_pretrained(BASE_MODEL)
-base_model = AutoModelForImageTextToText.from_pretrained(
-    BASE_MODEL,
-    torch_dtype=torch.bfloat16,
-    attn_implementation="sdpa",
-)
-print(f"Attaching adapter: {ADAPTER_REPO}/{ADAPTER_SUBFOLDER}")
-model = PeftModel.from_pretrained(base_model, ADAPTER_REPO, subfolder=ADAPTER_SUBFOLDER)
-model.eval()
-model.to("cuda")
-print("Model ready.")
-
-# CPU-only preprocessing models. Built lazily on first use so a cold boot that
-# never gets a request does not pay for them, then cached for the process.
-_signer_cropper = None
-_landmark_extractor = None
-
-
-def _get_signer_cropper():
-    global _signer_cropper
-    if _signer_cropper is None:
-        # sample_every_n=1: the caller hands us frames that are already strided.
-        _signer_cropper = pp.SignerCropper(
-            models_dir=MODELS_DIR, model_variant="full", sample_every_n=1
-        )
-    return _signer_cropper
-
-
-def _get_landmark_extractor():
-    global _landmark_extractor
-    if _landmark_extractor is None:
-        # "auto" prefers the CUDA execution provider — on CPU this model costs
-        # ~1s/frame, which would dominate the whole request.
-        _landmark_extractor = pp.LandmarkExtractor(mode="performance", device="auto")
-    return _landmark_extractor
-
-
-def _estimate_duration(video_path, *args, **kwargs) -> int:
-    """GPU reservation in seconds. Preprocessing dominates and scales with clip length.
-
-    Gradio passes extra arguments positionally, so the signature has to swallow them.
-    """
-    seconds = MAX_CLIP_SECONDS
-    try:
-        import cv2
-
-        cap = cv2.VideoCapture(video_path)
-        total = cap.get(cv2.CAP_PROP_FRAME_COUNT) or 0
-        native_fps = cap.get(cv2.CAP_PROP_FPS) or 25.0
-        cap.release()
-        if total > 0 and native_fps > 0:
-            seconds = min(MAX_CLIP_SECONDS, total / native_fps)
-    except Exception:  # noqa: BLE001
-        pass
-    return int(min(240, 60 + 12 * seconds))
-
-
-def _translate_impl(video_path, use_signer_crop, use_clahe, use_landmark_overlay):
-    if not video_path:
-        return "", None, "Upload or record a clip first."
-
-    timings = {}
-    notes = []
-    t_all = time.perf_counter()
-
-    # -- Native frames: needed for the pose-guided crop, which is computed in
-    #    source-video pixel space exactly as it was during training. Only every
-    #    4th frame is retained -- that is all the bbox pass samples.
-    t0 = time.perf_counter()
-    frames_bgr, native_fps, n_scanned = pp.read_video_frames_bgr(
-        video_path, stride=BBOX_FRAME_STRIDE, max_seconds=MAX_CLIP_SECONDS
-    )
-    timings["decode (native)"] = time.perf_counter() - t0
-
-    native_fps = native_fps if native_fps > 0 else 25.0
-    full_duration_s = pp.probe_duration_seconds(video_path) or (n_scanned / native_fps)
-    truncated = full_duration_s > MAX_CLIP_SECONDS + 0.5
-    duration_s = min(full_duration_s, MAX_CLIP_SECONDS)
-    if truncated:
-        notes.append(
-            f"Clip is {full_duration_s:.1f}s — only the first "
-            f"{MAX_CLIP_SECONDS:.0f}s were translated."
-        )
-
-    # -- Pose-guided signer bbox (MediaPipe). The frames are already strided, so
-    #    the cropper walks them one by one.
-    bbox = None
-    if use_signer_crop:
-        t0 = time.perf_counter()
-        try:
-            bbox = _get_signer_cropper().compute_bbox(frames_bgr)
-            if bbox.failed:
-                notes.append("No pose detected — the full frame was used instead of a crop.")
-                bbox = None
-            else:
-                notes.append(
-                    f"Signer crop: {bbox.x2 - bbox.x1}x{bbox.y2 - bbox.y1}px from "
-                    f"{bbox.frame_width}x{bbox.frame_height}px "
-                    f"(pose found in {bbox.detection_rate:.0%} of sampled frames)."
-                )
-        except Exception as exc:  # noqa: BLE001
-            notes.append(f"Signer crop unavailable ({exc!r}) — using the full frame.")
-        timings["signer crop (MediaPipe)"] = time.perf_counter() - t0
-    del frames_bgr
-
-    # -- Decode through qwen_vl_utils so the frame sampling and pixel budget
-    #    match training, then apply the crop to the decoded tensor.
-    video_content = {
-        "type": "video",
-        "video": video_path,
-        "fps": VIDEO_FPS,
-        "min_pixels": VIDEO_MIN_PIXELS,
-        "max_pixels": VIDEO_MAX_PIXELS,
-        "total_pixels": VIDEO_TOTAL_PIXELS,
-    }
-    if truncated:
-        # Bound what the model decodes too, not just the bbox pass.
-        video_content["video_end"] = MAX_CLIP_SECONDS
-
-    messages = [
-        {"role": "system", "content": [{"type": "text", "text": SYSTEM_PROMPT}]},
-        {
-            "role": "user",
-            "content": [video_content, {"type": "text", "text": USER_PROMPT}],
-        },
-    ]
-
-    text = processor.apply_chat_template(
-        messages, tokenize=False, add_generation_prompt=True, enable_thinking=False
-    )
-
-    t0 = time.perf_counter()
-    _, videos, video_kwargs = process_vision_info(
-        messages, image_patch_size=16, return_video_kwargs=True, return_video_metadata=True
-    )
-    timings["decode (Qwen sampler)"] = time.perf_counter() - t0
-
-    if not videos:
-        return "", None, "Could not decode any frames from that clip."
-
-    (video, video_metadata), = videos
-    if not isinstance(video, torch.Tensor):
-        video = torch.as_tensor(np.asarray(video))
-    if video.dtype != torch.uint8:
-        video = video.clamp(0, 255).to(torch.uint8)
-
-    if use_signer_crop and bbox is not None:
-        video = pp.apply_signer_crop(video, bbox)
-
-    if use_clahe:
-        t0 = time.perf_counter()
-        video = pp.apply_clahe(video)
-        timings["CLAHE"] = time.perf_counter() - t0
-
-    if use_landmark_overlay:
-        t0 = time.perf_counter()
-        try:
-            frames_rgb = video.permute(0, 2, 3, 1).contiguous().numpy()
-            frames_bgr_crop = np.ascontiguousarray(frames_rgb[..., ::-1])
-            cur_h, cur_w = int(video.shape[-2]), int(video.shape[-1])
-            pose, lh, rh = _get_landmark_extractor().extract(frames_bgr_crop, cur_w, cur_h)
-            pose, lh, rh = pp.postprocess_landmarks(pose, lh, rh)
-            video = pp.apply_landmark_overlay(video, pose, lh, rh)
-
-            hands_seen = int(np.mean(
-                [(~np.all(np.isnan(lh), axis=(1, 2))).mean(),
-                 (~np.all(np.isnan(rh), axis=(1, 2))).mean()]
-            ) * 100)
-            notes.append(f"Landmark overlay: hands tracked in ~{hands_seen}% of frames.")
-            del frames_rgb, frames_bgr_crop
-        except Exception as exc:  # noqa: BLE001
-            notes.append(
-                f"Landmark overlay unavailable ({exc!r}). The model expects it — "
-                "output quality will be degraded."
-            )
-        timings["landmarks (RTMPose)"] = time.perf_counter() - t0
-
-    n_frames, _, out_h, out_w = video.shape
-
-    # -- Preview of the exact tensor handed to the model.
-    preview_path = tempfile.NamedTemporaryFile(suffix=".mp4", delete=False).name
-    try:
-        pp.write_preview_mp4(video, preview_path, fps=VIDEO_FPS)
-    except Exception as exc:  # noqa: BLE001
-        notes.append(f"Could not render the preview clip ({exc!r}).")
-        preview_path = None
-
-    # -- Generate. do_resize=False: frames are already at the training resolution.
-    t0 = time.perf_counter()
-    inputs = processor(
-        text=[text],
-        videos=[video],
-        video_metadata=[video_metadata],
-        return_tensors="pt",
-        padding=True,
-        do_resize=False,
-        **video_kwargs,
-    ).to(model.device)
-
-    with torch.inference_mode():
-        generated = model.generate(**inputs, **GENERATION_KWARGS)
-
-    trimmed = generated[:, inputs.input_ids.shape[1]:]
-    translation = processor.batch_decode(trimmed, skip_special_tokens=True)[0].strip()
-    timings["generation"] = time.perf_counter() - t0
-    timings["total"] = time.perf_counter() - t_all
-
-    info = [
-        f"**Model input** — {n_frames} frames at {VIDEO_FPS} fps, {out_w}x{out_h} px "
-        f"({duration_s:.1f}s of signing).",
-        "",
-        "**Preprocessing**",
-    ]
-    info += [f"- {n}" for n in notes] or ["- (all stages disabled)"]
-    info += ["", "**Timings**"]
-    info += [f"- {k}: {v:.1f}s" for k, v in timings.items()]
-
-    return translation or "(empty output)", preview_path, "\n".join(info)
-
-
-@spaces.GPU(duration=_estimate_duration)
-def translate(video_path, use_signer_crop: bool = True, use_clahe: bool = True,
-              use_landmark_overlay: bool = True):
-    """Translate an American Sign Language video clip into English text.
-
-    Args:
-        video_path: Path to an ASL video clip, 1-15 seconds of continuous signing.
-        use_signer_crop: Crop to the signer using pose landmarks (training default: on).
-        use_clahe: Apply CLAHE contrast enhancement (training default: on).
-        use_landmark_overlay: Draw the pose/hand skeleton overlay (training default: on).
-
-    Returns:
-        The English translation, the preprocessed clip the model actually saw,
-        and a breakdown of the preprocessing pipeline that produced it.
-    """
-    try:
-        return _translate_impl(video_path, use_signer_crop, use_clahe, use_landmark_overlay)
-    except Exception as exc:  # noqa: BLE001 — surface errors in the UI, not as a stack trace
-        traceback.print_exc()
-        return "", None, f"**Something went wrong**\n\n```\n{exc!r}\n```"
-
-
-# ---------------------------------------------------------------------------
-# UI
-# ---------------------------------------------------------------------------
-DESCRIPTION = """
-# 🤟 sign-language-bridge — ASL → English
-
-Continuous **American Sign Language** translation with
-[`mamounyosef/sign-language-bridge`](https://huggingface.co/mamounyosef/sign-language-bridge):
-a multi-tier LoRA / RSLoRA fine-tune of
-[`Qwen3-VL-2B-Instruct`](https://huggingface.co/Qwen/Qwen3-VL-2B-Instruct)
-trained on How2Sign + OpenASL.
-
-Upload a clip or record one with your webcam. Clips of **1–15 seconds** of
-continuous signing, framed head-and-shoulders with good lighting, work best.
-
-> ⚠️ **Research preview.** BLEU-4 is 1.64 and WER is 112 % on the author's
-> How2Sign test partition — the model produces fluent English that is often
-> topically right but frequently disagrees with the reference word-for-word.
-> It can be confidently wrong. Do not use it where a mistranslation could cause
-> harm (medical, legal, safety-critical, or emergency settings).
-"""
-
-PIPELINE_NOTE = """
-### Why the processed clip looks like that
-
-The adapter was trained with three preprocessing stages applied to **every**
-clip, so inference has to reproduce them:
-
-1. **Pose-guided signer crop** — MediaPipe pose landmarks are unioned across the
-   clip and padded 25 %, giving one stable box around the signing space.
-2. **CLAHE** — contrast equalisation on the L channel in LAB (clip 2.0, 8×8 tiles).
-3. **Landmark overlay** — RTMPose Wholebody draws 6 upper-body joints (yellow)
-   and 21 keypoints per hand (green = left, blue = right) directly onto the pixels.
-
-Turning any of them off shows you how much the model leans on them — the output
-usually gets noticeably worse.
-"""
-
-with gr.Blocks(title="sign-language-bridge — ASL to English") as demo:
-    gr.Markdown(DESCRIPTION)
-
-    with gr.Row():
-        with gr.Column(scale=1):
-            video_in = gr.Video(
-                label="ASL clip",
-                sources=["upload", "webcam"],
-                include_audio=False,
-            )
-            with gr.Accordion("Preprocessing (training defaults: all on)", open=False):
-                crop_cb = gr.Checkbox(value=True, label="Pose-guided signer crop")
-                clahe_cb = gr.Checkbox(value=True, label="CLAHE contrast enhancement")
-                overlay_cb = gr.Checkbox(value=True, label="Landmark skeleton overlay")
-            run_btn = gr.Button("Translate", variant="primary")
-
-        with gr.Column(scale=1):
-            translation_out = gr.Textbox(
-                label="English translation",
-                lines=3,
-                show_copy_button=True,
-            )
-            preview_out = gr.Video(label="What the model actually saw", autoplay=True)
-            info_out = gr.Markdown()
-
-    gr.Markdown(PIPELINE_NOTE)
-
-    run_btn.click(
-        fn=translate,
-        inputs=[video_in, crop_cb, clahe_cb, overlay_cb],
-        outputs=[translation_out, preview_out, info_out],
-        api_name="translate",
-    )
-
-if __name__ == "__main__":
-    # SLB_OPEN_BROWSER is set by the local launcher; on a Space it is unset.
-    demo.queue(max_size=12).launch(
-        mcp_server=True,
-        inbrowser=os.environ.get("SLB_OPEN_BROWSER") == "1",
-    )
+"""
+sign-language-bridge — ASL to English translation demo.
+
+Runs `mamounyosef/sign-language-bridge` (a multi-tier LoRA/RSLoRA adapter on
+Qwen3-VL-2B-Instruct) on an uploaded or webcam-recorded ASL clip.
+
+The adapter was trained with three *always-on* preprocessing stages, so this
+Space reproduces all of them before the model sees a frame — see
+`preprocessing.py`. The processed clip is returned alongside the translation so
+you can see exactly what the model was shown.
+"""
+
+import os
+
+# Must precede every CUDA-touching / native-library import.
+# expandable_segments is a Linux/ZeroGPU memory-fragmentation fix. On Windows it
+# is unsupported by torch 2.6 and makes the allocator fail small allocations
+# while reporting gigabytes free, so it is scoped to non-Windows.
+if os.name != "nt":
+    os.environ.setdefault("PYTORCH_CUDA_ALLOC_CONF", "expandable_segments:True")
+os.environ.setdefault("GLOG_minloglevel", "2")
+os.environ.setdefault("TF_CPP_MIN_LOG_LEVEL", "3")
+# torchvision is the most reliable decoder in the Spaces image.
+os.environ.setdefault("FORCE_QWENVL_VIDEO_READER", "torchvision")
+
+import spaces  # noqa: E402  — must come before torch
+
+import gc
+import shutil
+import tempfile
+import time
+import traceback
+
+import gradio as gr
+import numpy as np
+import torch
+from huggingface_hub import snapshot_download
+from peft import PeftModel
+from qwen_vl_utils import process_vision_info
+from transformers import AutoModelForImageTextToText, AutoProcessor
+
+import preprocessing as pp
+
+# ---------------------------------------------------------------------------
+# Configuration — mirrors the evaluated checkpoint's config exactly.
+# Source: saved_metrics/test_results_qwen3vl/step_4610_optimized_final/summary.txt
+# ---------------------------------------------------------------------------
+BASE_MODEL = "Qwen/Qwen3-VL-2B-Instruct"
+ADAPTER_REPO = "mamounyosef/sign-language-bridge"
+ADAPTER_SUBFOLDER = "adapter"
+
+VIDEO_FPS = 20
+VIDEO_MIN_PIXELS = 4 * 32 * 32          # 4096
+VIDEO_MAX_PIXELS = 180 * 32 * 32        # 184320
+VIDEO_TOTAL_PIXELS = 20480 * 32 * 32    # 20971520
+
+SYSTEM_PROMPT = "You are a sign language translator."
+USER_PROMPT = "Translate this American Sign Language video into English."
+
+GENERATION_KWARGS = dict(
+    max_new_tokens=32,
+    num_beams=5,
+    length_penalty=0.6,
+    no_repeat_ngram_size=4,
+    repetition_penalty=1.1,
+    do_sample=False,
+)
+
+MAX_CLIP_SECONDS = 10.0   # keeps memory and preprocessing bounded; training clips were short
+BBOX_FRAME_STRIDE = 4     # the bbox pass only ever samples every 4th frame
+
+# Qwen3-VL requires frame dimensions to be a multiple of patch_size x merge_size.
+SNAP = 32
+
+# How large a frame the vision tower is asked to encode, as a fraction of the
+# training resolution. The tower encodes every temporal patch in one forward
+# pass, so this — not beam width — is what decides whether a clip fits in memory.
+#
+# Measured on a 12 GB RTX 3060 with ~6 GB of host commit free: full resolution
+# needs greedy decoding and often will not fit at all, while half resolution runs
+# the full 5-beam search comfortably. Picking the size up front matters: letting
+# a full-resolution attempt fail and retrying tends to leave the CUDA context in
+# a bad state ("CUDA error: unknown error") rather than recovering cleanly.
+RESOLUTION_PRESETS = {
+    "Full (training resolution — needs a large GPU)": 1.0,
+    "Half (recommended — fits a 12 GB card)": 0.5,
+    "Quarter (last resort)": 0.375,
+}
+DEFAULT_RESOLUTION = "Half (recommended — fits a 12 GB card)"
+
+# Beam search replicates the video tensor before the vision tower runs, so 5
+# beams means encoding the clip five times. That is by far the largest memory
+# consumer: on a 12 GB card, greedy decoding fits at full resolution while 5-beam
+# search does not. The evaluation numbers in the model card assume 5 beams.
+DECODING_PRESETS = {
+    "Greedy (recommended — fits in ~12 GB)": 1,
+    "Beam search x5 (matches the paper; needs a big GPU)": 5,
+}
+DEFAULT_DECODING = "Greedy (recommended — fits in ~12 GB)"
+
+# Where the MediaPipe .task file is cached. Overridable so the same code runs
+# on a Space (ephemeral /tmp) and on a local machine (persistent, any drive).
+MODELS_DIR = os.environ.get(
+    "SLB_MODELS_DIR", os.path.join(tempfile.gettempdir(), "slb_models")
+)
+
+
+def _check_ffmpeg_tooling() -> None:
+    """Report on the ffmpeg tooling Gradio's video component depends on.
+
+    Deliberately does NOT put a bare `ffmpeg` on PATH. Gradio only probes an
+    output video's codec when it finds `ffmpeg` there, and that probe shells out
+    to `ffprobe` -- so a *partial* install (ffmpeg but no ffprobe, which is
+    exactly what imageio-ffmpeg provides) makes every response fail with
+    FFExecutableNotFoundError. With neither on PATH, Gradio skips the check and
+    serves the file as-is, which is correct here: the preview is written as
+    H.264 / yuv420p in an .mp4, already browser-playable. The input side needs no
+    ffmpeg either, because the component is created with `format=None`.
+
+    Spaces images ship both binaries, so the full path runs there.
+    """
+    have_ffmpeg = shutil.which("ffmpeg") is not None
+    have_ffprobe = shutil.which("ffprobe") is not None
+    if have_ffmpeg and not have_ffprobe:
+        print(
+            "WARNING: `ffmpeg` is on PATH but `ffprobe` is not. Gradio needs both; "
+            "video responses may fail. Install a complete ffmpeg build, or remove "
+            "ffmpeg from PATH to make Gradio skip the codec probe.",
+            flush=True,
+        )
+
+
+_check_ffmpeg_tooling()
+
+# ---------------------------------------------------------------------------
+# Load once, at module scope. ZeroGPU intercepts .to("cuda") here and streams
+# the weights into VRAM on the first @spaces.GPU entry.
+# ---------------------------------------------------------------------------
+print(f"Loading processor + base model: {BASE_MODEL}")
+processor = AutoProcessor.from_pretrained(BASE_MODEL)
+
+# low_cpu_mem_usage memory-maps the checkpoint instead of materialising all
+# 4.3 GB in host RAM before the GPU copy. Without it, peak host usage is roughly
+# double, which on a 16 GB machine under memory pressure surfaces as a CUDA OOM
+# that confusingly reports gigabytes of VRAM still free.
+#
+# Note: NOT device_map="cuda". That routes through accelerate, which both
+# bypasses the ZeroGPU hijack on a Space and segfaults here in the meta-device
+# loader when host memory is tight.
+base_model = AutoModelForImageTextToText.from_pretrained(
+    BASE_MODEL,
+    dtype=torch.bfloat16,
+    attn_implementation="sdpa",
+    low_cpu_mem_usage=True,
+)
+# Place the base on the GPU BEFORE attaching the adapter. This adapter carries
+# `modules_to_save` (the embedding matrix and output head, ~721 MB), and PEFT
+# materialises those as real CPU tensors plus copies of the originals. Doing
+# that while the base is still resident in host RAM pushes a 16 GB machine over
+# its commit limit, which the CUDA driver reports as an OOM despite free VRAM.
+base_model.to("cuda")
+print(f"Attaching adapter: {ADAPTER_REPO}/{ADAPTER_SUBFOLDER}")
+# Resolve the adapter to a local directory rather than passing repo + subfolder
+# to PEFT. Two reasons:
+#   1. PEFT builds the remote filename with os.path.join, so on Windows the
+#      existence probe asks the Hub for "adapter\adapter_model.safetensors";
+#      that never matches, and it falls back to a .bin that isn't there.
+#   2. allow_patterns skips training_state.pt (~600 MB of optimizer/InfoNCE
+#      state) which is only needed to resume training, never for inference.
+_adapter_dir = os.path.join(
+    snapshot_download(ADAPTER_REPO, allow_patterns=[f"{ADAPTER_SUBFOLDER}/*"]),
+    ADAPTER_SUBFOLDER,
+)
+model = PeftModel.from_pretrained(base_model, _adapter_dir)
+model.eval()
+# On ZeroGPU this is intercepted at module scope and the weights are streamed
+# into VRAM on the first @spaces.GPU entry; locally it is a plain copy.
+model.to("cuda")
+print(f"Model ready on {next(model.parameters()).device}.")
+
+# CPU-only preprocessing models. Built lazily on first use so a cold boot that
+# never gets a request does not pay for them, then cached for the process.
+_signer_cropper = None
+_landmark_extractor = None
+
+
+def _get_signer_cropper():
+    global _signer_cropper
+    if _signer_cropper is None:
+        # sample_every_n=1: the caller hands us frames that are already strided.
+        _signer_cropper = pp.SignerCropper(
+            models_dir=MODELS_DIR, model_variant="full", sample_every_n=1
+        )
+    return _signer_cropper
+
+
+def _get_landmark_extractor():
+    """RTMPose Wholebody, deliberately pinned to the CPU.
+
+    Training used the `performance` model (x-large, 288x384 input). `balanced` is
+    the same backbone at 192x256, and measured here it is *faster on CPU*
+    (47 ms/frame) than `performance` is on the GPU (105 ms/frame) -- while
+    leaving the GPU entirely to the translation model.
+
+    Keeping ONNX Runtime off the GPU also avoids two real failures: its CUDA
+    arena competes with PyTorch for VRAM and host commit, and tearing the session
+    down mid-request left PyTorch unable to find cuDNN kernels
+    ("GET was unable to find an engine to execute this computation").
+    """
+    global _landmark_extractor
+    if _landmark_extractor is None:
+        _landmark_extractor = pp.LandmarkExtractor(mode="balanced", device="cpu")
+    return _landmark_extractor
+
+
+
+
+def _estimate_duration(video_path, *args, **kwargs) -> int:
+    """GPU reservation in seconds. Preprocessing dominates and scales with clip length.
+
+    Gradio passes extra arguments positionally, so the signature has to swallow them.
+    """
+    seconds = MAX_CLIP_SECONDS
+    try:
+        import cv2
+
+        cap = cv2.VideoCapture(video_path)
+        total = cap.get(cv2.CAP_PROP_FRAME_COUNT) or 0
+        native_fps = cap.get(cv2.CAP_PROP_FPS) or 25.0
+        cap.release()
+        if total > 0 and native_fps > 0:
+            seconds = min(MAX_CLIP_SECONDS, total / native_fps)
+    except Exception:  # noqa: BLE001
+        pass
+    return int(min(240, 60 + 12 * seconds))
+
+
+def _generate_once(inputs, beams: int):
+    """Run generation exactly once at the requested beam width.
+
+    Deliberately no retry-on-OOM. Beam search replicates `pixel_values_videos`
+    *before* the vision tower runs, so N beams means encoding the whole clip N
+    times -- it is the single largest memory consumer here, far more than the
+    weights. Catching an OOM mid-`generate` and retrying in the same process was
+    measured to leave the CUDA context unusable ("unknown error", "illegal
+    memory access") rather than recovering, so the caller picks a size that fits
+    up front and a failure is reported honestly instead of papered over.
+    """
+    if inputs.get("video_grid_thw") is not None and beams > 1:
+        # Qwen3-VL emits per-frame timestamps, so the beam-search input expansion
+        # splits video_grid_thw by a count equal to the number of temporal patches
+        # rather than the number of videos. Rewriting [[T,H,W]] as T rows of
+        # [1,H,W] makes the split line up. (Same workaround as the project's own
+        # eval script.) Greedy decoding does no expansion and needs it left alone.
+        vgt = inputs["video_grid_thw"]
+        vgt = torch.repeat_interleave(vgt, vgt[:, 0], dim=0).clone()
+        vgt[:, 0] = 1
+        inputs["video_grid_thw"] = vgt
+
+    kwargs = dict(GENERATION_KWARGS, num_beams=beams)
+    if beams == 1:
+        kwargs.pop("length_penalty", None)  # meaningless without beam search
+
+    with torch.inference_mode(), torch.amp.autocast("cuda", dtype=torch.bfloat16):
+        return model.generate(
+            **inputs,
+            **kwargs,
+            # temperature / top_p / top_k are baked into generation_config.json but
+            # unused here; passing None silences the invalid-flag warning.
+            temperature=None,
+            top_p=None,
+            top_k=None,
+            use_cache=True,
+        )
+
+
+def _translate_impl(video_path, use_signer_crop, use_clahe, use_landmark_overlay,
+                    resolution, decoding):
+    if not video_path:
+        return "", None, "Upload or record a clip first."
+
+    timings = {}
+    notes = []
+    t_all = time.perf_counter()
+
+    # -- Native frames: needed for the pose-guided crop, which is computed in
+    #    source-video pixel space exactly as it was during training. Only every
+    #    4th frame is retained -- that is all the bbox pass samples.
+    t0 = time.perf_counter()
+    frames_bgr, native_fps, n_scanned = pp.read_video_frames_bgr(
+        video_path, stride=BBOX_FRAME_STRIDE, max_seconds=MAX_CLIP_SECONDS
+    )
+    timings["decode (native)"] = time.perf_counter() - t0
+
+    native_fps = native_fps if native_fps > 0 else 25.0
+    full_duration_s = pp.probe_duration_seconds(video_path) or (n_scanned / native_fps)
+    truncated = full_duration_s > MAX_CLIP_SECONDS + 0.5
+    duration_s = min(full_duration_s, MAX_CLIP_SECONDS)
+    if truncated:
+        notes.append(
+            f"Clip is {full_duration_s:.1f}s — only the first "
+            f"{MAX_CLIP_SECONDS:.0f}s were translated."
+        )
+
+    # -- Pose-guided signer bbox (MediaPipe). The frames are already strided, so
+    #    the cropper walks them one by one.
+    bbox = None
+    if use_signer_crop:
+        t0 = time.perf_counter()
+        try:
+            bbox = _get_signer_cropper().compute_bbox(frames_bgr)
+            if bbox.failed:
+                notes.append("No pose detected — the full frame was used instead of a crop.")
+                bbox = None
+            else:
+                notes.append(
+                    f"Signer crop: {bbox.x2 - bbox.x1}x{bbox.y2 - bbox.y1}px from "
+                    f"{bbox.frame_width}x{bbox.frame_height}px "
+                    f"(pose found in {bbox.detection_rate:.0%} of sampled frames)."
+                )
+        except Exception as exc:  # noqa: BLE001
+            notes.append(f"Signer crop unavailable ({exc!r}) — using the full frame.")
+        timings["signer crop (MediaPipe)"] = time.perf_counter() - t0
+    del frames_bgr
+
+    # -- Decode + preprocess + generate, stepping the pixel budget down if the
+    #    vision tower cannot fit. The tower encodes every temporal patch in one
+    #    forward pass, so peak memory is driven by total_pixels far more than by
+    #    beam width; on a 12 GB card the training budget does not always fit.
+    #    Landmarks are normalised to the crop, so they are extracted once at the
+    #    first (largest) resolution and reused verbatim by every later attempt.
+    video_content = {
+        "type": "video",
+        "video": video_path,
+        "fps": VIDEO_FPS,
+        "min_pixels": VIDEO_MIN_PIXELS,
+        "max_pixels": VIDEO_MAX_PIXELS,
+        "total_pixels": VIDEO_TOTAL_PIXELS,
+    }
+    if truncated:
+        # Bound what the model decodes too, not just the bbox pass.
+        video_content["video_end"] = MAX_CLIP_SECONDS
+
+    messages = [
+        {"role": "system", "content": [{"type": "text", "text": SYSTEM_PROMPT}]},
+        {"role": "user", "content": [video_content, {"type": "text", "text": USER_PROMPT}]},
+    ]
+    text = processor.apply_chat_template(
+        messages, tokenize=False, add_generation_prompt=True, enable_thinking=False
+    )
+
+    t0 = time.perf_counter()
+    _, videos, video_kwargs = process_vision_info(
+        messages, image_patch_size=16, return_video_kwargs=True, return_video_metadata=True
+    )
+    timings["decode (Qwen sampler)"] = time.perf_counter() - t0
+    if not videos:
+        return "", None, "Could not decode any frames from that clip."
+
+    (video, video_metadata), = videos
+    if not isinstance(video, torch.Tensor):
+        video = torch.as_tensor(np.asarray(video))
+    if video.dtype != torch.uint8:
+        video = video.clamp(0, 255).to(torch.uint8)
+
+    if use_signer_crop and bbox is not None:
+        video = pp.apply_signer_crop(video, bbox)
+
+    if use_clahe:
+        t0 = time.perf_counter()
+        video = pp.apply_clahe(video)
+        timings["CLAHE"] = time.perf_counter() - t0
+
+    # Landmarks are normalised to the crop, so one extraction serves every scale
+    # in the retry ladder below. This is also by far the most expensive stage.
+    landmarks = None
+    if use_landmark_overlay:
+        t0 = time.perf_counter()
+        try:
+            frames_rgb = video.permute(0, 2, 3, 1).contiguous().numpy()
+            frames_bgr_crop = np.ascontiguousarray(frames_rgb[..., ::-1])
+            cur_h, cur_w = int(video.shape[-2]), int(video.shape[-1])
+
+            pose, lh, rh = _get_landmark_extractor().extract(frames_bgr_crop, cur_w, cur_h)
+            landmarks = pp.postprocess_landmarks(pose, lh, rh)
+            del frames_rgb, frames_bgr_crop
+
+            hands_seen = int(np.mean([
+                (~np.all(np.isnan(landmarks[1]), axis=(1, 2))).mean(),
+                (~np.all(np.isnan(landmarks[2]), axis=(1, 2))).mean(),
+            ]) * 100)
+            notes.append(f"Landmark overlay: hands tracked in ~{hands_seen}% of frames.")
+        except Exception as exc:  # noqa: BLE001
+            notes.append(
+                f"Landmark overlay unavailable ({exc!r}). The model expects it — "
+                "output quality will be degraded."
+            )
+        timings["landmarks (RTMPose)"] = time.perf_counter() - t0
+
+    base_h, base_w = int(video.shape[-2]), int(video.shape[-1])
+
+    # The vision tower encodes every temporal patch in one forward pass, and beam
+    # search replicates the pixel tensor before it does. Both are sized up front
+    # from the user's choices rather than discovered by failing.
+    scale = RESOLUTION_PRESETS.get(resolution, RESOLUTION_PRESETS[DEFAULT_RESOLUTION])
+    beams_used = DECODING_PRESETS.get(decoding, DECODING_PRESETS[DEFAULT_DECODING])
+
+    target_h = max(SNAP, (int(base_h * scale) // SNAP) * SNAP)
+    target_w = max(SNAP, (int(base_w * scale) // SNAP) * SNAP)
+
+    same_size = (target_h, target_w) == (base_h, base_w)
+    attempt = video if same_size else pp.resize_video(video, target_h, target_w)
+    del video
+    # Redraw the skeleton at the working resolution so the 1 px strokes stay
+    # crisp, exactly as training drew them onto already-resized frames.
+    if landmarks is not None:
+        attempt = pp.apply_landmark_overlay(attempt, *landmarks)
+
+    n_frames, _, out_h, out_w = attempt.shape
+
+    preview_path = tempfile.NamedTemporaryFile(suffix=".mp4", delete=False).name
+    try:
+        pp.write_preview_mp4(attempt, preview_path, fps=VIDEO_FPS)
+    except Exception as exc:  # noqa: BLE001
+        notes.append(f"Could not render the preview clip ({exc!r}).")
+        preview_path = None
+
+    # do_resize=False: frames are already at the intended resolution.
+    t0 = time.perf_counter()
+    inputs = processor(
+        text=[text],
+        videos=[attempt],
+        video_metadata=[video_metadata],
+        return_tensors="pt",
+        padding=True,
+        do_resize=False,
+        **video_kwargs,
+    ).to(model.device)
+
+    del attempt
+    gc.collect()
+    torch.cuda.empty_cache()
+
+    generated = _generate_once(inputs, beams_used)
+    trimmed = [out[len(inp):] for inp, out in zip(inputs["input_ids"], generated)]
+    translation = processor.batch_decode(
+        trimmed, skip_special_tokens=True, clean_up_tokenization_spaces=False
+    )[0].strip()
+    timings["generation"] = time.perf_counter() - t0
+    del inputs, generated
+    gc.collect()
+    torch.cuda.empty_cache()
+
+    downscaled = (out_h, out_w) != (base_h, base_w)
+
+    if beams_used != GENERATION_KWARGS["num_beams"]:
+        notes.append(
+            f"Decoded greedily rather than with the "
+            f"{GENERATION_KWARGS['num_beams']}-beam search the published metrics used."
+        )
+    if downscaled:
+        notes.append(
+            f"Frames downscaled from {base_w}x{base_h} to {out_w}x{out_h} — "
+            "raise *Input resolution* to feed the model the training resolution."
+        )
+    if beams_used != GENERATION_KWARGS["num_beams"] or downscaled:
+        notes.append(
+            "Output therefore differs from the published evaluation setup."
+        )
+
+    timings["total"] = time.perf_counter() - t_all
+
+    info = [
+        f"**Model input** — {n_frames} frames at {VIDEO_FPS} fps, {out_w}x{out_h} px "
+        f"({duration_s:.1f}s of signing), {beams_used} beam(s).",
+        "",
+        "**Preprocessing**",
+    ]
+    info += [f"- {n}" for n in notes] or ["- (all stages disabled)"]
+    info += ["", "**Timings**"]
+    info += [f"- {k}: {v:.1f}s" for k, v in timings.items()]
+
+    return translation or "(empty output)", preview_path, "\n".join(info)
+
+
+@spaces.GPU(duration=_estimate_duration)
+def translate(video_path, use_signer_crop: bool = True, use_clahe: bool = True,
+              use_landmark_overlay: bool = True, resolution: str = DEFAULT_RESOLUTION,
+              decoding: str = DEFAULT_DECODING):
+    """Translate an American Sign Language video clip into English text.
+
+    Args:
+        video_path: Path to an ASL video clip, 1-10 seconds of continuous signing.
+        use_signer_crop: Crop to the signer using pose landmarks (training default: on).
+        use_clahe: Apply CLAHE contrast enhancement (training default: on).
+        use_landmark_overlay: Draw the pose/hand skeleton overlay (training default: on).
+        resolution: How large a frame the vision tower encodes. Higher is closer
+            to the training setup but needs considerably more GPU memory.
+        decoding: Greedy, or the 5-beam search the published metrics used. Beam
+            search re-encodes the clip once per beam and needs far more memory.
+
+    Returns:
+        The English translation, the preprocessed clip the model actually saw,
+        and a breakdown of the preprocessing pipeline that produced it.
+    """
+    try:
+        return _translate_impl(
+            video_path, use_signer_crop, use_clahe, use_landmark_overlay,
+            resolution, decoding,
+        )
+    except torch.OutOfMemoryError:
+        traceback.print_exc()
+        return "", None, (
+            "**Out of GPU memory**\n\nEven the fallback size did not fit. Try a "
+            "shorter clip or a smaller setting under *Input resolution*, and close "
+            "other GPU or memory-heavy applications (browsers especially)."
+        )
+    except Exception as exc:  # noqa: BLE001 — surface errors in the UI, not as a stack trace
+        traceback.print_exc()
+        return "", None, f"**Something went wrong**\n\n```\n{exc!r}\n```"
+
+
+# ---------------------------------------------------------------------------
+# UI
+# ---------------------------------------------------------------------------
+DESCRIPTION = """
+# 🤟 sign-language-bridge — ASL → English
+
+Continuous **American Sign Language** translation with
+[`mamounyosef/sign-language-bridge`](https://huggingface.co/mamounyosef/sign-language-bridge):
+a multi-tier LoRA / RSLoRA fine-tune of
+[`Qwen3-VL-2B-Instruct`](https://huggingface.co/Qwen/Qwen3-VL-2B-Instruct)
+trained on How2Sign + OpenASL.
+
+Upload a clip or record one with your webcam. Clips of **1–15 seconds** of
+continuous signing, framed head-and-shoulders with good lighting, work best.
+
+> ⚠️ **Research preview.** BLEU-4 is 1.64 and WER is 112 % on the author's
+> How2Sign test partition — the model produces fluent English that is often
+> topically right but frequently disagrees with the reference word-for-word.
+> It can be confidently wrong. Do not use it where a mistranslation could cause
+> harm (medical, legal, safety-critical, or emergency settings).
+"""
+
+PIPELINE_NOTE = """
+### Why the processed clip looks like that
+
+The adapter was trained with three preprocessing stages applied to **every**
+clip, so inference has to reproduce them:
+
+1. **Pose-guided signer crop** — MediaPipe pose landmarks are unioned across the
+   clip and padded 25 %, giving one stable box around the signing space.
+2. **CLAHE** — contrast equalisation on the L channel in LAB (clip 2.0, 8×8 tiles).
+3. **Landmark overlay** — RTMPose Wholebody draws 6 upper-body joints (yellow)
+   and 21 keypoints per hand (green = left, blue = right) directly onto the pixels.
+
+Turning any of them off shows you how much the model leans on them — the output
+usually gets noticeably worse.
+"""
+
+with gr.Blocks(title="sign-language-bridge — ASL to English") as demo:
+    gr.Markdown(DESCRIPTION)
+
+    with gr.Row():
+        with gr.Column(scale=1):
+            video_in = gr.Video(
+                label="ASL clip",
+                sources=["upload", "webcam"],
+                include_audio=False,
+                format=None,   # skip Gradio's re-encode; we decode the original ourselves
+            )
+            resolution_dd = gr.Dropdown(
+                choices=list(RESOLUTION_PRESETS),
+                value=DEFAULT_RESOLUTION,
+                label="Input resolution",
+                info="Higher is closer to the training setup but needs much more GPU memory.",
+            )
+            decoding_dd = gr.Dropdown(
+                choices=list(DECODING_PRESETS),
+                value=DEFAULT_DECODING,
+                label="Decoding",
+                info="Beam search re-encodes the clip once per beam — accurate, but memory-hungry.",
+            )
+            with gr.Accordion("Preprocessing (training defaults: all on)", open=False):
+                crop_cb = gr.Checkbox(value=True, label="Pose-guided signer crop")
+                clahe_cb = gr.Checkbox(value=True, label="CLAHE contrast enhancement")
+                overlay_cb = gr.Checkbox(value=True, label="Landmark skeleton overlay")
+            run_btn = gr.Button("Translate", variant="primary")
+
+        with gr.Column(scale=1):
+            translation_out = gr.Textbox(
+                label="English translation",
+                lines=3,
+            )
+            preview_out = gr.Video(label="What the model actually saw", autoplay=True)
+            info_out = gr.Markdown()
+
+    gr.Markdown(PIPELINE_NOTE)
+
+    run_btn.click(
+        fn=translate,
+        inputs=[video_in, crop_cb, clahe_cb, overlay_cb, resolution_dd, decoding_dd],
+        outputs=[translation_out, preview_out, info_out],
+        api_name="translate",
+    )
+
+if __name__ == "__main__":
+    # SLB_OPEN_BROWSER is set by the local launcher; on a Space it is unset.
+    demo.queue(max_size=12).launch(
+        mcp_server=True,
+        show_error=True,
+        inbrowser=os.environ.get("SLB_OPEN_BROWSER") == "1",
+    )

preprocessing.py

@@ -542,6 +542,23 @@ def _download_once(url: str, dest: str) -> str:
     return dest
 
 
+def resize_video(video: torch.Tensor, target_h: int, target_w: int) -> torch.Tensor:
+    """Downscale a (T, C, H, W) uint8 clip, one frame at a time.
+
+    Deliberately not `F.interpolate`: that needs a float32 copy of the whole clip
+    (~200 MB for a few seconds of video) before it produces anything, which is
+    enough to fail on a memory-constrained machine. cv2 works per frame in uint8,
+    and INTER_AREA is the correct filter for downscaling.
+    """
+    T, C = video.shape[0], video.shape[1]
+    out = torch.empty((T, C, target_h, target_w), dtype=torch.uint8)
+    for t in range(T):
+        frame = video[t].permute(1, 2, 0).contiguous().numpy()
+        resized = cv2.resize(frame, (target_w, target_h), interpolation=cv2.INTER_AREA)
+        out[t] = torch.from_numpy(resized).permute(2, 0, 1)
+    return out
+
+
 def probe_duration_seconds(path: str) -> float:
     """Clip duration in seconds from container metadata, or 0.0 if unknown."""
     cap = cv2.VideoCapture(path)

requirements.txt

@@ -14,20 +14,16 @@ torchvision
 mediapipe
 rtmlib
 
-# NOTE — read before first GPU run.
-# rtmlib's default `onnxruntime` is the CPU build. Measured locally, the
-# `performance` wholebody model costs ~950 ms/frame on CPU versus ~105 ms/frame
-# on CUDA: a 7-second clip goes from 133 s to 15 s. On CPU this Space would be
-# unusable, so `onnxruntime-gpu` needs to be enabled once hardware is attached.
+# rtmlib brings the CPU build of onnxruntime, which is what this app wants.
+# It runs the `balanced` wholebody model: measured on a local RTX 3060 that is
+# 47 ms/frame on CPU, *faster* than the larger `performance` model on the GPU
+# (105 ms/frame), because it is the same backbone at 192x256 instead of 288x384.
 #
-# It is deliberately NOT pinned here: onnxruntime-gpu wheels target CUDA 12.x
-# while the ZeroGPU runtime is CUDA 13 (torch 2.11/cu130), and shipping an
-# untested pin would risk breaking the build for a Space that cannot currently
-# be tested. `LandmarkExtractor(device="auto")` falls back to CPU and says so in
-# the UI, so the app degrades rather than crashes.
-#
-# First thing to try when a GPU is available:
-#   onnxruntime-gpu
+# onnxruntime-gpu is deliberately NOT used. Sharing the GPU between ONNX Runtime
+# and PyTorch caused two failures locally: the ORT CUDA arena competed with the
+# model for memory, and releasing the session mid-request left PyTorch unable to
+# find cuDNN kernels ("GET was unable to find an engine to execute this
+# computation"). Keeping ONNX Runtime on the CPU leaves the GPU to the model.
 
 # Browser-playable H.264 preview of the processed clip.
 imageio