""" sign-language-bridge — ASL to English translation demo. Runs `mamounyosef/sign-language-bridge` (a multi-tier LoRA/RSLoRA adapter on Qwen3-VL-2B-Instruct) on an uploaded or webcam-recorded ASL clip. The adapter was trained with three *always-on* preprocessing stages, so this Space reproduces all of them before the model sees a frame — see `preprocessing.py`. The processed clip is returned alongside the translation so you can see exactly what the model was shown. """ import os # Must precede every CUDA-touching / native-library import. # expandable_segments is a Linux/ZeroGPU memory-fragmentation fix. On Windows it # is unsupported by torch 2.6 and makes the allocator fail small allocations # while reporting gigabytes free, so it is scoped to non-Windows. if os.name != "nt": os.environ.setdefault("PYTORCH_CUDA_ALLOC_CONF", "expandable_segments:True") os.environ.setdefault("GLOG_minloglevel", "2") os.environ.setdefault("TF_CPP_MIN_LOG_LEVEL", "3") # torchvision is the most reliable decoder in the Spaces image. os.environ.setdefault("FORCE_QWENVL_VIDEO_READER", "torchvision") import spaces # noqa: E402 — must come before torch import gc import shutil import tempfile import time import traceback import gradio as gr import numpy as np import torch from huggingface_hub import snapshot_download from peft import PeftModel from qwen_vl_utils import process_vision_info from transformers import AutoModelForImageTextToText, AutoProcessor import preprocessing as pp # --------------------------------------------------------------------------- # Configuration — mirrors the evaluated checkpoint's config exactly. # Source: saved_metrics/test_results_qwen3vl/step_4610_optimized_final/summary.txt # --------------------------------------------------------------------------- BASE_MODEL = "Qwen/Qwen3-VL-2B-Instruct" ADAPTER_REPO = "mamounyosef/sign-language-bridge" ADAPTER_SUBFOLDER = "adapter" VIDEO_FPS = 20 VIDEO_MIN_PIXELS = 4 * 32 * 32 # 4096 VIDEO_MAX_PIXELS = 180 * 32 * 32 # 184320 VIDEO_TOTAL_PIXELS = 20480 * 32 * 32 # 20971520 SYSTEM_PROMPT = "You are a sign language translator." USER_PROMPT = "Translate this American Sign Language video into English." GENERATION_KWARGS = dict( max_new_tokens=32, num_beams=5, length_penalty=0.6, no_repeat_ngram_size=4, repetition_penalty=1.1, do_sample=False, ) MAX_CLIP_SECONDS = 10.0 # keeps memory and preprocessing bounded; training clips were short BBOX_FRAME_STRIDE = 4 # the bbox pass only ever samples every 4th frame # Qwen3-VL requires frame dimensions to be a multiple of patch_size x merge_size. SNAP = 32 # How large a frame the vision tower is asked to encode, as a fraction of the # training resolution. The tower encodes every temporal patch in one forward # pass, so this — not beam width — is what decides whether a clip fits in memory. # # Measured on a 12 GB RTX 3060 with ~6 GB of host commit free: full resolution # needs greedy decoding and often will not fit at all, while half resolution runs # the full 5-beam search comfortably. Picking the size up front matters: letting # a full-resolution attempt fail and retrying tends to leave the CUDA context in # a bad state ("CUDA error: unknown error") rather than recovering cleanly. RESOLUTION_PRESETS = { "Full (training resolution — needs a large GPU)": 1.0, "Half (recommended — fits a 12 GB card)": 0.5, "Quarter (last resort)": 0.375, } DEFAULT_RESOLUTION = "Half (recommended — fits a 12 GB card)" # Beam search replicates the video tensor before the vision tower runs, so 5 # beams means encoding the clip five times. That is by far the largest memory # consumer: on a 12 GB card, greedy decoding fits at full resolution while 5-beam # search does not. The evaluation numbers in the model card assume 5 beams. DECODING_PRESETS = { "Greedy (recommended — fits in ~12 GB)": 1, "Beam search x5 (matches the paper; needs a big GPU)": 5, } DEFAULT_DECODING = "Greedy (recommended — fits in ~12 GB)" # Where the MediaPipe .task file is cached. Overridable so the same code runs # on a Space (ephemeral /tmp) and on a local machine (persistent, any drive). MODELS_DIR = os.environ.get( "SLB_MODELS_DIR", os.path.join(tempfile.gettempdir(), "slb_models") ) def _check_ffmpeg_tooling() -> None: """Report on the ffmpeg tooling Gradio's video component depends on. Deliberately does NOT put a bare `ffmpeg` on PATH. Gradio only probes an output video's codec when it finds `ffmpeg` there, and that probe shells out to `ffprobe` -- so a *partial* install (ffmpeg but no ffprobe, which is exactly what imageio-ffmpeg provides) makes every response fail with FFExecutableNotFoundError. With neither on PATH, Gradio skips the check and serves the file as-is, which is correct here: the preview is written as H.264 / yuv420p in an .mp4, already browser-playable. The input side needs no ffmpeg either, because the component is created with `format=None`. Spaces images ship both binaries, so the full path runs there. """ have_ffmpeg = shutil.which("ffmpeg") is not None have_ffprobe = shutil.which("ffprobe") is not None if have_ffmpeg and not have_ffprobe: print( "WARNING: `ffmpeg` is on PATH but `ffprobe` is not. Gradio needs both; " "video responses may fail. Install a complete ffmpeg build, or remove " "ffmpeg from PATH to make Gradio skip the codec probe.", flush=True, ) _check_ffmpeg_tooling() # --------------------------------------------------------------------------- # Load once, at module scope. ZeroGPU intercepts .to("cuda") here and streams # the weights into VRAM on the first @spaces.GPU entry. # --------------------------------------------------------------------------- print(f"Loading processor + base model: {BASE_MODEL}") processor = AutoProcessor.from_pretrained(BASE_MODEL) # low_cpu_mem_usage memory-maps the checkpoint instead of materialising all # 4.3 GB in host RAM before the GPU copy. Without it, peak host usage is roughly # double, which on a 16 GB machine under memory pressure surfaces as a CUDA OOM # that confusingly reports gigabytes of VRAM still free. # # Note: NOT device_map="cuda". That routes through accelerate, which both # bypasses the ZeroGPU hijack on a Space and segfaults here in the meta-device # loader when host memory is tight. base_model = AutoModelForImageTextToText.from_pretrained( BASE_MODEL, dtype=torch.bfloat16, attn_implementation="sdpa", low_cpu_mem_usage=True, ) # Place the base on the GPU BEFORE attaching the adapter. This adapter carries # `modules_to_save` (the embedding matrix and output head, ~721 MB), and PEFT # materialises those as real CPU tensors plus copies of the originals. Doing # that while the base is still resident in host RAM pushes a 16 GB machine over # its commit limit, which the CUDA driver reports as an OOM despite free VRAM. base_model.to("cuda") print(f"Attaching adapter: {ADAPTER_REPO}/{ADAPTER_SUBFOLDER}") # Resolve the adapter to a local directory rather than passing repo + subfolder # to PEFT. Two reasons: # 1. PEFT builds the remote filename with os.path.join, so on Windows the # existence probe asks the Hub for "adapter\adapter_model.safetensors"; # that never matches, and it falls back to a .bin that isn't there. # 2. allow_patterns skips training_state.pt (~600 MB of optimizer/InfoNCE # state) which is only needed to resume training, never for inference. _adapter_dir = os.path.join( snapshot_download(ADAPTER_REPO, allow_patterns=[f"{ADAPTER_SUBFOLDER}/*"]), ADAPTER_SUBFOLDER, ) model = PeftModel.from_pretrained(base_model, _adapter_dir) model.eval() # On ZeroGPU this is intercepted at module scope and the weights are streamed # into VRAM on the first @spaces.GPU entry; locally it is a plain copy. model.to("cuda") print(f"Model ready on {next(model.parameters()).device}.") # CPU-only preprocessing models. Built lazily on first use so a cold boot that # never gets a request does not pay for them, then cached for the process. _signer_cropper = None _landmark_extractor = None def _get_signer_cropper(): global _signer_cropper if _signer_cropper is None: # sample_every_n=1: the caller hands us frames that are already strided. _signer_cropper = pp.SignerCropper( models_dir=MODELS_DIR, model_variant="full", sample_every_n=1 ) return _signer_cropper def _get_landmark_extractor(): """RTMPose Wholebody, deliberately pinned to the CPU. Training used the `performance` model (x-large, 288x384 input). `balanced` is the same backbone at 192x256, and measured here it is *faster on CPU* (47 ms/frame) than `performance` is on the GPU (105 ms/frame) -- while leaving the GPU entirely to the translation model. Keeping ONNX Runtime off the GPU also avoids two real failures: its CUDA arena competes with PyTorch for VRAM and host commit, and tearing the session down mid-request left PyTorch unable to find cuDNN kernels ("GET was unable to find an engine to execute this computation"). """ global _landmark_extractor if _landmark_extractor is None: _landmark_extractor = pp.LandmarkExtractor(mode="balanced", device="cpu") return _landmark_extractor def _estimate_duration(video_path, *args, **kwargs) -> int: """GPU reservation in seconds. Preprocessing dominates and scales with clip length. Gradio passes extra arguments positionally, so the signature has to swallow them. """ seconds = MAX_CLIP_SECONDS try: import cv2 cap = cv2.VideoCapture(video_path) total = cap.get(cv2.CAP_PROP_FRAME_COUNT) or 0 native_fps = cap.get(cv2.CAP_PROP_FPS) or 25.0 cap.release() if total > 0 and native_fps > 0: seconds = min(MAX_CLIP_SECONDS, total / native_fps) except Exception: # noqa: BLE001 pass return int(min(240, 60 + 12 * seconds)) def _generate_once(inputs, beams: int): """Run generation exactly once at the requested beam width. Deliberately no retry-on-OOM. Beam search replicates `pixel_values_videos` *before* the vision tower runs, so N beams means encoding the whole clip N times -- it is the single largest memory consumer here, far more than the weights. Catching an OOM mid-`generate` and retrying in the same process was measured to leave the CUDA context unusable ("unknown error", "illegal memory access") rather than recovering, so the caller picks a size that fits up front and a failure is reported honestly instead of papered over. """ if inputs.get("video_grid_thw") is not None and beams > 1: # Qwen3-VL emits per-frame timestamps, so the beam-search input expansion # splits video_grid_thw by a count equal to the number of temporal patches # rather than the number of videos. Rewriting [[T,H,W]] as T rows of # [1,H,W] makes the split line up. (Same workaround as the project's own # eval script.) Greedy decoding does no expansion and needs it left alone. vgt = inputs["video_grid_thw"] vgt = torch.repeat_interleave(vgt, vgt[:, 0], dim=0).clone() vgt[:, 0] = 1 inputs["video_grid_thw"] = vgt kwargs = dict(GENERATION_KWARGS, num_beams=beams) if beams == 1: kwargs.pop("length_penalty", None) # meaningless without beam search with torch.inference_mode(), torch.amp.autocast("cuda", dtype=torch.bfloat16): return model.generate( **inputs, **kwargs, # temperature / top_p / top_k are baked into generation_config.json but # unused here; passing None silences the invalid-flag warning. temperature=None, top_p=None, top_k=None, use_cache=True, ) def _translate_impl(video_path, use_signer_crop, use_clahe, use_landmark_overlay, resolution, decoding): if not video_path: return "", None, "Upload or record a clip first." timings = {} notes = [] t_all = time.perf_counter() # -- Native frames: needed for the pose-guided crop, which is computed in # source-video pixel space exactly as it was during training. Only every # 4th frame is retained -- that is all the bbox pass samples. t0 = time.perf_counter() frames_bgr, native_fps, n_scanned = pp.read_video_frames_bgr( video_path, stride=BBOX_FRAME_STRIDE, max_seconds=MAX_CLIP_SECONDS ) timings["decode (native)"] = time.perf_counter() - t0 native_fps = native_fps if native_fps > 0 else 25.0 full_duration_s = pp.probe_duration_seconds(video_path) or (n_scanned / native_fps) truncated = full_duration_s > MAX_CLIP_SECONDS + 0.5 duration_s = min(full_duration_s, MAX_CLIP_SECONDS) if truncated: notes.append( f"Clip is {full_duration_s:.1f}s — only the first " f"{MAX_CLIP_SECONDS:.0f}s were translated." ) # -- Pose-guided signer bbox (MediaPipe). The frames are already strided, so # the cropper walks them one by one. bbox = None if use_signer_crop: t0 = time.perf_counter() try: bbox = _get_signer_cropper().compute_bbox(frames_bgr) if bbox.failed: notes.append("No pose detected — the full frame was used instead of a crop.") bbox = None else: notes.append( f"Signer crop: {bbox.x2 - bbox.x1}x{bbox.y2 - bbox.y1}px from " f"{bbox.frame_width}x{bbox.frame_height}px " f"(pose found in {bbox.detection_rate:.0%} of sampled frames)." ) except Exception as exc: # noqa: BLE001 notes.append(f"Signer crop unavailable ({exc!r}) — using the full frame.") timings["signer crop (MediaPipe)"] = time.perf_counter() - t0 del frames_bgr # -- Decode + preprocess + generate, stepping the pixel budget down if the # vision tower cannot fit. The tower encodes every temporal patch in one # forward pass, so peak memory is driven by total_pixels far more than by # beam width; on a 12 GB card the training budget does not always fit. # Landmarks are normalised to the crop, so they are extracted once at the # first (largest) resolution and reused verbatim by every later attempt. video_content = { "type": "video", "video": video_path, "fps": VIDEO_FPS, "min_pixels": VIDEO_MIN_PIXELS, "max_pixels": VIDEO_MAX_PIXELS, "total_pixels": VIDEO_TOTAL_PIXELS, } if truncated: # Bound what the model decodes too, not just the bbox pass. video_content["video_end"] = MAX_CLIP_SECONDS messages = [ {"role": "system", "content": [{"type": "text", "text": SYSTEM_PROMPT}]}, {"role": "user", "content": [video_content, {"type": "text", "text": USER_PROMPT}]}, ] text = processor.apply_chat_template( messages, tokenize=False, add_generation_prompt=True, enable_thinking=False ) t0 = time.perf_counter() _, videos, video_kwargs = process_vision_info( messages, image_patch_size=16, return_video_kwargs=True, return_video_metadata=True ) timings["decode (Qwen sampler)"] = time.perf_counter() - t0 if not videos: return "", None, "Could not decode any frames from that clip." (video, video_metadata), = videos if not isinstance(video, torch.Tensor): video = torch.as_tensor(np.asarray(video)) if video.dtype != torch.uint8: video = video.clamp(0, 255).to(torch.uint8) if use_signer_crop and bbox is not None: video = pp.apply_signer_crop(video, bbox) if use_clahe: t0 = time.perf_counter() video = pp.apply_clahe(video) timings["CLAHE"] = time.perf_counter() - t0 # Landmarks are normalised to the crop, so one extraction serves every scale # in the retry ladder below. This is also by far the most expensive stage. landmarks = None if use_landmark_overlay: t0 = time.perf_counter() try: frames_rgb = video.permute(0, 2, 3, 1).contiguous().numpy() frames_bgr_crop = np.ascontiguousarray(frames_rgb[..., ::-1]) cur_h, cur_w = int(video.shape[-2]), int(video.shape[-1]) pose, lh, rh = _get_landmark_extractor().extract(frames_bgr_crop, cur_w, cur_h) landmarks = pp.postprocess_landmarks(pose, lh, rh) del frames_rgb, frames_bgr_crop hands_seen = int(np.mean([ (~np.all(np.isnan(landmarks[1]), axis=(1, 2))).mean(), (~np.all(np.isnan(landmarks[2]), axis=(1, 2))).mean(), ]) * 100) notes.append(f"Landmark overlay: hands tracked in ~{hands_seen}% of frames.") except Exception as exc: # noqa: BLE001 notes.append( f"Landmark overlay unavailable ({exc!r}). The model expects it — " "output quality will be degraded." ) timings["landmarks (RTMPose)"] = time.perf_counter() - t0 base_h, base_w = int(video.shape[-2]), int(video.shape[-1]) # The vision tower encodes every temporal patch in one forward pass, and beam # search replicates the pixel tensor before it does. Both are sized up front # from the user's choices rather than discovered by failing. scale = RESOLUTION_PRESETS.get(resolution, RESOLUTION_PRESETS[DEFAULT_RESOLUTION]) beams_used = DECODING_PRESETS.get(decoding, DECODING_PRESETS[DEFAULT_DECODING]) target_h = max(SNAP, (int(base_h * scale) // SNAP) * SNAP) target_w = max(SNAP, (int(base_w * scale) // SNAP) * SNAP) same_size = (target_h, target_w) == (base_h, base_w) attempt = video if same_size else pp.resize_video(video, target_h, target_w) del video # Redraw the skeleton at the working resolution so the 1 px strokes stay # crisp, exactly as training drew them onto already-resized frames. if landmarks is not None: attempt = pp.apply_landmark_overlay(attempt, *landmarks) n_frames, _, out_h, out_w = attempt.shape preview_path = tempfile.NamedTemporaryFile(suffix=".mp4", delete=False).name try: pp.write_preview_mp4(attempt, preview_path, fps=VIDEO_FPS) except Exception as exc: # noqa: BLE001 notes.append(f"Could not render the preview clip ({exc!r}).") preview_path = None # do_resize=False: frames are already at the intended resolution. t0 = time.perf_counter() inputs = processor( text=[text], videos=[attempt], video_metadata=[video_metadata], return_tensors="pt", padding=True, do_resize=False, **video_kwargs, ).to(model.device) del attempt gc.collect() torch.cuda.empty_cache() generated = _generate_once(inputs, beams_used) trimmed = [out[len(inp):] for inp, out in zip(inputs["input_ids"], generated)] translation = processor.batch_decode( trimmed, skip_special_tokens=True, clean_up_tokenization_spaces=False )[0].strip() timings["generation"] = time.perf_counter() - t0 del inputs, generated gc.collect() torch.cuda.empty_cache() downscaled = (out_h, out_w) != (base_h, base_w) if beams_used != GENERATION_KWARGS["num_beams"]: notes.append( f"Decoded greedily rather than with the " f"{GENERATION_KWARGS['num_beams']}-beam search the published metrics used." ) if downscaled: notes.append( f"Frames downscaled from {base_w}x{base_h} to {out_w}x{out_h} — " "raise *Input resolution* to feed the model the training resolution." ) if beams_used != GENERATION_KWARGS["num_beams"] or downscaled: notes.append( "Output therefore differs from the published evaluation setup." ) timings["total"] = time.perf_counter() - t_all info = [ f"**Model input** — {n_frames} frames at {VIDEO_FPS} fps, {out_w}x{out_h} px " f"({duration_s:.1f}s of signing), {beams_used} beam(s).", "", "**Preprocessing**", ] info += [f"- {n}" for n in notes] or ["- (all stages disabled)"] info += ["", "**Timings**"] info += [f"- {k}: {v:.1f}s" for k, v in timings.items()] return translation or "(empty output)", preview_path, "\n".join(info) @spaces.GPU(duration=_estimate_duration) def translate(video_path, use_signer_crop: bool = True, use_clahe: bool = True, use_landmark_overlay: bool = True, resolution: str = DEFAULT_RESOLUTION, decoding: str = DEFAULT_DECODING): """Translate an American Sign Language video clip into English text. Args: video_path: Path to an ASL video clip, 1-10 seconds of continuous signing. use_signer_crop: Crop to the signer using pose landmarks (training default: on). use_clahe: Apply CLAHE contrast enhancement (training default: on). use_landmark_overlay: Draw the pose/hand skeleton overlay (training default: on). resolution: How large a frame the vision tower encodes. Higher is closer to the training setup but needs considerably more GPU memory. decoding: Greedy, or the 5-beam search the published metrics used. Beam search re-encodes the clip once per beam and needs far more memory. Returns: The English translation, the preprocessed clip the model actually saw, and a breakdown of the preprocessing pipeline that produced it. """ try: return _translate_impl( video_path, use_signer_crop, use_clahe, use_landmark_overlay, resolution, decoding, ) except torch.OutOfMemoryError: traceback.print_exc() return "", None, ( "**Out of GPU memory**\n\nEven the fallback size did not fit. Try a " "shorter clip or a smaller setting under *Input resolution*, and close " "other GPU or memory-heavy applications (browsers especially)." ) except Exception as exc: # noqa: BLE001 — surface errors in the UI, not as a stack trace traceback.print_exc() return "", None, f"**Something went wrong**\n\n```\n{exc!r}\n```" # --------------------------------------------------------------------------- # UI # --------------------------------------------------------------------------- DESCRIPTION = """ # 🤟 sign-language-bridge — ASL → English Continuous **American Sign Language** translation with [`mamounyosef/sign-language-bridge`](https://huggingface.co/mamounyosef/sign-language-bridge): a multi-tier LoRA / RSLoRA fine-tune of [`Qwen3-VL-2B-Instruct`](https://huggingface.co/Qwen/Qwen3-VL-2B-Instruct) trained on How2Sign + OpenASL. Upload a clip or record one with your webcam. Clips of **1–15 seconds** of continuous signing, framed head-and-shoulders with good lighting, work best. > ⚠️ **Research preview.** BLEU-4 is 1.64 and WER is 112 % on the author's > How2Sign test partition — the model produces fluent English that is often > topically right but frequently disagrees with the reference word-for-word. > It can be confidently wrong. Do not use it where a mistranslation could cause > harm (medical, legal, safety-critical, or emergency settings). """ PIPELINE_NOTE = """ ### Why the processed clip looks like that The adapter was trained with three preprocessing stages applied to **every** clip, so inference has to reproduce them: 1. **Pose-guided signer crop** — MediaPipe pose landmarks are unioned across the clip and padded 25 %, giving one stable box around the signing space. 2. **CLAHE** — contrast equalisation on the L channel in LAB (clip 2.0, 8×8 tiles). 3. **Landmark overlay** — RTMPose Wholebody draws 6 upper-body joints (yellow) and 21 keypoints per hand (green = left, blue = right) directly onto the pixels. Turning any of them off shows you how much the model leans on them — the output usually gets noticeably worse. """ with gr.Blocks(title="sign-language-bridge — ASL to English") as demo: gr.Markdown(DESCRIPTION) with gr.Row(): with gr.Column(scale=1): video_in = gr.Video( label="ASL clip", sources=["upload", "webcam"], include_audio=False, format=None, # skip Gradio's re-encode; we decode the original ourselves ) resolution_dd = gr.Dropdown( choices=list(RESOLUTION_PRESETS), value=DEFAULT_RESOLUTION, label="Input resolution", info="Higher is closer to the training setup but needs much more GPU memory.", ) decoding_dd = gr.Dropdown( choices=list(DECODING_PRESETS), value=DEFAULT_DECODING, label="Decoding", info="Beam search re-encodes the clip once per beam — accurate, but memory-hungry.", ) with gr.Accordion("Preprocessing (training defaults: all on)", open=False): crop_cb = gr.Checkbox(value=True, label="Pose-guided signer crop") clahe_cb = gr.Checkbox(value=True, label="CLAHE contrast enhancement") overlay_cb = gr.Checkbox(value=True, label="Landmark skeleton overlay") run_btn = gr.Button("Translate", variant="primary") with gr.Column(scale=1): translation_out = gr.Textbox( label="English translation", lines=3, ) preview_out = gr.Video(label="What the model actually saw", autoplay=True) info_out = gr.Markdown() gr.Markdown(PIPELINE_NOTE) run_btn.click( fn=translate, inputs=[video_in, crop_cb, clahe_cb, overlay_cb, resolution_dd, decoding_dd], outputs=[translation_out, preview_out, info_out], api_name="translate", ) if __name__ == "__main__": # SLB_OPEN_BROWSER is set by the local launcher; on a Space it is unset. demo.queue(max_size=12).launch( mcp_server=True, show_error=True, inbrowser=os.environ.get("SLB_OPEN_BROWSER") == "1", )