"""LTX-2.5 driven by a 3.1x compressed text encoder, on ZeroGPU. This is an end-to-end serving demonstration, not an isolated encoder-quality comparison. Outputs also depend on the third-party quantized DiT, custom AV guide handling, scheduling and optional refinement documented in the UI. The live prompt path uses the versioned raw-intermediate-slot conditioning contract. ZeroGPU shapes the code more than the models do. The GPU exists only inside a `@spaces.GPU` call, so everything expensive that does not need CUDA - cloning ComfyUI, pulling ~22 GB of weights - happens at import on CPU, and the models are built on first request and then kept. """ from __future__ import annotations import os import re import subprocess import sys import time from fractions import Fraction from pathlib import Path # Roughly 22 GB arrives on a cold start, and `huggingface_hub` reads this into # a module constant at import - setting it any later has no effect at all. os.environ.setdefault("HF_HUB_ENABLE_HF_TRANSFER", "1") import gradio as gr import spaces from huggingface_hub import hf_hub_download, snapshot_download # --------------------------------------------------------------------- layout ROOT = Path(__file__).resolve().parent WORK = Path(os.environ.get("LTX_WORK", "/tmp/ltx")) COMFY = WORK / "ComfyUI" MODELS = WORK / "models" OUT = WORK / "out" # Pinned. An unpinned clone means the Space silently changes behaviour the day # ComfyUI refactors a sampler, and the evidence in the model card stops # describing what is running here. COMFY_COMMIT = "b615af1c" GGUF_NODE_COMMIT = "6ea2651e" ENCODER_REPO = "topabaem/LTX-2.5-Text-Encoder-4bit-8GB" ENCODER_FILE = "A3.packed.safetensors" DIT_REPO = "realrebelai/LTX-2.5_GGUFs" DIT_FILE = "LTX-2.5-Distilled-Q3_K_M.gguf" VAE_REPO = "Lightricks/LTX-2.5" VAE_REVISION = "6c7e5e573ac1667efc83407806fe9b0b93730e60" VIDEO_VAE = "vae/ltx-2.5-video-vae-conv-bf16.safetensors" AUDIO_VAE = "vae/ltx-2.5-audio-vae-bf16.safetensors" UPSCALER = "latent_upscale_models/ltx-2.5-latent-spatial-upscaler-x2-bf16-1.0.safetensors" FPS = Fraction(24, 1) _STATE: dict = {} def _run(command: list[str], cwd: Path | None = None) -> None: result = subprocess.run(command, cwd=cwd, capture_output=True, text=True) if result.returncode != 0: raise RuntimeError(f"{' '.join(command)} failed:\n{result.stderr[-2000:]}") def _clone(url: str, dest: Path, commit: str) -> None: if dest.exists(): return dest.parent.mkdir(parents=True, exist_ok=True) _run(["git", "clone", "--filter=blob:none", url, str(dest)]) _run(["git", "checkout", commit], cwd=dest) #: `Lightricks/LTX-2.5` is gated, so the VAEs need an authenticated request from #: an account that has accepted the licence. Set HF_TOKEN as a Space secret. TOKEN = os.environ.get("HF_TOKEN") or None def install_comfy_requirements() -> None: """Install ComfyUI's own dependencies, minus the torch trio. ComfyUI's requirements list `torch`, `torchvision` and `torchaudio` unpinned, and letting pip act on those would replace the exact wheels this Space was built against - which is the failure the pins in `requirements.txt` exist to prevent. Everything else it asks for is real: `comfy_aimdo` and `comfy_kitchen` are imported while `comfy.ops` is still being imported, so a missing one does not degrade a feature, it stops the first generation with a ModuleNotFoundError from six frames deep. Done here rather than in `requirements.txt` because ComfyUI is cloned at runtime: this way the dependency list always matches COMFY_COMMIT instead of a hand-copied snapshot that goes stale the next time it is bumped. """ stamp = COMFY / ".requirements-installed" if stamp.exists(): return source = COMFY / "requirements.txt" skip = {"torch", "torchvision", "torchaudio"} wanted = [] for line in source.read_text().splitlines(): line = line.strip() if not line or line.startswith("#"): continue if re.split(r"[=<>~!\[;]", line)[0].strip().lower() in skip: continue wanted.append(line) _run([sys.executable, "-m", "pip", "install", "--no-cache-dir", *wanted]) stamp.touch() def bootstrap() -> dict[str, Path]: """Everything that does not need a GPU, done once at import.""" if TOKEN is None: raise RuntimeError( "HF_TOKEN is not set. Lightricks/LTX-2.5 is a gated repo and its " "VAEs cannot be fetched without a token from an account that has " "accepted the licence.") _clone("https://github.com/comfyanonymous/ComfyUI.git", COMFY, COMFY_COMMIT) _clone("https://github.com/city96/ComfyUI-GGUF.git", COMFY / "custom_nodes" / "ComfyUI-GGUF", GGUF_NODE_COMMIT) install_comfy_requirements() MODELS.mkdir(parents=True, exist_ok=True) OUT.mkdir(parents=True, exist_ok=True) def fetch(repo: str, filename: str, revision: str | None = None) -> Path: return Path(hf_hub_download(repo, filename, revision=revision, cache_dir=str(MODELS), token=TOKEN)) return { "samples": Path(snapshot_download( ENCODER_REPO, allow_patterns=["samples/*"], cache_dir=str(MODELS), token=TOKEN)) / "samples", "packed": fetch(ENCODER_REPO, ENCODER_FILE), "encoder_dir": Path(snapshot_download( ENCODER_REPO, allow_patterns=["encoder-hf/*"], cache_dir=str(MODELS), token=TOKEN)) / "encoder-hf", "gguf": fetch(DIT_REPO, DIT_FILE), "video_vae": fetch(VAE_REPO, VIDEO_VAE, VAE_REVISION), "audio_vae": fetch(VAE_REPO, AUDIO_VAE, VAE_REVISION), "upscaler": fetch(VAE_REPO, UPSCALER, VAE_REVISION), } PATHS = bootstrap() os.environ["LTX_COMFY_DIR"] = str(COMFY) sys.path.insert(0, str(ROOT / "scripts")) sys.path.insert(0, str(ROOT / "src")) def _load(): """Build the encoder and the DiT once, inside a GPU call.""" if "engine" in _STATE: return _STATE["encoder"], _STATE["engine"], _STATE["render"] from ltx_av_generate import LTXEngine from ltx_prompt_encoder import PackedPromptEncoder # H200 has room for the whole encoder, so nothing is pushed to CPU. os.environ.setdefault("LTX_PACKED_GPU_BUDGET", "40GiB") encoder = PackedPromptEncoder(PATHS["encoder_dir"], PATHS["packed"], device="cuda", resident=True) engine = LTXEngine(PATHS["gguf"], PATHS["video_vae"], PATHS["audio_vae"], comfy_args=["--reserve-vram", "1.0"], upscaler=PATHS["upscaler"]) import ltx_render_clips as render _STATE.update(encoder=encoder, engine=engine, render=render) return encoder, engine, render @spaces.GPU(duration=300) def generate(prompt, mode, image, video, width, height, seconds, seed, denoise, strength, refine=0.0, fit_positions=True, progress=gr.Progress()): if not prompt or not prompt.strip(): raise gr.Error("A prompt is required - the encoder is what this Space is showing.") progress(0.05, desc="loading models (first call only)") encoder, engine, render = _load() # LTX samples 8n+1 frames; anything else is silently rounded, so round here # where the number can be shown. length = max(9, int(round(float(seconds) * float(FPS))) // 8 * 8 + 1) progress(0.25, desc="encoding prompt") conditioning = encoder.encode(prompt.strip()) progress(0.45, desc=f"sampling {length} frames") frames, audio, rate = engine.generate( conditioning, mode=mode, width=int(width), height=int(height), length=length, fps=FPS, seed=int(seed), image=Path(image) if image else None, video=Path(video) if video else None, denoise=float(denoise), strength=float(strength), refine=float(refine), fit_positions=bool(fit_positions), tile=256, temporal=16) progress(0.9, desc="writing clip") out = OUT / f"{mode}-{int(seed)}-{int(time.time())}.mp4" render.write_clip(frames, audio, rate, FPS, out) scale = 2 if float(refine) > 0 else 1 note = (f"{mode} · {length} frames · {int(width)*scale}x{int(height)*scale}" f" · seed {int(seed)}") if scale == 2: note += f" · refined at {float(refine):.2f}" return str(out), note # ------------------------------------------------------------------------ ui INTRO = """[![Haverbex LTX2.5-4bit](https://huggingface.co/topabaem/LTX-2.5-Text-Encoder-4bit-8GB/resolve/main/banner.png)](https://buymeacoffee.com/choijjs83q) # LTX-2.5 on a 3.1x compressed text encoder The Gemma4-12B encoder that drives LTX-2.5, squeezed from **26.264 GB to 8.46 GB** with **no compute-capability floor** — it runs on a Volta V100 as happily as on an H200. Live prompts use the corrected **`gemma4-raw-intermediate-slots-v1`** conditioning contract: explicit BOS, a 1024-token left-padded Gemma forward, valid-token extraction, and raw intermediate hidden-state slots. This is an end-to-end demo: output also depends on the third-party quantized DiT, custom AV guide handling, scheduling and the optional refinement pass described below. Video and audio are generated together; the clips have their own soundtrack. [Model card](https://huggingface.co/topabaem/LTX-2.5-Text-Encoder-4bit-8GB) > **EN** — I'm a student researching ML quantization. Getting this one done > burned through so much in server bills that from here on I'll only be able to > afford *niu lai* movies. Thank you for using the model. > > **한국어** — 저는 ML 양자화를 연구하는 학생입니다. 양자화를 진행하면서 서버 > 비용을 너무 많이 써서, 앞으로 영화는 *niu lai*만 봐야 할 것 같습니다. 모델을 > 사용해 주셔서 감사합니다. > > **中文** — 我是一名研究机器学习量化的学生。做这次量化烧掉了太多服务器费用， > 以后看电影大概只能看 *niu lai* 了。感谢您使用这个模型。 > > ☕ **[Buy me a coffee](https://buymeacoffee.com/choijjs83q)** · [커피 한 잔 사주기](https://buymeacoffee.com/choijjs83q) · [请我喝杯咖啡](https://buymeacoffee.com/choijjs83q) """ NOTE = """ **Image- and video-to-video needed a fix ComfyUI does not ship.** `LTXVAddGuide` calls `torch.cat` on what is a `NestedTensor` for LTX-2.5, so it raises before it can do anything. The model, sampler and mask plumbing all support AV guides already — `ltx_av_guide.py` unwraps the pair, runs the stock node on the video half and re-wraps, so the guide arithmetic stays the vendor's. Measured on a 16 GB V100 at 512x320, 25 frames: t2v 98.8 s, i2v 62.7 s, v2v 86.5 s, at 5.84-6.80 GiB of *allocated* VRAM. Size a card from what `nvidia-smi` reports rather than that figure — it counts only live allocator blocks, and for the encoder alone the gap between the two is close to 2 GiB (8.48 GiB allocated against 9.70 GiB on the card). An i2v first frame lands relL2 0.0766 from its guide image, against 0.7061 for the same seed without the guide. **The DiT is a third-party `Q3_K_M` quantization**, not the one the model card's sample clips were rendered with — that exact file is no longer retrievable. Some of what you see is its doing, not the encoder's. """ #: The five that `samples/` was rendered from, in order. Kept here rather than #: parsed out of the sample folder, because a caption that silently goes missing #: is worse than one that is obviously wrong. SAMPLE_PROMPTS = [ ("Robot in the rain", "A humanoid robot steps out of a freight elevator into a rain-slick loading " "bay, sodium lights flaring off its wet white shell. Locked-off low camera " "as the robot turns toward the lens. Audio: heavy rain on metal, one servo " "whine, distant thunder."), ("Dune", "A lone astronaut in a scuffed white suit walks across a red dune at sunset, " "a long shadow stretching behind, dust curling off each bootfall. Low " "tracking camera moving with the walk. Audio: wind over sand, suit fan hum, " "muffled breathing."), ("Forge", "A blacksmith hammers glowing orange steel on an anvil inside a dark forge, " "sparks bursting outward with every strike, warm rim light on soot-streaked " "arms. Medium shot, camera static. Audio: ringing hammer strikes, roaring " "bellows, a hiss as the blade is quenched."), ("Night road", "A vintage sports car slides through a wet mountain hairpin at night, " "headlights sweeping across the guardrail, spray lit red by the tail lights. " "Camera pans to follow the car through the turn. Audio: engine roar, tyre " "squeal, rain on asphalt."), ("Smoke", "A battered rescue robot pushes through thick smoke in a collapsed corridor, " "headlamp cutting a cone through the dust, orange emergency light pulsing on " "the walls. Handheld camera following from behind. Audio: crackling fire, a " "muffled alarm, grinding treads."), ] PROMPT = ("A humanoid robot steps out of a freight elevator into a rain-slick " "loading bay, sodium lights flaring off its wet white shell. " "Locked-off low camera as the robot turns toward the lens. " "Audio: heavy rain on metal, one servo whine, distant thunder.") def _panel(mode: str): """The controls one tab needs. Image and video inputs are always created so the click handler has a stable signature; the unused one stays hidden.""" with gr.Row(): with gr.Column(scale=3): prompt = gr.Textbox(label="Prompt", value=PROMPT, lines=4) image = gr.Image(label="First frame", type="filepath", visible=mode == "i2v") video = gr.Video(label="Source clip", visible=mode == "v2v") with gr.Row(): width = gr.Slider(256, 1024, 768, step=32, label="Width") height = gr.Slider(256, 1024, 512, step=32, label="Height") with gr.Row(): # 15 s was measured at 6.26 GiB on a 16 GB V100 - no more than # 10 s cost - so the old 4 s ceiling was under-selling the model # rather than protecting anything. # Past 20 s the temporal RoPE leaves the range it was trained # on; the engine interpolates the positions back in, which keeps # it coherent at some cost in temporal detail. seconds = gr.Slider(0.5, 30.0, 2.0, step=0.5, label="Seconds") seed = gr.Number(20260813, label="Seed", precision=0) strength = gr.Slider(0.0, 1.0, 1.0, step=0.05, label="Guide strength", visible=mode == "i2v") denoise = gr.Slider(0.1, 1.0, 0.6, step=0.05, label="Denoise (lower keeps more of the source)", visible=mode == "v2v") # i2v is excluded: the upsampler drops the noise mask, so the guide # would have to be rebuilt at the new scale and it is not yet. fit = gr.Checkbox( value=True, label="Fit temporal positions past 20 s — off shows what the " "untreated RoPE does, and is only visible above 20 s") refine = gr.Slider( 0.0, 0.8, 0.4 if mode == "t2v" else 0.0, step=0.05, label="Second pass (0 = off) — upscales 2x and resamples, " "much sharper, roughly 2x the time", visible=mode != "i2v") go = gr.Button(f"Generate ({mode})", variant="primary") with gr.Column(scale=2): out = gr.Video(label="Result", autoplay=True) info = gr.Markdown() go.click( fn=generate, inputs=[prompt, gr.State(mode), image, video, width, height, seconds, seed, denoise, strength, refine, fit], outputs=[out, info], ) COMPARE_INTRO = """### Legacy sample pairs — before the conditioning correction Nothing below was generated on demand — these are the published clips from `samples/`, rendered with everything downstream of the encoder held identical: same DiT, same seed, same 8-step schedule, same VAE settings, one process, one V100. Within each old pair, the encoder checkpoint is the only variable. These clips used the same older hidden-state preprocessing on both sides and predate the live `gemma4-raw-intermediate-slots-v1` path. They are retained as historical checkpoint comparisons, not as evidence for the corrected live runtime or as a corrected BF16 oracle comparison. The **deterministic** column is the one that attributes a difference to the encoder. The vendor's default sampler, `euler_ancestral`, re-rolls noise at every step, so the two builds wander into different takes of the same scene and the pair stops being a controlled comparison. `euler` injects nothing, so the seed fixes the starting noise and the conditioning is all that is left to move a pixel. Four of the five legacy deterministic pairs hold together. **The dune does not** — BF16 renders a soldier in fatigues where the 4-bit renders a man in a business suit. Within that old controlled setup, the checkpoint change is the remaining variable. Neither build followed the requested astronaut, so this single pair should not be generalized into a current prompt-compliance claim. """ def _samples_tab(): """The published pairs, played rather than linked. Files are read from the snapshot fetched at bootstrap. If a clip is missing the component is left out and the caption says so, because a silently empty video player looks like a broken Space rather than a missing file. """ gr.Markdown(COMPARE_INTRO) root = PATHS["samples"] for index, (title, prompt) in enumerate(SAMPLE_PROMPTS): with gr.Accordion(f"{index:02d} — {title}", open=index == 2): gr.Markdown(f"*{prompt}*") for suffix, label in (("-det", "deterministic `euler`"), ("", "vendor default `euler_ancestral`")): bf = root / f"bf16{suffix}-{index:02d}.mp4" q4 = root / f"4bit{suffix}-{index:02d}.mp4" gr.Markdown(f"**{label}**") if not (bf.exists() and q4.exists()): gr.Markdown(f"_clips for {label} are not in the snapshot_") continue with gr.Row(): gr.Video(value=str(bf), label="BF16 original — 26.264 GB", interactive=False) gr.Video(value=str(q4), label="4-bit — 8.46 GB", interactive=False) with gr.Blocks(title="LTX-2.5 Text Encoder 4bit") as demo: gr.Markdown(INTRO) with gr.Tabs(): with gr.Tab("Text to video"): _panel("t2v") with gr.Tab("Image to video"): _panel("i2v") with gr.Tab("Video to video"): _panel("v2v") with gr.Tab("BF16 vs 4-bit"): _samples_tab() gr.Markdown(NOTE) if __name__ == "__main__": demo.queue(max_size=8).launch()