import os os.environ.setdefault("TORCH_COMPILE_DISABLE", "1") os.environ.setdefault("TORCHDYNAMO_DISABLE", "1") import math import random import tempfile import threading import time import numpy as np import spaces import torch import gradio as gr from PIL import Image, ImageOps from huggingface_hub import hf_hub_download from safetensors.torch import load_file from diffusers import LTX2InContextPipeline, LTX2LatentUpsamplePipeline from diffusers.pipelines.ltx2.pipeline_ltx2_ic_lora import LTX2ReferenceCondition from diffusers.pipelines.ltx2.latent_upsampler import LTX2LatentUpsamplerModel from diffusers.pipelines.ltx2.utils import DISTILLED_SIGMA_VALUES, STAGE_2_DISTILLED_SIGMA_VALUES from diffusers.utils import encode_video # --- Config ----------------------------------------------------------------- # FAST distilled variant of the ingredients (reference-sheet) IC-LoRA: 8-step schedule, CFG off. BASE_MODEL = "diffusers/LTX-2.3-Distilled-Diffusers" LORA_REPO = "Lightricks/LTX-2.3-22b-IC-LoRA-Ingredients" LORA_FILE = "ltx-2.3-22b-ic-lora-ingredients-0.9.safetensors" LORA_SCALE = 1.4 FPS = 24 WIDTH, HEIGHT = 768, 448 NUM_FRAMES = 121 NUM_STEPS = len(DISTILLED_SIGMA_VALUES) # 8 MAX_SEED = np.iinfo(np.int32).max HF_TOKEN = os.environ.get("HF_TOKEN") UPSAMPLER_REPO = "dg845/LTX-2.3-Spatial-Upsampler-Diffusers" # LTX-2.3 spatial x2 latent upsampler pipe = LTX2InContextPipeline.from_pretrained(BASE_MODEL, torch_dtype=torch.bfloat16) pipe.to("cuda") pipe.vae.enable_tiling() _lora_path = hf_hub_download(LORA_REPO, LORA_FILE, token=HF_TOKEN) # Kept as a togglable adapter (NOT fused) so Stage 2 can run on the bare distilled model. pipe.load_lora_weights(load_file(_lora_path), adapter_name="ingredients") # NOTE: AOTI temporarily disabled while validating 2-stage inference; re-enable once confirmed. # spaces.aoti_load(module=pipe.transformer, repo_id="ltx-community/LTX-2.3-Transformer-GroupA-sm120-cu130-r9e") # Stage-2 latent upsampler (spatial x2) — two-stage diffusers inference. _upsampler = LTX2LatentUpsamplerModel.from_pretrained( UPSAMPLER_REPO, subfolder="latent_upsampler", torch_dtype=torch.bfloat16) _upsampler.to("cuda") upsample_pipe = LTX2LatentUpsamplePipeline(vae=pipe.vae, latent_upsampler=_upsampler) def _gallery_paths(gallery): paths = [] for item in gallery or []: if isinstance(item, (list, tuple)): item = item[0] if isinstance(item, dict): item = item.get("path") or item.get("name") or item.get("image") if isinstance(item, str): paths.append(item) return paths def compose_sheet(gallery): paths = _gallery_paths(gallery) if not paths: raise gr.Error("Upload at least one subject image to build a sheet.") imgs = [Image.open(p).convert("RGB") for p in paths] if len(imgs) == 1: return imgs[0] CW, CH = 1536, 896 canvas = Image.new("RGB", (CW, CH), (0, 0, 0)) n = len(imgs) cols = math.ceil(math.sqrt(n)) rows = math.ceil(n / cols) g = 16 cw = (CW - g * (cols + 1)) // cols ch = (CH - g * (rows + 1)) // rows for i, im in enumerate(imgs): r, c = divmod(i, cols) canvas.paste(ImageOps.fit(im, (cw, ch), Image.LANCZOS), (g + c * (cw + g), g + r * (ch + g))) return canvas def _build_prompt(sheet, action): return f"Reference sheet: {sheet.strip()}\n\nGenerated video: {action.strip()}" def _export(video_np, audio, path): kw = {} if audio is not None: kw = dict(audio=audio[0].float().cpu(), audio_sample_rate=pipe.vocoder.config.output_sampling_rate) encode_video(video_np, fps=FPS, output_path=path, **kw) def _duration(*args, **kwargs): return 480 # two-stage (stage1 + x2 upsample + stage2 refine), AOTI off during validation @spaces.GPU(duration=_duration) def generate(sheet_image, sheet, action, lora_scale, seed, randomize, progress=gr.Progress(track_tqdm=True)): if sheet_image is None: raise gr.Error("Add a reference sheet (upload one, or build one from subject images in the other tab).") if not sheet.strip(): raise gr.Error("Describe the elements in the reference sheet (characters, props, location).") if not action.strip(): raise gr.Error("Describe the action / shot you want generated.") if randomize: seed = random.randint(0, MAX_SEED) seed = int(seed) sheet_img = sheet_image.convert("RGB").resize((WIDTH, HEIGHT), Image.LANCZOS) ref = [sheet_img] * NUM_FRAMES prompt = _build_prompt(sheet, action) gen = torch.Generator(device="cuda").manual_seed(seed) # --- Stage 1: base-res latents with the ingredients IC-LoRA (distilled 8-step) --- pipe.set_adapters("ingredients", float(lora_scale)) video_latent, audio_latent = pipe( prompt=prompt, negative_prompt="", reference_conditions=[LTX2ReferenceCondition(frames=ref, strength=1.0)], reference_downscale_factor=1, width=WIDTH, height=HEIGHT, num_frames=NUM_FRAMES, frame_rate=FPS, num_inference_steps=NUM_STEPS, sigmas=DISTILLED_SIGMA_VALUES, guidance_scale=1.0, stg_scale=0.0, audio_guidance_scale=1.0, audio_stg_scale=0.0, generator=gen, output_type="latent", return_dict=False, ) # --- Stage 2a: spatial x2 latent upsample --- up_latent = upsample_pipe(latents=video_latent, output_type="latent", return_dict=False)[0] # --- Stage 2b: short refine at 2x res on the bare distilled model (drop IC-LoRA + reference) --- pipe.disable_lora() video_out, audio_out = pipe( prompt=prompt, negative_prompt="", latents=up_latent, audio_latents=audio_latent, width=WIDTH * 2, height=HEIGHT * 2, num_frames=NUM_FRAMES, frame_rate=FPS, num_inference_steps=len(STAGE_2_DISTILLED_SIGMA_VALUES), sigmas=STAGE_2_DISTILLED_SIGMA_VALUES, noise_scale=STAGE_2_DISTILLED_SIGMA_VALUES[0], guidance_scale=1.0, stg_scale=0.0, audio_guidance_scale=1.0, audio_stg_scale=0.0, generator=gen, output_type="np", return_dict=False, ) pipe.enable_lora() out_path = tempfile.NamedTemporaryFile(suffix=".mp4", delete=False).name _export(video_out[0], audio_out, out_path) return out_path, seed with gr.Blocks(title="LTX-2.3 Ingredients (Fast)") as demo: gr.Markdown( "# ⚡ LTX-2.3 Ingredients — Fast (Distilled)\n" "Reference-sheet control, fast. Upload a ready sheet, or build one from individual subject images. Using " "[LTX 2.3 Distilled](https://huggingface.co/diffusers/LTX-2.3-Distilled-Diffusers) with the " "[Ingredients IC-LoRA](https://huggingface.co/Lightricks/LTX-2.3-22b-IC-LoRA-Ingredients), via diffusers 🧨. " "(For maximum fidelity, see the Dev demo.)" ) with gr.Row(): with gr.Column(): with gr.Tabs(): with gr.Tab("Reference sheet"): sheet_image = gr.Image(type="pil", label="Reference sheet (composite of characters / props / location)") with gr.Tab("Build from subject images"): gallery = gr.Gallery(label="Upload subject images (characters, props, location)", type="filepath", interactive=True, columns=4, height=240) build_btn = gr.Button("Build reference sheet ➜") gr.Markdown("*Tiles your images into one sheet and loads it into the **Reference sheet** tab.*") sheet = gr.Textbox(label="Reference sheet description", lines=3, placeholder="a young woman with red hair in a green jacket (face close-up + turnaround); a brass pocket watch; a cobblestone alley at night") action = gr.Textbox(label="Generated video — action / shot, plus any speech & sounds", lines=3, placeholder="the woman walks down the alley, checks the pocket watch and whispers 'almost time'; footsteps on cobblestone, distant city hum") with gr.Accordion("Settings", open=False): lora_scale = gr.Slider(0.8, 1.8, value=1.4, step=0.05, label="LoRA strength") randomize = gr.Checkbox(True, label="Randomize seed") seed = gr.Slider(0, MAX_SEED, value=42, step=1, label="Seed") run = gr.Button("Generate (fast)", variant="primary") with gr.Column(): video_out = gr.Video(label="Generated video") build_btn.click(compose_sheet, inputs=gallery, outputs=sheet_image) run.click(generate, inputs=[sheet_image, sheet, action, lora_scale, seed, randomize], outputs=[video_out, seed]) gr.Examples( examples=[ ["examples/sheet_garden.png", "a friendly cartoon hedgehog with rounded chestnut-brown fur, a cream face and belly, large expressive dark eyes, a small black nose and tiny rounded ears, shown in a face close-up and a full-body turnaround standing upright on two short legs; a cheerful cartoon rabbit with soft grey-and-white fur, long upright ears with pale pink inner lining, round amber eyes and a fluffy white tail, shown in a body turnaround; a green coiled garden hose neatly wound on a matching green wall-mounted reel; a row of green plastic spray bottles with trigger nozzles; the bright interior of a 'Greenfield Home & Garden' store with tall wooden shelves stocked with leafy potted plants, terracotta pots and gardening supplies, warm overhead lighting and a green-and-white storefront sign", "cheerful family-animation commercial scene, a lively medium shot inside the sunlit 'Greenfield Home & Garden' store with its tall shelves of leafy potted plants and terracotta pots. the rounded chestnut-brown hedgehog waddles briskly up toward the camera on its short legs, its cream belly bouncing, then stops, lifts a tiny paw in a friendly wave and beams with wide sparkling eyes, announcing in a warm, bright, sing-song voice: 'welcome to Greenfield!'. behind it the soft grey-and-white rabbit hops past in the aisle, long ears bobbing, cradling a green spray bottle against its chest; it pauses, gives the bottle a playful little squeeze that puffs a fine mist into a shaft of light, and adds in a chirpy, slightly higher voice: 'everything your garden needs!'. the hedgehog nods enthusiastically, gestures with both paws toward the laden shelves and finishes with a cosy chuckle: 'come on in!'. the animation is glossy and expressive with squash-and-stretch motion, rounded shapes and saturated greens; the camera pushes in gently and tilts up to reveal the green-and-white storefront sign. the audio is bright and immersive: the hedgehog's cheerful voice, the rabbit's lighter reply, the soft puff of the spray bottle, light pattering footsteps on the store floor and a warm, upbeat acoustic-ukulele jingle playing softly underneath", 1.4, 42, False], ["examples/sheet_hiker.png", "a young Asian woman with a warm skin tone, dark hair parted down the middle in two long braids resting on her chest, an olive-green short-sleeved t-shirt, khaki cargo pants, dark brown hiking boots and a black wristwatch on her left arm, with a serious natural expression; a large heavy-duty blue hiking backpack with an external silver metal frame, multiple side and top pouches, black adjustable straps and a brown leather square patch near the bottom; a simple thick natural wooden walking stick with rough bark texture and a slight fork near one end; a large sturdy yak with long shaggy white-and-blonde hair and curved grey horns, wearing an ornate saddle blanket with intricate blue, red and yellow patterns, a saddle with metal stirrups and colorful tassels near its ears and chest; a sweeping majestic mountain landscape where a dirt path winds through green rocky slopes toward towering snow-capped peaks under a bright blue sky with scattered white clouds; a small traditional square stone shrine with a flat slightly tiered roof and a bright yellow fabric valance along the roofline, bright blue window trim and a red wooden door, with a small stone stupa beside it", "cinematic adventure documentary scene, a dynamic medium wide shot of the young asian woman with her dark hair in two long braids, wearing an olive-green t-shirt and khaki cargo pants. she sits on a rock along a mountain dirt path, resting beside the massive white shaggy yak with curved horns and its ornate blue, red and yellow patterned saddle blanket with metal stirrups. leaning against a nearby small stone building with blue window trim, a red door and a yellow fabric roof valance are her large blue external-frame backpack and thick wooden walking stick; majestic snow-capped mountains tower in the distant background under a bright blue sky. she looks at the yak, chest heaving slightly from exertion, and says with a breathy, tired but gentle voice: 'we've got a long way to go...'. she pauses, extending her hand to gently pat the thick white fur on the yak's neck; the yak shifts its weight, the colorful tassels near its ears swaying, and a faint exhausted smile breaks across her face as she continues softly: '...big guy.'. lowering her hand she grabs her wooden walking stick, leaning her weight onto it as she turns her gaze up toward the distant snowy peaks, her expression shifting from exhaustion to quiet determination as she adds, her voice growing firmer: 'but the pass...', then takes a deep grounding breath: '...is just over that ridge.'. the camera is dynamically handheld, slowly orbiting the woman and the yak to reveal the depth of the valley and the towering mountains behind them; naturalistic breathtaking film aesthetic, bright crisp sunlight casting sharp shadows across the rocky path and stone shrine. clear immersive audio: her wind-swept voice, the heavy rhythmic breathing of the yak, the faint jingle of metal stirrups and the distant ambient howl of mountain winds, no background music", 1.4, 42, False], ["examples/subj_composite.png", "a smiling young woman with warm fair skin and shoulder-length curly dark-brown hair, soft brown eyes and a gentle open expression, wearing a cream cable-knit sweater and dark jeans, shown in a relaxed three-quarter pose; a dappled grey horse with a dark charcoal mane and tail, a soft mottled grey-and-white coat, dark intelligent eyes and a calm posture, wearing a simple brown leather halter; a green misty mountain meadow of tall dewy grass and scattered wildflowers, with faint pine-covered slopes dissolving into low morning fog under a pale silver sky", "tender naturalistic cinematic scene, a soft medium shot in a green misty mountain meadow at dawn, tall dewy grass glistening and low fog drifting between faint pine slopes. the young woman with curly dark-brown hair and a cream cable-knit sweater walks slowly up to the dappled grey horse, her breath faintly visible in the cool air, and raises a careful open hand. she gently strokes the horse's soft mottled neck, her face softening into a warm reassuring smile, and murmurs in a low, soothing, slightly breathy voice: 'good boy… easy now'. the horse lowers its head toward her, its charcoal mane shifting, flicks an ear and lets out a soft snort, its breath misting in the cold; she leans her forehead lightly against its cheek, closes her eyes for a moment, then whispers with quiet affection: 'there you go'. the camera drifts slowly in a gentle arc around the pair, the shallow-focus background of fog and wildflowers blurring softly behind them; the film aesthetic is delicate and breathtaking with cool silver dawn light and fine atmospheric haze. the audio is intimate and immersive: her gentle hushed voice, the horse's soft nicker and snort, the swish of dewy grass, light birdsong waking in the distance and a faint cool breeze, with no background music", 1.4, 42, False], ], inputs=[sheet_image, sheet, action, lora_scale, seed, randomize], outputs=[video_out, seed], fn=generate, cache_examples=True, cache_mode="lazy", ) if __name__ == "__main__": demo.launch(show_error=True)