import os os.environ.setdefault("TORCH_COMPILE_DISABLE", "1") os.environ.setdefault("TORCHDYNAMO_DISABLE", "1") import math import random import tempfile import numpy as np import spaces import torch import gradio as gr from PIL import Image, ImageOps from huggingface_hub import hf_hub_download from safetensors.torch import load_file from diffusers import LTX2InContextPipeline from diffusers.pipelines.ltx2.pipeline_ltx2_ic_lora import LTX2ReferenceCondition from diffusers.pipelines.ltx2.utils import DISTILLED_SIGMA_VALUES from diffusers.utils import encode_video # --- Config ----------------------------------------------------------------- # FAST distilled variant of the ingredients (reference-sheet) IC-LoRA: 8-step schedule, CFG off. BASE_MODEL = "diffusers/LTX-2.3-Distilled-Diffusers" LORA_REPO = "linoyts/LTX-2.3-loras" LORA_FILE = "ltx-2.3-22b-ic-lora-ingredients-0.9" # no .safetensors extension in the repo LORA_SCALE = 1.4 FPS = 24 WIDTH, HEIGHT = 768, 448 NUM_FRAMES = 121 NUM_STEPS = len(DISTILLED_SIGMA_VALUES) # 8 MAX_SEED = np.iinfo(np.int32).max HF_TOKEN = os.environ.get("HF_TOKEN") pipe = LTX2InContextPipeline.from_pretrained(BASE_MODEL, torch_dtype=torch.bfloat16) pipe.to("cuda") pipe.vae.enable_tiling() _lora_path = hf_hub_download(LORA_REPO, LORA_FILE, token=HF_TOKEN) pipe.load_lora_weights(load_file(_lora_path), adapter_name="ingredients") pipe.set_adapters("ingredients", LORA_SCALE) def _gallery_paths(gallery): paths = [] for item in gallery or []: if isinstance(item, (list, tuple)): item = item[0] if isinstance(item, dict): item = item.get("path") or item.get("name") or item.get("image") if isinstance(item, str): paths.append(item) return paths def compose_sheet(gallery): paths = _gallery_paths(gallery) if not paths: raise gr.Error("Upload at least one subject image to build a sheet.") imgs = [Image.open(p).convert("RGB") for p in paths] if len(imgs) == 1: return imgs[0] CW, CH = 1536, 896 canvas = Image.new("RGB", (CW, CH), (0, 0, 0)) n = len(imgs) cols = math.ceil(math.sqrt(n)) rows = math.ceil(n / cols) g = 16 cw = (CW - g * (cols + 1)) // cols ch = (CH - g * (rows + 1)) // rows for i, im in enumerate(imgs): r, c = divmod(i, cols) canvas.paste(ImageOps.fit(im, (cw, ch), Image.LANCZOS), (g + c * (cw + g), g + r * (ch + g))) return canvas def _build_prompt(sheet, action): return f"Reference sheet: {sheet.strip()}\n\nGenerated video: {action.strip()}" def _export(video_np, audio, path): kw = {} if audio is not None: kw = dict(audio=audio[0].float().cpu(), audio_sample_rate=pipe.vocoder.config.output_sampling_rate) encode_video(video_np, fps=FPS, output_path=path, **kw) def _duration(*args, **kwargs): return 200 @spaces.GPU(duration=_duration) def generate(sheet_image, sheet, action, lora_scale, seed, randomize, progress=gr.Progress()): if sheet_image is None: raise gr.Error("Add a reference sheet (upload one, or build one from subject images in the other tab).") if not sheet.strip(): raise gr.Error("Describe the elements in the reference sheet (characters, props, location).") if not action.strip(): raise gr.Error("Describe the action / shot you want generated.") if randomize: seed = random.randint(0, MAX_SEED) seed = int(seed) sheet_img = sheet_image.convert("RGB").resize((WIDTH, HEIGHT), Image.LANCZOS) ref = [sheet_img] * NUM_FRAMES pipe.set_adapters("ingredients", float(lora_scale)) prompt = _build_prompt(sheet, action) def _cb(p, i, t, kw): progress((i + 1) / NUM_STEPS, desc=f"Generating โ€” step {i + 1}/{NUM_STEPS}") return {} video_out, audio_out = pipe( prompt=prompt, negative_prompt="", reference_conditions=[LTX2ReferenceCondition(frames=ref, strength=1.0)], reference_downscale_factor=1, width=WIDTH, height=HEIGHT, num_frames=NUM_FRAMES, frame_rate=FPS, num_inference_steps=NUM_STEPS, sigmas=DISTILLED_SIGMA_VALUES, guidance_scale=1.0, stg_scale=0.0, audio_guidance_scale=1.0, audio_stg_scale=0.0, generator=torch.Generator(device="cuda").manual_seed(seed), output_type="np", return_dict=False, callback_on_step_end=_cb, ) out_path = tempfile.NamedTemporaryFile(suffix=".mp4", delete=False).name _export(video_out[0], audio_out, out_path) return out_path, seed with gr.Blocks(title="LTX-2.3 Ingredients (Fast)") as demo: gr.Markdown( "# โšก LTX-2.3 Ingredients โ€” Fast (Distilled)\n" "Reference-sheet control, fast. Upload a ready sheet, or build one from individual subject images. Using " "[LTX 2.3 Distilled](https://huggingface.co/diffusers/LTX-2.3-Distilled-Diffusers) with the " "[Ingredients IC-LoRA](https://huggingface.co/ltx-community/LTX-2.3-loras), via diffusers ๐Ÿงจ. " "(For maximum fidelity, see the Dev demo.)" ) with gr.Row(): with gr.Column(): with gr.Tabs(): with gr.Tab("Reference sheet"): sheet_image = gr.Image(type="pil", label="Reference sheet (composite of characters / props / location)") with gr.Tab("Build from subject images"): gallery = gr.Gallery(label="Upload subject images (characters, props, location)", type="filepath", interactive=True, columns=4, height=240) build_btn = gr.Button("Build reference sheet โžœ") gr.Markdown("*Tiles your images into one sheet and loads it into the **Reference sheet** tab.*") sheet = gr.Textbox(label="Reference sheet description", lines=3, placeholder="a young woman with red hair in a green jacket (face close-up + turnaround); a brass pocket watch; a cobblestone alley at night") action = gr.Textbox(label="Generated video โ€” action / shot, plus any speech & sounds", lines=3, placeholder="the woman walks down the alley, checks the pocket watch and whispers 'almost time'; footsteps on cobblestone, distant city hum") with gr.Accordion("Settings", open=False): lora_scale = gr.Slider(0.8, 1.8, value=1.4, step=0.05, label="LoRA strength") randomize = gr.Checkbox(True, label="Randomize seed") seed = gr.Slider(0, MAX_SEED, value=42, step=1, label="Seed") run = gr.Button("Generate (fast)", variant="primary") with gr.Column(): video_out = gr.Video(label="Generated video") used_seed = gr.Number(label="Seed used", interactive=False) build_btn.click(compose_sheet, inputs=gallery, outputs=sheet_image) run.click(generate, inputs=[sheet_image, sheet, action, lora_scale, seed, randomize], outputs=[video_out, used_seed]) gr.Examples( examples=[ ["examples/sheet_garden.png", "a cartoon hedgehog (face close-up and body turnaround) and a cartoon rabbit (turnaround); a green coiled garden hose reel and green spray bottles; the interior of a 'Greenfield Home & Garden' store with shelves of plants", "the hedgehog waddles up to the camera in the Greenfield Home & Garden store and says cheerfully 'Welcome to Greenfield!', while the rabbit hops past holding a green spray bottle; upbeat store music, the hedgehog's friendly voice and soft footsteps", 1.4, 42, False], ["examples/sheet_hiker.png", "a young woman hiker in a green shirt and khaki shorts (face close-up and body turnaround); a large blue hiking backpack; a wooden walking stick; a shaggy yak with a colorful woven saddle blanket; a Himalayan stone village with prayer flags and snowy mountains", "the woman loads the blue backpack onto the yak, pats its neck and says warmly 'almost at the summit, buddy', snowy peaks and a monastery behind her; wind, distant prayer bells, her voice and the yak's low grunt", 1.4, 42, False], ["examples/subj_composite.png", "a smiling young woman with curly dark hair; a dappled grey horse; a green misty mountain meadow", "the woman walks up to the grey horse in the misty meadow, gently strokes its neck and says softly 'good boy, easy now'; a soft horse nicker, her gentle voice, light wind and birdsong", 1.4, 42, False], ], inputs=[sheet_image, sheet, action, lora_scale, seed, randomize], outputs=[video_out, used_seed], fn=generate, cache_examples=True, cache_mode="lazy", ) if __name__ == "__main__": demo.launch(show_error=True)