Spaces:
Running on Zero
Running on Zero
| import os | |
| os.environ.setdefault("TORCH_COMPILE_DISABLE", "1") | |
| os.environ.setdefault("TORCHDYNAMO_DISABLE", "1") | |
| import math | |
| import random | |
| import tempfile | |
| import numpy as np | |
| import spaces | |
| import torch | |
| import gradio as gr | |
| from PIL import Image, ImageOps | |
| from huggingface_hub import hf_hub_download | |
| from safetensors.torch import load_file | |
| from diffusers import LTX2InContextPipeline | |
| from diffusers.pipelines.ltx2.pipeline_ltx2_ic_lora import LTX2ReferenceCondition | |
| from diffusers.pipelines.ltx2.utils import DISTILLED_SIGMA_VALUES | |
| from diffusers.utils import encode_video | |
| # --- Config ----------------------------------------------------------------- | |
| # FAST distilled variant of the ingredients (reference-sheet) IC-LoRA: 8-step schedule, CFG off. | |
| BASE_MODEL = "diffusers/LTX-2.3-Distilled-Diffusers" | |
| LORA_REPO = "linoyts/LTX-2.3-loras" | |
| LORA_FILE = "ltx-2.3-22b-ic-lora-ingredients-0.9" # no .safetensors extension in the repo | |
| LORA_SCALE = 1.4 | |
| FPS = 24 | |
| WIDTH, HEIGHT = 768, 448 | |
| NUM_FRAMES = 121 | |
| NUM_STEPS = len(DISTILLED_SIGMA_VALUES) # 8 | |
| MAX_SEED = np.iinfo(np.int32).max | |
| HF_TOKEN = os.environ.get("HF_TOKEN") | |
| pipe = LTX2InContextPipeline.from_pretrained(BASE_MODEL, torch_dtype=torch.bfloat16) | |
| pipe.to("cuda") | |
| pipe.vae.enable_tiling() | |
| _lora_path = hf_hub_download(LORA_REPO, LORA_FILE, token=HF_TOKEN) | |
| pipe.load_lora_weights(load_file(_lora_path), adapter_name="ingredients") | |
| pipe.set_adapters("ingredients", LORA_SCALE) | |
| def _compose_sheet(paths): | |
| imgs = [Image.open(p).convert("RGB") for p in paths] | |
| if len(imgs) == 1: | |
| return imgs[0] | |
| CW, CH = 1536, 896 | |
| canvas = Image.new("RGB", (CW, CH), (0, 0, 0)) | |
| n = len(imgs) | |
| cols = math.ceil(math.sqrt(n)) | |
| rows = math.ceil(n / cols) | |
| g = 16 | |
| cw = (CW - g * (cols + 1)) // cols | |
| ch = (CH - g * (rows + 1)) // rows | |
| for i, im in enumerate(imgs): | |
| r, c = divmod(i, cols) | |
| canvas.paste(ImageOps.fit(im, (cw, ch), Image.LANCZOS), (g + c * (cw + g), g + r * (ch + g))) | |
| return canvas | |
| def build_sheet_preview(files): | |
| if not files: | |
| return None | |
| paths = [f if isinstance(f, str) else f.get("path", f.get("name")) for f in files] | |
| return _compose_sheet(paths) | |
| def _build_prompt(sheet, action): | |
| return f"Reference sheet: {sheet.strip()}\n\nGenerated video: {action.strip()}" | |
| def _export(video_np, audio, path): | |
| kw = {} | |
| if audio is not None: | |
| kw = dict(audio=audio[0].float().cpu(), audio_sample_rate=pipe.vocoder.config.output_sampling_rate) | |
| encode_video(video_np, fps=FPS, output_path=path, **kw) | |
| def _duration(*args, **kwargs): | |
| return 200 | |
| def generate(files, sheet, action, lora_scale, seed, randomize, progress=gr.Progress(track_tqdm=True)): | |
| if not files: | |
| raise gr.Error("Upload a reference sheet image, or several subject images to build one.") | |
| if not sheet.strip(): | |
| raise gr.Error("Describe the elements in the reference sheet (characters, props, location).") | |
| if not action.strip(): | |
| raise gr.Error("Describe the action / shot you want generated.") | |
| if randomize: | |
| seed = random.randint(0, MAX_SEED) | |
| seed = int(seed) | |
| paths = [f if isinstance(f, str) else f.get("path", f.get("name")) for f in files] | |
| sheet_img = _compose_sheet(paths).resize((WIDTH, HEIGHT), Image.LANCZOS) | |
| ref = [sheet_img] * NUM_FRAMES | |
| pipe.set_adapters("ingredients", float(lora_scale)) | |
| prompt = _build_prompt(sheet, action) | |
| def _cb(p, i, t, kw): | |
| progress((i + 1) / NUM_STEPS, desc=f"Generating — step {i + 1}/{NUM_STEPS}") | |
| return {} | |
| video_out, audio_out = pipe( | |
| prompt=prompt, negative_prompt="", | |
| reference_conditions=[LTX2ReferenceCondition(frames=ref, strength=1.0)], | |
| reference_downscale_factor=1, | |
| width=WIDTH, height=HEIGHT, num_frames=NUM_FRAMES, frame_rate=FPS, | |
| num_inference_steps=NUM_STEPS, sigmas=DISTILLED_SIGMA_VALUES, | |
| guidance_scale=1.0, stg_scale=0.0, audio_guidance_scale=1.0, audio_stg_scale=0.0, | |
| generator=torch.Generator(device="cuda").manual_seed(seed), | |
| output_type="np", return_dict=False, callback_on_step_end=_cb, | |
| ) | |
| out_path = tempfile.NamedTemporaryFile(suffix=".mp4", delete=False).name | |
| _export(video_out[0], audio_out, out_path) | |
| return out_path, seed | |
| with gr.Blocks(title="LTX-2.3 Ingredients (Fast)") as demo: | |
| gr.Markdown( | |
| "# ⚡ LTX-2.3 Ingredients — Fast (Distilled)\n" | |
| "Reference-sheet control on the **distilled** checkpoint (8-step, fast). **Upload a ready reference " | |
| "sheet, or several subject images and we'll tile them into one.** Describe the sheet and the action " | |
| "(with speech/sounds for audio). For maximum fidelity use the non-distilled demo. " | |
| "IC-LoRA: [`linoyts/LTX-2.3-loras`](https://huggingface.co/linoyts/LTX-2.3-loras)." | |
| ) | |
| with gr.Row(): | |
| with gr.Column(): | |
| files = gr.File(label="Reference sheet (1 image) or subject images (several)", | |
| file_count="multiple", file_types=["image"], type="filepath") | |
| sheet_preview = gr.Image(label="Reference sheet used", type="pil", interactive=False) | |
| sheet = gr.Textbox(label="Reference sheet description", lines=3, | |
| placeholder="a young woman with red hair in a green jacket (face close-up + turnaround); a brass pocket watch; a cobblestone alley at night") | |
| action = gr.Textbox(label="Generated video — the action / shot, plus any speech & sounds", lines=3, | |
| placeholder="the woman walks down the alley and checks the pocket watch, slow dolly-in; footsteps on cobblestone, a soft voice saying 'almost time', distant city hum") | |
| with gr.Accordion("Settings", open=False): | |
| lora_scale = gr.Slider(0.8, 1.8, value=1.4, step=0.05, label="LoRA strength") | |
| randomize = gr.Checkbox(True, label="Randomize seed") | |
| seed = gr.Slider(0, MAX_SEED, value=42, step=1, label="Seed") | |
| run = gr.Button("Generate (fast)", variant="primary") | |
| with gr.Column(): | |
| video_out = gr.Video(label="Generated video") | |
| used_seed = gr.Number(label="Seed used", interactive=False) | |
| files.change(build_sheet_preview, inputs=files, outputs=sheet_preview) | |
| run.click(generate, inputs=[files, sheet, action, lora_scale, seed, randomize], outputs=[video_out, used_seed]) | |
| gr.Examples( | |
| examples=[ | |
| [["examples/sheet_garden.png"], | |
| "a cartoon hedgehog (face close-up and body turnaround) and a cartoon rabbit (turnaround); a green coiled garden hose reel and green spray bottles; the interior of a 'Greenfield Home & Garden' store with shelves of plants", | |
| "the hedgehog and the rabbit explore the Greenfield Home & Garden store among the plants and garden tools, the rabbit holding a green spray bottle, warm bright store lighting, playful slow camera; cheerful ambient store sounds and soft footsteps", | |
| 1.4, 42, False], | |
| [["examples/sheet_hiker.png"], | |
| "a young woman hiker in a green shirt and khaki shorts (face close-up and body turnaround); a large blue hiking backpack; a wooden walking stick; a shaggy yak with a colorful woven saddle blanket; a Himalayan stone village with prayer flags and snowy mountains", | |
| "the woman loads the blue backpack onto the yak in front of snowy Himalayan peaks and a monastery, gentle handheld camera, soft daylight; wind, distant prayer bells and the yak's low grunt", | |
| 1.4, 42, False], | |
| [["examples/subj_woman.jpg", "examples/subj_horse.jpg", "examples/subj_landscape.jpg"], | |
| "a smiling young woman with curly dark hair; a dappled grey horse; a green misty mountain meadow", | |
| "the woman walks up to the grey horse in the misty meadow and gently strokes its neck, soft daylight; gentle wind, a soft horse nicker and distant birdsong", | |
| 1.4, 42, False], | |
| ], | |
| inputs=[files, sheet, action, lora_scale, seed, randomize], | |
| outputs=[video_out, used_seed], fn=generate, cache_examples=True, cache_mode="lazy", | |
| ) | |
| if __name__ == "__main__": | |
| demo.launch(show_error=True) | |