ingredients rename + multi-image sheet builder + real example sheets/clips

.gitattributes

@@ -35,3 +35,7 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
 *tfevents* filter=lfs diff=lfs merge=lfs -text
 examples/sheet_camping.jpg filter=lfs diff=lfs merge=lfs -text
 examples/sheet_woman_horse.jpg filter=lfs diff=lfs merge=lfs -text
+examples/sheet_garden.png filter=lfs diff=lfs merge=lfs -text
+examples/sheet_hiker.png filter=lfs diff=lfs merge=lfs -text
+examples/subj_horse.jpg filter=lfs diff=lfs merge=lfs -text
+examples/subj_woman.jpg filter=lfs diff=lfs merge=lfs -text

app.py

@@ -1,9 +1,9 @@
 import os
 
-# ZeroGPU: torch.compile / dynamo are unsupported — disable before torch import.
 os.environ.setdefault("TORCH_COMPILE_DISABLE", "1")
 os.environ.setdefault("TORCHDYNAMO_DISABLE", "1")
 
+import math
 import random
 import tempfile
 
@@ -11,7 +11,7 @@ import numpy as np
 import spaces
 import torch
 import gradio as gr
-from PIL import Image
+from PIL import Image, ImageOps
 from huggingface_hub import hf_hub_download
 from safetensors.torch import load_file
 
@@ -21,7 +21,7 @@ from diffusers.pipelines.ltx2.utils import DISTILLED_SIGMA_VALUES
 from diffusers.utils import encode_video
 
 # --- Config -----------------------------------------------------------------
-# FAST distilled variant of the reference-sheet IC-LoRA: 8-step schedule, CFG off.
+# FAST distilled variant of the ingredients (reference-sheet) IC-LoRA: 8-step schedule, CFG off.
 BASE_MODEL = "diffusers/LTX-2.3-Distilled-Diffusers"
 LORA_REPO = "linoyts/LTX-2.3-loras"
 LORA_FILE = "ltx-2.3-22b-ic-lora-ingredients-0.9"  # no .safetensors extension in the repo
@@ -33,17 +33,39 @@ NUM_STEPS = len(DISTILLED_SIGMA_VALUES)  # 8
 MAX_SEED = np.iinfo(np.int32).max
 HF_TOKEN = os.environ.get("HF_TOKEN")
 
-# --- Load pipeline once at module scope (ZeroGPU registers it) ---------------
 pipe = LTX2InContextPipeline.from_pretrained(BASE_MODEL, torch_dtype=torch.bfloat16)
 pipe.to("cuda")
 pipe.vae.enable_tiling()
-
 _lora_path = hf_hub_download(LORA_REPO, LORA_FILE, token=HF_TOKEN)
-pipe.load_lora_weights(load_file(_lora_path), adapter_name="refsheet")
-pipe.set_adapters("refsheet", LORA_SCALE)
+pipe.load_lora_weights(load_file(_lora_path), adapter_name="ingredients")
+pipe.set_adapters("ingredients", LORA_SCALE)
+
+
+def _compose_sheet(paths):
+    imgs = [Image.open(p).convert("RGB") for p in paths]
+    if len(imgs) == 1:
+        return imgs[0]
+    CW, CH = 1536, 896
+    canvas = Image.new("RGB", (CW, CH), (0, 0, 0))
+    n = len(imgs)
+    cols = math.ceil(math.sqrt(n))
+    rows = math.ceil(n / cols)
+    g = 16
+    cw = (CW - g * (cols + 1)) // cols
+    ch = (CH - g * (rows + 1)) // rows
+    for i, im in enumerate(imgs):
+        r, c = divmod(i, cols)
+        canvas.paste(ImageOps.fit(im, (cw, ch), Image.LANCZOS), (g + c * (cw + g), g + r * (ch + g)))
+    return canvas
+
+
+def build_sheet_preview(files):
+    if not files:
+        return None
+    paths = [f if isinstance(f, str) else f.get("path", f.get("name")) for f in files]
+    return _compose_sheet(paths)
 
 
-# --- Helpers ----------------------------------------------------------------
 def _build_prompt(sheet, action):
     return f"Reference sheet: {sheet.strip()}\n\nGenerated video: {action.strip()}"
 
@@ -59,14 +81,12 @@ def _duration(*args, **kwargs):
     return 200
 
 
-# --- Inference --------------------------------------------------------------
 @spaces.GPU(duration=_duration)
-def generate(image, sheet, action, lora_scale, seed, randomize,
-             progress=gr.Progress(track_tqdm=True)):
-    if image is None:
-        raise gr.Error("Please upload a reference sheet image.")
+def generate(files, sheet, action, lora_scale, seed, randomize, progress=gr.Progress(track_tqdm=True)):
+    if not files:
+        raise gr.Error("Upload a reference sheet image, or several subject images to build one.")
     if not sheet.strip():
-        raise gr.Error("Describe the panels in the reference sheet (characters, props, location).")
+        raise gr.Error("Describe the elements in the reference sheet (characters, props, location).")
     if not action.strip():
         raise gr.Error("Describe the action / shot you want generated.")
 
@@ -74,9 +94,10 @@ def generate(image, sheet, action, lora_scale, seed, randomize,
         seed = random.randint(0, MAX_SEED)
     seed = int(seed)
 
-    sheet_img = image.convert("RGB").resize((WIDTH, HEIGHT), Image.LANCZOS)
+    paths = [f if isinstance(f, str) else f.get("path", f.get("name")) for f in files]
+    sheet_img = _compose_sheet(paths).resize((WIDTH, HEIGHT), Image.LANCZOS)
     ref = [sheet_img] * NUM_FRAMES
-    pipe.set_adapters("refsheet", float(lora_scale))
+    pipe.set_adapters("ingredients", float(lora_scale))
     prompt = _build_prompt(sheet, action)
 
     def _cb(p, i, t, kw):
@@ -84,8 +105,7 @@ def generate(image, sheet, action, lora_scale, seed, randomize,
         return {}
 
     video_out, audio_out = pipe(
-        prompt=prompt,
-        negative_prompt="",
+        prompt=prompt, negative_prompt="",
         reference_conditions=[LTX2ReferenceCondition(frames=ref, strength=1.0)],
         reference_downscale_factor=1,
         width=WIDTH, height=HEIGHT, num_frames=NUM_FRAMES, frame_rate=FPS,
@@ -94,24 +114,24 @@ def generate(image, sheet, action, lora_scale, seed, randomize,
         generator=torch.Generator(device="cuda").manual_seed(seed),
         output_type="np", return_dict=False, callback_on_step_end=_cb,
     )
-
     out_path = tempfile.NamedTemporaryFile(suffix=".mp4", delete=False).name
     _export(video_out[0], audio_out, out_path)
     return out_path, seed
 
 
-# --- UI ---------------------------------------------------------------------
-with gr.Blocks(title="LTX-2.3 Reference Sheet (Fast / Distilled)") as demo:
+with gr.Blocks(title="LTX-2.3 Ingredients (Fast)") as demo:
     gr.Markdown(
-        "# ⚡ LTX-2.3 Reference-Sheet Control — Fast (Distilled)\n"
-        "Same reference-sheet IC-LoRA on the **distilled** checkpoint with an 8-step schedule for fast "
-        "generation. Supply a composite reference sheet (characters / props / location) and an action prompt; "
-        "optionally describe the soundscape for generated audio. For maximum fidelity use the non-distilled demo. "
+        "# ⚡ LTX-2.3 Ingredients — Fast (Distilled)\n"
+        "Reference-sheet control on the **distilled** checkpoint (8-step, fast). **Upload a ready reference "
+        "sheet, or several subject images and we'll tile them into one.** Describe the sheet and the action "
+        "(with speech/sounds for audio). For maximum fidelity use the non-distilled demo. "
         "IC-LoRA: [`linoyts/LTX-2.3-loras`](https://huggingface.co/linoyts/LTX-2.3-loras)."
     )
     with gr.Row():
         with gr.Column():
-            image_in = gr.Image(type="pil", label="Reference sheet (one clean panel per element, black background)")
+            files = gr.File(label="Reference sheet (1 image) or subject images (several)",
+                            file_count="multiple", file_types=["image"], type="filepath")
+            sheet_preview = gr.Image(label="Reference sheet used", type="pil", interactive=False)
             sheet = gr.Textbox(label="Reference sheet description", lines=3,
                                placeholder="a young woman with red hair in a green jacket (face close-up + turnaround); a brass pocket watch; a cobblestone alley at night")
             action = gr.Textbox(label="Generated video — the action / shot, plus any speech & sounds", lines=3,
@@ -125,25 +145,25 @@ with gr.Blocks(title="LTX-2.3 Reference Sheet (Fast / Distilled)") as demo:
             video_out = gr.Video(label="Generated video")
             used_seed = gr.Number(label="Seed used", interactive=False)
 
-    run.click(generate, inputs=[image_in, sheet, action, lora_scale, seed, randomize],
-              outputs=[video_out, used_seed])
+    files.change(build_sheet_preview, inputs=files, outputs=sheet_preview)
+    run.click(generate, inputs=[files, sheet, action, lora_scale, seed, randomize], outputs=[video_out, used_seed])
 
     gr.Examples(
         examples=[
-            ["examples/sheet_camping.jpg",
-             "a young child in a yellow raincoat beside a golden retriever dog (large left panel); a cluster of small orange camping tents in a green field (top right); misty green mountains over a calm lake (bottom right)",
-             "the child and the golden retriever walk together across the grassy field toward the tents at dawn, gentle handheld camera, soft morning light; happy dog panting and a soft bark, a child giggling, birdsong and a gentle breeze",
+            [["examples/sheet_garden.png"],
+             "a cartoon hedgehog (face close-up and body turnaround) and a cartoon rabbit (turnaround); a green coiled garden hose reel and green spray bottles; the interior of a 'Greenfield Home & Garden' store with shelves of plants",
+             "the hedgehog and the rabbit explore the Greenfield Home & Garden store among the plants and garden tools, the rabbit holding a green spray bottle, warm bright store lighting, playful slow camera; cheerful ambient store sounds and soft footsteps",
              1.4, 42, False],
-            ["examples/sheet_astronaut.jpg",
-             "an astronaut in a white spacesuit holding the helmet under one arm (large left panel); a vast misty mountain landscape over still water (right panel)",
-             "the astronaut walks slowly across the misty shoreline, looking around in wonder, slow cinematic dolly-in; crunching footsteps on wet gravel, low wind, soft breathing and a whispered 'incredible'",
+            [["examples/sheet_hiker.png"],
+             "a young woman hiker in a green shirt and khaki shorts (face close-up and body turnaround); a large blue hiking backpack; a wooden walking stick; a shaggy yak with a colorful woven saddle blanket; a Himalayan stone village with prayer flags and snowy mountains",
+             "the woman loads the blue backpack onto the yak in front of snowy Himalayan peaks and a monastery, gentle handheld camera, soft daylight; wind, distant prayer bells and the yak's low grunt",
              1.4, 42, False],
-            ["examples/sheet_woman_horse.jpg",
-             "a smiling young woman with curly dark hair in a light top (large left panel); a dappled grey horse standing on grass (top right); a green misty mountain meadow (bottom right)",
-             "the woman walks up to the grey horse in the meadow and gently strokes its neck, smiling, soft daylight; a woman's warm voice saying 'hey there, good boy', a soft horse nicker, light wind and birdsong",
+            [["examples/subj_woman.jpg", "examples/subj_horse.jpg", "examples/subj_landscape.jpg"],
+             "a smiling young woman with curly dark hair; a dappled grey horse; a green misty mountain meadow",
+             "the woman walks up to the grey horse in the misty meadow and gently strokes its neck, soft daylight; gentle wind, a soft horse nicker and distant birdsong",
              1.4, 42, False],
         ],
-        inputs=[image_in, sheet, action, lora_scale, seed, randomize],
+        inputs=[files, sheet, action, lora_scale, seed, randomize],
         outputs=[video_out, used_seed], fn=generate, cache_examples=True, cache_mode="lazy",
     )