Switch backend to native LTX-2 (ICLoraPipeline) (#2)

- Switch backend to native LTX-2 (ICLoraPipeline) (68a3f584144acdf70a7213b3954a793582693a97)

README.md

@@ -11,7 +11,7 @@ pinned: false
 hardware: zero-a10g
 short_description: Fast reference-sheet to video with LTX-2.3 IC-LoRA
 models:
-  - diffusers/LTX-2.3-Distilled-Diffusers
+  - Lightricks/LTX-2.3
   - Lightricks/LTX-2.3-22b-IC-LoRA-Ingredients
 ---
 
@@ -23,5 +23,5 @@ recipe is non-distilled (30 steps, guidance 4.0) — see the companion non-disti
 maximum fidelity.
 
 Runs the IC-LoRA from [`linoyts/LTX-2.3-loras`](https://huggingface.co/linoyts/LTX-2.3-loras)
-on [`diffusers/LTX-2.3-Distilled-Diffusers`](https://huggingface.co/diffusers/LTX-2.3-Distilled-Diffusers)
-via `LTX2InContextPipeline`. 768×448 × 121 frames @ 24fps, LoRA scale 1.4.
+on [`Lightricks/LTX-2.3`](https://huggingface.co/Lightricks/LTX-2.3)
+via the native LTX-2 pipeline. 768×448 × 121 frames @ 24fps, LoRA scale 1.4.

app.py

@@ -1,57 +1,223 @@
 import os
+import subprocess
+import sys
 
-os.environ.setdefault("TORCH_COMPILE_DISABLE", "1")
-os.environ.setdefault("TORCHDYNAMO_DISABLE", "1")
+# ZeroGPU: torch.compile / dynamo unsupported — disable before any torch import.
+os.environ["TORCH_COMPILE_DISABLE"] = "1"
+os.environ["TORCHDYNAMO_DISABLE"] = "1"
 
-import math
+# memory-efficient attention
+subprocess.run([sys.executable, "-m", "pip", "install", "xformers==0.0.32.post2", "--no-build-isolation"], check=False)
+
+# --- clone + install the NATIVE LTX-2 codebase at the pinned commit the working ZeroGPU spaces use ---
+LTX_REPO_URL = "https://github.com/Lightricks/LTX-2.git"
+LTX_REPO_DIR = os.path.join(os.path.dirname(os.path.abspath(__file__)), "LTX-2")
+LTX_COMMIT = "ae855f8538843825f9015a419cf4ba5edaf5eec2"
+if not os.path.exists(LTX_REPO_DIR):
+    subprocess.run(["git", "clone", LTX_REPO_URL, LTX_REPO_DIR], check=True)
+    subprocess.run(["git", "-C", LTX_REPO_DIR, "checkout", LTX_COMMIT], check=True)
+subprocess.run([sys.executable, "-m", "pip", "install", "--force-reinstall", "--no-deps",
+                "-e", os.path.join(LTX_REPO_DIR, "packages", "ltx-core"),
+                "-e", os.path.join(LTX_REPO_DIR, "packages", "ltx-pipelines")], check=True)
+sys.path.insert(0, os.path.join(LTX_REPO_DIR, "packages", "ltx-pipelines", "src"))
+sys.path.insert(0, os.path.join(LTX_REPO_DIR, "packages", "ltx-core", "src"))
+
+import logging
 import random
 import tempfile
-import threading
-import time
 
 import numpy as np
-import spaces
+import imageio.v3 as iio
+from PIL import Image, ImageOps
+
 import torch
+torch._dynamo.config.suppress_errors = True
+torch._dynamo.config.disable = True
+
+import spaces
 import gradio as gr
-from PIL import Image, ImageOps
-from huggingface_hub import hf_hub_download
-from safetensors.torch import load_file
-
-from diffusers import LTX2InContextPipeline, LTX2LatentUpsamplePipeline
-from diffusers.pipelines.ltx2.pipeline_ltx2_ic_lora import LTX2ReferenceCondition
-from diffusers.pipelines.ltx2.latent_upsampler import LTX2LatentUpsamplerModel
-from diffusers.pipelines.ltx2.utils import DISTILLED_SIGMA_VALUES, STAGE_2_DISTILLED_SIGMA_VALUES
-from diffusers.utils import encode_video
-
-# --- Config -----------------------------------------------------------------
-# FAST distilled variant of the ingredients (reference-sheet) IC-LoRA: 8-step schedule, CFG off.
-BASE_MODEL = "diffusers/LTX-2.3-Distilled-Diffusers"
+from huggingface_hub import hf_hub_download, snapshot_download
+
+# Import LTX modules in the proven order — importing ltx_core.quantization/loader FIRST hits a
+# circular import (fp8_cast <-> loader.fuse_loras). Importing the model modules first forces the
+# correct init order (mirrors the working reference Space).
+from ltx_core.model.video_vae import TilingConfig, get_video_chunks_number, decode_video as _vae_decode_video  # noqa: F401
+from ltx_core.model.upsampler import upsample_video as _upsample_video  # noqa: F401
+from ltx_core.model.audio_vae import encode_audio as _vae_encode_audio  # noqa: F401
+from ltx_core.quantization import QuantizationPolicy
+from ltx_core.loader import LoraPathStrengthAndSDOps, LTXV_LORA_COMFY_RENAMING_MAP
+from ltx_pipelines.ic_lora import ICLoraPipeline
+from ltx_pipelines.utils.media_io import encode_video
+
+# --- ZeroGPU loader patch -------------------------------------------------------------
+# The native loader opens safetensors directly on the CUDA device
+# (safe_open(path, device="cuda")), doing the host->device copy in safetensors' own C++
+# (cudaMemcpy) — bypassing torch.Tensor.to, the call ZeroGPU patches to virtualise + pack
+# weights at module scope. Result: "No CUDA GPUs are available" at startup, nothing packs.
+# Patch it to open on CPU then move via torch.Tensor.to (ZeroGPU-virtualisable).
+import safetensors as _safetensors
+import ltx_core.loader.sft_loader as _sft
+from ltx_core.loader.primitives import StateDict as _StateDict
+
+def _zerogpu_safe_load(self, path, sd_ops, device=None):
+    device = device or torch.device("cpu")
+    sd, size, dtype = {}, 0, set()
+    model_paths = path if isinstance(path, list) else [path]
+    for shard_path in model_paths:
+        with _safetensors.safe_open(shard_path, framework="pt", device="cpu") as f:
+            for name in f.keys():
+                expected = name if sd_ops is None else sd_ops.apply_to_key(name)
+                if expected is None:
+                    continue
+                value = f.get_tensor(name).to(device=device)  # torch path -> ZeroGPU-virtualised
+                kvs = ((expected, value),)
+                if sd_ops is not None:
+                    kvs = sd_ops.apply_to_key_value(expected, value)
+                for k, v in kvs:
+                    size += v.nbytes
+                    dtype.add(v.dtype)
+                    sd[k] = v
+    return _StateDict(sd=sd, device=device, size=size, dtype=dtype)
+
+_sft.SafetensorsStateDictLoader.load = _zerogpu_safe_load
+print("[PATCH] safetensors loader -> CPU-open + torch.to (ZeroGPU-virtualisable)")
+# --------------------------------------------------------------------------------------
+
+# --- attention backend patch (FA3 crashes on Blackwell ZeroGPU; use xformers/SDPA) ---
+import torch.nn.functional as F
+from ltx_core.model.transformer import attention as _attn_mod
+
+def _sdpa_as_mea(query, key, value, attn_bias=None, scale=None, **kwargs):
+    q, k, v = query.transpose(1, 2), key.transpose(1, 2), value.transpose(1, 2)
+    return F.scaled_dot_product_attention(q, k, v, scale=scale).transpose(1, 2)
+
+# IMPORTANT (ZeroGPU): never query CUDA at module scope. SDPA works on every GPU (incl.
+# Blackwell ZeroGPU, where FA3 crashes), so patch it unconditionally.
+_attn_mod.memory_efficient_attention = _sdpa_as_mea
+print("[ATTN] SDPA (patched at module scope, no CUDA query)")
+
+logging.getLogger().setLevel(logging.INFO)
+
+# =========================== PER-LORA CONFIG (colorize) ===========================
+TITLE = "LTX-2.3 Ingredients (native LTX-2)"
 LORA_REPO = "Lightricks/LTX-2.3-22b-IC-LoRA-Ingredients"
 LORA_FILE = "ltx-2.3-22b-ic-lora-ingredients-0.9.safetensors"
 LORA_SCALE = 1.4
-FPS = 24
-WIDTH, HEIGHT = 768, 448
-NUM_FRAMES = 121
-NUM_STEPS = len(DISTILLED_SIGMA_VALUES)  # 8
+SKIP_STAGE_2 = True
+GRAYSCALE_REF = False
+RES_PRESETS = {"768×448": (768, 448), "960×544": (960, 544)}
+DEFAULT_PRESET = "768×448"
+FRAME_CHOICES = [49, 73, 97, 121]
+DEFAULT_FRAMES = 121
+
+def build_prompt(sheet, action):
+    return f"Reference sheet: {sheet.strip()}\n\nGenerated video: {action.strip()}"
+
+EXAMPLES = [
+    ["examples/sheet_garden.png",
+     "a friendly cartoon hedgehog with rounded chestnut-brown fur and big eyes, and a grey-and-white rabbit with long ears; a green coiled garden hose and green spray bottles; the bright interior of a 'Greenfield Home & Garden' store with leafy plants",
+     "the hedgehog waddles up and says 'welcome to Greenfield!' while the rabbit hops past with a spray bottle; warm acoustic store jingle, cheerful voice and soft footsteps",
+     "768×448", 121, 42, False],
+    ["examples/sheet_hiker.png",
+     "a young asian woman with two long braids in an olive t-shirt and khaki pants; a large blue external-frame backpack; a thick wooden walking stick; a shaggy white yak with an ornate blue-red-yellow saddle blanket; snowy mountains and a small stone shrine",
+     "she sits on a rock beside the yak, pats its neck and says in a tired gentle voice 'we've got a long way to go... big guy'; wind, the yak's low grunt and jingling stirrups, no music",
+     "768×448", 121, 42, False],
+]
+# =================================================================================
+
+FPS = 24.0
 MAX_SEED = np.iinfo(np.int32).max
 HF_TOKEN = os.environ.get("HF_TOKEN")
+LTX_MODEL_REPO = "Lightricks/LTX-2.3"
+GEMMA_REPO = "google/gemma-3-12b-it-qat-q4_0-unquantized"
+
+
+def _src_fps(path, default=FPS):
+    try:
+        return float(iio.immeta(path, plugin="pyav").get("fps", default)) or default
+    except Exception:
+        return default
+
+
+def _prep_reference(path, width, height, num_frames):
+    """Resample to 24fps, aspect-fit/crop to WxH, NF frames; (optionally grayscale); write temp mp4."""
+    vid = iio.imread(path, plugin="pyav")
+    src_fps = _src_fps(path)
+    n = len(vid)
+    out = []
+    for i in range(num_frames):
+        idx = min(int(round(i / FPS * src_fps)), n - 1)
+        im = Image.fromarray(vid[idx]).convert("RGB")
+        im = ImageOps.fit(im, (width, height), Image.LANCZOS)
+        if GRAYSCALE_REF:
+            im = im.convert("L").convert("RGB")
+        out.append(np.array(im))
+    tmp = tempfile.mktemp(suffix=".mp4")
+    iio.imwrite(tmp, np.stack(out), fps=FPS, plugin="pyav", codec="libx264")
+    return tmp
+
+
+def _pick_resolution(path, preset):
+    w, h = RES_PRESETS[preset]
+    try:
+        f0 = iio.imread(path, plugin="pyav", index=0)
+        if f0.shape[0] > f0.shape[1]:  # portrait
+            w, h = h, w
+    except Exception:
+        pass
+    return w, h
+
+
+# --- Load native pipeline + IC-LoRA once at module scope (ZeroGPU packs weights here) ---
+print("Downloading checkpoints…")
+checkpoint_path = hf_hub_download(LTX_MODEL_REPO, "ltx-2.3-22b-distilled-1.1.safetensors", token=HF_TOKEN)
+spatial_upsampler_path = hf_hub_download(LTX_MODEL_REPO, "ltx-2.3-spatial-upscaler-x2-1.1.safetensors", token=HF_TOKEN)
+gemma_root = snapshot_download(GEMMA_REPO, token=HF_TOKEN)
+lora_path = hf_hub_download(LORA_REPO, LORA_FILE, token=HF_TOKEN)
+
+print("Building ICLoraPipeline…")
+pipeline = ICLoraPipeline(
+    distilled_checkpoint_path=checkpoint_path,
+    spatial_upsampler_path=spatial_upsampler_path,
+    gemma_root=gemma_root,
+    loras=[LoraPathStrengthAndSDOps(lora_path, LORA_SCALE, LTXV_LORA_COMFY_RENAMING_MAP)],
+    # bf16 (NOT fp8): the IC-LoRA is fused into the transformer at MODULE SCOPE (the GPU
+    # worker can't re-open the checkpoint file). fp8_cast()'s fusion runs a custom CUDA kernel
+    # that can't be ZeroGPU-virtualised; the bf16 fuse rule is pure torch -> virtualisable.
+    quantization=None,
+)
 
-UPSAMPLER_REPO = "dg845/LTX-2.3-Spatial-Upsampler-Diffusers"  # LTX-2.3 spatial x2 latent upsampler
 
-pipe = LTX2InContextPipeline.from_pretrained(BASE_MODEL, torch_dtype=torch.bfloat16)
-pipe.to("cuda")
-pipe.vae.enable_tiling()
-_lora_path = hf_hub_download(LORA_REPO, LORA_FILE, token=HF_TOKEN)
-# Kept as a togglable adapter (NOT fused) so Stage 2 can run on the bare distilled model.
-pipe.load_lora_weights(load_file(_lora_path), adapter_name="ingredients")
-# NOTE: AOTI temporarily disabled while validating 2-stage inference; re-enable once confirmed.
-# spaces.aoti_load(module=pipe.transformer, repo_id="ltx-community/LTX-2.3-Transformer-GroupA-sm120-cu130-r9e")
+def _preload_pin(ledger, tag):
+    if ledger is None:
+        return
+    for name in ["transformer", "video_encoder", "video_decoder", "audio_encoder",
+                 "audio_decoder", "vocoder", "spatial_upsampler", "text_encoder",
+                 "gemma_embeddings_processor"]:
+        fn = getattr(ledger, name, None)
+        if callable(fn):
+            try:
+                obj = fn()
+                setattr(ledger, name, (lambda o=obj: o))
+                print(f"[preload {tag}] {name} ✓")
+            except Exception as e:
+                print(f"[preload {tag}] {name} skipped: {e}")
 
-# Stage-2 latent upsampler (spatial x2) — two-stage diffusers inference.
-_upsampler = LTX2LatentUpsamplerModel.from_pretrained(
-    UPSAMPLER_REPO, subfolder="latent_upsampler", torch_dtype=torch.bfloat16)
-_upsampler.to("cuda")
-upsample_pipe = LTX2LatentUpsamplePipeline(vae=pipe.vae, latent_upsampler=_upsampler)
+# Preload stage 1 always; preload stage 2 only when two-stage is used (skip_stage_2=False).
+# Eagerly pinning both ledgers materializes TWO ~46GB transformers — too big for the ZeroGPU pack.
+_preload_pin(getattr(pipeline, "stage_1_model_ledger", None), "stage1")
+if not SKIP_STAGE_2:
+    _preload_pin(getattr(pipeline, "stage_2_model_ledger", None), "stage2")
+print("Pipeline ready.")
+
+
+def _sheet_to_video(img, width, height, num_frames):
+    """Repeat the reference-sheet image into an NF-frame video for video_conditioning."""
+    im = img.convert("RGB").resize((width, height), Image.LANCZOS)
+    vid = np.stack([np.array(im)] * num_frames)
+    tmp = tempfile.mktemp(suffix=".mp4")
+    iio.imwrite(tmp, vid, fps=FPS, plugin="pyav", codec="libx264")
+    return tmp
 
 
 def _gallery_paths(gallery):
@@ -67,6 +233,7 @@ def _gallery_paths(gallery):
 
 
 def compose_sheet(gallery):
+    import math
     paths = _gallery_paths(gallery)
     if not paths:
         raise gr.Error("Upload at least one subject image to build a sheet.")
@@ -75,79 +242,45 @@ def compose_sheet(gallery):
         return imgs[0]
     CW, CH = 1536, 896
     canvas = Image.new("RGB", (CW, CH), (0, 0, 0))
-    n = len(imgs)
-    cols = math.ceil(math.sqrt(n))
-    rows = math.ceil(n / cols)
-    g = 16
-    cw = (CW - g * (cols + 1)) // cols
-    ch = (CH - g * (rows + 1)) // rows
+    cols = math.ceil(math.sqrt(len(imgs))); rows = math.ceil(len(imgs) / cols); g = 16
+    cw = (CW - g * (cols + 1)) // cols; ch = (CH - g * (rows + 1)) // rows
     for i, im in enumerate(imgs):
         r, c = divmod(i, cols)
         canvas.paste(ImageOps.fit(im, (cw, ch), Image.LANCZOS), (g + c * (cw + g), g + r * (ch + g)))
     return canvas
 
 
-def _build_prompt(sheet, action):
-    return f"Reference sheet: {sheet.strip()}\n\nGenerated video: {action.strip()}"
-
-
-def _export(video_np, audio, path):
-    kw = {}
-    if audio is not None:
-        kw = dict(audio=audio[0].float().cpu(), audio_sample_rate=pipe.vocoder.config.output_sampling_rate)
-    encode_video(video_np, fps=FPS, output_path=path, **kw)
-
-
 def _duration(*args, **kwargs):
-    return 480  # two-stage (stage1 + x2 upsample + stage2 refine), AOTI off during validation
+    nf = next((a for a in args if isinstance(a, int) and a in FRAME_CHOICES), DEFAULT_FRAMES)
+    return int(60 + nf * 1.2)
 
 
 @spaces.GPU(duration=_duration)
-def generate(sheet_image, sheet, action, lora_scale, seed, randomize, progress=gr.Progress(track_tqdm=True)):
+@torch.inference_mode()
+def generate(sheet_image, sheet, action, seed, randomize, progress=gr.Progress(track_tqdm=True)):
     if sheet_image is None:
         raise gr.Error("Add a reference sheet (upload one, or build one from subject images in the other tab).")
     if not sheet.strip():
         raise gr.Error("Describe the elements in the reference sheet (characters, props, location).")
     if not action.strip():
         raise gr.Error("Describe the action / shot you want generated.")
-
-    if randomize:
-        seed = random.randint(0, MAX_SEED)
-    seed = int(seed)
-
-    sheet_img = sheet_image.convert("RGB").resize((WIDTH, HEIGHT), Image.LANCZOS)
-    ref = [sheet_img] * NUM_FRAMES
-    prompt = _build_prompt(sheet, action)
-    gen = torch.Generator(device="cuda").manual_seed(seed)
-
-    # --- Stage 1: base-res latents with the ingredients IC-LoRA (distilled 8-step) ---
-    pipe.set_adapters("ingredients", float(lora_scale))
-    video_latent, audio_latent = pipe(
-        prompt=prompt, negative_prompt="",
-        reference_conditions=[LTX2ReferenceCondition(frames=ref, strength=1.0)],
-        reference_downscale_factor=1,
-        width=WIDTH, height=HEIGHT, num_frames=NUM_FRAMES, frame_rate=FPS,
-        num_inference_steps=NUM_STEPS, sigmas=DISTILLED_SIGMA_VALUES,
-        guidance_scale=1.0, stg_scale=0.0, audio_guidance_scale=1.0, audio_stg_scale=0.0,
-        generator=gen, output_type="latent", return_dict=False,
+    seed = random.randint(0, MAX_SEED) if randomize else int(seed)
+    # Fixed generation geometry (matches the public Space). The native IC-LoRA is fused at a
+    # fixed strength (LORA_SCALE) at module scope.
+    width, height, num_frames = 768, 448, 121
+    ref_path = _sheet_to_video(sheet_image, width, height, num_frames)
+    tiling = TilingConfig.default()
+    gen_w, gen_h = (width * 2, height * 2) if SKIP_STAGE_2 else (width, height)
+    video_out, audio_out = pipeline(
+        prompt=build_prompt(sheet, action),
+        seed=seed, height=gen_h, width=gen_w,
+        num_frames=num_frames, frame_rate=FPS,
+        images=[], video_conditioning=[(ref_path, 1.0)],
+        skip_stage_2=SKIP_STAGE_2, tiling_config=tiling,
     )
-    # --- Stage 2a: spatial x2 latent upsample ---
-    up_latent = upsample_pipe(latents=video_latent, output_type="latent", return_dict=False)[0]
-    # --- Stage 2b: short refine at 2x res on the bare distilled model (drop IC-LoRA + reference) ---
-    pipe.disable_lora()
-    video_out, audio_out = pipe(
-        prompt=prompt, negative_prompt="",
-        latents=up_latent, audio_latents=audio_latent,
-        width=WIDTH * 2, height=HEIGHT * 2, num_frames=NUM_FRAMES, frame_rate=FPS,
-        num_inference_steps=len(STAGE_2_DISTILLED_SIGMA_VALUES),
-        sigmas=STAGE_2_DISTILLED_SIGMA_VALUES, noise_scale=STAGE_2_DISTILLED_SIGMA_VALUES[0],
-        guidance_scale=1.0, stg_scale=0.0, audio_guidance_scale=1.0, audio_stg_scale=0.0,
-        generator=gen, output_type="np", return_dict=False,
-    )
-    pipe.enable_lora()
-
-    out_path = tempfile.NamedTemporaryFile(suffix=".mp4", delete=False).name
-    _export(video_out[0], audio_out, out_path)
+    out_path = tempfile.mktemp(suffix=".mp4")
+    encode_video(video=video_out, fps=FPS, audio=audio_out, output_path=out_path,
+                 video_chunks_number=get_video_chunks_number(num_frames, tiling))
     return out_path, seed
 
 
@@ -155,8 +288,8 @@ with gr.Blocks(title="LTX-2.3 Ingredients (Fast)") as demo:
     gr.Markdown(
         "# ⚡ LTX-2.3 Ingredients — Fast (Distilled)\n"
         "Reference-sheet control, fast. Upload a ready sheet, or build one from individual subject images. Using "
-        "[LTX 2.3 Distilled](https://huggingface.co/diffusers/LTX-2.3-Distilled-Diffusers) with the "
-        "[Ingredients IC-LoRA](https://huggingface.co/Lightricks/LTX-2.3-22b-IC-LoRA-Ingredients), via diffusers 🧨. "
+        "[LTX 2.3 Distilled](https://huggingface.co/Lightricks/LTX-2.3) with the "
+        "[Ingredients IC-LoRA](https://huggingface.co/Lightricks/LTX-2.3-22b-IC-LoRA-Ingredients). "
         "(For maximum fidelity, see the Dev demo.)"
     )
     with gr.Row():
@@ -174,7 +307,6 @@ with gr.Blocks(title="LTX-2.3 Ingredients (Fast)") as demo:
             action = gr.Textbox(label="Generated video — action / shot, plus any speech & sounds", lines=3,
                                 placeholder="the woman walks down the alley, checks the pocket watch and whispers 'almost time'; footsteps on cobblestone, distant city hum")
             with gr.Accordion("Settings", open=False):
-                lora_scale = gr.Slider(0.8, 1.8, value=1.4, step=0.05, label="LoRA strength")
                 randomize = gr.Checkbox(True, label="Randomize seed")
                 seed = gr.Slider(0, MAX_SEED, value=42, step=1, label="Seed")
             run = gr.Button("Generate (fast)", variant="primary")
@@ -182,24 +314,15 @@ with gr.Blocks(title="LTX-2.3 Ingredients (Fast)") as demo:
             video_out = gr.Video(label="Generated video")
 
     build_btn.click(compose_sheet, inputs=gallery, outputs=sheet_image)
-    run.click(generate, inputs=[sheet_image, sheet, action, lora_scale, seed, randomize], outputs=[video_out, seed])
+    run.click(generate, inputs=[sheet_image, sheet, action, seed, randomize], outputs=[video_out, seed])
 
     gr.Examples(
         examples=[
-            ["examples/sheet_garden.png",
-             "a friendly cartoon hedgehog with rounded chestnut-brown fur, a cream face and belly, large expressive dark eyes, a small black nose and tiny rounded ears, shown in a face close-up and a full-body turnaround standing upright on two short legs; a cheerful cartoon rabbit with soft grey-and-white fur, long upright ears with pale pink inner lining, round amber eyes and a fluffy white tail, shown in a body turnaround; a green coiled garden hose neatly wound on a matching green wall-mounted reel; a row of green plastic spray bottles with trigger nozzles; the bright interior of a 'Greenfield Home & Garden' store with tall wooden shelves stocked with leafy potted plants, terracotta pots and gardening supplies, warm overhead lighting and a green-and-white storefront sign",
-             "cheerful family-animation commercial scene, a lively medium shot inside the sunlit 'Greenfield Home & Garden' store with its tall shelves of leafy potted plants and terracotta pots. the rounded chestnut-brown hedgehog waddles briskly up toward the camera on its short legs, its cream belly bouncing, then stops, lifts a tiny paw in a friendly wave and beams with wide sparkling eyes, announcing in a warm, bright, sing-song voice: 'welcome to Greenfield!'. behind it the soft grey-and-white rabbit hops past in the aisle, long ears bobbing, cradling a green spray bottle against its chest; it pauses, gives the bottle a playful little squeeze that puffs a fine mist into a shaft of light, and adds in a chirpy, slightly higher voice: 'everything your garden needs!'. the hedgehog nods enthusiastically, gestures with both paws toward the laden shelves and finishes with a cosy chuckle: 'come on in!'. the animation is glossy and expressive with squash-and-stretch motion, rounded shapes and saturated greens; the camera pushes in gently and tilts up to reveal the green-and-white storefront sign. the audio is bright and immersive: the hedgehog's cheerful voice, the rabbit's lighter reply, the soft puff of the spray bottle, light pattering footsteps on the store floor and a warm, upbeat acoustic-ukulele jingle playing softly underneath",
-             1.4, 42, False],
-            ["examples/sheet_hiker.png",
-             "a young Asian woman with a warm skin tone, dark hair parted down the middle in two long braids resting on her chest, an olive-green short-sleeved t-shirt, khaki cargo pants, dark brown hiking boots and a black wristwatch on her left arm, with a serious natural expression; a large heavy-duty blue hiking backpack with an external silver metal frame, multiple side and top pouches, black adjustable straps and a brown leather square patch near the bottom; a simple thick natural wooden walking stick with rough bark texture and a slight fork near one end; a large sturdy yak with long shaggy white-and-blonde hair and curved grey horns, wearing an ornate saddle blanket with intricate blue, red and yellow patterns, a saddle with metal stirrups and colorful tassels near its ears and chest; a sweeping majestic mountain landscape where a dirt path winds through green rocky slopes toward towering snow-capped peaks under a bright blue sky with scattered white clouds; a small traditional square stone shrine with a flat slightly tiered roof and a bright yellow fabric valance along the roofline, bright blue window trim and a red wooden door, with a small stone stupa beside it",
-             "cinematic adventure documentary scene, a dynamic medium wide shot of the young asian woman with her dark hair in two long braids, wearing an olive-green t-shirt and khaki cargo pants. she sits on a rock along a mountain dirt path, resting beside the massive white shaggy yak with curved horns and its ornate blue, red and yellow patterned saddle blanket with metal stirrups. leaning against a nearby small stone building with blue window trim, a red door and a yellow fabric roof valance are her large blue external-frame backpack and thick wooden walking stick; majestic snow-capped mountains tower in the distant background under a bright blue sky. she looks at the yak, chest heaving slightly from exertion, and says with a breathy, tired but gentle voice: 'we've got a long way to go...'. she pauses, extending her hand to gently pat the thick white fur on the yak's neck; the yak shifts its weight, the colorful tassels near its ears swaying, and a faint exhausted smile breaks across her face as she continues softly: '...big guy.'. lowering her hand she grabs her wooden walking stick, leaning her weight onto it as she turns her gaze up toward the distant snowy peaks, her expression shifting from exhaustion to quiet determination as she adds, her voice growing firmer: 'but the pass...', then takes a deep grounding breath: '...is just over that ridge.'. the camera is dynamically handheld, slowly orbiting the woman and the yak to reveal the depth of the valley and the towering mountains behind them; naturalistic breathtaking film aesthetic, bright crisp sunlight casting sharp shadows across the rocky path and stone shrine. clear immersive audio: her wind-swept voice, the heavy rhythmic breathing of the yak, the faint jingle of metal stirrups and the distant ambient howl of mountain winds, no background music",
-             1.4, 42, False],
-            ["examples/subj_composite.png",
-             "a smiling young woman with warm fair skin and shoulder-length curly dark-brown hair, soft brown eyes and a gentle open expression, wearing a cream cable-knit sweater and dark jeans, shown in a relaxed three-quarter pose; a dappled grey horse with a dark charcoal mane and tail, a soft mottled grey-and-white coat, dark intelligent eyes and a calm posture, wearing a simple brown leather halter; a green misty mountain meadow of tall dewy grass and scattered wildflowers, with faint pine-covered slopes dissolving into low morning fog under a pale silver sky",
-             "tender naturalistic cinematic scene, a soft medium shot in a green misty mountain meadow at dawn, tall dewy grass glistening and low fog drifting between faint pine slopes. the young woman with curly dark-brown hair and a cream cable-knit sweater walks slowly up to the dappled grey horse, her breath faintly visible in the cool air, and raises a careful open hand. she gently strokes the horse's soft mottled neck, her face softening into a warm reassuring smile, and murmurs in a low, soothing, slightly breathy voice: 'good boy… easy now'. the horse lowers its head toward her, its charcoal mane shifting, flicks an ear and lets out a soft snort, its breath misting in the cold; she leans her forehead lightly against its cheek, closes her eyes for a moment, then whispers with quiet affection: 'there you go'. the camera drifts slowly in a gentle arc around the pair, the shallow-focus background of fog and wildflowers blurring softly behind them; the film aesthetic is delicate and breathtaking with cool silver dawn light and fine atmospheric haze. the audio is intimate and immersive: her gentle hushed voice, the horse's soft nicker and snort, the swish of dewy grass, light birdsong waking in the distance and a faint cool breeze, with no background music",
-             1.4, 42, False],
+            ['examples/sheet_garden.png', "a friendly cartoon hedgehog with rounded chestnut-brown fur, a cream face and belly, large expressive dark eyes, a small black nose and tiny rounded ears, shown in a face close-up and a full-body turnaround standing upright on two short legs; a cheerful cartoon rabbit with soft grey-and-white fur, long upright ears with pale pink inner lining, round amber eyes and a fluffy white tail, shown in a body turnaround; a green coiled garden hose neatly wound on a matching green wall-mounted reel; a row of green plastic spray bottles with trigger nozzles; the bright interior of a 'Greenfield Home & Garden' store with tall wooden shelves stocked with leafy potted plants, terracotta pots and gardening supplies, warm overhead lighting and a green-and-white storefront sign", "cheerful family-animation commercial scene, a lively medium shot inside the sunlit 'Greenfield Home & Garden' store with its tall shelves of leafy potted plants and terracotta pots. the rounded chestnut-brown hedgehog waddles briskly up toward the camera on its short legs, its cream belly bouncing, then stops, lifts a tiny paw in a friendly wave and beams with wide sparkling eyes, announcing in a warm, bright, sing-song voice: 'welcome to Greenfield!'. behind it the soft grey-and-white rabbit hops past in the aisle, long ears bobbing, cradling a green spray bottle against its chest; it pauses, gives the bottle a playful little squeeze that puffs a fine mist into a shaft of light, and adds in a chirpy, slightly higher voice: 'everything your garden needs!'. the hedgehog nods enthusiastically, gestures with both paws toward the laden shelves and finishes with a cosy chuckle: 'come on in!'. the animation is glossy and expressive with squash-and-stretch motion, rounded shapes and saturated greens; the camera pushes in gently and tilts up to reveal the green-and-white storefront sign. the audio is bright and immersive: the hedgehog's cheerful voice, the rabbit's lighter reply, the soft puff of the spray bottle, light pattering footsteps on the store floor and a warm, upbeat acoustic-ukulele jingle playing softly underneath", 42, False],
+            ['examples/sheet_hiker.png', 'a young Asian woman with a warm skin tone, dark hair parted down the middle in two long braids resting on her chest, an olive-green short-sleeved t-shirt, khaki cargo pants, dark brown hiking boots and a black wristwatch on her left arm, with a serious natural expression; a large heavy-duty blue hiking backpack with an external silver metal frame, multiple side and top pouches, black adjustable straps and a brown leather square patch near the bottom; a simple thick natural wooden walking stick with rough bark texture and a slight fork near one end; a large sturdy yak with long shaggy white-and-blonde hair and curved grey horns, wearing an ornate saddle blanket with intricate blue, red and yellow patterns, a saddle with metal stirrups and colorful tassels near its ears and chest; a sweeping majestic mountain landscape where a dirt path winds through green rocky slopes toward towering snow-capped peaks under a bright blue sky with scattered white clouds; a small traditional square stone shrine with a flat slightly tiered roof and a bright yellow fabric valance along the roofline, bright blue window trim and a red wooden door, with a small stone stupa beside it', "cinematic adventure documentary scene, a dynamic medium wide shot of the young asian woman with her dark hair in two long braids, wearing an olive-green t-shirt and khaki cargo pants. she sits on a rock along a mountain dirt path, resting beside the massive white shaggy yak with curved horns and its ornate blue, red and yellow patterned saddle blanket with metal stirrups. leaning against a nearby small stone building with blue window trim, a red door and a yellow fabric roof valance are her large blue external-frame backpack and thick wooden walking stick; majestic snow-capped mountains tower in the distant background under a bright blue sky. she looks at the yak, chest heaving slightly from exertion, and says with a breathy, tired but gentle voice: 'we've got a long way to go...'. she pauses, extending her hand to gently pat the thick white fur on the yak's neck; the yak shifts its weight, the colorful tassels near its ears swaying, and a faint exhausted smile breaks across her face as she continues softly: '...big guy.'. lowering her hand she grabs her wooden walking stick, leaning her weight onto it as she turns her gaze up toward the distant snowy peaks, her expression shifting from exhaustion to quiet determination as she adds, her voice growing firmer: 'but the pass...', then takes a deep grounding breath: '...is just over that ridge.'. the camera is dynamically handheld, slowly orbiting the woman and the yak to reveal the depth of the valley and the towering mountains behind them; naturalistic breathtaking film aesthetic, bright crisp sunlight casting sharp shadows across the rocky path and stone shrine. clear immersive audio: her wind-swept voice, the heavy rhythmic breathing of the yak, the faint jingle of metal stirrups and the distant ambient howl of mountain winds, no background music", 42, False],
+            ['examples/subj_composite.png', 'a smiling young woman with warm fair skin and shoulder-length curly dark-brown hair, soft brown eyes and a gentle open expression, wearing a cream cable-knit sweater and dark jeans, shown in a relaxed three-quarter pose; a dappled grey horse with a dark charcoal mane and tail, a soft mottled grey-and-white coat, dark intelligent eyes and a calm posture, wearing a simple brown leather halter; a green misty mountain meadow of tall dewy grass and scattered wildflowers, with faint pine-covered slopes dissolving into low morning fog under a pale silver sky', "tender naturalistic cinematic scene, a soft medium shot in a green misty mountain meadow at dawn, tall dewy grass glistening and low fog drifting between faint pine slopes. the young woman with curly dark-brown hair and a cream cable-knit sweater walks slowly up to the dappled grey horse, her breath faintly visible in the cool air, and raises a careful open hand. she gently strokes the horse's soft mottled neck, her face softening into a warm reassuring smile, and murmurs in a low, soothing, slightly breathy voice: 'good boy… easy now'. the horse lowers its head toward her, its charcoal mane shifting, flicks an ear and lets out a soft snort, its breath misting in the cold; she leans her forehead lightly against its cheek, closes her eyes for a moment, then whispers with quiet affection: 'there you go'. the camera drifts slowly in a gentle arc around the pair, the shallow-focus background of fog and wildflowers blurring softly behind them; the film aesthetic is delicate and breathtaking with cool silver dawn light and fine atmospheric haze. the audio is intimate and immersive: her gentle hushed voice, the horse's soft nicker and snort, the swish of dewy grass, light birdsong waking in the distance and a faint cool breeze, with no background music", 42, False],
         ],
-        inputs=[sheet_image, sheet, action, lora_scale, seed, randomize],
+        inputs=[sheet_image, sheet, action, seed, randomize],
         outputs=[video_out, seed], fn=generate, cache_examples=True, cache_mode="lazy",
     )
 

requirements.txt

@@ -1,9 +1,11 @@
-git+https://github.com/huggingface/diffusers
-transformers
+transformers==4.57.6
 accelerate
-peft
-safetensors
-sentencepiece
-imageio
-imageio-ffmpeg
+torch==2.8.0
+torchaudio==2.8.0
+einops
+scipy
 av
+scikit-image>=0.25.2
+flashpack==0.1.2
+imageio[ffmpeg]
+pillow