multimodalart HF Staff commited on
Commit
4962d7f
·
verified ·
1 Parent(s): 9e1e035

Upload app.py with huggingface_hub

Browse files
Files changed (1) hide show
  1. app.py +128 -133
app.py CHANGED
@@ -1,18 +1,14 @@
1
  """LTX-2.3 Cinemagraph LoRA — image-to-video demo.
2
 
3
- Turns a still image into a looping cinemagraph (selective motion, frozen background)
4
- using the `Lightricks/LTX-2.3-22b-LoRA-Cinemagraph` LoRA on top of the LTX-2.3-22B base.
5
-
6
- Architecture (the 22B base is too large to hold Gemma + the video model together):
7
- * Text encoding (Gemma-3-12B + LTX embeddings connectors) runs on a SEPARATE
8
- Space `linoyts/ltx23-gemma-encoder-api`, called here via gradio_client. It
9
- returns the final positive/negative EmbeddingsProcessorOutput tensors as a .pt.
10
- * This Space runs the TI2VidTwoStagesPipeline (dev stage-1 with CFG/STG + distilled
11
- LoRA stage-2 with x2 spatial upsample). `encode_prompts` is monkeypatched with the
12
- precomputed embeddings so Gemma never loads here.
13
- * The Cinemagraph LoRA is loaded into `loras` so it applies to both stages.
14
-
15
- Mirrors linoyts/ltx23-dev-api, extended to i2v + the cinemagraph LoRA.
16
  """
17
 
18
  import os
@@ -21,7 +17,6 @@ import sys
21
 
22
  os.environ["TORCH_COMPILE_DISABLE"] = "1"
23
  os.environ["TORCHDYNAMO_DISABLE"] = "1"
24
- os.environ.setdefault("PYTORCH_CUDA_ALLOC_CONF", "expandable_segments:True")
25
 
26
  subprocess.run([sys.executable, "-m", "pip", "install", "xformers==0.0.32.post2",
27
  "--no-build-isolation"], check=False)
@@ -61,11 +56,11 @@ import numpy as np
61
  import spaces
62
  from gradio_client import Client, handle_file
63
  from huggingface_hub import hf_hub_download
64
- from PIL import Image
65
 
66
  import ltx_pipelines.ti2vid_two_stages as ti2vid_module
67
  from ltx_core.components.guiders import MultiModalGuiderParams
68
- from ltx_core.loader import LTXV_LORA_COMFY_RENAMING_MAP, LoraPathStrengthAndSDOps
69
  from ltx_core.model.video_vae import TilingConfig, get_video_chunks_number
70
  from ltx_core.quantization import QuantizationPolicy
71
  from ltx_core.text_encoders.gemma.embeddings_processor import EmbeddingsProcessorOutput
@@ -127,22 +122,23 @@ def _patched_load(self, path, sd_ops, device=None):
127
 
128
  SafetensorsStateDictLoader.load = _patched_load
129
 
130
- # --- LoRA-fusion patch for ZeroGPU startup ---
131
- # On ZeroGPU, module-scope `.to("cuda")` is intercepted (no real GPU is attached
132
- # yet), but fp8 weights still *report* device="cuda". The cast-only-fp8 LoRA fusion
133
- # path (`_fuse_delta_with_cast_fp8`) then launches a Triton CUDA kernel at import
134
- # time, which crashes with "no CUDA-capable device is detected". Force the
135
- # CPU-style fusion (plain bf16 add + deterministic fp8 downcast) instead no GPU
136
- # kernel needed, and the ZeroGPU packing step handles the real device transfer.
137
  import ltx_core.loader.fuse_loras as _fuse_mod
138
 
139
 
140
- def _fuse_delta_with_cast_fp8_nogpu(deltas, weight, key, target_dtype, device):
141
- deltas.add_(weight.to(dtype=deltas.dtype, device=deltas.device))
142
- return {key: deltas.to(dtype=target_dtype)}
143
 
144
 
145
- _fuse_mod._fuse_delta_with_cast_fp8 = _fuse_delta_with_cast_fp8_nogpu
 
146
 
147
  logging.getLogger().setLevel(logging.INFO)
148
  MAX_SEED = np.iinfo(np.int32).max
@@ -156,39 +152,35 @@ DEFAULT_NEGATIVE = (
156
  "crawling texture, distorted, blurry, low quality"
157
  )
158
 
159
- # LTX-2.3 Cinemagraph guider defaults (per model card: stg_v scale 1.0, block 29, CFG 4.0).
160
  VIDEO_STG, VIDEO_RESCALE, A2V_SCALE, STG_BLOCKS = 1.0, 0.7, 3.0, [29]
161
  AUDIO_CFG, AUDIO_STG, AUDIO_RESCALE, V2A_SCALE = 7.0, 1.0, 0.7, 3.0
162
 
163
  LTX_REPO = os.environ.get("LTX_REPO", "Lightricks/LTX-2.3")
164
- LORA_REPO = os.environ.get("LORA_REPO", "Lightricks/LTX-2.3-22b-LoRA-Cinemagraph")
165
- LORA_FILE = "ltx-2.3-22b-lora-cinemagraph-0.9.safetensors"
166
  DEV_FILE = "ltx-2.3-22b-dev.safetensors"
167
  DISTILLED_LORA_FILE = "ltx-2.3-22b-distilled-lora-384-1.1.safetensors"
168
  UPSCALER_FILE = "ltx-2.3-spatial-upscaler-x2-1.1.safetensors"
 
 
 
169
  TOKEN = os.environ.get("HF_TOKEN")
170
  TEXT_ENCODER_SPACE = os.environ.get("TEXT_ENCODER_SPACE", "linoyts/ltx23-gemma-encoder-api")
171
 
172
- # Cinemagraph LoRA strength (0.7-3.0 per model card; 1.0 is a solid default).
173
- CINEMAGRAPH_LORA_STRENGTH = float(os.environ.get("CINEMAGRAPH_LORA_STRENGTH", "1.0"))
174
-
175
  print("Downloading checkpoints…")
176
  checkpoint_path = hf_hub_download(LTX_REPO, DEV_FILE, token=TOKEN)
177
  distilled_lora_path = hf_hub_download(LTX_REPO, DISTILLED_LORA_FILE, token=TOKEN)
178
  upsampler_path = hf_hub_download(LTX_REPO, UPSCALER_FILE, token=TOKEN)
179
- cinemagraph_lora_path = hf_hub_download(LORA_REPO, LORA_FILE, token=TOKEN)
180
 
181
  pipeline = TI2VidTwoStagesPipeline(
182
  checkpoint_path=checkpoint_path,
183
  distilled_lora=[LoraPathStrengthAndSDOps(path=distilled_lora_path, strength=1.0, sd_ops=None)],
184
  spatial_upsampler_path=upsampler_path,
185
  gemma_root=None, # text encoding happens on TEXT_ENCODER_SPACE
186
- # Cinemagraph LoRA (ComfyUI-format) applied to BOTH stages via `loras`.
187
  loras=[LoraPathStrengthAndSDOps(
188
- path=cinemagraph_lora_path,
189
- strength=CINEMAGRAPH_LORA_STRENGTH,
190
- sd_ops=LTXV_LORA_COMFY_RENAMING_MAP,
191
- )],
192
  quantization=QuantizationPolicy.fp8_cast(),
193
  )
194
 
@@ -209,15 +201,19 @@ print("Pipeline ready.")
209
  _emb_cache = {}
210
 
211
 
212
- def fetch_embeddings(prompt, negative_prompt, enhance_prompt, seed):
213
- """Call the Gemma encoder Space; returns {'positive', 'negative', 'final_prompt'}."""
214
- key = (prompt, negative_prompt, bool(enhance_prompt), int(seed) if enhance_prompt else 0)
 
 
 
 
215
  if key in _emb_cache:
216
  return _emb_cache[key]
217
- client = Client(TEXT_ENCODER_SPACE, token=TOKEN)
218
  emb_file, final_prompt, status = client.predict(
219
  prompt=prompt, negative_prompt=negative_prompt or "", encode_negative=True,
220
- enhance_prompt=enhance_prompt, seed=int(seed), api_name="/encode",
221
  )
222
  print(f"[encoder] {status} | final prompt: {final_prompt[:80]}…")
223
  data = torch.load(emb_file, map_location="cpu", weights_only=True)
@@ -235,21 +231,30 @@ def _to_output(pack, device):
235
  )
236
 
237
 
238
- def gpu_duration(embeddings, image_path, prompt, negative_prompt="", duration=4.0, *args, **kwargs):
239
- return min(600, 150 + int(duration) * 45)
 
 
 
 
 
 
 
 
 
240
 
241
 
242
  @spaces.GPU(duration=gpu_duration)
243
  @torch.inference_mode()
244
  def _generate_gpu(embeddings, image_path, prompt, negative_prompt, duration, used_seed,
245
- num_inference_steps, video_cfg_scale, width, height,
246
  progress=gr.Progress(track_tqdm=True)):
247
  num_frames = ((int(duration * FRAME_RATE) + 1 - 1 + 7) // 8) * 8 + 1
248
  tiling_config = TilingConfig.default()
249
  video_chunks_number = get_video_chunks_number(num_frames, tiling_config)
250
 
251
- # i2v: condition on the input image at frame 0 with full strength.
252
- images = [ImageConditioningInput(path=image_path, frame_idx=0, strength=1.0)]
253
 
254
  precomputed = [_to_output(embeddings["positive"], "cuda"),
255
  _to_output(embeddings["negative"], "cuda")]
@@ -273,7 +278,7 @@ def _generate_gpu(embeddings, image_path, prompt, negative_prompt, duration, use
273
  modality_scale=V2A_SCALE, skip_step=0, stg_blocks=STG_BLOCKS),
274
  images=images,
275
  tiling_config=tiling_config,
276
- enhance_prompt=False, # already enhanced on the encoder Space
277
  )
278
  finally:
279
  ti2vid_module.encode_prompts = original
@@ -284,127 +289,125 @@ def _generate_gpu(embeddings, image_path, prompt, negative_prompt, duration, use
284
  return output_path
285
 
286
 
287
- def _round64(x):
288
- return max(64, int(round(x / 64)) * 64)
 
 
 
289
 
290
 
291
  def generate(
292
- image,
293
  prompt: str,
294
  negative_prompt: str = DEFAULT_NEGATIVE,
295
  duration: float = 4.0,
296
  seed: int = 42,
297
- randomize_seed: bool = False,
298
  num_inference_steps: int = 30,
299
  video_cfg_scale: float = 4.0,
 
300
  width: int = 704,
301
  height: int = 512,
302
- enhance_prompt: bool = True,
303
  progress=gr.Progress(track_tqdm=True),
304
  ):
305
- """Generate a looping cinemagraph video from a still image.
306
 
307
  Args:
308
- image: input still image (filepath) to animate.
309
- prompt: describes what should move and what should stay frozen. The
310
- trigger word CINEMAGRAPH_MOTION is auto-prepended if missing.
311
- negative_prompt: things to avoid (camera motion, global movement, etc.).
312
- duration: video length in seconds.
313
  seed: RNG seed.
314
- randomize_seed: pick a new random seed each run.
315
- num_inference_steps: stage-1 denoising steps.
316
- video_cfg_scale: classifier-free guidance scale.
317
- width: output width (rounded to a multiple of 64).
318
- height: output height (rounded to a multiple of 64).
319
- enhance_prompt: run Gemma prompt enhancement on the encoder Space.
320
 
321
  Returns:
322
- (video_path, used_seed)
323
  """
324
  if image is None:
325
- raise gr.Error("Please provide an input image to animate.")
326
- if not prompt or not prompt.strip():
327
- raise gr.Error("Please provide a prompt describing the motion.")
328
-
329
- # Ensure the LoRA trigger word is present.
330
- if TRIGGER.lower() not in prompt.lower():
331
- prompt = f"{TRIGGER}, {prompt.strip()}"
332
-
333
- width, height = _round64(width), _round64(height)
334
  used_seed = random.randint(0, MAX_SEED) if randomize_seed else int(seed)
335
-
336
- # Normalize the input image to a local temp file path (RGB jpeg).
337
- if isinstance(image, str):
338
- image_path = image
339
- else:
340
- image_path = tempfile.mktemp(suffix=".png")
341
- Image.fromarray(np.asarray(image)).convert("RGB").save(image_path)
342
-
343
- progress(0.05, desc="encoding prompt (Gemma Space)…")
344
- embeddings = fetch_embeddings(prompt, negative_prompt, enhance_prompt, used_seed)
345
- progress(0.15, desc="generating cinemagraph…")
346
- output_path = _generate_gpu(embeddings, image_path, prompt, negative_prompt, duration,
347
- used_seed, num_inference_steps, video_cfg_scale, width, height)
348
  return output_path, used_seed
349
 
350
 
351
  CSS = """
352
- #col-container { max-width: 1100px; margin: 0 auto; }
353
  .dark .gradio-container { color: var(--body-text-color); }
354
  """
355
 
356
- EX_G = ("CINEMAGRAPH_MOTION, tripod locked-off static camera, zero camera movement, the man, "
357
- "face, hair, clothing, beach, sky, and background remain completely frozen, only the "
358
- "beach reflection inside his sunglasses moves, reflected time-lapse of ocean waves roll "
359
- "and shimmer in the lenses, everything outside the glasses stays still, seamless natural loop")
360
- EX_J = ("CINEMAGRAPH_MOTION, tripod locked-off static camera, zero camera movement, the man and "
361
- "cow remain completely frozen, only the clouds in the sky move in fast time-lapse, clouds "
362
- "drift and roll across the sky, grass and foreground remain still, seamless natural loop.")
363
- EX_T = ("CINEMAGRAPH_MOTION, tripod locked-off static camera, zero camera movement, the woman is "
364
- "frozen, her face, eyes, hair, clothing, seat, and airplane interior remain completely "
365
- "frozen, only the illustrated tear drops move downward along her cheeks like a cutout "
366
- "collage animation, seamless natural loop.")
367
-
368
- with gr.Blocks(theme=gr.themes.Citrus(), css=CSS, title="LTX-2.3 Cinemagraph") as demo:
369
  with gr.Column(elem_id="col-container"):
370
  gr.Markdown(
371
  "# 🎞️ LTX-2.3 Cinemagraph\n"
372
- "Turn a **still image** into a looping **cinemagraph** — selective motion "
373
- "(water, light, reflections, clouds) while the rest of the scene stays frozen. "
374
- "Powered by the [`LTX-2.3-22b-LoRA-Cinemagraph`](https://huggingface.co/Lightricks/LTX-2.3-22b-LoRA-Cinemagraph) "
375
- "LoRA on LTX-2.3-22B. Text encoding runs on a separate Gemma encoder Space.\n\n"
376
- "💡 Describe **what moves** and **what stays still**. The `CINEMAGRAPH_MOTION` "
377
- "trigger word is added automatically."
 
378
  )
379
  with gr.Row():
380
  with gr.Column():
381
- image = gr.Image(label="Input image", type="filepath")
382
- prompt = gr.Textbox(label="Prompt", lines=3,
383
- placeholder="only the neon sign flickers, everything else stays perfectly still, seamless natural loop")
384
- btn = gr.Button("Generate cinemagraph", variant="primary")
 
 
385
  with gr.Accordion("Advanced settings", open=False):
386
  negative_prompt = gr.Textbox(label="Negative prompt", value=DEFAULT_NEGATIVE, lines=2)
387
  duration = gr.Slider(1, 8, value=4, step=0.5, label="Duration (s)")
388
  with gr.Row():
389
  seed = gr.Number(label="Seed", value=42, precision=0)
390
- randomize_seed = gr.Checkbox(label="Randomize seed", value=False)
391
  with gr.Row():
392
  num_inference_steps = gr.Slider(10, 50, value=30, step=1, label="Steps")
393
- video_cfg_scale = gr.Slider(1, 8, value=4.0, step=0.1, label="CFG scale")
 
 
394
  with gr.Row():
395
- width = gr.Slider(320, 1024, value=704, step=64, label="Width")
396
- height = gr.Slider(320, 1024, value=512, step=64, label="Height")
397
- enhance_prompt = gr.Checkbox(label="Enhance prompt (Gemma)", value=True)
398
  with gr.Column():
399
  video = gr.Video(label="Cinemagraph", autoplay=True, loop=True)
400
  used_seed = gr.Number(label="Used seed", precision=0)
401
 
 
 
 
 
402
  gr.Examples(
403
- examples=[
404
- ["examples/Glasses.jpeg", EX_G],
405
- ["examples/Jump.jpeg", EX_J],
406
- ["examples/Tears.jpeg", EX_T],
407
- ],
408
  inputs=[image, prompt],
409
  outputs=[video, used_seed],
410
  fn=generate,
@@ -412,13 +415,5 @@ with gr.Blocks(theme=gr.themes.Citrus(), css=CSS, title="LTX-2.3 Cinemagraph") a
412
  cache_mode="lazy",
413
  )
414
 
415
- btn.click(
416
- generate,
417
- [image, prompt, negative_prompt, duration, seed, randomize_seed,
418
- num_inference_steps, video_cfg_scale, width, height, enhance_prompt],
419
- [video, used_seed],
420
- api_name="generate",
421
- )
422
-
423
  if __name__ == "__main__":
424
- demo.launch(mcp_server=True)
 
1
  """LTX-2.3 Cinemagraph LoRA — image-to-video demo.
2
 
3
+ Turns a still image into a looping cinemagraph: only the element you describe moves
4
+ while the rest of the frame stays frozen (locked-off static camera). Built on the native
5
+ LTX-2.3 `TI2VidTwoStagesPipeline` (dev/base stage 1 with CFG/STG guidance, distilled-LoRA
6
+ stage 2 + x2 spatial upsampler) with the Cinemagraph LoRA fused into stage 1.
7
+
8
+ Text encoding (positive + negative, for CFG) is offloaded to the Gemma encoder Space
9
+ (TEXT_ENCODER_SPACE) via gradio_client `encode_prompts` is monkeypatched with the
10
+ precomputed [ctx_p, ctx_n], so this Space never loads Gemma. Mirrors the working
11
+ `linoyts/ltx23-dev-api` backend pattern.
 
 
 
 
12
  """
13
 
14
  import os
 
17
 
18
  os.environ["TORCH_COMPILE_DISABLE"] = "1"
19
  os.environ["TORCHDYNAMO_DISABLE"] = "1"
 
20
 
21
  subprocess.run([sys.executable, "-m", "pip", "install", "xformers==0.0.32.post2",
22
  "--no-build-isolation"], check=False)
 
56
  import spaces
57
  from gradio_client import Client, handle_file
58
  from huggingface_hub import hf_hub_download
59
+ from PIL import Image, ImageOps
60
 
61
  import ltx_pipelines.ti2vid_two_stages as ti2vid_module
62
  from ltx_core.components.guiders import MultiModalGuiderParams
63
+ from ltx_core.loader import LoraPathStrengthAndSDOps, LTXV_LORA_COMFY_RENAMING_MAP
64
  from ltx_core.model.video_vae import TilingConfig, get_video_chunks_number
65
  from ltx_core.quantization import QuantizationPolicy
66
  from ltx_core.text_encoders.gemma.embeddings_processor import EmbeddingsProcessorOutput
 
122
 
123
  SafetensorsStateDictLoader.load = _patched_load
124
 
125
+ # ZeroGPU LoRA-into-fp8 fusion patch:
126
+ # fuse_loras._fuse_delta_with_cast_fp8 upcasts the fp8 base weight + adds the LoRA delta via a
127
+ # Triton CUDA kernel (calculate_weight_float8 -> fused_add_round_kernel). That kernel launches a
128
+ # *real* CUDA op, which cannot be ZeroGPU-virtualised at module scope and dies with
129
+ # "CUDA error: no CUDA-capable device is detected". Replace it with a pure-torch equivalent
130
+ # (upcast to bf16, add delta, re-cast to fp8) that goes through torch.Tensor ops ZeroGPU can
131
+ # virtualise. dev-api avoids this only because it fuses no user LoRA into the fp8 stage-1 base.
132
  import ltx_core.loader.fuse_loras as _fuse_mod
133
 
134
 
135
+ def _fuse_delta_with_cast_fp8_torch(deltas, weight, key, target_dtype, device):
136
+ fused = (deltas.to(torch.bfloat16) + weight.to(torch.bfloat16)).to(dtype=target_dtype)
137
+ return {key: fused}
138
 
139
 
140
+ _fuse_mod._fuse_delta_with_cast_fp8 = _fuse_delta_with_cast_fp8_torch
141
+ print("[PATCH] fp8 LoRA fusion -> pure-torch (ZeroGPU-virtualisable, no Triton kernel)")
142
 
143
  logging.getLogger().setLevel(logging.INFO)
144
  MAX_SEED = np.iinfo(np.int32).max
 
152
  "crawling texture, distorted, blurry, low quality"
153
  )
154
 
155
+ # LTX-2.3 guider defaults. Cinemagraph card recommends stg_v scale 1.0 @ block 29, guidance 4.0.
156
  VIDEO_STG, VIDEO_RESCALE, A2V_SCALE, STG_BLOCKS = 1.0, 0.7, 3.0, [29]
157
  AUDIO_CFG, AUDIO_STG, AUDIO_RESCALE, V2A_SCALE = 7.0, 1.0, 0.7, 3.0
158
 
159
  LTX_REPO = os.environ.get("LTX_REPO", "Lightricks/LTX-2.3")
 
 
160
  DEV_FILE = "ltx-2.3-22b-dev.safetensors"
161
  DISTILLED_LORA_FILE = "ltx-2.3-22b-distilled-lora-384-1.1.safetensors"
162
  UPSCALER_FILE = "ltx-2.3-spatial-upscaler-x2-1.1.safetensors"
163
+ CINEMAGRAPH_REPO = os.environ.get("CINEMAGRAPH_REPO", "Lightricks/LTX-2.3-22b-LoRA-Cinemagraph")
164
+ CINEMAGRAPH_FILE = "ltx-2.3-22b-lora-cinemagraph-0.9.safetensors"
165
+ DEFAULT_LORA_STRENGTH = 1.0
166
  TOKEN = os.environ.get("HF_TOKEN")
167
  TEXT_ENCODER_SPACE = os.environ.get("TEXT_ENCODER_SPACE", "linoyts/ltx23-gemma-encoder-api")
168
 
 
 
 
169
  print("Downloading checkpoints…")
170
  checkpoint_path = hf_hub_download(LTX_REPO, DEV_FILE, token=TOKEN)
171
  distilled_lora_path = hf_hub_download(LTX_REPO, DISTILLED_LORA_FILE, token=TOKEN)
172
  upsampler_path = hf_hub_download(LTX_REPO, UPSCALER_FILE, token=TOKEN)
173
+ cinemagraph_lora_path = hf_hub_download(CINEMAGRAPH_REPO, CINEMAGRAPH_FILE, token=TOKEN)
174
 
175
  pipeline = TI2VidTwoStagesPipeline(
176
  checkpoint_path=checkpoint_path,
177
  distilled_lora=[LoraPathStrengthAndSDOps(path=distilled_lora_path, strength=1.0, sd_ops=None)],
178
  spatial_upsampler_path=upsampler_path,
179
  gemma_root=None, # text encoding happens on TEXT_ENCODER_SPACE
180
+ # Cinemagraph LoRA fused into stage 1 (ComfyUI-format keys -> comfy renaming map).
181
  loras=[LoraPathStrengthAndSDOps(
182
+ path=cinemagraph_lora_path, strength=DEFAULT_LORA_STRENGTH,
183
+ sd_ops=LTXV_LORA_COMFY_RENAMING_MAP)],
 
 
184
  quantization=QuantizationPolicy.fp8_cast(),
185
  )
186
 
 
201
  _emb_cache = {}
202
 
203
 
204
+ def fetch_embeddings(prompt, negative_prompt, seed):
205
+ """Call the Gemma encoder Space; returns {'positive', 'negative', 'final_prompt'}.
206
+
207
+ enhance_prompt is disabled: cinemagraph prompting relies on an explicit
208
+ 'only X moves, everything else frozen' structure that prompt-enhancement can dilute.
209
+ """
210
+ key = (prompt, negative_prompt)
211
  if key in _emb_cache:
212
  return _emb_cache[key]
213
+ client = Client(TEXT_ENCODER_SPACE, token=TOKEN, httpx_kwargs={"timeout": 300})
214
  emb_file, final_prompt, status = client.predict(
215
  prompt=prompt, negative_prompt=negative_prompt or "", encode_negative=True,
216
+ enhance_prompt=False, seed=int(seed), api_name="/encode",
217
  )
218
  print(f"[encoder] {status} | final prompt: {final_prompt[:80]}…")
219
  data = torch.load(emb_file, map_location="cpu", weights_only=True)
 
231
  )
232
 
233
 
234
+ def _prep_image(image_path, width, height):
235
+ """Aspect-fit/crop the still to WxH and save a temp file the VAE encoder can read."""
236
+ im = Image.open(image_path).convert("RGB")
237
+ im = ImageOps.fit(im, (width, height), Image.LANCZOS)
238
+ tmp = tempfile.mktemp(suffix=".png")
239
+ im.save(tmp)
240
+ return tmp
241
+
242
+
243
+ def gpu_duration(embeddings, image_path, prompt, negative_prompt="", duration=3.0, *args, **kwargs):
244
+ return min(600, 150 + int(duration) * 40)
245
 
246
 
247
  @spaces.GPU(duration=gpu_duration)
248
  @torch.inference_mode()
249
  def _generate_gpu(embeddings, image_path, prompt, negative_prompt, duration, used_seed,
250
+ num_inference_steps, video_cfg_scale, lora_strength, width, height,
251
  progress=gr.Progress(track_tqdm=True)):
252
  num_frames = ((int(duration * FRAME_RATE) + 1 - 1 + 7) // 8) * 8 + 1
253
  tiling_config = TilingConfig.default()
254
  video_chunks_number = get_video_chunks_number(num_frames, tiling_config)
255
 
256
+ cond_img = _prep_image(image_path, int(width), int(height))
257
+ images = [ImageConditioningInput(path=cond_img, frame_idx=0, strength=1.0)]
258
 
259
  precomputed = [_to_output(embeddings["positive"], "cuda"),
260
  _to_output(embeddings["negative"], "cuda")]
 
278
  modality_scale=V2A_SCALE, skip_step=0, stg_blocks=STG_BLOCKS),
279
  images=images,
280
  tiling_config=tiling_config,
281
+ enhance_prompt=False, # already handled remotely; cinemagraph wants literal prompts
282
  )
283
  finally:
284
  ti2vid_module.encode_prompts = original
 
289
  return output_path
290
 
291
 
292
+ def _build_prompt(user_prompt: str) -> str:
293
+ p = (user_prompt or "").strip()
294
+ if TRIGGER.lower() in p.lower():
295
+ return p
296
+ return f"{TRIGGER}, tripod locked-off static camera, zero camera movement, {p}"
297
 
298
 
299
  def generate(
300
+ image: str,
301
  prompt: str,
302
  negative_prompt: str = DEFAULT_NEGATIVE,
303
  duration: float = 4.0,
304
  seed: int = 42,
305
+ randomize_seed: bool = True,
306
  num_inference_steps: int = 30,
307
  video_cfg_scale: float = 4.0,
308
+ lora_strength: float = DEFAULT_LORA_STRENGTH,
309
  width: int = 704,
310
  height: int = 512,
 
311
  progress=gr.Progress(track_tqdm=True),
312
  ):
313
+ """Turn a still image into a looping cinemagraph (selective motion, everything else frozen).
314
 
315
  Args:
316
+ image: path to the still input image (i2v conditioning at frame 0).
317
+ prompt: what should move (e.g. "only the ocean waves shimmer in his sunglasses").
318
+ The CINEMAGRAPH_MOTION trigger + static-camera framing are added automatically.
319
+ negative_prompt: things to suppress (camera motion, whole-image movement, artifacts).
320
+ duration: clip length in seconds.
321
  seed: RNG seed.
322
+ randomize_seed: draw a fresh random seed instead of `seed`.
323
+ num_inference_steps: stage-1 diffusion steps.
324
+ video_cfg_scale: classifier-free guidance scale (cinemagraph default 4.0).
325
+ lora_strength: Cinemagraph LoRA strength note (fixed at load; shown for reference).
326
+ width: output width (multiple of 64).
327
+ height: output height (multiple of 64).
328
 
329
  Returns:
330
+ (video_path, used_seed): the generated .mp4 and the seed actually used.
331
  """
332
  if image is None:
333
+ raise gr.Error("Please upload a still image to animate.")
334
+ if not (prompt or "").strip():
335
+ raise gr.Error("Describe what should move (e.g. 'only the neon sign flickers').")
 
 
 
 
 
 
336
  used_seed = random.randint(0, MAX_SEED) if randomize_seed else int(seed)
337
+ full_prompt = _build_prompt(prompt)
338
+ if TRIGGER.lower() not in full_prompt.lower():
339
+ full_prompt = f"{TRIGGER}, {full_prompt}"
340
+ if "seamless natural loop" not in full_prompt.lower():
341
+ full_prompt = f"{full_prompt}, seamless natural loop"
342
+ progress(0.05, desc="encoding prompts (Gemma Space)…")
343
+ embeddings = fetch_embeddings(full_prompt, negative_prompt, used_seed)
344
+ output_path = _generate_gpu(
345
+ embeddings, image, full_prompt, negative_prompt, duration, used_seed,
346
+ num_inference_steps, video_cfg_scale, lora_strength, width, height)
 
 
 
347
  return output_path, used_seed
348
 
349
 
350
  CSS = """
351
+ #col-container { max-width: 1200px; margin: 0 auto; }
352
  .dark .gradio-container { color: var(--body-text-color); }
353
  """
354
 
355
+ EX_DIR = os.path.join(os.path.dirname(os.path.abspath(__file__)), "examples")
356
+ EXAMPLES = [
357
+ [os.path.join(EX_DIR, "Glasses.jpeg"),
358
+ "the man, face, hair, clothing, beach, sky, and background remain completely frozen, only the beach reflection inside his sunglasses moves, reflected time-lapse of ocean waves roll and shimmer in the lenses, everything outside the glasses stays still"],
359
+ [os.path.join(EX_DIR, "Jump.jpeg"),
360
+ "the man and cow remain completely frozen, only the clouds in the sky move in fast time-lapse, clouds drift and roll across the sky, grass and foreground remain still"],
361
+ [os.path.join(EX_DIR, "Motel.jpeg"),
362
+ "only the neon sign flickers, the starburst, Desert, MOTEL, and NO VACANCY neon tubes subtly pulse and flicker like a real vintage neon sign, everything else stays perfectly still"],
363
+ [os.path.join(EX_DIR, "Tears.jpeg"),
364
+ "the woman is frozen, her face, eyes, hair, clothing, seat, and airplane interior remain completely frozen, only the illustrated tear drops move downward along her cheeks like a cutout collage animation"],
365
+ ]
366
+
367
+ with gr.Blocks(title="LTX-2.3 Cinemagraph LoRA", theme=gr.themes.Citrus(), css=CSS) as demo:
368
  with gr.Column(elem_id="col-container"):
369
  gr.Markdown(
370
  "# 🎞️ LTX-2.3 Cinemagraph\n"
371
+ "Turn a **still image** into a looping **cinemagraph** — only the element you "
372
+ "describe moves (water, reflections, neon, clouds, rain) while the rest of the "
373
+ "frame stays frozen. Built on [LTX-2.3](https://huggingface.co/Lightricks/LTX-2.3) "
374
+ "with the [Cinemagraph LoRA](https://huggingface.co/Lightricks/LTX-2.3-22b-LoRA-Cinemagraph). "
375
+ "Text encoding is offloaded to a remote Gemma encoder Space.\n\n"
376
+ "**Tip:** be explicit *\"only the [element] moves; everything else frozen.\"* "
377
+ "The `CINEMAGRAPH_MOTION` trigger and static-camera framing are added for you."
378
  )
379
  with gr.Row():
380
  with gr.Column():
381
+ image = gr.Image(label="Still image", type="filepath")
382
+ prompt = gr.Textbox(
383
+ label="What should move?", lines=3,
384
+ placeholder="only the ocean waves reflected in his sunglasses shimmer; everything else stays frozen",
385
+ )
386
+ run = gr.Button("Animate", variant="primary")
387
  with gr.Accordion("Advanced settings", open=False):
388
  negative_prompt = gr.Textbox(label="Negative prompt", value=DEFAULT_NEGATIVE, lines=2)
389
  duration = gr.Slider(1, 8, value=4, step=0.5, label="Duration (s)")
390
  with gr.Row():
391
  seed = gr.Number(label="Seed", value=42, precision=0)
392
+ randomize_seed = gr.Checkbox(label="Randomize seed", value=True)
393
  with gr.Row():
394
  num_inference_steps = gr.Slider(10, 50, value=30, step=1, label="Steps")
395
+ video_cfg_scale = gr.Slider(1, 8, value=4.0, step=0.1, label="Guidance (CFG)")
396
+ lora_strength = gr.Slider(0.5, 3.0, value=DEFAULT_LORA_STRENGTH, step=0.1,
397
+ label="LoRA strength (set at load)")
398
  with gr.Row():
399
+ width = gr.Slider(384, 1024, value=704, step=64, label="Width")
400
+ height = gr.Slider(384, 1024, value=512, step=64, label="Height")
 
401
  with gr.Column():
402
  video = gr.Video(label="Cinemagraph", autoplay=True, loop=True)
403
  used_seed = gr.Number(label="Used seed", precision=0)
404
 
405
+ inputs = [image, prompt, negative_prompt, duration, seed, randomize_seed,
406
+ num_inference_steps, video_cfg_scale, lora_strength, width, height]
407
+ run.click(generate, inputs, [video, used_seed], api_name="generate")
408
+
409
  gr.Examples(
410
+ examples=EXAMPLES,
 
 
 
 
411
  inputs=[image, prompt],
412
  outputs=[video, used_seed],
413
  fn=generate,
 
415
  cache_mode="lazy",
416
  )
417
 
 
 
 
 
 
 
 
 
418
  if __name__ == "__main__":
419
+ demo.launch(mcp_server=True, show_error=True)