dream2589632147 commited on
Commit
7eee8c8
·
verified ·
1 Parent(s): bd867f6

Upload 9 files

Browse files
Files changed (5) hide show
  1. README.md +65 -5
  2. aoti.py +30 -17
  3. app.py +945 -608
  4. packages.txt +1 -1
  5. requirements.txt +13 -10
README.md CHANGED
@@ -1,13 +1,73 @@
1
  ---
2
- title: Wan2.2 14B Fast Preview
3
- emoji: 🐌
4
- colorFrom: yellow
 
5
  colorTo: pink
6
  sdk: gradio
7
  sdk_version: 6.0.1
8
  app_file: app.py
9
  pinned: false
10
- short_description: generate a video from an image with a text prompt
 
 
11
  ---
12
 
13
- Check out the configuration reference at https://huggingface.co/docs/hub/spaces-config-reference
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
  ---
2
+ title: Dream Motion Pro - Wan 2.2
3
+ author: dream2589632147
4
+ emoji: 🎬
5
+ colorFrom: indigo
6
  colorTo: pink
7
  sdk: gradio
8
  sdk_version: 6.0.1
9
  app_file: app.py
10
  pinned: false
11
+ short_description: Fast Wan 2.2 image-to-video with first/last frames
12
+ models:
13
+ - TestOrganizationPleaseIgnore/WAMU-Merge-MotionBoost_WAN2.2_I2V_LIGHTNING
14
  ---
15
 
16
+ # Dream Motion Pro Wan 2.2
17
+
18
+ Create cinematic videos from a single image or guide the animation with both a **first frame and a last frame**.
19
+
20
+ ## Main features
21
+
22
+ - Single-image Image-to-Video mode
23
+ - First + Last Frame mode
24
+ - Wan 2.2 14B Lightning/MotionBoost checkpoint
25
+ - FP8 transformer quantization
26
+ - AOTI-compiled transformer blocks for ZeroGPU
27
+ - Fast, Balanced and High Quality profiles
28
+ - Natural, Cinematic, Portrait, Product Ad, Dynamic and Anime motion styles
29
+ - Identity-preservation prompt controls
30
+ - Automatic professional prompt enhancement
31
+ - Landscape, portrait, square and social-media aspect ratios
32
+ - Optional RIFE interpolation to 32 or 64 FPS
33
+ - Seed control and repeatable generations
34
+ - Extract any generated video frame and use it as the next first image
35
+
36
+ ## Space variables
37
+
38
+ Add these in **Settings → Variables and secrets**:
39
+
40
+ | Variable | Required | Purpose |
41
+ |---|---:|---|
42
+ | `REPO_ID` | Recommended | Model repository. Default: `TestOrganizationPleaseIgnore/WAMU-Merge-MotionBoost_WAN2.2_I2V_LIGHTNING` |
43
+ | `ENABLE_AOTI` | No | Set to `false` only for troubleshooting. Default: `true` |
44
+ | `AOTI_REPO` | No | AOTI package repository |
45
+ | `ENABLE_VAE_TILING` | No | Set to `true` if VAE memory becomes a problem |
46
+ | `MAX_GPU_SECONDS` | No | Maximum requested ZeroGPU duration. Default: `300` |
47
+ | `RUNPOD_URL` | No | Shows a RunPod button beneath the generated video |
48
+ | `SUPPORT_URL` | No | Shows a support/PayPal button beneath the generated video |
49
+
50
+ ## Recommended defaults
51
+
52
+ - **Quality Mode:** Balanced
53
+ - **Duration:** 3.5 seconds
54
+ - **Output FPS:** 16
55
+ - **Steps:** 6
56
+ - **Guidance:** 1.0 / 1.0
57
+ - **Scheduler:** UniPCMultistep
58
+ - **Flow Shift:** 3.0
59
+
60
+ Lightning checkpoints usually perform best at approximately **4–8 inference steps**. Raising the steps or guidance unnecessarily increases GPU time and may reduce motion quality.
61
+
62
+ ## First + Last Frame mode
63
+
64
+ Both images are normalized to the same dimensions. The last image is center-cropped to match the prepared first frame so the pipeline receives a consistent resolution.
65
+
66
+ This release intentionally supports **one image** or **first + last frames only**. It does not include a three-frame mode.
67
+
68
+ ## Notes
69
+
70
+ - 32 and 64 FPS outputs use RIFE after the base 16 FPS generation.
71
+ - Higher FPS improves playback smoothness but does not create additional semantic motion.
72
+ - A clear, high-resolution source image and one main action generally produce the most stable result.
73
+ - Generate multiple seeds when choosing a final result.
aoti.py CHANGED
@@ -1,13 +1,13 @@
1
- """
2
- """
3
 
4
  from typing import cast
5
 
6
  import torch
7
  from huggingface_hub import hf_hub_download
8
- from spaces.zero.torch.aoti import ZeroGPUCompiledModel
9
- from spaces.zero.torch.aoti import ZeroGPUWeights
10
- from torch._functorch._aot_autograd.subclass_parametrization import unwrap_tensor_subclass_parameters
 
11
 
12
 
13
  def _shallow_clone_module(module: torch.nn.Module) -> torch.nn.Module:
@@ -15,21 +15,34 @@ def _shallow_clone_module(module: torch.nn.Module) -> torch.nn.Module:
15
  clone.__dict__ = module.__dict__.copy()
16
  clone._parameters = module._parameters.copy()
17
  clone._buffers = module._buffers.copy()
18
- clone._modules = {k: _shallow_clone_module(v) for k, v in module._modules.items() if v is not None}
 
 
 
 
19
  return clone
20
 
21
 
22
- def aoti_blocks_load(module: torch.nn.Module, repo_id: str, variant: str | None = None):
 
 
 
 
23
  repeated_blocks = cast(list[str], module._repeated_blocks)
24
- aoti_files = {name: hf_hub_download(
25
- repo_id=repo_id,
26
- filename='package.pt2',
27
- subfolder=name if variant is None else f'{name}.{variant}',
28
- ) for name in repeated_blocks}
 
 
 
 
29
  for block_name, aoti_file in aoti_files.items():
30
  for block in module.modules():
31
- if block.__class__.__name__ == block_name:
32
- block_ = _shallow_clone_module(block)
33
- unwrap_tensor_subclass_parameters(block_)
34
- weights = ZeroGPUWeights(block_.state_dict())
35
- block.forward = ZeroGPUCompiledModel(aoti_file, weights)
 
 
1
+ """Compatibility helper for loading AOTI-compiled repeated transformer blocks."""
 
2
 
3
  from typing import cast
4
 
5
  import torch
6
  from huggingface_hub import hf_hub_download
7
+ from spaces.zero.torch.aoti import ZeroGPUCompiledModel, ZeroGPUWeights
8
+ from torch._functorch._aot_autograd.subclass_parametrization import (
9
+ unwrap_tensor_subclass_parameters,
10
+ )
11
 
12
 
13
  def _shallow_clone_module(module: torch.nn.Module) -> torch.nn.Module:
 
15
  clone.__dict__ = module.__dict__.copy()
16
  clone._parameters = module._parameters.copy()
17
  clone._buffers = module._buffers.copy()
18
+ clone._modules = {
19
+ name: _shallow_clone_module(child)
20
+ for name, child in module._modules.items()
21
+ if child is not None
22
+ }
23
  return clone
24
 
25
 
26
+ def aoti_blocks_load(
27
+ module: torch.nn.Module,
28
+ repo_id: str,
29
+ variant: str | None = None,
30
+ ) -> None:
31
  repeated_blocks = cast(list[str], module._repeated_blocks)
32
+ aoti_files = {
33
+ name: hf_hub_download(
34
+ repo_id=repo_id,
35
+ filename="package.pt2",
36
+ subfolder=name if variant is None else f"{name}.{variant}",
37
+ )
38
+ for name in repeated_blocks
39
+ }
40
+
41
  for block_name, aoti_file in aoti_files.items():
42
  for block in module.modules():
43
+ if block.__class__.__name__ != block_name:
44
+ continue
45
+ cloned_block = _shallow_clone_module(block)
46
+ unwrap_tensor_subclass_parameters(cloned_block)
47
+ weights = ZeroGPUWeights(cloned_block.state_dict())
48
+ block.forward = ZeroGPUCompiledModel(aoti_file, weights)
app.py CHANGED
@@ -1,273 +1,95 @@
1
- import os; os.system('pip install --upgrade --no-deps spaces')
2
- import spaces
3
- import shutil
4
- import subprocess
5
- import sys
6
  import copy
 
 
7
  import random
 
8
  import tempfile
9
- import warnings
10
  import time
11
- import gc
12
  import uuid
13
- from tqdm import tqdm
 
 
 
 
 
 
 
14
  import cv2
 
15
  import numpy as np
 
16
  import torch
17
  import torch._dynamo
18
- from huggingface_hub import list_models
 
19
  from torch.nn import functional as F
20
- from PIL import Image
 
 
 
 
 
21
 
22
- import gradio as gr
23
  from diffusers import (
24
- FlowMatchEulerDiscreteScheduler,
25
- SASolverScheduler,
26
  DEISMultistepScheduler,
27
  DPMSolverMultistepInverseScheduler,
28
- UniPCMultistepScheduler,
29
  DPMSolverMultistepScheduler,
30
  DPMSolverSinglestepScheduler,
 
 
 
31
  )
32
  from diffusers.pipelines.wan.pipeline_wan_i2v import WanImageToVideoPipeline
33
  from diffusers.utils.export_utils import export_to_video
34
 
35
- from torchao.quantization import quantize_, Float8DynamicActivationFloat8WeightConfig, Int8WeightOnlyConfig
36
- import aoti
37
-
38
- os.environ["TOKENIZERS_PARALLELISM"] = "true"
39
  warnings.filterwarnings("ignore")
40
- IS_ZERO_GPU = bool(os.getenv("SPACES_ZERO_GPU"))
41
-
42
- # if IS_ZERO_GPU:
43
- # print("Loading...")
44
- # subprocess.run("rm -rf /data-nvme/zerogpu-offload/*", env={}, shell=True)
45
 
46
- # --- FRAME EXTRACTION JS & LOGIC ---
 
 
47
 
48
- # JS to grab timestamp from the output video
49
- get_timestamp_js = """
50
- function() {
51
- // Select the video element specifically inside the component with id 'generated-video'
52
- const video = document.querySelector('#generated-video video');
53
-
54
- if (video) {
55
- console.log("Video found! Time: " + video.currentTime);
56
- return video.currentTime;
57
- } else {
58
- console.log("No video element found.");
59
- return 0;
60
- }
61
  }
62
- """
63
-
64
-
65
- def extract_frame(video_path, timestamp):
66
- # Safety check: if no video is present
67
- if not video_path:
68
- return None
69
-
70
- print(f"Extracting frame at timestamp: {timestamp}")
71
-
72
- cap = cv2.VideoCapture(video_path)
73
-
74
- if not cap.isOpened():
75
- return None
76
-
77
- # Calculate frame number
78
- fps = cap.get(cv2.CAP_PROP_FPS)
79
- target_frame_num = int(float(timestamp) * fps)
80
-
81
- # Cap total frames to prevent errors at the very end of video
82
- total_frames = int(cap.get(cv2.CAP_PROP_FRAME_COUNT))
83
- if target_frame_num >= total_frames:
84
- target_frame_num = total_frames - 1
85
-
86
- # Set position
87
- cap.set(cv2.CAP_PROP_POS_FRAMES, target_frame_num)
88
- ret, frame = cap.read()
89
- cap.release()
90
-
91
- if ret:
92
- # Convert from BGR (OpenCV) to RGB (Gradio)
93
- # Gradio Image component handles Numpy array -> PIL conversion automatically
94
- return cv2.cvtColor(frame, cv2.COLOR_BGR2RGB)
95
-
96
- return None
97
-
98
- # --- END FRAME EXTRACTION LOGIC ---
99
-
100
-
101
- def clear_vram():
102
- gc.collect()
103
- torch.cuda.empty_cache()
104
-
105
-
106
- # RIFE
107
- if not os.path.exists("RIFEv4.26_0921.zip"):
108
- print("Downloading RIFE Model...")
109
- subprocess.run([
110
- "wget", "-q",
111
- "https://huggingface.co/r3gm/RIFE/resolve/main/RIFEv4.26_0921.zip",
112
- "-O", "RIFEv4.26_0921.zip"
113
- ], check=True)
114
- subprocess.run(["unzip", "-o", "RIFEv4.26_0921.zip"], check=True)
115
-
116
- # sys.path.append(os.getcwd())
117
-
118
- from train_log.RIFE_HDv3 import Model
119
- device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
120
- rife_model = Model()
121
- rife_model.load_model("train_log", -1)
122
- rife_model.eval()
123
-
124
 
125
- @torch.no_grad()
126
- def interpolate_bits(frames_np, multiplier=2, scale=1.0):
127
- """
128
- Interpolation maintaining Numpy Float 0-1 format.
129
- Args:
130
- frames_np: Numpy Array (Time, Height, Width, Channels) - Float32 [0.0, 1.0]
131
- multiplier: int (2, 4, 8)
132
- Returns:
133
- List of Numpy Arrays (Height, Width, Channels) - Float32 [0.0, 1.0]
134
- """
135
-
136
- # Handle input shape
137
- if isinstance(frames_np, list):
138
- # Convert list of arrays to one big array for easier shape handling if needed,
139
- # but here we just grab dims from first frame
140
- T = len(frames_np)
141
- H, W, C = frames_np[0].shape
142
- else:
143
- T, H, W, C = frames_np.shape
144
-
145
- # 1. No Interpolation Case
146
- if multiplier < 2:
147
- # Just convert 4D array to list of 3D arrays
148
- if isinstance(frames_np, np.ndarray):
149
- return list(frames_np)
150
- return frames_np
151
-
152
- n_interp = multiplier - 1
153
-
154
- # Pre-calc padding for RIFE (requires dimensions divisible by 32/scale)
155
- tmp = max(128, int(128 / scale))
156
- ph = ((H - 1) // tmp + 1) * tmp
157
- pw = ((W - 1) // tmp + 1) * tmp
158
- padding = (0, pw - W, 0, ph - H)
159
-
160
- # Helper: Numpy (H, W, C) Float -> Tensor (1, C, H, W) Half
161
- def to_tensor(frame_np):
162
- # frame_np is float32 0-1
163
- t = torch.from_numpy(frame_np).to(device)
164
- # HWC -> CHW
165
- t = t.permute(2, 0, 1).unsqueeze(0)
166
- return F.pad(t, padding).half()
167
-
168
- # Helper: Tensor (1, C, H, W) Half -> Numpy (H, W, C) Float
169
- def from_tensor(tensor):
170
- # Crop padding
171
- t = tensor[0, :, :H, :W]
172
- # CHW -> HWC
173
- t = t.permute(1, 2, 0)
174
- # Keep as float32, range 0-1
175
- return t.float().cpu().numpy()
176
-
177
- def make_inference(I0, I1, n):
178
- if rife_model.version >= 3.9:
179
- res = []
180
- for i in range(n):
181
- res.append(rife_model.inference(I0, I1, (i+1) * 1. / (n+1), scale))
182
- return res
183
- else:
184
- middle = rife_model.inference(I0, I1, scale)
185
- if n == 1:
186
- return [middle]
187
- first_half = make_inference(I0, middle, n=n//2)
188
- second_half = make_inference(middle, I1, n=n//2)
189
- if n % 2:
190
- return [*first_half, middle, *second_half]
191
- else:
192
- return [*first_half, *second_half]
193
-
194
- output_frames = []
195
-
196
- # Process Frames
197
- # Load first frame into GPU
198
- I1 = to_tensor(frames_np[0])
199
-
200
- total_steps = T - 1
201
-
202
- with tqdm(total=total_steps, desc="Interpolating", unit="frame") as pbar:
203
-
204
- for i in range(total_steps):
205
- I0 = I1
206
- # Add original frame to output
207
- output_frames.append(from_tensor(I0))
208
-
209
- # Load next frame
210
- I1 = to_tensor(frames_np[i+1])
211
-
212
- # Generate intermediate frames
213
- mid_tensors = make_inference(I0, I1, n_interp)
214
-
215
- # Append intermediate frames
216
- for mid in mid_tensors:
217
- output_frames.append(from_tensor(mid))
218
-
219
- if (i + 1) % 50 == 0:
220
- pbar.update(50)
221
- pbar.update(total_steps % 50)
222
-
223
- # Add the very last frame
224
- output_frames.append(from_tensor(I1))
225
-
226
- # Cleanup
227
- del I0, I1, mid_tensors
228
- torch.cuda.empty_cache()
229
-
230
- return output_frames
231
-
232
-
233
- # WAN
234
-
235
- ORG_NAME = "TestOrganizationPleaseIgnore"
236
- # MODEL_ID = "Wan-AI/Wan2.2-I2V-A14B-Diffusers"
237
- MODEL_ID = os.getenv("REPO_ID") or random.choice(
238
- list(list_models(author=ORG_NAME, filter='diffusers:WanImageToVideoPipeline'))
239
- ).modelId
240
- CACHE_DIR = os.path.expanduser("~/.cache/huggingface/")
241
-
242
- LORA_MODELS = [
243
- # {
244
- # "repo_id": "lkzd7/WAN2.2_LoraSet_NSFW",
245
- # "high_tr": "wan2.2_i2v_high_ulitmate_pussy_asshole.safetensors",
246
- # "low_tr": "wan2.2_i2v_low_ulitmate_pussy_asshole.safetensors",
247
- # "high_scale": 0.5,
248
- # "low_scale": 0.5
249
- # },
250
- {
251
- "repo_id": "jsoren/test_lora",
252
- "high_tr": "DR34ML4Y_I2V_14B_HIGH_V2.safetensors",
253
- "low_tr": "DR34ML4Y_I2V_14B_LOW_V2.safetensors",
254
- "high_scale": 1.2,
255
- "low_scale": 0.8
256
- },
257
- ]
258
 
259
  MAX_DIM = 832
260
  MIN_DIM = 480
261
  SQUARE_DIM = 640
262
  MULTIPLE_OF = 16
263
- MAX_SEED = np.iinfo(np.int32).max
264
-
265
- FIXED_FPS = 16
266
- MIN_FRAMES_MODEL = 8
267
- MAX_FRAMES_MODEL = 160
268
-
269
- MIN_DURATION = round(MIN_FRAMES_MODEL / FIXED_FPS, 1)
270
- MAX_DURATION = round(MAX_FRAMES_MODEL / FIXED_FPS, 1)
 
 
 
 
 
 
 
271
 
272
  SCHEDULER_MAP = {
273
  "FlowMatchEulerDiscrete": FlowMatchEulerDiscreteScheduler,
@@ -279,454 +101,969 @@ SCHEDULER_MAP = {
279
  "DPMSolverSinglestep": DPMSolverSinglestepScheduler,
280
  }
281
 
282
- pipe = WanImageToVideoPipeline.from_pretrained(
283
- MODEL_ID,
284
- torch_dtype=torch.bfloat16,
285
- ).to('cuda')
286
- original_scheduler = copy.deepcopy(pipe.scheduler)
287
-
288
- for i, lora in enumerate(LORA_MODELS):
289
- name_high_tr = lora["high_tr"].split(".")[0].split("/")[-1] + "Hh"
290
- name_low_tr = lora["low_tr"].split(".")[0].split("/")[-1] + "Ll"
291
-
292
- try:
293
- pipe.load_lora_weights(
294
- lora["repo_id"],
295
- weight_name=lora["high_tr"],
296
- adapter_name=name_high_tr
297
- )
298
-
299
- kwargs_lora = {"load_into_transformer_2": True}
300
- pipe.load_lora_weights(
301
- lora["repo_id"],
302
- weight_name=lora["low_tr"],
303
- adapter_name=name_low_tr,
304
- **kwargs_lora
305
- )
306
-
307
- pipe.set_adapters([name_high_tr, name_low_tr], adapter_weights=[1.0, 1.0])
308
-
309
- pipe.fuse_lora(adapter_names=[name_high_tr], lora_scale=lora["high_scale"], components=["transformer"])
310
- pipe.fuse_lora(adapter_names=[name_low_tr], lora_scale=lora["low_scale"], components=["transformer_2"])
311
-
312
- pipe.unload_lora_weights()
313
-
314
- print(f"Applied: {lora['high_tr']}, hs={lora['high_scale']}/ls={lora['low_scale']}, {i+1}/{len(LORA_MODELS)}")
315
- except Exception as e:
316
- print("Error:", str(e))
317
- print("Failed LoRA:", name_high_tr)
318
- pipe.unload_lora_weights()
319
-
320
- # if os.path.exists(CACHE_DIR):
321
- # shutil.rmtree(CACHE_DIR)
322
- # print("Deleted Hugging Face cache.")
323
- # else:
324
- # print("No hub cache found.")
325
-
326
- quantize_(pipe.text_encoder, Int8WeightOnlyConfig())
327
- torch._dynamo.reset()
328
- quantize_(pipe.transformer, Float8DynamicActivationFloat8WeightConfig())
329
- torch._dynamo.reset()
330
- quantize_(pipe.transformer_2, Float8DynamicActivationFloat8WeightConfig())
331
- torch._dynamo.reset()
332
-
333
- spaces.aoti_load(
334
- module=pipe.transformer,
335
- repo_id='cbensimon/WanTransformer3DModel-sm120-cu130-raa',
336
  )
337
- spaces.aoti_load(
338
- module=pipe.transformer_2,
339
- repo_id='cbensimon/WanTransformer3DModel-sm120-cu130-raa',
340
  )
341
 
342
- # pipe.vae.enable_slicing()
343
- # pipe.vae.enable_tiling()
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
344
 
345
- default_prompt_i2v = "make this image come alive, cinematic motion, smooth animation"
346
- default_negative_prompt = "色调艳丽, 过曝, 静态, 细节模糊不清, 字幕, 风格, 作品, 画作, 画面, 静止, 整体发灰, 最差质量, 低质量, JPEG压缩残留, 丑陋的, 残缺的, 多余的手指, 画得不好的手部, 画得不好的脸部, 畸形的, 毁容的, 形态畸形的肢体, 手指融合, 静止不动的画面, 杂乱的背景, 三条腿, 背景人很多, 倒着走"
 
347
 
 
 
 
348
 
349
- def model_title():
350
- repo_name = MODEL_ID.split('/')[-1].replace("_", " ")
351
- url = f"https://huggingface.co/{MODEL_ID}"
352
- return f"## This space is currently running [{repo_name}]({url}) 🐢"
353
 
 
 
 
 
354
 
355
- def resize_image(image: Image.Image) -> Image.Image:
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
356
  width, height = image.size
 
357
  if width == height:
358
- return image.resize((SQUARE_DIM, SQUARE_DIM), Image.LANCZOS)
359
-
360
  aspect_ratio = width / height
361
- MAX_ASPECT_RATIO = MAX_DIM / MIN_DIM
362
- MIN_ASPECT_RATIO = MIN_DIM / MAX_DIM
 
363
 
364
- image_to_resize = image
365
- if aspect_ratio > MAX_ASPECT_RATIO:
366
- target_w, target_h = MAX_DIM, MIN_DIM
367
- crop_width = int(round(height * MAX_ASPECT_RATIO))
368
  left = (width - crop_width) // 2
369
- image_to_resize = image.crop((left, 0, left + crop_width, height))
370
- elif aspect_ratio < MIN_ASPECT_RATIO:
371
- target_w, target_h = MIN_DIM, MAX_DIM
372
- crop_height = int(round(width / MIN_ASPECT_RATIO))
373
  top = (height - crop_height) // 2
374
- image_to_resize = image.crop((0, top, width, top + crop_height))
 
 
 
375
  else:
376
- if width > height:
377
- target_w = MAX_DIM
378
- target_h = int(round(target_w / aspect_ratio))
379
- else:
380
- target_h = MAX_DIM
381
- target_w = int(round(target_h * aspect_ratio))
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
382
 
383
- final_w = round(target_w / MULTIPLE_OF) * MULTIPLE_OF
384
- final_h = round(target_h / MULTIPLE_OF) * MULTIPLE_OF
385
- final_w = max(MIN_DIM, min(MAX_DIM, final_w))
386
- final_h = max(MIN_DIM, min(MAX_DIM, final_h))
387
- return image_to_resize.resize((final_w, final_h), Image.LANCZOS)
388
 
 
 
389
 
390
- def resize_and_crop_to_match(target_image, reference_image):
391
- ref_width, ref_height = reference_image.size
392
- target_width, target_height = target_image.size
393
- scale = max(ref_width / target_width, ref_height / target_height)
394
- new_width, new_height = int(target_width * scale), int(target_height * scale)
395
- resized = target_image.resize((new_width, new_height), Image.Resampling.LANCZOS)
396
- left, top = (new_width - ref_width) // 2, (new_height - ref_height) // 2
397
- return resized.crop((left, top, left + ref_width, top + ref_height))
398
 
 
 
 
 
 
 
 
399
 
400
- def get_num_frames(duration_seconds: float):
401
- return 1 + int(np.clip(
402
- int(round(duration_seconds * FIXED_FPS)),
403
- MIN_FRAMES_MODEL,
404
- MAX_FRAMES_MODEL,
405
- ))
406
 
407
 
408
  def get_inference_duration(
409
- resized_image,
410
- processed_last_image,
411
- prompt,
412
- steps,
413
- negative_prompt,
414
- num_frames,
415
- guidance_scale,
416
- guidance_scale_2,
417
- current_seed,
418
- scheduler_name,
419
- flow_shift,
420
- frame_multiplier,
421
- quality,
422
- duration_seconds,
423
- safe_mode,
424
- progress
425
- ):
426
- BASE_FRAMES_HEIGHT_WIDTH = 81 * 832 * 624
427
- BASE_STEP_DURATION = 5.
428
- width, height = resized_image.size
429
- factor = num_frames * width * height / BASE_FRAMES_HEIGHT_WIDTH
430
- step_duration = BASE_STEP_DURATION * factor ** 1.5
431
- gen_time = int(steps) * step_duration
432
-
433
- if guidance_scale > 1:
434
- gen_time = gen_time * 2.4
435
-
436
- frame_factor = frame_multiplier // FIXED_FPS
437
- if frame_factor > 1:
438
- total_out_frames = (num_frames * frame_factor) - num_frames
439
- inter_time = (total_out_frames * 0.02)
440
- gen_time += inter_time
441
-
442
- total_time = 15 + gen_time
443
  if safe_mode:
444
- total_time = total_time * 1.30
445
 
446
- return total_time
447
 
448
 
449
- @spaces.GPU(duration=get_inference_duration, size='xlarge')
450
  def run_inference(
451
- resized_image,
452
- processed_last_image,
453
- prompt,
454
- steps,
455
- negative_prompt,
456
- num_frames,
457
- guidance_scale,
458
- guidance_scale_2,
459
- current_seed,
460
- scheduler_name,
461
- flow_shift,
462
- frame_multiplier,
463
- quality,
464
- duration_seconds,
465
- safe_mode=False,
466
  progress=gr.Progress(track_tqdm=True),
467
- ):
468
- scheduler_class = SCHEDULER_MAP.get(scheduler_name)
469
- if scheduler_class.__name__ != pipe.scheduler.config._class_name or flow_shift != pipe.scheduler.config.get("flow_shift", "shift"):
470
- config = copy.deepcopy(original_scheduler.config)
471
- if scheduler_class == FlowMatchEulerDiscreteScheduler:
472
- config['shift'] = flow_shift
473
- else:
474
- config['flow_shift'] = flow_shift
475
- pipe.scheduler = scheduler_class.from_config(config)
476
-
477
  clear_vram()
 
478
 
479
- task_name = str(uuid.uuid4())[:8]
480
- print(f"Generating {num_frames} frames, task: {task_name}, {duration_seconds}, {resized_image.size}")
481
- start = time.time()
482
- result = pipe(
483
- image=resized_image,
484
- last_image=processed_last_image,
485
- prompt=prompt,
486
- negative_prompt=negative_prompt,
487
- height=resized_image.height,
488
- width=resized_image.width,
489
- num_frames=num_frames,
490
- guidance_scale=float(guidance_scale),
491
- guidance_scale_2=float(guidance_scale_2),
492
- num_inference_steps=int(steps),
493
- generator=torch.Generator(device="cuda").manual_seed(current_seed),
494
- output_type="np"
495
  )
496
- print("gen time passed:", time.time() - start)
497
-
498
- raw_frames_np = result.frames[0] # Returns (T, H, W, C) float32
499
- pipe.scheduler = original_scheduler
500
-
501
- frame_factor = frame_multiplier // FIXED_FPS
502
- if frame_factor > 1:
503
- start = time.time()
504
- print(f"Processing frames (RIFE Multiplier: {frame_factor}x)...")
505
- rife_model.device()
506
- rife_model.flownet = rife_model.flownet.half()
507
- final_frames = interpolate_bits(raw_frames_np, multiplier=int(frame_factor))
508
- print("Interpolation time passed:", time.time() - start)
509
- else:
510
- final_frames = list(raw_frames_np)
511
 
512
- final_fps = FIXED_FPS * int(frame_factor)
 
 
 
 
 
 
 
 
 
 
 
 
 
 
513
 
514
- with tempfile.NamedTemporaryFile(suffix=".mp4", delete=False) as tmpfile:
515
- video_path = tmpfile.name
 
 
 
 
 
516
 
517
- start = time.time()
518
- with tqdm(total=3, desc="Rendering Media", unit="clip") as pbar:
519
- pbar.update(2)
520
- export_to_video(final_frames, video_path, fps=final_fps, quality=quality)
521
- pbar.update(1)
522
- print(f"Export time passed, {final_fps} FPS:", time.time() - start)
523
 
524
- return video_path, task_name
 
 
 
 
 
 
 
 
 
 
 
525
 
526
 
527
  def generate_video(
528
- input_image,
529
- last_image,
530
- prompt,
531
- steps=4,
532
- negative_prompt=default_negative_prompt,
533
- duration_seconds=MAX_DURATION,
534
- guidance_scale=1,
535
- guidance_scale_2=1,
536
- seed=42,
537
- randomize_seed=False,
538
- quality=5,
539
- scheduler="UniPCMultistep",
540
- flow_shift=6.0,
541
- frame_multiplier=16,
542
- safe_mode=False,
543
- video_component=True,
 
 
 
 
 
 
 
 
544
  progress=gr.Progress(track_tqdm=True),
545
  ):
546
- """
547
- Generate a video from an input image using the Wan 2.2 14B I2V model with Lightning LoRA.
548
- This function takes an input image and generates a video animation based on the provided
549
- prompt and parameters. It uses an FP8 qunatized Wan 2.2 14B Image-to-Video model in with Lightning LoRA
550
- for fast generation in 4-8 steps.
551
- Args:
552
- input_image (PIL.Image): The input image to animate. Will be resized to target dimensions.
553
- last_image (PIL.Image, optional): The optional last image for the video.
554
- prompt (str): Text prompt describing the desired animation or motion.
555
- steps (int, optional): Number of inference steps. More steps = higher quality but slower.
556
- Defaults to 4. Range: 1-30.
557
- negative_prompt (str, optional): Negative prompt to avoid unwanted elements.
558
- Defaults to default_negative_prompt (contains unwanted visual artifacts).
559
- duration_seconds (float, optional): Duration of the generated video in seconds.
560
- Defaults to 2. Clamped between MIN_FRAMES_MODEL/FIXED_FPS and MAX_FRAMES_MODEL/FIXED_FPS.
561
- guidance_scale (float, optional): Controls adherence to the prompt. Higher values = more adherence.
562
- Defaults to 1.0. Range: 0.0-20.0.
563
- guidance_scale_2 (float, optional): Controls adherence to the prompt. Higher values = more adherence.
564
- Defaults to 1.0. Range: 0.0-20.0.
565
- seed (int, optional): Random seed for reproducible results. Defaults to 42.
566
- Range: 0 to MAX_SEED (2147483647).
567
- randomize_seed (bool, optional): Whether to use a random seed instead of the provided seed.
568
- Defaults to False.
569
- quality (float, optional): Video output quality. Default is 5. Uses variable bit rate.
570
- Highest quality is 10, lowest is 1.
571
- scheduler (str, optional): The name of the scheduler to use for inference. Defaults to "UniPCMultistep".
572
- flow_shift (float, optional): The flow shift value for compatible schedulers. Defaults to 6.0.
573
- frame_multiplier (int, optional): The int value for fps enhancer
574
- video_component(bool, optional): Show video player in output.
575
- Defaults to True.
576
- progress (gr.Progress, optional): Gradio progress tracker. Defaults to gr.Progress(track_tqdm=True).
577
- Returns:
578
- tuple: A tuple containing:
579
- - video_path (str): Path for the video component.
580
- - video_path (str): Path for the file download component. Attempt to avoid reconversion in video component.
581
- - current_seed (int): The seed used for generation.
582
- Raises:
583
- gr.Error: If input_image is None (no image uploaded).
584
- Note:
585
- - Frame count is calculated as duration_seconds * FIXED_FPS (24)
586
- - Output dimensions are adjusted to be multiples of MOD_VALUE (32)
587
- - The function uses GPU acceleration via the @spaces.GPU decorator
588
- - Generation time varies based on steps and duration (see get_duration function)
589
- """
590
-
591
- if input_image is None:
592
- raise gr.Error("Please upload an input image.")
593
 
594
  num_frames = get_num_frames(duration_seconds)
595
  current_seed = random.randint(0, MAX_SEED) if randomize_seed else int(seed)
596
- resized_image = resize_image(input_image)
597
 
598
- processed_last_image = None
599
- if last_image:
600
- processed_last_image = resize_and_crop_to_match(last_image, resized_image)
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
601
 
602
- video_path, task_n = run_inference(
603
- resized_image,
604
- processed_last_image,
605
- prompt,
606
- steps,
607
- negative_prompt,
608
- num_frames,
609
- guidance_scale,
610
- guidance_scale_2,
611
  current_seed,
612
- scheduler,
613
- flow_shift,
614
- frame_multiplier,
615
- quality,
616
- duration_seconds,
617
- safe_mode,
618
- progress,
619
  )
620
- print(f"GPU complete: {task_n}")
621
 
622
- return (video_path if video_component else None), video_path, current_seed
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
623
 
624
 
625
  CSS = """
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
626
  #hidden-timestamp {
627
  opacity: 0;
628
- height: 0px;
629
- width: 0px;
630
- margin: 0px;
631
- padding: 0px;
632
  overflow: hidden;
633
  position: absolute;
634
  pointer-events: none;
635
  }
 
 
 
 
 
 
636
  """
637
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
638
 
639
- with gr.Blocks(delete_cache=(3600, 10800)) as demo:
640
- gr.Markdown(model_title())
641
- gr.Markdown("Run Wan 2.2 in just 4-8 steps, fp8 quantization & AoT compilation - compatible with 🧨 diffusers and ZeroGPU")
642
-
643
- with gr.Row():
644
- with gr.Column():
645
- input_image_component = gr.Image(type="pil", label="Input Image", sources=["upload", "clipboard"])
646
- prompt_input = gr.Textbox(label="Prompt", value=default_prompt_i2v)
647
- duration_seconds_input = gr.Slider(minimum=MIN_DURATION, maximum=MAX_DURATION, step=0.1, value=3.5, label="Duration (seconds)", info=f"Clamped to model's {MIN_FRAMES_MODEL}-{MAX_FRAMES_MODEL} frames at {FIXED_FPS}fps.")
648
- frame_multi = gr.Dropdown(
649
- choices=[FIXED_FPS, FIXED_FPS*2, FIXED_FPS*4, FIXED_FPS*8],
650
- value=FIXED_FPS,
651
- label="Video Fluidity (Frames per Second)",
652
- info="Extra frames will be generated using flow estimation, which estimates motion between frames to make the video smoother."
653
  )
654
- safe_mode_checkbox = gr.Checkbox(
655
- label="🛠️ Safe Mode",
656
- value=True,
657
- info="Requests 30% extra processing time to try to prevent unfinished tasks when the server is busy."
 
658
  )
659
- with gr.Accordion("Advanced Settings", open=False):
660
- last_image_component = gr.Image(type="pil", label="Last Image (Optional)", sources=["upload", "clipboard"])
661
- negative_prompt_input = gr.Textbox(label="Negative Prompt", value=default_negative_prompt, info="Used if any Guidance Scale > 1.", lines=3)
662
- quality_slider = gr.Slider(minimum=1, maximum=10, step=1, value=6, label="Video Quality", info="If set to 10, the generated video may be too large and won't play in the Gradio preview.")
663
- seed_input = gr.Slider(label="Seed", minimum=0, maximum=MAX_SEED, step=1, value=42, interactive=True)
664
- randomize_seed_checkbox = gr.Checkbox(label="Randomize seed", value=True, interactive=True)
665
- steps_slider = gr.Slider(minimum=1, maximum=30, step=1, value=6, label="Inference Steps")
666
- guidance_scale_input = gr.Slider(minimum=0.0, maximum=10.0, step=0.5, value=1, label="Guidance Scale - high noise stage", info="Values above 1 increase GPU usage and may take longer to process.")
667
- guidance_scale_2_input = gr.Slider(minimum=0.0, maximum=10.0, step=0.5, value=1, label="Guidance Scale 2 - low noise stage")
668
- scheduler_dropdown = gr.Dropdown(
669
- label="Scheduler",
670
- choices=list(SCHEDULER_MAP.keys()),
671
- value="UniPCMultistep",
672
- info="Select a custom scheduler."
673
  )
674
- flow_shift_slider = gr.Slider(minimum=0.5, maximum=15.0, step=0.1, value=3.0, label="Flow Shift")
675
- play_result_video = gr.Checkbox(label="Display result", value=True, interactive=True)
676
- gr.Markdown(f"[ZeroGPU help, tips and troubleshooting](https://huggingface.co/datasets/{ORG_NAME}/help/blob/main/gpu_help.md)")
677
- gr.Markdown( # TestOrganizationPleaseIgnore/wamu-tools
678
- "To use a different model, **duplicate this Space** first, then change the `REPO_ID` environment variable. "
679
- "[See compatible models here](https://huggingface.co/models?other=diffusers:WanImageToVideoPipeline&sort=trending&search=WAN2.2_I2V_LIGHTNING)."
680
  )
681
 
682
- generate_button = gr.Button("Generate Video", variant="primary")
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
683
 
684
- with gr.Column():
685
- # ASSIGNED elem_id="generated-video" so JS can find it
686
- video_output = gr.Video(label="Generated Video", autoplay=True, sources=["upload"], buttons=["download", "share"], interactive=True, elem_id="generated-video")
687
-
688
- # --- Frame Grabbing UI ---
689
  with gr.Row():
690
- grab_frame_btn = gr.Button("📸 Use Current Frame as Input", variant="secondary")
691
- timestamp_box = gr.Number(value=0, label="Timestamp", visible=True, elem_id="hidden-timestamp")
692
- # -------------------------
693
-
694
- file_output = gr.File(label="Download Video")
695
-
696
- ui_inputs = [
697
- input_image_component, last_image_component, prompt_input, steps_slider,
698
- negative_prompt_input, duration_seconds_input,
699
- guidance_scale_input, guidance_scale_2_input, seed_input, randomize_seed_checkbox,
700
- quality_slider, scheduler_dropdown, flow_shift_slider, frame_multi,
701
- safe_mode_checkbox,
702
- play_result_video
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
703
  ]
704
-
705
  generate_button.click(
706
- fn=generate_video,
707
- inputs=ui_inputs,
708
- outputs=[video_output, file_output, seed_input]
 
 
 
 
 
 
709
  )
710
-
711
- # --- Frame Grabbing Events ---
712
- # 1. Click button -> JS runs -> puts time in hidden number box
713
- grab_frame_btn.click(
714
  fn=None,
715
  inputs=None,
716
  outputs=[timestamp_box],
717
- js=get_timestamp_js
718
  )
719
-
720
- # 2. Hidden number box changes -> Python runs -> puts frame in Input Image
721
  timestamp_box.change(
722
  fn=extract_frame,
723
  inputs=[video_output, timestamp_box],
724
- outputs=[input_image_component]
725
  )
726
 
727
  if __name__ == "__main__":
728
- demo.queue().launch(
729
  mcp_server=True,
730
- css=CSS,
731
  show_error=True,
732
- )
 
1
+ from __future__ import annotations
2
+
 
 
 
3
  import copy
4
+ import gc
5
+ import os
6
  import random
7
+ import sys
8
  import tempfile
 
9
  import time
 
10
  import uuid
11
+ import warnings
12
+ import zipfile
13
+ from pathlib import Path
14
+ from typing import Any
15
+
16
+ os.environ.setdefault("TOKENIZERS_PARALLELISM", "false")
17
+ os.environ.setdefault("PYTORCH_CUDA_ALLOC_CONF", "expandable_segments:True")
18
+
19
  import cv2
20
+ import gradio as gr
21
  import numpy as np
22
+ import spaces
23
  import torch
24
  import torch._dynamo
25
+ from huggingface_hub import hf_hub_download
26
+ from PIL import Image, ImageOps
27
  from torch.nn import functional as F
28
+ from torchao.quantization import (
29
+ Float8DynamicActivationFloat8WeightConfig,
30
+ Int8WeightOnlyConfig,
31
+ quantize_,
32
+ )
33
+ from tqdm import tqdm
34
 
 
35
  from diffusers import (
 
 
36
  DEISMultistepScheduler,
37
  DPMSolverMultistepInverseScheduler,
 
38
  DPMSolverMultistepScheduler,
39
  DPMSolverSinglestepScheduler,
40
+ FlowMatchEulerDiscreteScheduler,
41
+ SASolverScheduler,
42
+ UniPCMultistepScheduler,
43
  )
44
  from diffusers.pipelines.wan.pipeline_wan_i2v import WanImageToVideoPipeline
45
  from diffusers.utils.export_utils import export_to_video
46
 
 
 
 
 
47
  warnings.filterwarnings("ignore")
 
 
 
 
 
48
 
49
+ # -----------------------------------------------------------------------------
50
+ # Configuration
51
+ # -----------------------------------------------------------------------------
52
 
53
+ DEFAULT_MODEL_ID = (
54
+ "TestOrganizationPleaseIgnore/"
55
+ "WAMU-Merge-MotionBoost_WAN2.2_I2V_LIGHTNING"
56
+ )
57
+ MODEL_ID = os.getenv("REPO_ID", DEFAULT_MODEL_ID).strip() or DEFAULT_MODEL_ID
58
+ AOTI_REPO = os.getenv(
59
+ "AOTI_REPO", "cbensimon/WanTransformer3DModel-sm120-cu130-raa"
60
+ ).strip()
61
+ ENABLE_AOTI = os.getenv("ENABLE_AOTI", "true").lower() not in {"0", "false", "no"}
62
+ ENABLE_VAE_TILING = os.getenv("ENABLE_VAE_TILING", "false").lower() in {
63
+ "1",
64
+ "true",
65
+ "yes",
66
  }
67
+ RUNPOD_URL = os.getenv("RUNPOD_URL", "").strip()
68
+ SUPPORT_URL = os.getenv("SUPPORT_URL", "").strip()
69
+ MAX_GPU_SECONDS = int(os.getenv("MAX_GPU_SECONDS", "300"))
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
70
 
71
+ IS_ZERO_GPU = bool(os.getenv("SPACES_ZERO_GPU"))
72
+ DEVICE = torch.device("cuda" if torch.cuda.is_available() else "cpu")
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
73
 
74
  MAX_DIM = 832
75
  MIN_DIM = 480
76
  SQUARE_DIM = 640
77
  MULTIPLE_OF = 16
78
+ BASE_FPS = 16
79
+ MIN_MODEL_FRAMES = 9
80
+ MAX_MODEL_FRAMES = 161
81
+ MIN_DURATION = round((MIN_MODEL_FRAMES - 1) / BASE_FPS, 1)
82
+ MAX_DURATION = round((MAX_MODEL_FRAMES - 1) / BASE_FPS, 1)
83
+ MAX_SEED = int(np.iinfo(np.int32).max)
84
+
85
+ ASPECT_DIMENSIONS: dict[str, tuple[int, int] | None] = {
86
+ "Auto (keep source ratio)": None,
87
+ "Landscape 16:9": (832, 480),
88
+ "Portrait 9:16": (480, 832),
89
+ "Square 1:1": (640, 640),
90
+ "Portrait 4:5": (512, 640),
91
+ "Landscape 3:2": (768, 512),
92
+ }
93
 
94
  SCHEDULER_MAP = {
95
  "FlowMatchEulerDiscrete": FlowMatchEulerDiscreteScheduler,
 
101
  "DPMSolverSinglestep": DPMSolverSinglestepScheduler,
102
  }
103
 
104
+ QUALITY_PROFILES = {
105
+ "Fast Preview": {"steps": 4, "quality": 5, "flow_shift": 3.0},
106
+ "Balanced": {"steps": 6, "quality": 7, "flow_shift": 3.0},
107
+ "High Quality": {"steps": 8, "quality": 8, "flow_shift": 3.5},
108
+ }
109
+
110
+ MOTION_STYLE_PROMPTS = {
111
+ "Natural": "natural realistic movement, physically plausible motion, stable composition",
112
+ "Cinematic": "cinematic motion, elegant pacing, dramatic depth, premium film look",
113
+ "Portrait": "subtle facial expression, natural blinking, gentle head movement, stable facial identity",
114
+ "Product Ad": "premium product commercial, controlled motion, clean composition, polished advertising look",
115
+ "Dynamic": "energetic movement, stronger motion, dramatic action, coherent fast pacing",
116
+ "Anime": "fluid anime-style animation, clean line consistency, expressive but stable movement",
117
+ }
118
+
119
+ CAMERA_PROMPTS = {
120
+ "Static": "locked-off camera, stable framing",
121
+ "Slow Push In": "slow smooth camera push-in",
122
+ "Slow Pull Back": "slow smooth camera pull-back",
123
+ "Pan Left": "slow cinematic camera pan to the left",
124
+ "Pan Right": "slow cinematic camera pan to the right",
125
+ "Orbit": "smooth controlled camera orbit around the subject",
126
+ "Handheld": "subtle realistic handheld camera movement",
127
+ }
128
+
129
+ MOTION_LEVEL_PROMPTS = {
130
+ "Low": "minimal subtle motion, preserve the original composition",
131
+ "Medium": "moderate natural motion with stable details",
132
+ "High": "strong visible motion while keeping the subject coherent",
133
+ }
134
+
135
+ DEFAULT_PROMPT = "Make this image come alive with smooth, realistic motion."
136
+ DEFAULT_NEGATIVE_PROMPT = (
137
+ "overexposed, oversaturated, static frame, blurry details, low quality, worst quality, "
138
+ "jpeg artifacts, text, subtitles, watermark, logo, deformed face, identity drift, "
139
+ "bad anatomy, malformed limbs, extra fingers, fused fingers, duplicated person, "
140
+ "warped body, flicker, jitter, unstable background, abrupt camera movement, reverse walking, "
141
+ "色调艳丽, 过曝, 静态, 细节模糊不清, 字幕, 最差质量, 低质量, 多余的手指, "
142
+ "画得不好的手部, 画得不好的脸部, 畸形的, 静止不动的画面, 杂乱的背景"
143
+ )
144
+
145
+ IDENTITY_PROMPT = (
146
+ "preserve the exact facial identity, facial structure, hairstyle, skin tone, clothing, "
147
+ "body proportions and distinguishing features throughout the video"
 
 
 
 
 
 
 
 
 
 
148
  )
149
+ IDENTITY_NEGATIVE = (
150
+ "face morphing, changing identity, changing hairstyle, changing clothes, distorted eyes, "
151
+ "asymmetric face, duplicated facial features"
152
  )
153
 
154
+ # -----------------------------------------------------------------------------
155
+ # Model loading: Wan 2.2 + FP8 + AOTI
156
+ # -----------------------------------------------------------------------------
157
+
158
+ if torch.cuda.is_available():
159
+ torch.backends.cuda.matmul.allow_tf32 = True
160
+ torch.set_float32_matmul_precision("high")
161
+
162
+
163
+ def _load_aoti_component(module: torch.nn.Module, repo_id: str) -> None:
164
+ """Load an AOTI package while remaining compatible with different spaces builds."""
165
+ if hasattr(spaces, "aoti_load"):
166
+ spaces.aoti_load(module=module, repo_id=repo_id)
167
+ return
168
+
169
+ from aoti import aoti_blocks_load
170
+
171
+ aoti_blocks_load(module=module, repo_id=repo_id)
172
+
173
+
174
+ def load_pipeline() -> WanImageToVideoPipeline:
175
+ print(f"Loading model: {MODEL_ID}")
176
+ pipeline = WanImageToVideoPipeline.from_pretrained(
177
+ MODEL_ID,
178
+ torch_dtype=torch.bfloat16,
179
+ ).to("cuda")
180
+
181
+ # The selected WAMU repository is already a merged Lightning/MotionBoost model.
182
+ # Do not fuse a second Lightning LoRA here; that can reduce quality and consistency.
183
+ print("Quantizing text encoder to INT8...")
184
+ quantize_(pipeline.text_encoder, Int8WeightOnlyConfig())
185
+ torch._dynamo.reset()
186
+
187
+ print("Quantizing high-noise transformer to FP8...")
188
+ quantize_(pipeline.transformer, Float8DynamicActivationFloat8WeightConfig())
189
+ torch._dynamo.reset()
190
+
191
+ print("Quantizing low-noise transformer to FP8...")
192
+ quantize_(pipeline.transformer_2, Float8DynamicActivationFloat8WeightConfig())
193
+ torch._dynamo.reset()
194
+
195
+ if ENABLE_AOTI:
196
+ print(f"Loading AOTI packages from: {AOTI_REPO}")
197
+ _load_aoti_component(pipeline.transformer, AOTI_REPO)
198
+ _load_aoti_component(pipeline.transformer_2, AOTI_REPO)
199
+
200
+ if ENABLE_VAE_TILING:
201
+ try:
202
+ pipeline.vae.enable_slicing()
203
+ pipeline.vae.enable_tiling()
204
+ print("VAE slicing and tiling enabled.")
205
+ except Exception as exc:
206
+ print(f"VAE tiling could not be enabled: {exc}")
207
+
208
+ return pipeline
209
+
210
+
211
+ pipe = load_pipeline()
212
+ ORIGINAL_SCHEDULER = copy.deepcopy(pipe.scheduler)
213
 
214
+ # -----------------------------------------------------------------------------
215
+ # Optional RIFE interpolation, loaded lazily only when 32/64 FPS is requested
216
+ # -----------------------------------------------------------------------------
217
 
218
+ RIFE_REPO = "r3gm/RIFE"
219
+ RIFE_FILENAME = "RIFEv4.26_0921.zip"
220
+ RIFE_MODEL: Any | None = None
221
 
 
 
 
 
222
 
223
+ def ensure_rife_model() -> Any:
224
+ global RIFE_MODEL
225
+ if RIFE_MODEL is not None:
226
+ return RIFE_MODEL
227
 
228
+ if not Path("train_log/RIFE_HDv3.py").exists():
229
+ print("Downloading RIFE frame-interpolation model...")
230
+ archive = hf_hub_download(repo_id=RIFE_REPO, filename=RIFE_FILENAME)
231
+ with zipfile.ZipFile(archive, "r") as zip_ref:
232
+ zip_ref.extractall(".")
233
+
234
+ if os.getcwd() not in sys.path:
235
+ sys.path.insert(0, os.getcwd())
236
+
237
+ from train_log.RIFE_HDv3 import Model
238
+
239
+ model = Model()
240
+ model.load_model("train_log", -1)
241
+ model.eval()
242
+ RIFE_MODEL = model
243
+ return RIFE_MODEL
244
+
245
+
246
+ @torch.no_grad()
247
+ def interpolate_frames(
248
+ frames_np: np.ndarray | list[np.ndarray],
249
+ multiplier: int = 2,
250
+ scale: float = 1.0,
251
+ ) -> list[np.ndarray]:
252
+ if multiplier < 2:
253
+ return list(frames_np)
254
+
255
+ model = ensure_rife_model()
256
+ model.device()
257
+ model.flownet = model.flownet.half()
258
+
259
+ if isinstance(frames_np, list):
260
+ total_frames = len(frames_np)
261
+ height, width, _ = frames_np[0].shape
262
+ else:
263
+ total_frames, height, width, _ = frames_np.shape
264
+
265
+ interpolation_count = multiplier - 1
266
+ block = max(128, int(128 / scale))
267
+ padded_height = ((height - 1) // block + 1) * block
268
+ padded_width = ((width - 1) // block + 1) * block
269
+ padding = (0, padded_width - width, 0, padded_height - height)
270
+
271
+ def to_tensor(frame: np.ndarray) -> torch.Tensor:
272
+ tensor = torch.from_numpy(frame).to(DEVICE)
273
+ tensor = tensor.permute(2, 0, 1).unsqueeze(0)
274
+ return F.pad(tensor, padding).half()
275
+
276
+ def to_numpy(tensor: torch.Tensor) -> np.ndarray:
277
+ tensor = tensor[0, :, :height, :width].permute(1, 2, 0)
278
+ return tensor.float().cpu().numpy()
279
+
280
+ def infer_between(first: torch.Tensor, second: torch.Tensor, count: int) -> list[torch.Tensor]:
281
+ if model.version >= 3.9:
282
+ return [
283
+ model.inference(first, second, (index + 1) / (count + 1), scale)
284
+ for index in range(count)
285
+ ]
286
+
287
+ middle = model.inference(first, second, scale)
288
+ if count == 1:
289
+ return [middle]
290
+ first_half = infer_between(first, middle, count // 2)
291
+ second_half = infer_between(middle, second, count // 2)
292
+ if count % 2:
293
+ return [*first_half, middle, *second_half]
294
+ return [*first_half, *second_half]
295
+
296
+ output: list[np.ndarray] = []
297
+ next_tensor = to_tensor(frames_np[0])
298
+
299
+ with tqdm(total=total_frames - 1, desc="Frame interpolation", unit="frame") as bar:
300
+ for index in range(total_frames - 1):
301
+ current_tensor = next_tensor
302
+ output.append(to_numpy(current_tensor))
303
+ next_tensor = to_tensor(frames_np[index + 1])
304
+ for middle_tensor in infer_between(
305
+ current_tensor, next_tensor, interpolation_count
306
+ ):
307
+ output.append(to_numpy(middle_tensor))
308
+ bar.update(1)
309
+
310
+ output.append(to_numpy(next_tensor))
311
+
312
+ try:
313
+ model.flownet.to("cpu")
314
+ except Exception:
315
+ pass
316
+ clear_vram()
317
+ return output
318
+
319
+ # -----------------------------------------------------------------------------
320
+ # Image, prompt and scheduler helpers
321
+ # -----------------------------------------------------------------------------
322
+
323
+
324
+ def clear_vram() -> None:
325
+ gc.collect()
326
+ if torch.cuda.is_available():
327
+ torch.cuda.empty_cache()
328
+
329
+
330
+ def normalize_image(image: Image.Image) -> Image.Image:
331
+ return ImageOps.exif_transpose(image).convert("RGB")
332
+
333
+
334
+ def center_crop_resize(image: Image.Image, size: tuple[int, int]) -> Image.Image:
335
+ image = normalize_image(image)
336
+ target_width, target_height = size
337
+ width, height = image.size
338
+ scale = max(target_width / width, target_height / height)
339
+ resized_width = max(target_width, round(width * scale))
340
+ resized_height = max(target_height, round(height * scale))
341
+ image = image.resize((resized_width, resized_height), Image.Resampling.LANCZOS)
342
+ left = (resized_width - target_width) // 2
343
+ top = (resized_height - target_height) // 2
344
+ return image.crop((left, top, left + target_width, top + target_height))
345
+
346
+
347
+ def resize_auto(image: Image.Image) -> Image.Image:
348
+ image = normalize_image(image)
349
  width, height = image.size
350
+
351
  if width == height:
352
+ return image.resize((SQUARE_DIM, SQUARE_DIM), Image.Resampling.LANCZOS)
353
+
354
  aspect_ratio = width / height
355
+ maximum_ratio = MAX_DIM / MIN_DIM
356
+ minimum_ratio = MIN_DIM / MAX_DIM
357
+ source = image
358
 
359
+ if aspect_ratio > maximum_ratio:
360
+ target_width, target_height = MAX_DIM, MIN_DIM
361
+ crop_width = round(height * maximum_ratio)
 
362
  left = (width - crop_width) // 2
363
+ source = image.crop((left, 0, left + crop_width, height))
364
+ elif aspect_ratio < minimum_ratio:
365
+ target_width, target_height = MIN_DIM, MAX_DIM
366
+ crop_height = round(width / minimum_ratio)
367
  top = (height - crop_height) // 2
368
+ source = image.crop((0, top, width, top + crop_height))
369
+ elif width > height:
370
+ target_width = MAX_DIM
371
+ target_height = round(target_width / aspect_ratio)
372
  else:
373
+ target_height = MAX_DIM
374
+ target_width = round(target_height * aspect_ratio)
375
+
376
+ final_width = round(target_width / MULTIPLE_OF) * MULTIPLE_OF
377
+ final_height = round(target_height / MULTIPLE_OF) * MULTIPLE_OF
378
+ final_width = max(MIN_DIM, min(MAX_DIM, final_width))
379
+ final_height = max(MIN_DIM, min(MAX_DIM, final_height))
380
+ return source.resize((final_width, final_height), Image.Resampling.LANCZOS)
381
+
382
+
383
+ def prepare_first_frame(image: Image.Image, aspect_ratio: str) -> Image.Image:
384
+ dimensions = ASPECT_DIMENSIONS.get(aspect_ratio)
385
+ if dimensions is None:
386
+ return resize_auto(image)
387
+ return center_crop_resize(image, dimensions)
388
+
389
+
390
+ def prepare_last_frame(image: Image.Image, reference: Image.Image) -> Image.Image:
391
+ return center_crop_resize(image, reference.size)
392
+
393
+
394
+ def get_num_frames(duration_seconds: float) -> int:
395
+ target = int(round(float(duration_seconds) * BASE_FPS)) + 1
396
+ # Wan video frame counts are most reliable as 4n + 1.
397
+ target = 4 * round((target - 1) / 4) + 1
398
+ return int(np.clip(target, MIN_MODEL_FRAMES, MAX_MODEL_FRAMES))
399
+
400
+
401
+ def build_prompts(
402
+ prompt: str,
403
+ motion_style: str,
404
+ camera_motion: str,
405
+ motion_strength: str,
406
+ enhance_prompt: bool,
407
+ preserve_identity: bool,
408
+ negative_prompt: str,
409
+ ) -> tuple[str, str]:
410
+ prompt = (prompt or "").strip()
411
+ if not prompt:
412
+ prompt = DEFAULT_PROMPT
413
+
414
+ additions: list[str] = []
415
+ if enhance_prompt:
416
+ additions.extend(
417
+ [
418
+ MOTION_STYLE_PROMPTS.get(motion_style, ""),
419
+ CAMERA_PROMPTS.get(camera_motion, ""),
420
+ MOTION_LEVEL_PROMPTS.get(motion_strength, ""),
421
+ "smooth temporal consistency, coherent details, natural motion blur, no flicker",
422
+ ]
423
+ )
424
+ if preserve_identity:
425
+ additions.append(IDENTITY_PROMPT)
426
+
427
+ final_prompt = ", ".join(part for part in [prompt, *additions] if part)
428
+ final_negative = (negative_prompt or DEFAULT_NEGATIVE_PROMPT).strip()
429
+ if preserve_identity:
430
+ final_negative = f"{final_negative}, {IDENTITY_NEGATIVE}"
431
+ return final_prompt, final_negative
432
+
433
+
434
+ def configure_scheduler(name: str, flow_shift: float) -> None:
435
+ scheduler_class = SCHEDULER_MAP.get(name, UniPCMultistepScheduler)
436
+ current_name = pipe.scheduler.config.get("_class_name", pipe.scheduler.__class__.__name__)
437
+ current_shift = pipe.scheduler.config.get(
438
+ "flow_shift", pipe.scheduler.config.get("shift", None)
439
+ )
440
+
441
+ if current_name == scheduler_class.__name__ and current_shift == flow_shift:
442
+ return
443
+
444
+ config = copy.deepcopy(ORIGINAL_SCHEDULER.config)
445
+ if scheduler_class is FlowMatchEulerDiscreteScheduler:
446
+ config["shift"] = float(flow_shift)
447
+ else:
448
+ config["flow_shift"] = float(flow_shift)
449
+ pipe.scheduler = scheduler_class.from_config(config)
450
+
451
+
452
+ def profile_settings(profile: str):
453
+ values = QUALITY_PROFILES.get(profile, QUALITY_PROFILES["Balanced"])
454
+ return (
455
+ gr.update(value=values["steps"]),
456
+ gr.update(value=values["quality"]),
457
+ gr.update(value=values["flow_shift"]),
458
+ )
459
 
 
 
 
 
 
460
 
461
+ def swap_images(first: Image.Image | None, last: Image.Image | None):
462
+ return last, first
463
 
 
 
 
 
 
 
 
 
464
 
465
+ def format_external_links() -> str:
466
+ links: list[str] = []
467
+ if RUNPOD_URL:
468
+ links.append(f"[⚡ Run on Private GPU via RunPod]({RUNPOD_URL})")
469
+ if SUPPORT_URL:
470
+ links.append(f"[❤️ Support This Free Tool]({SUPPORT_URL})")
471
+ return " &nbsp; • &nbsp; ".join(links)
472
 
473
+ # -----------------------------------------------------------------------------
474
+ # GPU generation
475
+ # -----------------------------------------------------------------------------
 
 
 
476
 
477
 
478
  def get_inference_duration(
479
+ first_frame: Image.Image,
480
+ last_frame: Image.Image | None,
481
+ prompt: str,
482
+ negative_prompt: str,
483
+ num_frames: int,
484
+ steps: int,
485
+ guidance_scale: float,
486
+ guidance_scale_2: float,
487
+ seed: int,
488
+ scheduler_name: str,
489
+ flow_shift: float,
490
+ output_fps: int,
491
+ video_quality: int,
492
+ safe_mode: bool,
493
+ progress,
494
+ ) -> int:
495
+ del last_frame, prompt, negative_prompt, guidance_scale_2, seed
496
+ del scheduler_name, flow_shift, video_quality, progress
497
+
498
+ base_pixels = 81 * 832 * 624
499
+ width, height = first_frame.size
500
+ factor = max(0.2, num_frames * width * height / base_pixels)
501
+ seconds_per_step = 5.0 * factor**1.5
502
+ estimated = 15 + int(steps) * seconds_per_step
503
+
504
+ if float(guidance_scale) > 1:
505
+ estimated *= 2.4
506
+
507
+ interpolation_multiplier = max(1, int(output_fps) // BASE_FPS)
508
+ if interpolation_multiplier > 1:
509
+ extra_frames = num_frames * interpolation_multiplier - num_frames
510
+ estimated += extra_frames * 0.025
511
+
 
512
  if safe_mode:
513
+ estimated *= 1.25
514
 
515
+ return int(max(30, min(MAX_GPU_SECONDS, round(estimated))))
516
 
517
 
518
+ @spaces.GPU(duration=get_inference_duration, size="xlarge")
519
  def run_inference(
520
+ first_frame: Image.Image,
521
+ last_frame: Image.Image | None,
522
+ prompt: str,
523
+ negative_prompt: str,
524
+ num_frames: int,
525
+ steps: int,
526
+ guidance_scale: float,
527
+ guidance_scale_2: float,
528
+ seed: int,
529
+ scheduler_name: str,
530
+ flow_shift: float,
531
+ output_fps: int,
532
+ video_quality: int,
533
+ safe_mode: bool,
 
534
  progress=gr.Progress(track_tqdm=True),
535
+ ) -> tuple[str, str, float]:
536
+ del safe_mode
 
 
 
 
 
 
 
 
537
  clear_vram()
538
+ configure_scheduler(scheduler_name, flow_shift)
539
 
540
+ task_id = str(uuid.uuid4())[:8]
541
+ started = time.time()
542
+ print(
543
+ f"Task {task_id}: {num_frames} frames, {first_frame.size}, "
544
+ f"{steps} steps, seed={seed}"
 
 
 
 
 
 
 
 
 
 
 
545
  )
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
546
 
547
+ try:
548
+ result = pipe(
549
+ image=first_frame,
550
+ last_image=last_frame,
551
+ prompt=prompt,
552
+ negative_prompt=negative_prompt,
553
+ height=first_frame.height,
554
+ width=first_frame.width,
555
+ num_frames=int(num_frames),
556
+ guidance_scale=float(guidance_scale),
557
+ guidance_scale_2=float(guidance_scale_2),
558
+ num_inference_steps=int(steps),
559
+ generator=torch.Generator(device="cuda").manual_seed(int(seed)),
560
+ output_type="np",
561
+ )
562
 
563
+ raw_frames = result.frames[0]
564
+ multiplier = max(1, int(output_fps) // BASE_FPS)
565
+ if multiplier > 1:
566
+ print(f"Task {task_id}: applying RIFE {multiplier}x interpolation")
567
+ final_frames = interpolate_frames(raw_frames, multiplier=multiplier)
568
+ else:
569
+ final_frames = list(raw_frames)
570
 
571
+ with tempfile.NamedTemporaryFile(suffix=".mp4", delete=False) as temp_file:
572
+ video_path = temp_file.name
 
 
 
 
573
 
574
+ export_to_video(
575
+ final_frames,
576
+ video_path,
577
+ fps=int(output_fps),
578
+ quality=int(video_quality),
579
+ )
580
+ elapsed = time.time() - started
581
+ print(f"Task {task_id}: completed in {elapsed:.1f}s")
582
+ return video_path, task_id, elapsed
583
+ finally:
584
+ pipe.scheduler = ORIGINAL_SCHEDULER
585
+ clear_vram()
586
 
587
 
588
  def generate_video(
589
+ first_image: Image.Image | None,
590
+ last_image: Image.Image | None,
591
+ generation_mode: str,
592
+ prompt: str,
593
+ motion_style: str,
594
+ camera_motion: str,
595
+ motion_strength: str,
596
+ preserve_identity: bool,
597
+ enhance_prompt: bool,
598
+ aspect_ratio: str,
599
+ duration_seconds: float,
600
+ output_fps: int,
601
+ quality_profile: str,
602
+ steps: int,
603
+ negative_prompt: str,
604
+ video_quality: int,
605
+ seed: int,
606
+ randomize_seed: bool,
607
+ guidance_scale: float,
608
+ guidance_scale_2: float,
609
+ scheduler_name: str,
610
+ flow_shift: float,
611
+ safe_mode: bool,
612
+ show_preview: bool,
613
  progress=gr.Progress(track_tqdm=True),
614
  ):
615
+ if first_image is None:
616
+ raise gr.Error("Please upload a first image.")
617
+
618
+ use_last_frame = generation_mode == "First + Last Frame"
619
+ if use_last_frame and last_image is None:
620
+ raise gr.Error("Please upload a last image or switch to Single Image mode.")
621
+
622
+ first_frame = prepare_first_frame(first_image, aspect_ratio)
623
+ final_last_frame = (
624
+ prepare_last_frame(last_image, first_frame)
625
+ if use_last_frame and last_image is not None
626
+ else None
627
+ )
628
+
629
+ final_prompt, final_negative = build_prompts(
630
+ prompt=prompt,
631
+ motion_style=motion_style,
632
+ camera_motion=camera_motion,
633
+ motion_strength=motion_strength,
634
+ enhance_prompt=enhance_prompt,
635
+ preserve_identity=preserve_identity,
636
+ negative_prompt=negative_prompt,
637
+ )
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
638
 
639
  num_frames = get_num_frames(duration_seconds)
640
  current_seed = random.randint(0, MAX_SEED) if randomize_seed else int(seed)
 
641
 
642
+ try:
643
+ video_path, task_id, elapsed = run_inference(
644
+ first_frame,
645
+ final_last_frame,
646
+ final_prompt,
647
+ final_negative,
648
+ num_frames,
649
+ int(steps),
650
+ float(guidance_scale),
651
+ float(guidance_scale_2),
652
+ current_seed,
653
+ scheduler_name,
654
+ float(flow_shift),
655
+ int(output_fps),
656
+ int(video_quality),
657
+ bool(safe_mode),
658
+ progress,
659
+ )
660
+ except gr.Error:
661
+ raise
662
+ except Exception as exc:
663
+ clear_vram()
664
+ message = str(exc).strip() or exc.__class__.__name__
665
+ print(f"Generation failed: {message}")
666
+ raise gr.Error(f"Generation failed: {message}") from exc
667
+
668
+ mode_label = "First → Last" if use_last_frame else "Single Image"
669
+ status = (
670
+ f"### ✅ Generation complete\n"
671
+ f"**Task:** `{task_id}` &nbsp; • &nbsp; **Mode:** {mode_label} &nbsp; • &nbsp; "
672
+ f"**Resolution:** {first_frame.width}×{first_frame.height} &nbsp; • &nbsp; "
673
+ f"**Frames:** {num_frames} at {output_fps} FPS &nbsp; • &nbsp; "
674
+ f"**Seed:** `{current_seed}` &nbsp; • &nbsp; **Time:** {elapsed:.1f}s"
675
+ )
676
 
677
+ return (
678
+ video_path if show_preview else None,
679
+ video_path,
 
 
 
 
 
 
680
  current_seed,
681
+ status,
682
+ final_prompt,
 
 
 
 
 
683
  )
 
684
 
685
+ # -----------------------------------------------------------------------------
686
+ # Frame extraction from generated video
687
+ # -----------------------------------------------------------------------------
688
+
689
+ GET_TIMESTAMP_JS = """
690
+ function() {
691
+ const video = document.querySelector('#generated-video video');
692
+ return video ? video.currentTime : 0;
693
+ }
694
+ """
695
+
696
+
697
+ def extract_frame(video_path: str | None, timestamp: float) -> np.ndarray | None:
698
+ if not video_path:
699
+ raise gr.Error("Generate or upload a video first.")
700
+
701
+ capture = cv2.VideoCapture(video_path)
702
+ if not capture.isOpened():
703
+ raise gr.Error("The video could not be opened.")
704
+
705
+ fps = capture.get(cv2.CAP_PROP_FPS) or BASE_FPS
706
+ total_frames = max(1, int(capture.get(cv2.CAP_PROP_FRAME_COUNT)))
707
+ frame_number = int(max(0, float(timestamp)) * fps)
708
+ frame_number = min(frame_number, total_frames - 1)
709
+ capture.set(cv2.CAP_PROP_POS_FRAMES, frame_number)
710
+ success, frame = capture.read()
711
+ capture.release()
712
+
713
+ if not success:
714
+ raise gr.Error("The selected frame could not be extracted.")
715
+ return cv2.cvtColor(frame, cv2.COLOR_BGR2RGB)
716
+
717
+ # -----------------------------------------------------------------------------
718
+ # Interface
719
+ # -----------------------------------------------------------------------------
720
+
721
+
722
+ def model_heading() -> str:
723
+ name = MODEL_ID.split("/")[-1].replace("_", " ")
724
+ return (
725
+ "<div class='hero'>"
726
+ "<div class='hero-badge'>WAN 2.2 • FP8 • AOTI • Lightning</div>"
727
+ "<h1>Dream Motion Pro</h1>"
728
+ "<p>Turn a single image—or a first and last frame—into a smooth cinematic video.</p>"
729
+ f"<p class='model-line'>Running <a href='https://huggingface.co/{MODEL_ID}' target='_blank'>{name}</a></p>"
730
+ "</div>"
731
+ )
732
 
733
 
734
  CSS = """
735
+ .gradio-container { max-width: 1380px !important; }
736
+ .hero {
737
+ padding: 26px 28px;
738
+ margin-bottom: 18px;
739
+ border: 1px solid var(--border-color-primary);
740
+ border-radius: 20px;
741
+ background: linear-gradient(135deg, rgba(79,70,229,.15), rgba(6,182,212,.09));
742
+ }
743
+ .hero h1 { margin: 8px 0 5px; font-size: 2.15rem; }
744
+ .hero p { margin: 4px 0; opacity: .88; }
745
+ .hero-badge {
746
+ display: inline-block;
747
+ padding: 6px 11px;
748
+ border-radius: 999px;
749
+ font-weight: 700;
750
+ font-size: .78rem;
751
+ letter-spacing: .04em;
752
+ border: 1px solid var(--border-color-primary);
753
+ }
754
+ .model-line { font-size: .9rem; }
755
+ #generate-button { min-height: 52px; font-weight: 800; font-size: 1.02rem; }
756
+ #generated-video video { border-radius: 16px; }
757
  #hidden-timestamp {
758
  opacity: 0;
759
+ height: 0;
760
+ width: 0;
761
+ margin: 0;
762
+ padding: 0;
763
  overflow: hidden;
764
  position: absolute;
765
  pointer-events: none;
766
  }
767
+ .link-row { text-align: center; padding: 8px 0 2px; font-weight: 650; }
768
+ .tip-box {
769
+ border: 1px solid var(--border-color-primary);
770
+ border-radius: 14px;
771
+ padding: 12px 14px;
772
+ }
773
  """
774
 
775
+ with gr.Blocks(
776
+ css=CSS,
777
+ theme=gr.themes.Soft(),
778
+ delete_cache=(3600, 10800),
779
+ title="Dream Motion Pro",
780
+ ) as demo:
781
+ gr.HTML(model_heading())
782
+
783
+ with gr.Row(equal_height=False):
784
+ with gr.Column(scale=6):
785
+ generation_mode = gr.Radio(
786
+ ["Single Image", "First + Last Frame"],
787
+ value="Single Image",
788
+ label="Generation Mode",
789
+ info="The second mode guides both the beginning and ending of the video.",
790
+ )
791
+
792
+ with gr.Row():
793
+ first_image = gr.Image(
794
+ type="pil",
795
+ label="First Image",
796
+ sources=["upload", "clipboard"],
797
+ height=320,
798
+ )
799
+ last_image = gr.Image(
800
+ type="pil",
801
+ label="Last Image (used only in First + Last Frame mode)",
802
+ sources=["upload", "clipboard"],
803
+ height=320,
804
+ )
805
+
806
+ with gr.Row():
807
+ swap_button = gr.Button("⇄ Swap First / Last", variant="secondary")
808
 
809
+ prompt_input = gr.Textbox(
810
+ label="Describe the motion",
811
+ value=DEFAULT_PROMPT,
812
+ lines=3,
813
+ placeholder="Example: The subject looks toward the camera while the camera slowly pushes in.",
 
 
 
 
 
 
 
 
 
814
  )
815
+
816
+ motion_style = gr.Radio(
817
+ choices=list(MOTION_STYLE_PROMPTS.keys()),
818
+ value="Cinematic",
819
+ label="Motion Style",
820
  )
821
+
822
+ with gr.Row():
823
+ quality_profile = gr.Dropdown(
824
+ choices=list(QUALITY_PROFILES.keys()),
825
+ value="Balanced",
826
+ label="Quality Mode",
 
 
 
 
 
 
 
 
827
  )
828
+ aspect_ratio = gr.Dropdown(
829
+ choices=list(ASPECT_DIMENSIONS.keys()),
830
+ value="Auto (keep source ratio)",
831
+ label="Aspect Ratio",
 
 
832
  )
833
 
834
+ with gr.Row():
835
+ duration_seconds = gr.Slider(
836
+ minimum=MIN_DURATION,
837
+ maximum=MAX_DURATION,
838
+ step=0.5,
839
+ value=3.5,
840
+ label="Duration (seconds)",
841
+ )
842
+ output_fps = gr.Dropdown(
843
+ choices=[16, 32, 64],
844
+ value=16,
845
+ label="Output FPS",
846
+ info="32/64 FPS uses RIFE interpolation after generation.",
847
+ )
848
+
849
+ with gr.Row():
850
+ preserve_identity = gr.Checkbox(
851
+ value=True,
852
+ label="Preserve Face & Identity",
853
+ )
854
+ enhance_prompt = gr.Checkbox(
855
+ value=True,
856
+ label="Professional Prompt Enhancement",
857
+ )
858
+
859
+ with gr.Accordion("Advanced Controls", open=False):
860
+ with gr.Row():
861
+ camera_motion = gr.Dropdown(
862
+ choices=list(CAMERA_PROMPTS.keys()),
863
+ value="Slow Push In",
864
+ label="Camera Movement",
865
+ )
866
+ motion_strength = gr.Radio(
867
+ choices=list(MOTION_LEVEL_PROMPTS.keys()),
868
+ value="Medium",
869
+ label="Motion Strength",
870
+ )
871
+
872
+ negative_prompt = gr.Textbox(
873
+ label="Negative Prompt",
874
+ value=DEFAULT_NEGATIVE_PROMPT,
875
+ lines=4,
876
+ )
877
+
878
+ with gr.Row():
879
+ steps = gr.Slider(
880
+ minimum=1,
881
+ maximum=10,
882
+ step=1,
883
+ value=6,
884
+ label="Inference Steps",
885
+ info="Lightning models normally perform best around 4–8 steps.",
886
+ )
887
+ video_quality = gr.Slider(
888
+ minimum=1,
889
+ maximum=10,
890
+ step=1,
891
+ value=7,
892
+ label="MP4 Quality",
893
+ )
894
+
895
+ with gr.Row():
896
+ seed = gr.Slider(
897
+ minimum=0,
898
+ maximum=MAX_SEED,
899
+ step=1,
900
+ value=42,
901
+ label="Seed",
902
+ )
903
+ randomize_seed = gr.Checkbox(
904
+ value=True,
905
+ label="Randomize Seed",
906
+ )
907
+
908
+ with gr.Row():
909
+ guidance_scale = gr.Slider(
910
+ minimum=0.0,
911
+ maximum=6.0,
912
+ step=0.5,
913
+ value=1.0,
914
+ label="High-noise Guidance",
915
+ )
916
+ guidance_scale_2 = gr.Slider(
917
+ minimum=0.0,
918
+ maximum=6.0,
919
+ step=0.5,
920
+ value=1.0,
921
+ label="Low-noise Guidance",
922
+ )
923
+
924
+ with gr.Row():
925
+ scheduler_name = gr.Dropdown(
926
+ choices=list(SCHEDULER_MAP.keys()),
927
+ value="UniPCMultistep",
928
+ label="Scheduler",
929
+ )
930
+ flow_shift = gr.Slider(
931
+ minimum=0.5,
932
+ maximum=12.0,
933
+ step=0.1,
934
+ value=3.0,
935
+ label="Flow Shift",
936
+ )
937
+
938
+ with gr.Row():
939
+ safe_mode = gr.Checkbox(
940
+ value=True,
941
+ label="Extra ZeroGPU Time Buffer",
942
+ )
943
+ show_preview = gr.Checkbox(
944
+ value=True,
945
+ label="Show Video Preview",
946
+ )
947
+
948
+ generate_button = gr.Button(
949
+ "✨ Generate Video",
950
+ variant="primary",
951
+ elem_id="generate-button",
952
+ )
953
+
954
+ gr.Markdown(
955
+ "**Best results:** use a clear image, describe one main action, keep camera movement controlled, "
956
+ "and generate 2–3 variations with different seeds."
957
+ )
958
+
959
+ with gr.Column(scale=6):
960
+ video_output = gr.Video(
961
+ label="Generated Video",
962
+ autoplay=True,
963
+ buttons=["download", "share"],
964
+ interactive=False,
965
+ elem_id="generated-video",
966
+ )
967
+ status_output = gr.Markdown("Your generation details will appear here.")
968
 
 
 
 
 
 
969
  with gr.Row():
970
+ grab_frame_button = gr.Button(
971
+ "📸 Use Current Video Frame as First Image",
972
+ variant="secondary",
973
+ )
974
+ timestamp_box = gr.Number(
975
+ value=0,
976
+ visible=True,
977
+ elem_id="hidden-timestamp",
978
+ )
979
+
980
+ file_output = gr.File(label="Download Original MP4")
981
+
982
+ with gr.Accordion("Prompt used for this result", open=False):
983
+ final_prompt_output = gr.Textbox(
984
+ label="Enhanced Prompt",
985
+ interactive=False,
986
+ lines=5,
987
+ )
988
+
989
+ external_links = format_external_links()
990
+ if external_links:
991
+ gr.Markdown(
992
+ f"<div class='link-row'>{external_links}</div>",
993
+ sanitize_html=False,
994
+ )
995
+
996
+ gr.Markdown(
997
+ "### Why this Space is fast\n"
998
+ "The Wan 2.2 high-noise and low-noise transformers run with FP8 quantization and AOTI-compiled blocks. "
999
+ "The selected WAMU checkpoint already includes Lightning and MotionBoost behavior."
1000
+ )
1001
+
1002
+ quality_profile.change(
1003
+ fn=profile_settings,
1004
+ inputs=[quality_profile],
1005
+ outputs=[steps, video_quality, flow_shift],
1006
+ )
1007
+
1008
+ swap_button.click(
1009
+ fn=swap_images,
1010
+ inputs=[first_image, last_image],
1011
+ outputs=[first_image, last_image],
1012
+ )
1013
+
1014
+ inputs = [
1015
+ first_image,
1016
+ last_image,
1017
+ generation_mode,
1018
+ prompt_input,
1019
+ motion_style,
1020
+ camera_motion,
1021
+ motion_strength,
1022
+ preserve_identity,
1023
+ enhance_prompt,
1024
+ aspect_ratio,
1025
+ duration_seconds,
1026
+ output_fps,
1027
+ quality_profile,
1028
+ steps,
1029
+ negative_prompt,
1030
+ video_quality,
1031
+ seed,
1032
+ randomize_seed,
1033
+ guidance_scale,
1034
+ guidance_scale_2,
1035
+ scheduler_name,
1036
+ flow_shift,
1037
+ safe_mode,
1038
+ show_preview,
1039
  ]
1040
+
1041
  generate_button.click(
1042
+ fn=generate_video,
1043
+ inputs=inputs,
1044
+ outputs=[
1045
+ video_output,
1046
+ file_output,
1047
+ seed,
1048
+ status_output,
1049
+ final_prompt_output,
1050
+ ],
1051
  )
1052
+
1053
+ grab_frame_button.click(
 
 
1054
  fn=None,
1055
  inputs=None,
1056
  outputs=[timestamp_box],
1057
+ js=GET_TIMESTAMP_JS,
1058
  )
 
 
1059
  timestamp_box.change(
1060
  fn=extract_frame,
1061
  inputs=[video_output, timestamp_box],
1062
+ outputs=[first_image],
1063
  )
1064
 
1065
  if __name__ == "__main__":
1066
+ demo.queue(default_concurrency_limit=1, max_size=20).launch(
1067
  mcp_server=True,
 
1068
  show_error=True,
1069
+ )
packages.txt CHANGED
@@ -1 +1 @@
1
- ffmpeg
 
1
+ ffmpeg
requirements.txt CHANGED
@@ -1,19 +1,22 @@
 
 
 
 
1
  diffusers==0.38.0
2
  transformers==4.57.6
3
- accelerate===1.13.0
4
  safetensors
5
  sentencepiece
6
  peft==0.19.1
7
  ftfy
8
- imageio
9
- imageio-ffmpeg
10
- opencv-python
11
- torchao==0.17.0
12
 
13
- numpy>=1.16, <=1.23.5
14
- # tqdm>=4.35.0
15
- # sk-video>=1.1.10
16
- # opencv-python>=4.1.2
17
- # moviepy>=1.0.3
18
  torch==2.11.0
19
  torchvision==0.26.0
 
 
 
 
 
 
 
 
 
1
+ gradio==6.0.1
2
+ spaces
3
+ huggingface_hub
4
+
5
  diffusers==0.38.0
6
  transformers==4.57.6
7
+ accelerate==1.13.0
8
  safetensors
9
  sentencepiece
10
  peft==0.19.1
11
  ftfy
 
 
 
 
12
 
 
 
 
 
 
13
  torch==2.11.0
14
  torchvision==0.26.0
15
+ torchao==0.17.0
16
+
17
+ numpy>=1.23.5,<3
18
+ Pillow
19
+ opencv-python-headless
20
+ imageio
21
+ imageio-ffmpeg
22
+ tqdm