import os # ZeroGPU: torch.compile / dynamo are unsupported โ€” disable before torch import. os.environ.setdefault("TORCH_COMPILE_DISABLE", "1") os.environ.setdefault("TORCHDYNAMO_DISABLE", "1") import random import tempfile import numpy as np import spaces import torch import gradio as gr from PIL import Image, ImageFilter from huggingface_hub import hf_hub_download from safetensors.torch import load_file from diffusers import LTX2InContextPipeline from diffusers.pipelines.ltx2.pipeline_ltx2_ic_lora import LTX2ReferenceCondition from diffusers.pipelines.ltx2.utils import DISTILLED_SIGMA_VALUES from diffusers.utils import load_video, encode_video # --- Config ----------------------------------------------------------------- BASE_MODEL = "diffusers/LTX-2.3-Distilled-Diffusers" LORA_REPO = "linoyts/ltx2.3-inpainting-lora" LORA_FILE = "ltx-2.3-22b-ic-lora-inpainting.safetensors" LORA_SCALE = 1.0 FPS = 24 NUM_STEPS = len(DISTILLED_SIGMA_VALUES) # 8 MASK_FILL = 128 # masked region painted neutral grey in the reference the model fills MAX_SEED = np.iinfo(np.int32).max HF_TOKEN = os.environ.get("HF_TOKEN") RES_PRESETS = {"Fast (768ร—448)": (768, 448), "Quality (960ร—544)": (960, 544)} FRAME_CHOICES = [49, 73, 97, 121] # --- Load pipeline once at module scope (ZeroGPU registers it) --------------- pipe = LTX2InContextPipeline.from_pretrained(BASE_MODEL, torch_dtype=torch.bfloat16) pipe.to("cuda") pipe.vae.enable_tiling() _lora_path = hf_hub_download(LORA_REPO, LORA_FILE, token=HF_TOKEN) pipe.load_lora_weights(load_file(_lora_path), adapter_name="inpaint") pipe.set_adapters("inpaint", LORA_SCALE) # --- Helpers ---------------------------------------------------------------- def _resample(frames, n): idx = np.linspace(0, len(frames) - 1, n).round().astype(int) return [frames[i] for i in idx] def _pick_resolution(first_frame: Image.Image, preset: str): w, h = RES_PRESETS[preset] if first_frame.height > first_frame.width: w, h = h, w return w, h def first_frame(video): """Populate the mask editor with the uploaded video's first frame.""" if video is None: return None frames = load_video(video) return np.array(frames[0].convert("RGB")) if frames else None def _mask_from_editor(editor_value, width, height): """Extract a binary mask (H,W) from a gr.ImageEditor value โ€” union of painted layers.""" if not editor_value: return None layers = editor_value.get("layers") or [] bg = editor_value.get("background") if bg is None: return None H0, W0 = np.asarray(bg).shape[:2] acc = np.zeros((H0, W0), dtype=bool) for layer in layers: arr = np.asarray(layer) if arr.ndim == 3 and arr.shape[2] == 4: acc |= arr[..., 3] > 10 elif arr.ndim == 3: acc |= arr.sum(axis=2) > 10 if not acc.any(): return None m = Image.fromarray((acc * 255).astype(np.uint8)).resize((width, height), Image.NEAREST) return np.array(m) > 127 def _duration(*args, **kwargs): preset = args[3] if len(args) > 3 else "Fast" num_frames = args[4] if len(args) > 4 else 73 per_frame = 1.6 if "Quality" in str(preset) else 1.0 return int(50 + int(num_frames) * per_frame) # --- Inference -------------------------------------------------------------- @spaces.GPU(duration=_duration) def inpaint(video, mask_editor, prompt, preset, num_frames, seed, randomize, progress=gr.Progress(track_tqdm=True)): if video is None: raise gr.Error("Please upload a video.") if not prompt.strip(): raise gr.Error("Describe what should fill the masked region.") if randomize: seed = random.randint(0, MAX_SEED) seed = int(seed) frames = load_video(video) if not frames: raise gr.Error("Could not read any frames from that video.") width, height = _pick_resolution(frames[0], preset) num_frames = int(num_frames) mask = _mask_from_editor(mask_editor, width, height) if mask is None: raise gr.Error("Draw a mask over the region to inpaint (use the brush on the frame).") orig = [np.array(f.convert("RGB").resize((width, height), Image.LANCZOS)) for f in _resample(frames, num_frames)] # Reference = video with the masked region painted neutral grey; the model fills it. ref = [] for fr in orig: m = fr.copy() m[mask] = MASK_FILL ref.append(Image.fromarray(m)) ref_cond = LTX2ReferenceCondition(frames=ref, strength=1.0) video_out, _audio = pipe( prompt=prompt, negative_prompt="", reference_conditions=[ref_cond], reference_downscale_factor=1, width=width, height=height, num_frames=num_frames, frame_rate=FPS, num_inference_steps=NUM_STEPS, sigmas=DISTILLED_SIGMA_VALUES, guidance_scale=1.0, stg_scale=0.0, audio_guidance_scale=1.0, audio_stg_scale=0.0, generator=torch.Generator(device="cuda").manual_seed(seed), output_type="np", return_dict=False, ) # Composite: keep original pixels outside the mask, generated pixels inside (feathered). gen = (np.clip(video_out[0], 0, 1) * 255).astype(np.uint8) soft = np.array(Image.fromarray((mask * 255).astype(np.uint8)).filter(ImageFilter.GaussianBlur(3))) / 255.0 soft = soft[None, :, :, None] orig_arr = np.stack(orig).astype(np.float32) n = min(len(gen), len(orig_arr)) out = (gen[:n].astype(np.float32) * soft + orig_arr[:n] * (1 - soft)).astype(np.uint8) out_path = tempfile.NamedTemporaryFile(suffix=".mp4", delete=False).name encode_video(out, fps=FPS, output_path=out_path) return out_path, seed # --- UI --------------------------------------------------------------------- with gr.Blocks(title="LTX-2.3 Video Inpainting") as demo: gr.Markdown( "# ๐Ÿช„ LTX-2.3 Video Inpainting\n" "Mask a region of a video and regenerate it from a prompt, keeping the rest of the frame intact. " "Upload a clip, **brush over the area to replace** on the first frame, describe what should appear there. " "The mask is applied across all frames. " "IC-LoRA: [`linoyts/ltx2.3-inpainting-lora`](https://huggingface.co/linoyts/ltx2.3-inpainting-lora) ยท " "base: distilled LTX-2.3." ) with gr.Row(): with gr.Column(): video_in = gr.Video(label="Input video") mask_editor = gr.ImageEditor( label="Brush the region to inpaint (loads from the video's first frame)", type="numpy", layers=False, brush=gr.Brush(colors=["#ff2d55"], color_mode="fixed"), ) prompt = gr.Textbox( label="What should fill the masked region", placeholder="a lush green bush with small white flowers", lines=2, ) with gr.Accordion("Settings", open=False): preset = gr.Dropdown(list(RES_PRESETS), value="Fast (768ร—448)", label="Resolution") num_frames = gr.Dropdown(FRAME_CHOICES, value=73, label="Frames (24fps)") randomize = gr.Checkbox(True, label="Randomize seed") seed = gr.Slider(0, MAX_SEED, value=42, step=1, label="Seed") run = gr.Button("Inpaint", variant="primary") with gr.Column(): video_out = gr.Video(label="Inpainted result") used_seed = gr.Number(label="Seed used", interactive=False) video_in.change(first_frame, inputs=video_in, outputs=mask_editor) run.click( inpaint, inputs=[video_in, mask_editor, prompt, preset, num_frames, seed, randomize], outputs=[video_out, used_seed], ) if __name__ == "__main__": demo.launch(show_error=True)