# Copyright (c) 2025 Lightricks. All rights reserved. # Created by Amit Pintz. import torch from ltx_core.components.diffusion_steps import EulerDiffusionStep from ltx_core.loader.sd_ops import LTXV_LORA_COMFY_RENAMING_MAP from ltx_core.components.noisers import GaussianNoiser from ltx_core.components.protocols import DiffusionStepProtocol from ltx_core.loader import LoraPathStrengthAndSDOps from ltx_core.model.audio_vae import decode_audio as vae_decode_audio from ltx_core.model.upsampler import upsample_video from ltx_core.model.video_vae import TilingConfig, get_video_chunks_number from ltx_core.model.video_vae import decode_video as vae_decode_video from ltx_core.text_encoders.gemma import encode_text from ltx_core.types import LatentState, VideoPixelShape from ltx_pipelines import utils from ltx_pipelines.utils import ModelLedger from ltx_pipelines.utils.args import default_2_stage_distilled_arg_parser from ltx_pipelines.utils.constants import ( AUDIO_SAMPLE_RATE, DEFAULT_LORA_STRENGTH, DISTILLED_SIGMA_VALUES, STAGE_2_DISTILLED_SIGMA_VALUES, ) from ltx_pipelines.utils.helpers import ( assert_resolution, cleanup_memory, denoise_audio_video, euler_denoising_loop, generate_enhanced_prompt, get_device, image_conditionings_by_replacing_latent, simple_denoising_func, ) from ltx_pipelines.utils.media_io import encode_video from ltx_pipelines.utils.types import PipelineComponents device = get_device() class DistilledPipeline: def __init__( self, checkpoint_path: str, gemma_root: str, spatial_upsampler_path: str, loras: list[LoraPathStrengthAndSDOps], device: torch.device = device, fp8transformer: bool = False, local_files_only: bool = True, ): self.device = device self.dtype = torch.bfloat16 self.model_ledger = ModelLedger( dtype=self.dtype, device=device, checkpoint_path=checkpoint_path, spatial_upsampler_path=spatial_upsampler_path, gemma_root_path=gemma_root, loras=loras, fp8transformer=fp8transformer, local_files_only=local_files_only ) self.pipeline_components = PipelineComponents( dtype=self.dtype, device=device, ) # Cached models to avoid reloading self._video_encoder = None self._transformer = None @torch.inference_mode() def __call__( self, prompt: str, output_path: str, seed: int, height: int, width: int, num_frames: int, frame_rate: float, images: list[tuple[str, int, float]], tiling_config: TilingConfig | None = None, video_context: torch.Tensor | None = None, audio_context: torch.Tensor | None = None, ) -> None: generator = torch.Generator(device=self.device).manual_seed(seed) noiser = GaussianNoiser(generator=generator) stepper = EulerDiffusionStep() dtype = torch.bfloat16 # Use pre-computed embeddings if provided, otherwise encode text if video_context is None or audio_context is None: text_encoder = self.model_ledger.text_encoder() context_p = encode_text(text_encoder, prompts=[prompt])[0] video_context, audio_context = context_p torch.cuda.synchronize() del text_encoder utils.cleanup_memory() else: # Move pre-computed embeddings to device if needed video_context = video_context.to(self.device) audio_context = audio_context.to(self.device) # Stage 1: Initial low resolution video generation. # Load models only if not already cached if self._video_encoder is None: self._video_encoder = self.model_ledger.video_encoder() video_encoder = self._video_encoder if self._transformer is None: self._transformer = self.model_ledger.transformer() transformer = self._transformer stage_1_sigmas = torch.Tensor(DISTILLED_SIGMA_VALUES).to(self.device) def denoising_loop( sigmas: torch.Tensor, video_state: LatentState, audio_state: LatentState, stepper: DiffusionStepProtocol ) -> tuple[LatentState, LatentState]: return euler_denoising_loop( sigmas=sigmas, video_state=video_state, audio_state=audio_state, stepper=stepper, denoise_fn=simple_denoising_func( video_context=video_context, audio_context=audio_context, transformer=transformer, # noqa: F821 ), ) stage_1_output_shape = VideoPixelShape(batch=1, frames=num_frames, width=width, height=height, fps=frame_rate) stage_1_conditionings = image_conditionings_by_replacing_latent( images=images, height=stage_1_output_shape.height, width=stage_1_output_shape.width, video_encoder=video_encoder, dtype=dtype, device=self.device, ) video_state, audio_state = denoise_audio_video( output_shape=stage_1_output_shape, conditionings=stage_1_conditionings, noiser=noiser, sigmas=stage_1_sigmas, stepper=stepper, denoising_loop_fn=denoising_loop, components=self.pipeline_components, dtype=dtype, device=self.device, ) # Stage 2: Upsample and refine the video at higher resolution with distilled LORA. upscaled_video_latent = upsample_video( latent=video_state.latent[:1], video_encoder=video_encoder, upsampler=self.model_ledger.spatial_upsampler() ) torch.cuda.synchronize() cleanup_memory() stage_2_sigmas = torch.Tensor(STAGE_2_DISTILLED_SIGMA_VALUES).to(self.device) stage_2_output_shape = VideoPixelShape( batch=1, frames=num_frames, width=width * 2, height=height * 2, fps=frame_rate ) stage_2_conditionings = image_conditionings_by_replacing_latent( images=images, height=stage_2_output_shape.height, width=stage_2_output_shape.width, video_encoder=video_encoder, dtype=dtype, device=self.device, ) video_chunks_number = get_video_chunks_number(num_frames, tiling_config) video_state, audio_state = denoise_audio_video( output_shape=stage_2_output_shape, conditionings=stage_2_conditionings, noiser=noiser, sigmas=stage_2_sigmas, stepper=stepper, denoising_loop_fn=denoising_loop, components=self.pipeline_components, dtype=dtype, device=self.device, noise_scale=stage_2_sigmas[0], initial_video_latent=upscaled_video_latent, initial_audio_latent=audio_state.latent, ) torch.cuda.synchronize() # del transformer # del video_encoder # utils.cleanup_memory() decoded_video = vae_decode_video(video_state.latent, self.model_ledger.video_decoder(), tiling_config) decoded_audio = vae_decode_audio(audio_state.latent, self.model_ledger.audio_decoder(), self.model_ledger.vocoder()) encode_video( video=decoded_video, fps=frame_rate, audio=decoded_audio, audio_sample_rate=AUDIO_SAMPLE_RATE, output_path=output_path, video_chunks_number=video_chunks_number, )