"""AudioDiT model configuration""" # transformers >= 5.0 renamed PretrainedConfig → PreTrainedConfig; tolerate both. try: from transformers import PreTrainedConfig except ImportError: from transformers import PretrainedConfig as PreTrainedConfig from transformers import logging from transformers.models.umt5.configuration_umt5 import UMT5Config logger = logging.get_logger(__name__) class AudioDiTVaeConfig(PreTrainedConfig): r""" Configuration class for the AudioDiT WAV-VAE audio autoencoder. Args: in_channels (`int`, *optional*, defaults to 1): Number of input audio channels (mono=1). channels (`int`, *optional*, defaults to 128): Base channel count for encoder/decoder. c_mults (`list[int]`, *optional*, defaults to `[1, 2, 4, 8, 16]`): Channel multipliers for each encoder/decoder stage. strides (`list[int]`, *optional*, defaults to `[2, 4, 4, 8, 8]`): Downsampling strides for each encoder stage. latent_dim (`int`, *optional*, defaults to 64): Dimensionality of the latent space (after VAE bottleneck: encoder outputs 128, split to mean+scale → 64). encoder_latent_dim (`int`, *optional*, defaults to 128): Dimensionality of the encoder output before VAE bottleneck. use_snake (`bool`, *optional*, defaults to `True`): Whether to use Snake activation instead of ELU. downsample_shortcut (`str`, *optional*, defaults to `"averaging"`): Shortcut type for encoder downsampling blocks. upsample_shortcut (`str`, *optional*, defaults to `"duplicating"`): Shortcut type for decoder upsampling blocks. out_shortcut (`str`, *optional*, defaults to `"averaging"`): Shortcut type for encoder output projection. in_shortcut (`str`, *optional*, defaults to `"duplicating"`): Shortcut type for decoder input projection. final_tanh (`bool`, *optional*, defaults to `False`): Whether to apply tanh to the decoder output. downsampling_ratio (`int`, *optional*, defaults to 2048): Total downsampling ratio from audio samples to latent frames. sample_rate (`int`, *optional*, defaults to 24000): Audio sample rate. scale (`float`, *optional*, defaults to 0.71): Scale factor for the latent space. """ model_type = "audiodit_vae" def __init__( self, in_channels: int = 1, channels: int = 128, c_mults: list[int] | None = None, strides: list[int] | None = None, latent_dim: int = 64, encoder_latent_dim: int = 128, use_snake: bool = True, downsample_shortcut: str = "averaging", upsample_shortcut: str = "duplicating", out_shortcut: str = "averaging", in_shortcut: str = "duplicating", final_tanh: bool = False, downsampling_ratio: int = 2048, sample_rate: int = 24000, scale: float = 0.71, **kwargs, ): super().__init__(**kwargs) self.in_channels = in_channels self.channels = channels self.c_mults = c_mults if c_mults is not None else [1, 2, 4, 8, 16] self.strides = strides if strides is not None else [2, 4, 4, 8, 8] self.latent_dim = latent_dim self.encoder_latent_dim = encoder_latent_dim self.use_snake = use_snake self.downsample_shortcut = downsample_shortcut self.upsample_shortcut = upsample_shortcut self.out_shortcut = out_shortcut self.in_shortcut = in_shortcut self.final_tanh = final_tanh self.downsampling_ratio = downsampling_ratio self.sample_rate = sample_rate self.scale = scale class AudioDiTConfig(PreTrainedConfig): r""" Configuration class for AudioDiT, a Conditional Flow Matching TTS model based on DiT architecture. Args: dit_dim (`int`, *optional*, defaults to 1536): Hidden dimension of the DiT transformer. dit_depth (`int`, *optional*, defaults to 24): Number of transformer layers. dit_heads (`int`, *optional*, defaults to 24): Number of attention heads. dit_ff_mult (`float`, *optional*, defaults to 4.0): Feed-forward network multiplier. dit_text_dim (`int`, *optional*, defaults to 768): Dimension of the text encoder output (UMT5-base). dit_dropout (`float`, *optional*, defaults to 0.0): Dropout rate. dit_bias (`bool`, *optional*, defaults to `True`): Whether to use bias in linear layers. dit_cross_attn (`bool`, *optional*, defaults to `True`): Whether to use cross-attention layers. dit_adaln_type (`str`, *optional*, defaults to `"global"`): Type of adaptive layer norm (`"global"` or `"local"`). dit_adaln_use_text_cond (`bool`, *optional*, defaults to `True`): Whether to condition AdaLN on text embeddings. dit_long_skip (`bool`, *optional*, defaults to `True`): Whether to use long skip connection (input added to output). dit_text_conv (`bool`, *optional*, defaults to `True`): Whether to apply ConvNeXt blocks on text embeddings. dit_qk_norm (`bool`, *optional*, defaults to `True`): Whether to apply RMS normalization to Q and K. dit_cross_attn_norm (`bool`, *optional*, defaults to `False`): Whether to apply layer normalization in cross-attention. dit_eps (`float`, *optional*, defaults to 1e-6): Epsilon for normalization layers. dit_use_latent_condition (`bool`, *optional*, defaults to `True`): Whether to use latent conditioning (for prompt audio). repa_dit_layer (`int`, *optional*, defaults to 8): Layer index for representation alignment. latent_dim (`int`, *optional*, defaults to 64): Dimensionality of the audio latent space. sigma (`float`, *optional*, defaults to 0.0): Noise level for conditional flow matching. sampling_rate (`int`, *optional*, defaults to 24000): Audio sample rate. latent_hop (`int`, *optional*, defaults to 2048): Hop size in audio samples per latent frame. max_wav_duration (`float`, *optional*, defaults to 30.0): Maximum audio duration in seconds. text_encoder_model (`str`, *optional*, defaults to `"google/umt5-base"`): HuggingFace model identifier for the text encoder. text_add_embed (`bool`, *optional*, defaults to `True`): Whether to add the first hidden state to the last hidden state in text encoding. text_norm_feat (`bool`, *optional*, defaults to `True`): Whether to apply layer normalization to text features. vae_config (`AudioDiTVaeConfig` or `dict`, *optional*): Configuration for the WAV-VAE audio autoencoder. Example: ```python >>> from transformers import AudioDiTConfig, AudioDiTModel >>> configuration = AudioDiTConfig() >>> model = AudioDiTModel(configuration) >>> configuration = model.config ``` """ model_type = "audiodit" sub_configs = {"vae_config": AudioDiTVaeConfig, "text_encoder_config": UMT5Config} def __init__( self, dit_dim: int = 1536, dit_depth: int = 24, dit_heads: int = 24, dit_ff_mult: float = 4.0, dit_text_dim: int = 768, dit_dropout: float = 0.0, dit_bias: bool = True, dit_cross_attn: bool = True, dit_adaln_type: str = "global", dit_adaln_use_text_cond: bool = True, dit_long_skip: bool = True, dit_text_conv: bool = True, dit_qk_norm: bool = True, dit_cross_attn_norm: bool = False, dit_eps: float = 1e-6, dit_use_latent_condition: bool = True, repa_dit_layer: int = 8, latent_dim: int = 64, sigma: float = 0.0, sampling_rate: int = 24000, latent_hop: int = 2048, max_wav_duration: float = 30.0, text_encoder_model: str = "google/umt5-base", text_add_embed: bool = True, text_norm_feat: bool = True, vae_config: AudioDiTVaeConfig | dict | None = None, text_encoder_config: UMT5Config | dict | None = None, **kwargs, ): super().__init__(**kwargs) self.dit_dim = dit_dim self.dit_depth = dit_depth self.dit_heads = dit_heads self.dit_ff_mult = dit_ff_mult self.dit_text_dim = dit_text_dim self.dit_dropout = dit_dropout self.dit_bias = dit_bias self.dit_cross_attn = dit_cross_attn self.dit_adaln_type = dit_adaln_type self.dit_adaln_use_text_cond = dit_adaln_use_text_cond self.dit_long_skip = dit_long_skip self.dit_text_conv = dit_text_conv self.dit_qk_norm = dit_qk_norm self.dit_cross_attn_norm = dit_cross_attn_norm self.dit_eps = dit_eps self.dit_use_latent_condition = dit_use_latent_condition self.repa_dit_layer = repa_dit_layer self.latent_dim = latent_dim self.sigma = sigma self.sampling_rate = sampling_rate self.latent_hop = latent_hop self.max_wav_duration = max_wav_duration self.text_encoder_model = text_encoder_model self.text_add_embed = text_add_embed self.text_norm_feat = text_norm_feat if isinstance(vae_config, dict): vae_config = AudioDiTVaeConfig(**vae_config) self.vae_config = vae_config if vae_config is not None else AudioDiTVaeConfig() if isinstance(text_encoder_config, dict): text_encoder_config = UMT5Config(**text_encoder_config) self.text_encoder_config = text_encoder_config __all__ = ["AudioDiTConfig", "AudioDiTVaeConfig"]