q4/embedded_config.json

{
  "transformer": {
    "_class_name": "AVTransformer3DModel",
    "attention_head_dim": 128,
    "attention_type": "default",
    "caption_channels": 3840,
    "cross_attention_dim": 4096,
    "in_channels": 128,
    "norm_eps": 1e-06,
    "num_attention_heads": 32,
    "num_layers": 48,
    "out_channels": 128,
    "audio_num_attention_heads": 32,
    "audio_attention_head_dim": 64,
    "audio_out_channels": 128,
    "audio_cross_attention_dim": 2048,
    "audio_positional_embedding_max_pos": [
      20
    ],
    "use_embeddings_connector": true,
    "connector_attention_head_dim": 128,
    "connector_num_attention_heads": 32,
    "connector_num_layers": 8,
    "connector_positional_embedding_max_pos": [
      4096
    ],
    "connector_num_learnable_registers": 128,
    "use_middle_indices_grid": true,
    "apply_gated_attention": true,
    "connector_apply_gated_attention": true,
    "caption_projection_first_linear": false,
    "caption_projection_second_linear": false,
    "audio_connector_attention_head_dim": 64,
    "audio_connector_num_attention_heads": 32,
    "cross_attention_adaln": true,
    "text_encoder_norm_type": "per_token_rms",
    "rope_type": "split",
    "frequencies_precision": "float64",
    "positional_embedding_theta": 10000.0,
    "positional_embedding_max_pos": [
      20,
      2048,
      2048
    ],
    "timestep_scale_multiplier": 1000,
    "av_ca_timestep_scale_multiplier": 1000.0
  },
  "vae": {
    "_class_name": "CausalVideoAutoencoder",
    "dims": 3,
    "in_channels": 3,
    "out_channels": 3,
    "latent_channels": 128,
    "patch_size": 4,
    "norm_layer": "pixel_norm",
    "spatial_padding_mode": "zeros",
    "timestep_conditioning": false,
    "decoder_blocks": [
      [
        "res_x",
        {
          "num_layers": 4
        }
      ],
      [
        "compress_space",
        {
          "multiplier": 2
        }
      ],
      [
        "res_x",
        {
          "num_layers": 6
        }
      ],
      [
        "compress_time",
        {
          "multiplier": 2
        }
      ],
      [
        "res_x",
        {
          "num_layers": 4
        }
      ],
      [
        "compress_all",
        {
          "multiplier": 1
        }
      ],
      [
        "res_x",
        {
          "num_layers": 2
        }
      ],
      [
        "compress_all",
        {
          "multiplier": 2
        }
      ],
      [
        "res_x",
        {
          "num_layers": 2
        }
      ]
    ],
    "encoder_blocks": [
      [
        "res_x",
        {
          "num_layers": 4
        }
      ],
      [
        "compress_space_res",
        {
          "multiplier": 2
        }
      ],
      [
        "res_x",
        {
          "num_layers": 6
        }
      ],
      [
        "compress_time_res",
        {
          "multiplier": 2
        }
      ],
      [
        "res_x",
        {
          "num_layers": 4
        }
      ],
      [
        "compress_all_res",
        {
          "multiplier": 2
        }
      ],
      [
        "res_x",
        {
          "num_layers": 2
        }
      ],
      [
        "compress_all_res",
        {
          "multiplier": 1
        }
      ],
      [
        "res_x",
        {
          "num_layers": 2
        }
      ]
    ]
  },
  "audio_vae": {
    "model": {
      "params": {
        "ddconfig": {
          "double_z": true,
          "mel_bins": 64,
          "z_channels": 8,
          "resolution": 256,
          "in_channels": 2,
          "out_ch": 2,
          "ch": 128,
          "ch_mult": [
            1,
            2,
            4
          ],
          "num_res_blocks": 2,
          "dropout": 0.0,
          "mid_block_add_attention": false,
          "norm_type": "pixel",
          "causality_axis": "height"
        },
        "sampling_rate": 16000
      }
    }
  },
  "scheduler": {
    "_class_name": "RectifiedFlowScheduler",
    "_diffusers_version": "0.25.1",
    "num_train_timesteps": 1000,
    "sampler": "LinearQuadratic"
  },
  "vocoder": {
    "vocoder": {
      "upsample_initial_channel": 1536,
      "resblock": "AMP1",
      "upsample_rates": [
        5,
        2,
        2,
        2,
        2,
        2
      ],
      "resblock_kernel_sizes": [
        3,
        7,
        11
      ],
      "upsample_kernel_sizes": [
        11,
        4,
        4,
        4,
        4,
        4
      ],
      "resblock_dilation_sizes": [
        [
          1,
          3,
          5
        ],
        [
          1,
          3,
          5
        ],
        [
          1,
          3,
          5
        ]
      ],
      "stereo": true,
      "use_tanh_at_final": false,
      "activation": "snakebeta",
      "use_bias_at_final": false
    },
    "bwe": {
      "upsample_initial_channel": 512,
      "resblock": "AMP1",
      "upsample_rates": [
        6,
        5,
        2,
        2,
        2
      ],
      "resblock_kernel_sizes": [
        3,
        7,
        11
      ],
      "upsample_kernel_sizes": [
        12,
        11,
        4,
        4,
        4
      ],
      "resblock_dilation_sizes": [
        [
          1,
          3,
          5
        ],
        [
          1,
          3,
          5
        ],
        [
          1,
          3,
          5
        ]
      ],
      "stereo": true,
      "use_tanh_at_final": false,
      "activation": "snakebeta",
      "use_bias_at_final": false,
      "apply_final_activation": false,
      "input_sampling_rate": 16000,
      "output_sampling_rate": 48000,
      "hop_length": 80,
      "n_fft": 512,
      "win_size": 512,
      "num_mels": 64
    }
  }
}