{
  "architectures": [
    "Dots3NoteForCausalLM"
  ],
  "apply_mla_qkv_lora_rescale": true,
  "attention_bias": false,
  "attention_dropout": 0.0,
  "attention_gate_type": "headwise",
  "bos_token_id": 151643,
  "eos_token_id": 151668,
  "first_k_dense_replace": 1,
  "hidden_act": "silu",
  "hidden_size": 5120,
  "index_head_dim": 128,
  "index_n_heads": 64,
  "index_topk": 2048,
  "initializer_range": 0.02,
  "intermediate_size": 13824,
  "kv_lora_rank": 512,
  "layer_types": [
    "full_attention",
    "full_attention",
    "sliding_attention",
    "sliding_attention",
    "sliding_attention",
    "full_attention",
    "sliding_attention",
    "sliding_attention",
    "sliding_attention",
    "full_attention",
    "sliding_attention",
    "sliding_attention",
    "sliding_attention",
    "full_attention",
    "sliding_attention",
    "sliding_attention",
    "sliding_attention",
    "full_attention",
    "sliding_attention",
    "sliding_attention",
    "sliding_attention",
    "full_attention",
    "sliding_attention",
    "sliding_attention",
    "sliding_attention",
    "full_attention",
    "sliding_attention",
    "sliding_attention",
    "sliding_attention",
    "full_attention",
    "sliding_attention",
    "sliding_attention",
    "sliding_attention",
    "full_attention",
    "sliding_attention",
    "sliding_attention",
    "sliding_attention",
    "full_attention",
    "sliding_attention",
    "sliding_attention",
    "sliding_attention",
    "full_attention",
    "sliding_attention",
    "sliding_attention",
    "sliding_attention",
    "full_attention"
  ],
  "max_position_embeddings": 524288,
  "model_type": "dots3_note",
  "moe_intermediate_size": 1536,
  "moe_layer_freq": 1,
  "n_routed_experts": 256,
  "n_shared_experts": 1,
  "norm_topk_prob": true,
  "num_attention_heads": 128,
  "num_experts_per_tok": 8,
  "num_hidden_layers": 46,
  "num_key_value_heads": 128,
  "pretraining_tp": 1,
  "q_lora_rank": 1024,
  "qk_nope_head_dim": 128,
  "qk_rope_head_dim": 64,
  "quantization_config": {
    "config_groups": {
      "group_0": {
        "format": "nvfp4-pack-quantized",
        "input_activations": {
          "actorder": null,
          "block_structure": null,
          "dynamic": "local",
          "group_size": 16,
          "num_bits": 4,
          "observer": "static_minmax",
          "observer_kwargs": {},
          "scale_dtype": "torch.float8_e4m3fn",
          "strategy": "tensor_group",
          "symmetric": true,
          "type": "float",
          "zp_dtype": null
        },
        "output_activations": null,
        "targets": [
          "re:.*\\.mlp\\.experts\\.\\d+\\.(gate_proj|up_proj|down_proj)$",
          "re:.*\\.mlp\\.shared_experts\\.(gate_proj|up_proj|down_proj)$",
          "FusedMoE"
        ],
        "weights": {
          "actorder": null,
          "block_structure": null,
          "dynamic": false,
          "group_size": 16,
          "num_bits": 4,
          "observer": "memoryless_minmax",
          "observer_kwargs": {},
          "scale_dtype": "torch.float8_e4m3fn",
          "strategy": "tensor_group",
          "symmetric": true,
          "type": "float",
          "zp_dtype": null
        }
      },
      "group_1": {
        "format": "float-quantized",
        "input_activations": {
          "actorder": null,
          "block_structure": null,
          "dynamic": true,
          "group_size": null,
          "num_bits": 8,
          "observer": null,
          "observer_kwargs": {},
          "scale_dtype": null,
          "strategy": "token",
          "symmetric": true,
          "type": "float",
          "zp_dtype": null
        },
        "output_activations": null,
        "targets": [
          "re:.*self_attn\\.(q_a_proj|q_b_proj|kv_a_proj_with_mqa|kv_b_proj|o_proj|g_proj)$",
          "re:.*self_attn\\.indexer\\.(weights_proj|wk|wq_b)$",
          "re:.*layers\\.(0|46)\\.mlp\\.(gate_proj|up_proj|down_proj)$",
          "re:.*eh_proj$"
        ],
        "weights": {
          "actorder": null,
          "block_structure": [
            128,
            128
          ],
          "dynamic": false,
          "group_size": null,
          "num_bits": 8,
          "observer": "minmax",
          "observer_kwargs": {},
          "scale_dtype": null,
          "strategy": "block",
          "symmetric": true,
          "type": "float",
          "zp_dtype": null
        }
      }
    },
    "format": "mixed-precision",
    "global_compression_ratio": null,
    "ignore": [
      "lm_head",
      "model.embed_tokens",
      "re:.*embed_tokens",
      "re:^mtp\\..*",
      "re:^model\\.mtp\\..*",
      "re:.*vision.*",
      "re:.*visual.*",
      "re:.*audio.*",
      "re:.*\\.mlp\\.gate$",
      "re:.*e_score_correction",
      "re:.*norm.*"
    ],
    "kv_cache_scheme": null,
    "quant_method": "compressed-tensors",
    "quantization_status": "compressed",
    "sparsity_config": {},
    "transform_config": {},
    "version": "0.17.0"
  },
  "rms_norm_eps": 1e-05,
  "rope_scaling": null,
  "rope_theta": 80000000.0,
  "routed_scaling_factor": 1.0,
  "scoring_func": "sigmoid",
  "sliding_window_size": 513,
  "swa_attention_gate_type": "headwise",
  "swa_kv_lora_rank": 1024,
  "swa_num_attention_heads": 64,
  "swa_num_key_value_heads": 64,
  "swa_q_lora_rank": 1024,
  "swa_qk_nope_head_dim": 192,
  "swa_qk_rope_head_dim": 64,
  "swa_rope_theta": 50000.0,
  "swa_v_head_dim": 128,
  "tie_word_embeddings": false,
  "topk_method": "noaux_tc",
  "torch_dtype": "bfloat16",
  "transformers_version": "4.36.0",
  "use_cache": true,
  "v_head_dim": 128,
  "vocab_size": 152064,
  "vision_config": {
    "embed_dim": 1536,
    "hidden_size": 5120,
    "intermediate_size": 4224,
    "moe_intermediate_size": 2112,
    "num_hidden_layers": 42,
    "num_attention_heads": 24,
    "num_channels": 3,
    "patch_size": 14,
    "spatial_merge_size": 2,
    "temporal_patch_size": 1,
    "rms_norm_eps": 1e-05,
    "use_bias": false,
    "use_qk_norm": true,
    "attn_implementation": "flash_attention_3",
    "initializer_range": 0.02,
    "is_causal": false,
    "post_norm": true,
    "pre_pixel_shuffle": true,
    "pyramid_num_routed": [
      -1,
      -1,
      -1,
      -1,
      -1,
      -1,
      -1,
      -1,
      -1,
      -1,
      -1,
      -1,
      -1,
      -1,
      -1,
      -1,
      -1,
      -1,
      -1,
      -1,
      -1,
      -1,
      -1,
      -1,
      -1,
      4,
      8,
      12,
      16,
      20,
      24,
      28,
      32,
      36,
      40,
      44,
      48,
      52,
      56,
      60,
      64,
      64
    ],
    "capacity_factor": 2,
    "router_scoring_func": "sigmoid",
    "router_scale": 1.0,
    "adapter_type": "patch_merger",
    "adapter_in_dim": 1536,
    "adapter_out_dim": 5120,
    "adapter_merge_size": 2
  },
  "audio_config": {
    "encoder_type": "dots",
    "whisper_config": {
      "d_model": 1280,
      "encoder_attention_heads": 20,
      "encoder_ffn_dim": 5120,
      "encoder_layers": 32,
      "num_mel_bins": 128,
      "max_source_positions": 6000,
      "activation_function": "swiglu"
    },
    "use_conv2d_stem": true,
    "use_rope": true,
    "use_rms_norm": true,
    "use_causal": false,
    "downsample_hidden_size": 480,
    "conv_chunksize": 500,
    "conv_stem_gradient_checkpointing": true,
    "conv_bucket_step": 10,
    "conv_bucket_max_elements": 20000,
    "rope_parameters": {
      "partial_rotary_factor": 0.5,
      "rope_theta": 10000.0
    },
    "merge_factor": 1,
    "chunk_seconds": 60,
    "whisper_adapter_in_dim": 1280,
    "whisper_adapter_out_dim": 5120,
    "sampling_rate": 16000,
    "audio_comp_start": "<|audio_comp_start|>",
    "audio_comp_span": "<|audio_comp_pad|>",
    "audio_comp_end": "<|audio_comp_end|>"
  }
}