{ "architectures": [ "Zamba2ForCausalLM" ], "adapter_rank": 128, "add_bias_linear": false, "attention_dropout": 0.0, "attention_head_dim": 224, "attention_hidden_size": 7168, "bos_token_id": 1, "chunk_size": 256, "eos_token_id": 2, "ffn_hidden_size": 14336, "hidden_act": "gelu", "hidden_size": 3584, "hybrid_layer_ids": [ 6, 11, 17, 23, 29, 35, 41, 47, 53, 59, 65, 71, 77 ], "initializer_range": 0.02, "intermediate_size": 14336, "kv_channels": 112, "layers_block_type": [ "mamba", "mamba", "mamba", "mamba", "mamba", "mamba", "hybrid", "mamba", "mamba", "mamba", "mamba", "hybrid", "mamba", "mamba", "mamba", "mamba", "mamba", "hybrid", "mamba", "mamba", "mamba", "mamba", "mamba", "hybrid", "mamba", "mamba", "mamba", "mamba", "mamba", "hybrid", "mamba", "mamba", "mamba", "mamba", "mamba", "hybrid", "mamba", "mamba", "mamba", "mamba", "mamba", "hybrid", "mamba", "mamba", "mamba", "mamba", "mamba", "hybrid", "mamba", "mamba", "mamba", "mamba", "mamba", "hybrid", "mamba", "mamba", "mamba", "mamba", "mamba", "hybrid", "mamba", "mamba", "mamba", "mamba", "mamba", "hybrid", "mamba", "mamba", "mamba", "mamba", "mamba", "hybrid", "mamba", "mamba", "mamba", "mamba", "mamba", "hybrid", "mamba", "mamba", "mamba" ], "mamba_d_conv": 4, "mamba_d_state": 64, "mamba_expand": 2, "mamba_headdim": 64, "mamba_ngroups": 2, "max_position_embeddings": 4096, "model_type": "zamba2", "n_mamba_heads": 112, "num_attention_heads": 32, "num_hidden_layers": 81, "num_key_value_heads": 32, "num_logits_to_keep": 1, "num_mem_blocks": 2, "num_query_groups": 32, "pad_token_id": 0, "rms_norm_eps": 1e-05, "rope_theta": 10000, "time_step_floor": 0.0001, "time_step_limit": null, "time_step_max": 0.1, "time_step_min": 0.001, "transformers_version": "4.49.0.dev0", "use_cache": true, "use_conv_bias": true, "use_long_context": false, "use_mem_rope": true, "use_shared_attention_adapter": false, "use_shared_mlp_adapter": true, "vocab_size": 32000, "quantization_config": { "quant_method": "hxq", "codebook_size": 256, "sidecar_enabled": true, "exact_patterns": [ "embed_tokens", "embed_positions", "wte", "wpe", "lm_head", "layernorm", "layer_norm", "norm", "backbone.embedding" ], "compressed_modules": [ "model.layers.0.mamba.in_proj", "model.layers.0.mamba.out_proj", "model.layers.10.mamba.in_proj", "model.layers.10.mamba.out_proj", "model.layers.11.linear", "model.layers.11.mamba_decoder.mamba.in_proj", "model.layers.11.mamba_decoder.mamba.out_proj", "model.layers.11.shared_transformer.feed_forward.down_proj", "model.layers.11.shared_transformer.feed_forward.gate_up_proj_adapter_list.11.0", "model.layers.11.shared_transformer.feed_forward.gate_up_proj_adapter_list.11.1", "model.layers.11.shared_transformer.feed_forward.gate_up_proj_adapter_list.1.0", "model.layers.11.shared_transformer.feed_forward.gate_up_proj_adapter_list.1.1", "model.layers.11.shared_transformer.feed_forward.gate_up_proj_adapter_list.3.0", "model.layers.11.shared_transformer.feed_forward.gate_up_proj_adapter_list.3.1", "model.layers.11.shared_transformer.feed_forward.gate_up_proj_adapter_list.5.0", "model.layers.11.shared_transformer.feed_forward.gate_up_proj_adapter_list.5.1", "model.layers.11.shared_transformer.feed_forward.gate_up_proj_adapter_list.7.0", "model.layers.11.shared_transformer.feed_forward.gate_up_proj_adapter_list.7.1", "model.layers.11.shared_transformer.feed_forward.gate_up_proj_adapter_list.9.0", "model.layers.11.shared_transformer.feed_forward.gate_up_proj_adapter_list.9.1", "model.layers.11.shared_transformer.feed_forward.gate_up_proj", "model.layers.11.shared_transformer.self_attn.k_proj", "model.layers.11.shared_transformer.self_attn.o_proj", "model.layers.11.shared_transformer.self_attn.q_proj", "model.layers.11.shared_transformer.self_attn.v_proj", "model.layers.12.mamba.in_proj", "model.layers.12.mamba.out_proj", "model.layers.13.mamba.in_proj", "model.layers.13.mamba.out_proj", "model.layers.14.mamba.in_proj", "model.layers.14.mamba.out_proj", "model.layers.15.mamba.in_proj", "model.layers.15.mamba.out_proj", "model.layers.16.mamba.in_proj", "model.layers.16.mamba.out_proj", "model.layers.17.linear", "model.layers.17.mamba_decoder.mamba.in_proj", "model.layers.17.mamba_decoder.mamba.out_proj", "model.layers.18.mamba.in_proj", "model.layers.18.mamba.out_proj", "model.layers.19.mamba.in_proj", "model.layers.19.mamba.out_proj", "model.layers.1.mamba.in_proj", "model.layers.1.mamba.out_proj", "model.layers.20.mamba.in_proj", "model.layers.20.mamba.out_proj", "model.layers.21.mamba.in_proj", "model.layers.21.mamba.out_proj", "model.layers.22.mamba.in_proj", "model.layers.22.mamba.out_proj", "model.layers.23.linear", "model.layers.23.mamba_decoder.mamba.in_proj", "model.layers.23.mamba_decoder.mamba.out_proj", "model.layers.24.mamba.in_proj", "model.layers.24.mamba.out_proj", "model.layers.25.mamba.in_proj", "model.layers.25.mamba.out_proj", "model.layers.26.mamba.in_proj", "model.layers.26.mamba.out_proj", "model.layers.27.mamba.in_proj", "model.layers.27.mamba.out_proj", "model.layers.28.mamba.in_proj", "model.layers.28.mamba.out_proj", "model.layers.29.linear", "model.layers.29.mamba_decoder.mamba.in_proj", "model.layers.29.mamba_decoder.mamba.out_proj", "model.layers.2.mamba.in_proj", "model.layers.2.mamba.out_proj", "model.layers.30.mamba.in_proj", "model.layers.30.mamba.out_proj", "model.layers.31.mamba.in_proj", "model.layers.31.mamba.out_proj", "model.layers.32.mamba.in_proj", "model.layers.32.mamba.out_proj", "model.layers.33.mamba.in_proj", "model.layers.33.mamba.out_proj", "model.layers.34.mamba.in_proj", "model.layers.34.mamba.out_proj", "model.layers.35.linear", "model.layers.35.mamba_decoder.mamba.in_proj", "model.layers.35.mamba_decoder.mamba.out_proj", "model.layers.36.mamba.in_proj", "model.layers.36.mamba.out_proj", "model.layers.37.mamba.in_proj", "model.layers.37.mamba.out_proj", "model.layers.38.mamba.in_proj", "model.layers.38.mamba.out_proj", "model.layers.39.mamba.in_proj", "model.layers.39.mamba.out_proj", "model.layers.3.mamba.in_proj", "model.layers.3.mamba.out_proj", "model.layers.40.mamba.in_proj", "model.layers.40.mamba.out_proj", "model.layers.41.linear", "model.layers.41.mamba_decoder.mamba.in_proj", "model.layers.41.mamba_decoder.mamba.out_proj", "model.layers.42.mamba.in_proj", "model.layers.42.mamba.out_proj", "model.layers.43.mamba.in_proj", "model.layers.43.mamba.out_proj", "model.layers.44.mamba.in_proj", "model.layers.44.mamba.out_proj", "model.layers.45.mamba.in_proj", "model.layers.45.mamba.out_proj", "model.layers.46.mamba.in_proj", "model.layers.46.mamba.out_proj", "model.layers.47.linear", "model.layers.47.mamba_decoder.mamba.in_proj", "model.layers.47.mamba_decoder.mamba.out_proj", "model.layers.48.mamba.in_proj", "model.layers.48.mamba.out_proj", "model.layers.49.mamba.in_proj", "model.layers.49.mamba.out_proj", "model.layers.4.mamba.in_proj", "model.layers.4.mamba.out_proj", "model.layers.50.mamba.in_proj", "model.layers.50.mamba.out_proj", "model.layers.51.mamba.in_proj", "model.layers.51.mamba.out_proj", "model.layers.52.mamba.in_proj", "model.layers.52.mamba.out_proj", "model.layers.53.linear", "model.layers.53.mamba_decoder.mamba.in_proj", "model.layers.53.mamba_decoder.mamba.out_proj", "model.layers.54.mamba.in_proj", "model.layers.54.mamba.out_proj", "model.layers.55.mamba.in_proj", "model.layers.55.mamba.out_proj", "model.layers.56.mamba.in_proj", "model.layers.56.mamba.out_proj", "model.layers.57.mamba.in_proj", "model.layers.57.mamba.out_proj", "model.layers.58.mamba.in_proj", "model.layers.58.mamba.out_proj", "model.layers.59.linear", "model.layers.59.mamba_decoder.mamba.in_proj", "model.layers.59.mamba_decoder.mamba.out_proj", "model.layers.5.mamba.in_proj", "model.layers.5.mamba.out_proj", "model.layers.60.mamba.in_proj", "model.layers.60.mamba.out_proj", "model.layers.61.mamba.in_proj", "model.layers.61.mamba.out_proj", "model.layers.62.mamba.in_proj", "model.layers.62.mamba.out_proj", "model.layers.63.mamba.in_proj", "model.layers.63.mamba.out_proj", "model.layers.64.mamba.in_proj", "model.layers.64.mamba.out_proj", "model.layers.65.linear", "model.layers.65.mamba_decoder.mamba.in_proj", "model.layers.65.mamba_decoder.mamba.out_proj", "model.layers.66.mamba.in_proj", "model.layers.66.mamba.out_proj", "model.layers.67.mamba.in_proj", "model.layers.67.mamba.out_proj", "model.layers.68.mamba.in_proj", "model.layers.68.mamba.out_proj", "model.layers.69.mamba.in_proj", "model.layers.69.mamba.out_proj", "model.layers.6.linear", "model.layers.6.mamba_decoder.mamba.in_proj", "model.layers.6.mamba_decoder.mamba.out_proj", "model.layers.6.shared_transformer.feed_forward.down_proj", "model.layers.6.shared_transformer.feed_forward.gate_up_proj_adapter_list.0.0", "model.layers.6.shared_transformer.feed_forward.gate_up_proj_adapter_list.0.1", "model.layers.6.shared_transformer.feed_forward.gate_up_proj_adapter_list.10.0", "model.layers.6.shared_transformer.feed_forward.gate_up_proj_adapter_list.10.1", "model.layers.6.shared_transformer.feed_forward.gate_up_proj_adapter_list.12.0", "model.layers.6.shared_transformer.feed_forward.gate_up_proj_adapter_list.12.1", "model.layers.6.shared_transformer.feed_forward.gate_up_proj_adapter_list.2.0", "model.layers.6.shared_transformer.feed_forward.gate_up_proj_adapter_list.2.1", "model.layers.6.shared_transformer.feed_forward.gate_up_proj_adapter_list.4.0", "model.layers.6.shared_transformer.feed_forward.gate_up_proj_adapter_list.4.1", "model.layers.6.shared_transformer.feed_forward.gate_up_proj_adapter_list.6.0", "model.layers.6.shared_transformer.feed_forward.gate_up_proj_adapter_list.6.1", "model.layers.6.shared_transformer.feed_forward.gate_up_proj_adapter_list.8.0", "model.layers.6.shared_transformer.feed_forward.gate_up_proj_adapter_list.8.1", "model.layers.6.shared_transformer.feed_forward.gate_up_proj", "model.layers.6.shared_transformer.self_attn.k_proj", "model.layers.6.shared_transformer.self_attn.o_proj", "model.layers.6.shared_transformer.self_attn.q_proj", "model.layers.6.shared_transformer.self_attn.v_proj", "model.layers.70.mamba.in_proj", "model.layers.70.mamba.out_proj", "model.layers.71.linear", "model.layers.71.mamba_decoder.mamba.in_proj", "model.layers.71.mamba_decoder.mamba.out_proj", "model.layers.72.mamba.in_proj", "model.layers.72.mamba.out_proj", "model.layers.73.mamba.in_proj", "model.layers.73.mamba.out_proj", "model.layers.74.mamba.in_proj", "model.layers.74.mamba.out_proj", "model.layers.75.mamba.in_proj", "model.layers.75.mamba.out_proj", "model.layers.76.mamba.in_proj", "model.layers.76.mamba.out_proj", "model.layers.77.linear", "model.layers.77.mamba_decoder.mamba.in_proj", "model.layers.77.mamba_decoder.mamba.out_proj", "model.layers.78.mamba.in_proj", "model.layers.78.mamba.out_proj", "model.layers.79.mamba.in_proj", "model.layers.79.mamba.out_proj", "model.layers.7.mamba.in_proj", "model.layers.7.mamba.out_proj", "model.layers.80.mamba.in_proj", "model.layers.80.mamba.out_proj", "model.layers.8.mamba.in_proj", "model.layers.8.mamba.out_proj", "model.layers.9.mamba.in_proj", "model.layers.9.mamba.out_proj" ], "n_clusters": 4096, "vector_dim": 2, "index_packing": "12bit" } }