| { | |
| "d_model": 1536, | |
| "d_intermediate": 2800, | |
| "n_layer": 24, | |
| "vocab_size": 128256, | |
| "ssm_cfg": { | |
| "layer": "Mamba3", | |
| "d_state": 128, | |
| "expand": 2, | |
| "headdim": 64, | |
| "ngroups": 1, | |
| "rope_fraction": 0.5, | |
| "dt_min": 0.001, | |
| "dt_max": 0.1, | |
| "dt_init_floor": 0.0001, | |
| "A_floor": 0.0001, | |
| "chunk_size": 16, | |
| "is_mimo": true, | |
| "mimo_rank": 4, | |
| "is_outproj_norm": false | |
| }, | |
| "attn_layer_idx": [], | |
| "attn_cfg": {}, | |
| "rms_norm": true, | |
| "residual_in_fp32": true, | |
| "fused_add_norm": false, | |
| "pad_vocab_size_multiple": 16, | |
| "tie_embeddings": true | |
| } | |