arrangement: backloaded
attn_ratio: 1
bias: false
d_ff: 3072
d_head: 64
d_model: 768
dropout: 0.0
irab_w_delta_init: -0.1
irab_w_shared_init: 0.05
layer_types:
- mamba
- mamba
- mamba
- mamba
- mamba
- mamba
- mamba
- mamba
- mamba
- attn
- attn
- attn
mamba_chunk_size: 64
mamba_d_state: 64
mamba_expand: 2
mamba_headdim: 64
mamba_ngroups: 1
mamba_ratio: 3
n_heads: 12
n_kv_heads: 4
n_layers: 12
rms_norm_eps: 1.0e-05
rope_base: 10000.0
seq_len: 4096
sgff_gate_dropout: 0.0
sgff_max_depth: 16
tie_embeddings: true
tope_base_struct: 10.0
tope_blend_init: 0.0
tope_max_struct_depth: 64
use_irab: false
use_sgff: false
use_tope: false
vocab_size: 32768