name: veridian-2b-bpe131k
vocab_size: 131072
hidden_size: 2048
intermediate_size: 5632
num_hidden_layers: 20
num_attention_heads: 16
num_key_value_heads: 4
num_local_experts: 4
num_shared_experts: 1
num_experts_per_tok: 2
max_position_embeddings: 1024
target_max_position_embeddings: 1024
pad_token_id: 0
bos_token_id: 1
eos_token_id: 2
attention_dropout: 0.0
residual_dropout: 0.0
rms_norm_eps: 1.0e-6
rope_theta: 100000.0
router_aux_loss_coef: 0.01
router_z_loss_coef: 1.0e-4
capacity_factor_train: 1.0
capacity_factor_eval: 1.0
initializer_range: 0.02
tie_word_embeddings: true
use_bias: false
use_cache: true