veridian_beta_0_config.yaml

name: veridian-1b-bootstrap
vocab_size: 32000
pad_token_id: 0
bos_token_id: 1
eos_token_id: 2
max_position_embeddings: 2048
target_max_position_embeddings: 2048
hidden_size: 1536
num_hidden_layers: 16
num_attention_heads: 12
num_key_value_heads: 4
intermediate_size: 4096
num_local_experts: 4
num_shared_experts: 1
num_experts_per_tok: 2
attention_dropout: 0.0
residual_dropout: 0.0
rms_norm_eps: 1.0e-6
rope_theta: 100000.0
router_aux_loss_coef: 0.01
router_z_loss_coef: 1.0e-4
capacity_factor_train: 1.25
capacity_factor_eval: 1.0
initializer_range: 0.02
tie_word_embeddings: true
use_bias: false
use_cache: true