| name: veridian-1b-bootstrap | |
| vocab_size: 32000 | |
| pad_token_id: 0 | |
| bos_token_id: 1 | |
| eos_token_id: 2 | |
| max_position_embeddings: 2048 | |
| target_max_position_embeddings: 2048 | |
| hidden_size: 1536 | |
| num_hidden_layers: 16 | |
| num_attention_heads: 12 | |
| num_key_value_heads: 4 | |
| intermediate_size: 4096 | |
| num_local_experts: 4 | |
| num_shared_experts: 1 | |
| num_experts_per_tok: 2 | |
| attention_dropout: 0.0 | |
| residual_dropout: 0.0 | |
| rms_norm_eps: 1.0e-6 | |
| rope_theta: 100000.0 | |
| router_aux_loss_coef: 0.01 | |
| router_z_loss_coef: 1.0e-4 | |
| capacity_factor_train: 1.25 | |
| capacity_factor_eval: 1.0 | |
| initializer_range: 0.02 | |
| tie_word_embeddings: true | |
| use_bias: false | |
| use_cache: true | |