{
  "vocab_size": 49152,
  "n_layer": 16,
  "n_embd": 1024,
  "n_head": 16,
  "kv_heads": 4,
  "intermediate_size": 2752,
  "kernel_size": 4,
  "attn_dropout_p": 0.1,
  "resid_dropout_p": 0.05,
  "n_conv_layers": 10,
  "max_seq_len": 16384,
  "rope_theta": 10000.0,
  "use_native_gqa": true,
  "activation_checkpointing": true
}