{
  "architectures": ["TweaktronOmniForCausalLM"],
  "model_type": "custom_model",
  "auto_map": {
    "AutoConfig": "configuration_tweaktron.TweaktronOmniConfig",
    "AutoModel": "modeling_tweaktron.TweaktronOmniModel",
    "AutoModelForCausalLM": "modeling_tweaktron.TweaktronOmniForCausalLM"
  },

  "hidden_size": 2048,
  "num_heads": 14,
  "num_hidden_layers": 3,
  "hidden_ratio": 4,
  "intermediate_size": null,
  "hidden_act": "swish",
  "norm_eps": 1e-06,
  "max_position_embeddings": 4096,

  "comba_num_heads": 6,
  "comba_head_dim": 256,
  "comba_expand_v": 2.0,
  "comba_use_output_gate": true,
  "comba_use_output_correction": true,
  "comba_use_inner_decay": true,
  "comba_correction_factor": 1.0,
  "comba_conv_size": 4,

  "mom_num_heads": 4,
  "mom_head_dim": 256,
  "mom_expand_v": 1.0,
  "mom_num_memories": 8,
  "mom_topk": 2,
  "mom_capacity": 1.0,
  "mom_shared_mem": true,
  "mom_single_kv_proj": false,
  "mom_use_output_gate": true,
  "mom_conv_size": 4,
  "aux_loss_scale": 0.01,

  "raven_num_heads": 4,
  "raven_num_kv_heads": null,
  "raven_num_slots": 64,
  "raven_expand_k": 1.0,
  "raven_expand_v": 1.0,
  "raven_feature_map": "swish",
  "raven_use_output_gate": false,
  "raven_decay_type": "Mamba2",
  "raven_topk": 32,
  "raven_bias_rmm": false,
  "raven_add_gumbel_noise": true,
  "raven_router_score": "sigmoid",
  "raven_router_type": "lin",
  "raven_gate_logit_normalizer": 8,

  "vocab_size": 32016,
  "bos_token_id": 1,
  "eos_token_id": 2,
  "pad_token_id": null,
  "tie_word_embeddings": false,
  "initializer_range": 0.02,
  "use_cache": true,

  "fuse_norm": true,
  "fuse_swiglu": true,
  "fuse_cross_entropy": true,
  "fuse_linear_cross_entropy": false,

  "torch_dtype": "bfloat16",
  "transformers_version": "5.0.0"
}