_name_or_path=LiquidAI/LFM2.5-8B-A1B
architectures=Lfm2MoeForCausalLM
attention_head_dim=64
attention_heads=32
attention_kv_heads=8
bos_token_id=124894
chunk_size_feed_forward=0
context_length=128000
conv_L_cache=3
conv_bias=false
decoder_start_token_id=124894
dtype=bfloat16
eos_token_id=124900
ffn_intermediate_dim=7168
hidden_dim=2048
hidden_size=2048
initializer_range=0.02
intermediate_size=7168
is_encoder_decoder=false
layer_norm_eps=1e-05
layer_types=conv,conv,full_attention,conv,conv,conv,full_attention,conv,conv,conv,full_attention,conv,conv,conv,full_attention,conv,conv,conv,full_attention,conv,conv,full_attention,conv,conv
max_position_embeddings=128000
model_type=lfm2
moe_every_n_layers=0
moe_intermediate_size=1792
norm_eps=1e-05
norm_topk_prob=true
num_attention_heads=32
num_dense_layers=2
num_experts=32
num_experts_per_tok=4
num_hidden_layers=24
num_key_value_heads=8
num_layers=24
num_shared_experts=0
num_top_experts=4
output_attentions=false
output_hidden_states=false
pad_token_id=124893
return_dict=true
rope_theta=5000000
routed_scaling_factor=1.0
tie_word_embeddings=true
transformers_version=5.5.4
use_cache=true
use_expert_bias=true
vocab_size=128000