{ "architectures": [ "NemotronHPuzzleForCausalLM" ], "attention_bias": false, "attention_dropout": 0.0, "auto_map": { "AutoConfig": "configuration_nemotron_h_puzzle.NemotronHPuzzleConfig", "AutoModelForCausalLM": "modeling_nemotron_h_puzzle.NemotronHPuzzleForCausalLM" }, "block_configs": [ { "block_type": "mamba" }, { "block_type": "moe", "moe_intermediate_size": 1280, "num_experts_per_tok": 4 }, { "block_type": "mamba" }, { "block_type": "moe", "moe_intermediate_size": 1280, "num_experts_per_tok": 8 }, { "block_type": "mamba" }, { "block_type": "moe", "moe_intermediate_size": 1280, "num_experts_per_tok": 10 }, { "block_type": "mamba" }, { "block_type": "attention" }, { "block_type": "moe", "moe_intermediate_size": 1280, "num_experts_per_tok": 8 }, { "block_type": "mamba" }, { "block_type": "moe", "moe_intermediate_size": 1280, "num_experts_per_tok": 8 }, { "block_type": "mamba" }, { "block_type": "moe", "moe_intermediate_size": 1280, "num_experts_per_tok": 8 }, { "block_type": "mamba" }, { "block_type": "moe", "moe_intermediate_size": 1280, "num_experts_per_tok": 12 }, { "block_type": "mamba" }, { "block_type": "attention" }, { "block_type": "moe", "moe_intermediate_size": 1280, "num_experts_per_tok": 8 }, { "block_type": "mamba" }, { "block_type": "moe", "moe_intermediate_size": 1280, "num_experts_per_tok": 10 }, { "block_type": "mamba" }, { "block_type": "moe", "moe_intermediate_size": 1280, "num_experts_per_tok": 8 }, { "block_type": "mamba" }, { "block_type": "moe", "moe_intermediate_size": 2688, "num_experts_per_tok": 12 }, { "block_type": "mamba" }, { "block_type": "attention" }, { "block_type": "moe", "moe_intermediate_size": 1536, "num_experts_per_tok": 14 }, { "block_type": "mamba" }, { "block_type": "moe", "moe_intermediate_size": 2688, "num_experts_per_tok": 12 }, { "block_type": "mamba" }, { "block_type": "moe", "moe_intermediate_size": 1536, "num_experts_per_tok": 12 }, { "block_type": "mamba" }, { "block_type": "moe", "moe_intermediate_size": 1536, "num_experts_per_tok": 12 }, { "block_type": "mamba" }, { "block_type": "moe", "moe_intermediate_size": 2688, "num_experts_per_tok": 12 }, { "block_type": "mamba" }, { "block_type": "attention" }, { "block_type": "moe", "moe_intermediate_size": 2688, "num_experts_per_tok": 12 }, { "block_type": "mamba" }, { "block_type": "moe", "moe_intermediate_size": 2688, "num_experts_per_tok": 12 }, { "block_type": "mamba" }, { "block_type": "moe", "moe_intermediate_size": 1536, "num_experts_per_tok": 10 }, { "block_type": "mamba" }, { "block_type": "moe", "moe_intermediate_size": 2688, "num_experts_per_tok": 12 }, { "block_type": "mamba" }, { "block_type": "moe", "moe_intermediate_size": 2688, "num_experts_per_tok": 12 }, { "block_type": "mamba" }, { "block_type": "attention" }, { "block_type": "moe", "moe_intermediate_size": 1792, "num_experts_per_tok": 12 }, { "block_type": "mamba" }, { "block_type": "moe", "moe_intermediate_size": 1792, "num_experts_per_tok": 14 }, { "block_type": "mamba" }, { "block_type": "moe", "moe_intermediate_size": 1280, "num_experts_per_tok": 10 }, { "block_type": "mamba" }, { "block_type": "moe", "moe_intermediate_size": 1280, "num_experts_per_tok": 10 }, { "block_type": "mamba" }, { "block_type": "moe", "moe_intermediate_size": 1280, "num_experts_per_tok": 12 }, { "block_type": "mamba" }, { "block_type": "attention" }, { "block_type": "moe", "moe_intermediate_size": 1280, "num_experts_per_tok": 8 }, { "block_type": "mamba" }, { "block_type": "moe", "moe_intermediate_size": 1280, "num_experts_per_tok": 12 }, { "block_type": "mamba" }, { "block_type": "moe", "moe_intermediate_size": 1280, "num_experts_per_tok": 10 }, { "block_type": "mamba" }, { "block_type": "moe", "moe_intermediate_size": 1280, "num_experts_per_tok": 8 }, { "block_type": "mamba" }, { "block_type": "moe", "moe_intermediate_size": 1280, "num_experts_per_tok": 8 }, { "block_type": "mamba" }, { "block_type": "attention" }, { "block_type": "moe", "moe_intermediate_size": 1280, "num_experts_per_tok": 10 }, { "block_type": "mamba" }, { "block_type": "moe", "moe_intermediate_size": 1280, "num_experts_per_tok": 10 }, { "block_type": "mamba" }, { "block_type": "moe", "moe_intermediate_size": 1280, "num_experts_per_tok": 10 }, { "block_type": "mamba" }, { "block_type": "moe", "moe_intermediate_size": 1280, "num_experts_per_tok": 12 }, { "block_type": "mamba" }, { "block_type": "attention" }, { "block_type": "moe", "moe_intermediate_size": 1280, "num_experts_per_tok": 12 }, { "block_type": "mamba" }, { "block_type": "moe", "moe_intermediate_size": 1280, "num_experts_per_tok": 14 }, { "block_type": "mamba" }, { "block_type": "moe", "moe_intermediate_size": 1280, "num_experts_per_tok": 16 }, { "block_type": "mamba" }, { "block_type": "moe", "moe_intermediate_size": 1792, "num_experts_per_tok": 18 }, { "block_type": "mamba" }, { "block_type": "moe", "moe_intermediate_size": 2048, "num_experts_per_tok": 18 } ], "blockwise_members": [ "num_experts_per_tok", "moe_intermediate_size", "block_type" ], "bos_token_id": 1, "chunk_size": 128, "conv_kernel": 4, "dtype": "bfloat16", "eos_token_id": 2, "expand": 2, "head_dim": 128, "hidden_dropout": 0.0, "hidden_size": 4096, "initializer_range": 0.02, "intermediate_size": 21504, "layer_norm_epsilon": 1e-05, "layers_block_type": [ "mamba", "moe", "mamba", "moe", "mamba", "moe", "mamba", "attention", "moe", "mamba", "moe", "mamba", "moe", "mamba", "moe", "mamba", "attention", "moe", "mamba", "moe", "mamba", "moe", "mamba", "moe", "mamba", "attention", "moe", "mamba", "moe", "mamba", "moe", "mamba", "moe", "mamba", "moe", "mamba", "attention", "moe", "mamba", "moe", "mamba", "moe", "mamba", "moe", "mamba", "moe", "mamba", "attention", "moe", "mamba", "moe", "mamba", "moe", "mamba", "moe", "mamba", "moe", "mamba", "attention", "moe", "mamba", "moe", "mamba", "moe", "mamba", "moe", "mamba", "moe", "mamba", "attention", "moe", "mamba", "moe", "mamba", "moe", "mamba", "moe", "mamba", "attention", "moe", "mamba", "moe", "mamba", "moe", "mamba", "moe", "mamba", "moe" ], "mamba_head_dim": 64, "mamba_hidden_act": "silu", "mamba_num_heads": 128, "mamba_proj_bias": false, "mamba_ssm_cache_dtype": "float32", "max_position_embeddings": 262144, "mlp_bias": false, "mlp_hidden_act": "relu2", "model_type": "nemotron_h_puzzle", "moe_latent_size": 1024, "moe_shared_expert_intermediate_size": 5376, "moe_shared_expert_overlap": true, "mtp_block_configs": [ { "block_type": "attention" }, { "block_type": "moe", "moe_intermediate_size": 2688, "num_experts_per_tok": 22 } ], "mtp_layers_block_type": [ "attention", "moe" ], "n_group": 1, "n_groups": 8, "n_routed_experts": 512, "n_shared_experts": 1, "norm_eps": 1e-05, "norm_topk_prob": true, "num_attention_heads": 32, "num_key_value_heads": 2, "num_logits_to_keep": 1, "num_nextn_predict_layers": 1, "pad_token_id": 0, "quantization_config": { "config_groups": { "group_0": { "input_activations": { "dynamic": false, "num_bits": 8, "type": "float" }, "targets": [ "Linear" ], "weights": { "dynamic": false, "num_bits": 8, "type": "float" } } }, "ignore": [ "backbone.layers.0.mixer.conv1d", "backbone.layers.1.mixer.fc1_latent_proj", "backbone.layers.1.mixer.fc2_latent_proj", "backbone.layers.10.mixer.fc1_latent_proj", "backbone.layers.10.mixer.fc2_latent_proj", "backbone.layers.11.mixer.conv1d", "backbone.layers.12.mixer.fc1_latent_proj", "backbone.layers.12.mixer.fc2_latent_proj", "backbone.layers.13.mixer.conv1d", "backbone.layers.14.mixer.fc1_latent_proj", "backbone.layers.14.mixer.fc2_latent_proj", "backbone.layers.15.mixer.conv1d", "backbone.layers.16*", "backbone.layers.17.mixer.fc1_latent_proj", "backbone.layers.17.mixer.fc2_latent_proj", "backbone.layers.18.mixer.conv1d", "backbone.layers.19.mixer.fc1_latent_proj", "backbone.layers.19.mixer.fc2_latent_proj", "backbone.layers.2.mixer.conv1d", "backbone.layers.20.mixer.conv1d", "backbone.layers.21.mixer.fc1_latent_proj", "backbone.layers.21.mixer.fc2_latent_proj", "backbone.layers.22.mixer.conv1d", "backbone.layers.23.mixer.fc1_latent_proj", "backbone.layers.23.mixer.fc2_latent_proj", "backbone.layers.24.mixer.conv1d", "backbone.layers.25*", "backbone.layers.26.mixer.fc1_latent_proj", "backbone.layers.26.mixer.fc2_latent_proj", "backbone.layers.27.mixer.conv1d", "backbone.layers.28.mixer.fc1_latent_proj", "backbone.layers.28.mixer.fc2_latent_proj", "backbone.layers.29.mixer.conv1d", "backbone.layers.3.mixer.fc1_latent_proj", "backbone.layers.3.mixer.fc2_latent_proj", "backbone.layers.30.mixer.fc1_latent_proj", "backbone.layers.30.mixer.fc2_latent_proj", "backbone.layers.31.mixer.conv1d", "backbone.layers.32.mixer.fc1_latent_proj", "backbone.layers.32.mixer.fc2_latent_proj", "backbone.layers.33.mixer.conv1d", "backbone.layers.34.mixer.fc1_latent_proj", "backbone.layers.34.mixer.fc2_latent_proj", "backbone.layers.35.mixer.conv1d", "backbone.layers.36*", "backbone.layers.37.mixer.fc1_latent_proj", "backbone.layers.37.mixer.fc2_latent_proj", "backbone.layers.38.mixer.conv1d", "backbone.layers.39.mixer.fc1_latent_proj", "backbone.layers.39.mixer.fc2_latent_proj", "backbone.layers.4.mixer.conv1d", "backbone.layers.40.mixer.conv1d", "backbone.layers.41.mixer.fc1_latent_proj", "backbone.layers.41.mixer.fc2_latent_proj", "backbone.layers.42.mixer.conv1d", "backbone.layers.43.mixer.fc1_latent_proj", "backbone.layers.43.mixer.fc2_latent_proj", "backbone.layers.44.mixer.conv1d", "backbone.layers.45.mixer.fc1_latent_proj", "backbone.layers.45.mixer.fc2_latent_proj", "backbone.layers.46.mixer.conv1d", "backbone.layers.47*", "backbone.layers.48.mixer.fc1_latent_proj", "backbone.layers.48.mixer.fc2_latent_proj", "backbone.layers.49.mixer.conv1d", "backbone.layers.5.mixer.fc1_latent_proj", "backbone.layers.5.mixer.fc2_latent_proj", "backbone.layers.50.mixer.fc1_latent_proj", "backbone.layers.50.mixer.fc2_latent_proj", "backbone.layers.51.mixer.conv1d", "backbone.layers.52.mixer.fc1_latent_proj", "backbone.layers.52.mixer.fc2_latent_proj", "backbone.layers.53.mixer.conv1d", "backbone.layers.54.mixer.fc1_latent_proj", "backbone.layers.54.mixer.fc2_latent_proj", "backbone.layers.55.mixer.conv1d", "backbone.layers.56.mixer.fc1_latent_proj", "backbone.layers.56.mixer.fc2_latent_proj", "backbone.layers.57.mixer.conv1d", "backbone.layers.58*", "backbone.layers.59.mixer.fc1_latent_proj", "backbone.layers.59.mixer.fc2_latent_proj", "backbone.layers.6.mixer.conv1d", "backbone.layers.60.mixer.conv1d", "backbone.layers.61.mixer.fc1_latent_proj", "backbone.layers.61.mixer.fc2_latent_proj", "backbone.layers.62.mixer.conv1d", "backbone.layers.63.mixer.fc1_latent_proj", "backbone.layers.63.mixer.fc2_latent_proj", "backbone.layers.64.mixer.conv1d", "backbone.layers.65.mixer.fc1_latent_proj", "backbone.layers.65.mixer.fc2_latent_proj", "backbone.layers.66.mixer.conv1d", "backbone.layers.67.mixer.fc1_latent_proj", "backbone.layers.67.mixer.fc2_latent_proj", "backbone.layers.68.mixer.conv1d", "backbone.layers.69*", "backbone.layers.7.*", "backbone.layers.70.mixer.fc1_latent_proj", "backbone.layers.70.mixer.fc2_latent_proj", "backbone.layers.71.mixer.conv1d", "backbone.layers.72.mixer.fc1_latent_proj", "backbone.layers.72.mixer.fc2_latent_proj", "backbone.layers.73.mixer.conv1d", "backbone.layers.74.mixer.fc1_latent_proj", "backbone.layers.74.mixer.fc2_latent_proj", "backbone.layers.75.mixer.conv1d", "backbone.layers.76.mixer.fc1_latent_proj", "backbone.layers.76.mixer.fc2_latent_proj", "backbone.layers.77.mixer.conv1d", "backbone.layers.78*", "backbone.layers.79.mixer.fc1_latent_proj", "backbone.layers.79.mixer.fc2_latent_proj", "backbone.layers.8.mixer.fc1_latent_proj", "backbone.layers.8.mixer.fc2_latent_proj", "backbone.layers.80.mixer.conv1d", "backbone.layers.81.mixer.fc1_latent_proj", "backbone.layers.81.mixer.fc2_latent_proj", "backbone.layers.82.mixer.conv1d", "backbone.layers.83.mixer.fc1_latent_proj", "backbone.layers.83.mixer.fc2_latent_proj", "backbone.layers.84.mixer.conv1d", "backbone.layers.85.mixer.fc1_latent_proj", "backbone.layers.85.mixer.fc2_latent_proj", "backbone.layers.86.mixer.conv1d", "backbone.layers.87.mixer.fc1_latent_proj", "backbone.layers.87.mixer.fc2_latent_proj", "backbone.layers.9.mixer.conv1d", "lm_head", "mtp", "mtp.layers.1.mixer.experts" ], "kv_cache_scheme": { "dynamic": false, "num_bits": 8, "type": "float" }, "producer": { "name": "modelopt", "version": "0.21.0" }, "quant_algo": "FP8", "quant_method": "modelopt" }, "rescale_prenorm_residual": true, "residual_in_fp32": false, "routed_scaling_factor": 5.0, "sliding_window": null, "ssm_state_size": 96, "tie_word_embeddings": false, "time_step_floor": 0.0001, "time_step_max": 0.1, "time_step_min": 0.001, "topk_group": 1, "transformers_version": "4.45.0", "use_bias": false, "use_cache": true, "use_conv_bias": true, "use_mamba_kernels": true, "vocab_size": 131072 }