{ "architectures": [ "NemotronHTwoTowerForCausalLM" ], "attention_bias": false, "attention_dropout": 0.0, "auto_map": { "AutoConfig": "configuration_nemotron_h.NemotronHConfig", "AutoModelForCausalLM": "modeling_nemotron_twotower.NemotronHTwoTowerForCausalLM" }, "bos_token_id": 1, "chunk_size": 128, "conv_kernel": 4, "dtype": "bfloat16", "eos_token_id": 2, "expand": 2, "head_dim": 128, "hidden_dropout": 0.0, "hidden_size": 2688, "hybrid_override_pattern": "MEMEM*EMEMEM*EMEMEM*EMEMEM*EMEMEM*EMEMEMEM*EMEMEMEME", "initializer_range": 0.02, "intermediate_size": 1856, "layer_norm_epsilon": 1e-05, "mamba_head_dim": 64, "mamba_hidden_act": "silu", "mamba_num_heads": 64, "mamba_proj_bias": false, "max_position_embeddings": 262144, "mlp_bias": false, "mlp_hidden_act": "relu2", "model_type": "nemotron_h", "moe_intermediate_size": 1856, "moe_shared_expert_intermediate_size": 3712, "n_group": 1, "n_groups": 8, "n_routed_experts": 128, "n_shared_experts": 1, "norm_eps": 1e-05, "norm_topk_prob": true, "num_attention_heads": 32, "num_experts_per_tok": 6, "num_hidden_layers": 52, "num_key_value_heads": 2, "num_logits_to_keep": 1, "pad_token_id": 0, "partial_rotary_factor": 1.0, "rescale_prenorm_residual": true, "residual_in_fp32": false, "rope_theta": 10000, "routed_scaling_factor": 2.5, "sliding_window": null, "ssm_state_size": 128, "tie_word_embeddings": false, "time_step_floor": 0.0001, "time_step_limit": [ 0.0, Infinity ], "time_step_max": 0.1, "time_step_min": 0.001, "topk_group": 1, "transformers_version": "4.57.1", "use_bias": false, "use_cache": true, "use_conv_bias": true, "use_mamba_kernels": true, "vocab_size": 131072, "quantization_config": { "config_groups": { "group_0": { "input_activations": { "dynamic": false, "num_bits": 4, "type": "float", "group_size": 16 }, "weights": { "dynamic": false, "num_bits": 4, "type": "float", "group_size": 16 }, "targets": [ "Linear" ] } }, "ignore": [ "context_lm_head", "context_tower.embeddings", "context_tower.layers.0*", "context_tower.layers.1.mixer.gate", "context_tower.layers.1.mixer.shared_experts*", "context_tower.layers.10.mixer.gate", "context_tower.layers.10.mixer.shared_experts*", "context_tower.layers.11*", "context_tower.layers.12*", "context_tower.layers.13.mixer.gate", "context_tower.layers.13.mixer.shared_experts*", "context_tower.layers.14*", "context_tower.layers.15.mixer.gate", "context_tower.layers.15.mixer.shared_experts*", "context_tower.layers.16*", "context_tower.layers.17.mixer.gate", "context_tower.layers.17.mixer.shared_experts*", "context_tower.layers.18*", "context_tower.layers.19*", "context_tower.layers.2.*", "context_tower.layers.20.mixer.gate", "context_tower.layers.20.mixer.shared_experts*", "context_tower.layers.21*", "context_tower.layers.22.mixer.gate", "context_tower.layers.22.mixer.shared_experts*", "context_tower.layers.23*", "context_tower.layers.24.mixer.gate", "context_tower.layers.24.mixer.shared_experts*", "context_tower.layers.25*", "context_tower.layers.26*", "context_tower.layers.27.mixer.gate", "context_tower.layers.27.mixer.shared_experts*", "context_tower.layers.28*", "context_tower.layers.29.mixer.gate", "context_tower.layers.29.mixer.shared_experts*", "context_tower.layers.3.mixer.gate", "context_tower.layers.3.mixer.shared_experts*", "context_tower.layers.30*", "context_tower.layers.31.mixer.gate", "context_tower.layers.31.mixer.shared_experts*", "context_tower.layers.32*", "context_tower.layers.33*", "context_tower.layers.34.mixer.gate", "context_tower.layers.34.mixer.shared_experts*", "context_tower.layers.35*", "context_tower.layers.36.mixer.gate", "context_tower.layers.36.mixer.shared_experts*", "context_tower.layers.37*", "context_tower.layers.38.mixer.gate", "context_tower.layers.38.mixer.shared_experts*", "context_tower.layers.39*", "context_tower.layers.4.*", "context_tower.layers.40.mixer.gate", "context_tower.layers.40.mixer.shared_experts*", "context_tower.layers.41*", "context_tower.layers.42*", "context_tower.layers.43.mixer.gate", "context_tower.layers.43.mixer.shared_experts*", "context_tower.layers.44*", "context_tower.layers.45.mixer.gate", "context_tower.layers.45.mixer.shared_experts*", "context_tower.layers.46*", "context_tower.layers.47.mixer.gate", "context_tower.layers.47.mixer.shared_experts*", "context_tower.layers.48*", "context_tower.layers.49.mixer.gate", "context_tower.layers.49.mixer.shared_experts*", "context_tower.layers.5.*", "context_tower.layers.50*", "context_tower.layers.51.mixer.gate", "context_tower.layers.51.mixer.shared_experts*", "context_tower.layers.6.mixer.gate", "context_tower.layers.6.mixer.shared_experts*", "context_tower.layers.7*", "context_tower.layers.8.mixer.gate", "context_tower.layers.8.mixer.shared_experts*", "context_tower.layers.9*", "denoiser_tower.embeddings", "denoiser_tower.layers.0*", "denoiser_tower.layers.1.mixer.gate", "denoiser_tower.layers.1.mixer.shared_experts*", "denoiser_tower.layers.10.mixer.gate", "denoiser_tower.layers.10.mixer.shared_experts*", "denoiser_tower.layers.11*", "denoiser_tower.layers.12*", "denoiser_tower.layers.13.mixer.gate", "denoiser_tower.layers.13.mixer.shared_experts*", "denoiser_tower.layers.14*", "denoiser_tower.layers.15.mixer.gate", "denoiser_tower.layers.15.mixer.shared_experts*", "denoiser_tower.layers.16*", "denoiser_tower.layers.17.mixer.gate", "denoiser_tower.layers.17.mixer.shared_experts*", "denoiser_tower.layers.18*", "denoiser_tower.layers.19*", "denoiser_tower.layers.2.*", "denoiser_tower.layers.20.mixer.gate", "denoiser_tower.layers.20.mixer.shared_experts*", "denoiser_tower.layers.21*", "denoiser_tower.layers.22.mixer.gate", "denoiser_tower.layers.22.mixer.shared_experts*", "denoiser_tower.layers.23*", "denoiser_tower.layers.24.mixer.gate", "denoiser_tower.layers.24.mixer.shared_experts*", "denoiser_tower.layers.25*", "denoiser_tower.layers.26*", "denoiser_tower.layers.27.mixer.gate", "denoiser_tower.layers.27.mixer.shared_experts*", "denoiser_tower.layers.28*", "denoiser_tower.layers.29.mixer.gate", "denoiser_tower.layers.29.mixer.shared_experts*", "denoiser_tower.layers.3.mixer.gate", "denoiser_tower.layers.3.mixer.shared_experts*", "denoiser_tower.layers.30*", "denoiser_tower.layers.31.mixer.gate", "denoiser_tower.layers.31.mixer.shared_experts*", "denoiser_tower.layers.32*", "denoiser_tower.layers.33*", "denoiser_tower.layers.34.mixer.gate", "denoiser_tower.layers.34.mixer.shared_experts*", "denoiser_tower.layers.35*", "denoiser_tower.layers.36.mixer.gate", "denoiser_tower.layers.36.mixer.shared_experts*", "denoiser_tower.layers.37*", "denoiser_tower.layers.38.mixer.gate", "denoiser_tower.layers.38.mixer.shared_experts*", "denoiser_tower.layers.39*", "denoiser_tower.layers.4.*", "denoiser_tower.layers.40.mixer.gate", "denoiser_tower.layers.40.mixer.shared_experts*", "denoiser_tower.layers.41*", "denoiser_tower.layers.42*", "denoiser_tower.layers.43.mixer.gate", "denoiser_tower.layers.43.mixer.shared_experts*", "denoiser_tower.layers.44*", "denoiser_tower.layers.45.mixer.gate", "denoiser_tower.layers.45.mixer.shared_experts*", "denoiser_tower.layers.46*", "denoiser_tower.layers.47.mixer.gate", "denoiser_tower.layers.47.mixer.shared_experts*", "denoiser_tower.layers.48*", "denoiser_tower.layers.49.mixer.gate", "denoiser_tower.layers.49.mixer.shared_experts*", "denoiser_tower.layers.5.*", "denoiser_tower.layers.50*", "denoiser_tower.layers.51.mixer.gate", "denoiser_tower.layers.51.mixer.shared_experts*", "denoiser_tower.layers.6.mixer.gate", "denoiser_tower.layers.6.mixer.shared_experts*", "denoiser_tower.layers.7*", "denoiser_tower.layers.8.mixer.gate", "denoiser_tower.layers.8.mixer.shared_experts*", "denoiser_tower.layers.9*", "lm_head", "t_block*", "t_embedder*" ], "quant_algo": "NVFP4", "producer": { "name": "modelopt", "version": "0.46.0.dev129+g973cb09cb" }, "quant_method": "modelopt" } }